Repository: LLNL/spack Branch: develop Commit: 7e864787bd15 Files: 1696 Total size: 12.2 MB Directory structure: gitextract_njnmw550/ ├── .ci/ │ ├── gitlab/ │ │ └── forward_dotenv_variables.py │ └── gitlab-ci.yml ├── .codecov.yml ├── .devcontainer/ │ ├── postCreateCommand.sh │ ├── ubuntu20.04/ │ │ └── devcontainer.json │ └── ubuntu22.04/ │ └── devcontainer.json ├── .dockerignore ├── .flake8 ├── .git-blame-ignore-revs ├── .gitattributes ├── .github/ │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yml │ │ ├── config.yml │ │ └── feature_request.yml │ ├── dependabot.yml │ ├── labeler.yml │ ├── pull_request_template.md │ └── workflows/ │ ├── bin/ │ │ ├── canonicalize.py │ │ ├── execute_installer.ps1 │ │ ├── format-rst.py │ │ ├── generate_spack_yaml_containerize.sh │ │ ├── setup_git.ps1 │ │ ├── setup_git.sh │ │ └── system_shortcut_check.ps1 │ ├── bootstrap.yml │ ├── build-containers.yml │ ├── ci.yaml │ ├── coverage.yml │ ├── import-check.yaml │ ├── prechecks.yml │ ├── requirements/ │ │ ├── coverage/ │ │ │ └── requirements.txt │ │ ├── style/ │ │ │ └── requirements.txt │ │ └── unit_tests/ │ │ └── requirements.txt │ ├── stale.yaml │ ├── triage.yml │ └── unit_tests.yaml ├── .gitignore ├── .mailmap ├── .readthedocs.yml ├── CHANGELOG.md ├── CITATION.cff ├── COPYRIGHT ├── LICENSE-APACHE ├── LICENSE-MIT ├── NEWS.md ├── NOTICE ├── README.md ├── SECURITY.md ├── bin/ │ ├── haspywin.py │ ├── sbang │ ├── spack │ ├── spack-python │ ├── spack-tmpconfig │ ├── spack.bat │ ├── spack.ps1 │ ├── spack_cmd.bat │ └── spack_pwsh.ps1 ├── etc/ │ └── spack/ │ └── defaults/ │ ├── base/ │ │ ├── concretizer.yaml │ │ ├── config.yaml │ │ ├── mirrors.yaml │ │ ├── modules.yaml │ │ ├── packages.yaml │ │ └── repos.yaml │ ├── bootstrap.yaml │ ├── darwin/ │ │ ├── modules.yaml │ │ └── packages.yaml │ ├── include.yaml │ ├── linux/ │ │ └── modules.yaml │ └── windows/ │ ├── config.yaml │ └── packages.yaml ├── lib/ │ └── spack/ │ ├── _vendoring/ │ │ └── __init__.py │ ├── docs/ │ │ ├── .gitignore │ │ ├── .spack/ │ │ │ └── repos.yaml │ │ ├── Makefile │ │ ├── _gh_pages_redirect/ │ │ │ ├── .nojekyll │ │ │ └── index.html │ │ ├── _static/ │ │ │ ├── css/ │ │ │ │ └── custom.css │ │ │ └── js/ │ │ │ └── versions.js │ │ ├── _templates/ │ │ │ ├── base.html │ │ │ └── sidebar/ │ │ │ └── brand.html │ │ ├── advanced_topics.rst │ │ ├── binary_caches.rst │ │ ├── bootstrapping.rst │ │ ├── build_settings.rst │ │ ├── build_systems/ │ │ │ ├── autotoolspackage.rst │ │ │ ├── bundlepackage.rst │ │ │ ├── cachedcmakepackage.rst │ │ │ ├── cmakepackage.rst │ │ │ ├── cudapackage.rst │ │ │ ├── custompackage.rst │ │ │ ├── inteloneapipackage.rst │ │ │ ├── luapackage.rst │ │ │ ├── makefilepackage.rst │ │ │ ├── mavenpackage.rst │ │ │ ├── mesonpackage.rst │ │ │ ├── octavepackage.rst │ │ │ ├── perlpackage.rst │ │ │ ├── pythonpackage.rst │ │ │ ├── qmakepackage.rst │ │ │ ├── racketpackage.rst │ │ │ ├── rocmpackage.rst │ │ │ ├── rpackage.rst │ │ │ ├── rubypackage.rst │ │ │ ├── sconspackage.rst │ │ │ ├── sippackage.rst │ │ │ ├── sourceforgepackage.rst │ │ │ └── wafpackage.rst │ │ ├── build_systems.rst │ │ ├── chain.rst │ │ ├── command_index.in │ │ ├── conf.py │ │ ├── config_yaml.rst │ │ ├── configuration.rst │ │ ├── configuring_compilers.rst │ │ ├── containers.rst │ │ ├── contribution_guide.rst │ │ ├── developer_guide.rst │ │ ├── env_vars_yaml.rst │ │ ├── environments.rst │ │ ├── environments_basics.rst │ │ ├── extensions.rst │ │ ├── features.rst │ │ ├── frequently_asked_questions.rst │ │ ├── getting_help.rst │ │ ├── getting_started.rst │ │ ├── google5fda5f94b4ffb8de.html │ │ ├── gpu_configuration.rst │ │ ├── images/ │ │ │ └── packaging.excalidrawlib │ │ ├── include_yaml.rst │ │ ├── index.rst │ │ ├── installing.rst │ │ ├── installing_prerequisites.rst │ │ ├── mirrors.rst │ │ ├── module_file_support.rst │ │ ├── package_api.rst │ │ ├── package_fundamentals.rst │ │ ├── package_review_guide.rst │ │ ├── packages_yaml.rst │ │ ├── packaging_guide_advanced.rst │ │ ├── packaging_guide_build.rst │ │ ├── packaging_guide_creation.rst │ │ ├── packaging_guide_testing.rst │ │ ├── pipelines.rst │ │ ├── repositories.rst │ │ ├── requirements.txt │ │ ├── roles_and_responsibilities.rst │ │ ├── signing.rst │ │ ├── spack.yaml │ │ ├── spec_syntax.rst │ │ ├── toolchains_yaml.rst │ │ └── windows.rst │ ├── llnl/ │ │ └── __init__.py │ └── spack/ │ ├── __init__.py │ ├── aliases.py │ ├── archspec.py │ ├── audit.py │ ├── binary_distribution.py │ ├── bootstrap/ │ │ ├── __init__.py │ │ ├── _common.py │ │ ├── clingo.py │ │ ├── config.py │ │ ├── core.py │ │ ├── environment.py │ │ ├── prototypes/ │ │ │ ├── clingo-darwin-aarch64.json │ │ │ ├── clingo-darwin-x86_64.json │ │ │ ├── clingo-freebsd-amd64.json │ │ │ ├── clingo-linux-aarch64.json │ │ │ ├── clingo-linux-ppc64le.json │ │ │ ├── clingo-linux-x86_64.json │ │ │ └── clingo-windows-x86_64.json │ │ └── status.py │ ├── build_environment.py │ ├── buildcache_migrate.py │ ├── buildcache_prune.py │ ├── builder.py │ ├── caches.py │ ├── ci/ │ │ ├── README.md │ │ ├── __init__.py │ │ ├── common.py │ │ ├── generator_registry.py │ │ └── gitlab.py │ ├── cmd/ │ │ ├── __init__.py │ │ ├── add.py │ │ ├── arch.py │ │ ├── audit.py │ │ ├── blame.py │ │ ├── bootstrap.py │ │ ├── build_env.py │ │ ├── buildcache.py │ │ ├── cd.py │ │ ├── change.py │ │ ├── checksum.py │ │ ├── ci.py │ │ ├── clean.py │ │ ├── commands.py │ │ ├── common/ │ │ │ ├── __init__.py │ │ │ ├── arguments.py │ │ │ ├── confirmation.py │ │ │ ├── env_utility.py │ │ │ └── spec_strings.py │ │ ├── compiler.py │ │ ├── compilers.py │ │ ├── concretize.py │ │ ├── config.py │ │ ├── containerize.py │ │ ├── create.py │ │ ├── debug.py │ │ ├── deconcretize.py │ │ ├── dependencies.py │ │ ├── dependents.py │ │ ├── deprecate.py │ │ ├── dev_build.py │ │ ├── develop.py │ │ ├── diff.py │ │ ├── docs.py │ │ ├── edit.py │ │ ├── env.py │ │ ├── extensions.py │ │ ├── external.py │ │ ├── fetch.py │ │ ├── find.py │ │ ├── gc.py │ │ ├── gpg.py │ │ ├── graph.py │ │ ├── help.py │ │ ├── info.py │ │ ├── install.py │ │ ├── installer/ │ │ │ ├── CMakeLists.txt │ │ │ ├── README.md │ │ │ ├── bundle.wxs.in │ │ │ ├── patch.xml │ │ │ └── spack.wxs.in │ │ ├── license.py │ │ ├── list.py │ │ ├── load.py │ │ ├── location.py │ │ ├── log_parse.py │ │ ├── logs.py │ │ ├── maintainers.py │ │ ├── make_installer.py │ │ ├── mark.py │ │ ├── mirror.py │ │ ├── module.py │ │ ├── modules/ │ │ │ ├── __init__.py │ │ │ ├── lmod.py │ │ │ └── tcl.py │ │ ├── patch.py │ │ ├── pkg.py │ │ ├── providers.py │ │ ├── pydoc.py │ │ ├── python.py │ │ ├── reindex.py │ │ ├── remove.py │ │ ├── repo.py │ │ ├── resource.py │ │ ├── restage.py │ │ ├── solve.py │ │ ├── spec.py │ │ ├── stage.py │ │ ├── style.py │ │ ├── tags.py │ │ ├── test.py │ │ ├── test_env.py │ │ ├── tutorial.py │ │ ├── undevelop.py │ │ ├── uninstall.py │ │ ├── unit_test.py │ │ ├── unload.py │ │ ├── url.py │ │ ├── verify.py │ │ ├── versions.py │ │ └── view.py │ ├── compilers/ │ │ ├── __init__.py │ │ ├── adaptor.py │ │ ├── config.py │ │ ├── error.py │ │ ├── flags.py │ │ └── libraries.py │ ├── concretize.py │ ├── config.py │ ├── container/ │ │ ├── __init__.py │ │ ├── images.json │ │ ├── images.py │ │ └── writers.py │ ├── context.py │ ├── cray_manifest.py │ ├── database.py │ ├── dependency.py │ ├── deptypes.py │ ├── detection/ │ │ ├── __init__.py │ │ ├── common.py │ │ ├── path.py │ │ └── test.py │ ├── directives.py │ ├── directives_meta.py │ ├── directory_layout.py │ ├── enums.py │ ├── environment/ │ │ ├── __init__.py │ │ ├── depfile.py │ │ ├── environment.py │ │ ├── list.py │ │ └── shell.py │ ├── error.py │ ├── extensions.py │ ├── externals.py │ ├── fetch_strategy.py │ ├── filesystem_view.py │ ├── graph.py │ ├── hash_types.py │ ├── hooks/ │ │ ├── __init__.py │ │ ├── absolutify_elf_sonames.py │ │ ├── autopush.py │ │ ├── drop_redundant_rpaths.py │ │ ├── licensing.py │ │ ├── module_file_generation.py │ │ ├── permissions_setters.py │ │ ├── resolve_shared_libraries.py │ │ ├── sbang.py │ │ ├── windows_runtime_linkage.py │ │ └── write_install_manifest.py │ ├── install_test.py │ ├── installer.py │ ├── installer_dispatch.py │ ├── llnl/ │ │ ├── __init__.py │ │ ├── path.py │ │ ├── string.py │ │ ├── url.py │ │ └── util/ │ │ ├── __init__.py │ │ ├── argparsewriter.py │ │ ├── filesystem.py │ │ ├── lang.py │ │ ├── link_tree.py │ │ ├── lock.py │ │ ├── symlink.py │ │ └── tty/ │ │ ├── __init__.py │ │ ├── colify.py │ │ ├── color.py │ │ └── log.py │ ├── main.py │ ├── mirrors/ │ │ ├── __init__.py │ │ ├── layout.py │ │ ├── mirror.py │ │ └── utils.py │ ├── mixins.py │ ├── modules/ │ │ ├── __init__.py │ │ ├── common.py │ │ ├── lmod.py │ │ └── tcl.py │ ├── multimethod.py │ ├── new_installer.py │ ├── oci/ │ │ ├── __init__.py │ │ ├── image.py │ │ ├── oci.py │ │ └── opener.py │ ├── operating_systems/ │ │ ├── __init__.py │ │ ├── _operating_system.py │ │ ├── freebsd.py │ │ ├── linux_distro.py │ │ ├── mac_os.py │ │ └── windows_os.py │ ├── package.py │ ├── package_base.py │ ├── package_completions.py │ ├── package_prefs.py │ ├── package_test.py │ ├── patch.py │ ├── paths.py │ ├── phase_callbacks.py │ ├── platforms/ │ │ ├── __init__.py │ │ ├── _functions.py │ │ ├── _platform.py │ │ ├── cray.py │ │ ├── darwin.py │ │ ├── freebsd.py │ │ ├── linux.py │ │ ├── test.py │ │ └── windows.py │ ├── projections.py │ ├── provider_index.py │ ├── relocate.py │ ├── relocate_text.py │ ├── repo.py │ ├── repo_migrate.py │ ├── report.py │ ├── reporters/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── cdash.py │ │ ├── extract.py │ │ └── junit.py │ ├── resource.py │ ├── rewiring.py │ ├── schema/ │ │ ├── __init__.py │ │ ├── bootstrap.py │ │ ├── buildcache_spec.py │ │ ├── cdash.py │ │ ├── ci.py │ │ ├── compilers.py │ │ ├── concretizer.py │ │ ├── config.py │ │ ├── container.py │ │ ├── cray_manifest.py │ │ ├── database_index.py │ │ ├── definitions.py │ │ ├── develop.py │ │ ├── env.py │ │ ├── env_vars.py │ │ ├── environment.py │ │ ├── include.py │ │ ├── merged.py │ │ ├── mirrors.py │ │ ├── modules.py │ │ ├── packages.py │ │ ├── projections.py │ │ ├── repos.py │ │ ├── spec.py │ │ ├── spec_list.py │ │ ├── toolchains.py │ │ ├── upstreams.py │ │ ├── url_buildcache_manifest.py │ │ └── view.py │ ├── solver/ │ │ ├── __init__.py │ │ ├── asp.py │ │ ├── concretize.lp │ │ ├── core.py │ │ ├── direct_dependency.lp │ │ ├── display.lp │ │ ├── error_messages.lp │ │ ├── heuristic.lp │ │ ├── input_analysis.py │ │ ├── libc_compatibility.lp │ │ ├── os_compatibility.lp │ │ ├── requirements.py │ │ ├── reuse.py │ │ ├── runtimes.py │ │ ├── splices.lp │ │ ├── splicing.py │ │ ├── versions.py │ │ └── when_possible.lp │ ├── spec.py │ ├── spec_filter.py │ ├── spec_parser.py │ ├── stage.py │ ├── store.py │ ├── subprocess_context.py │ ├── tag.py │ ├── tengine.py │ ├── test/ │ │ ├── __init__.py │ │ ├── architecture.py │ │ ├── audit.py │ │ ├── binary_distribution.py │ │ ├── bootstrap.py │ │ ├── build_distribution.py │ │ ├── build_environment.py │ │ ├── build_system_guess.py │ │ ├── builder.py │ │ ├── buildrequest.py │ │ ├── buildtask.py │ │ ├── cache_fetch.py │ │ ├── cc.py │ │ ├── ci.py │ │ ├── cmd/ │ │ │ ├── __init__.py │ │ │ ├── arch.py │ │ │ ├── audit.py │ │ │ ├── blame.py │ │ │ ├── bootstrap.py │ │ │ ├── build_env.py │ │ │ ├── buildcache.py │ │ │ ├── cd.py │ │ │ ├── checksum.py │ │ │ ├── ci.py │ │ │ ├── clean.py │ │ │ ├── commands.py │ │ │ ├── common/ │ │ │ │ ├── __init__.py │ │ │ │ ├── arguments.py │ │ │ │ └── spec_strings.py │ │ │ ├── compiler.py │ │ │ ├── concretize.py │ │ │ ├── config.py │ │ │ ├── create.py │ │ │ ├── debug.py │ │ │ ├── deconcretize.py │ │ │ ├── dependencies.py │ │ │ ├── dependents.py │ │ │ ├── deprecate.py │ │ │ ├── dev_build.py │ │ │ ├── develop.py │ │ │ ├── diff.py │ │ │ ├── edit.py │ │ │ ├── env.py │ │ │ ├── extensions.py │ │ │ ├── external.py │ │ │ ├── fetch.py │ │ │ ├── find.py │ │ │ ├── gc.py │ │ │ ├── gpg.py │ │ │ ├── graph.py │ │ │ ├── help.py │ │ │ ├── info.py │ │ │ ├── init_py_functions.py │ │ │ ├── install.py │ │ │ ├── is_git_repo.py │ │ │ ├── license.py │ │ │ ├── list.py │ │ │ ├── load.py │ │ │ ├── location.py │ │ │ ├── logs.py │ │ │ ├── maintainers.py │ │ │ ├── mark.py │ │ │ ├── mirror.py │ │ │ ├── module.py │ │ │ ├── pkg.py │ │ │ ├── print_shell_vars.py │ │ │ ├── providers.py │ │ │ ├── python.py │ │ │ ├── reindex.py │ │ │ ├── repo.py │ │ │ ├── resource.py │ │ │ ├── spec.py │ │ │ ├── stage.py │ │ │ ├── style.py │ │ │ ├── tags.py │ │ │ ├── test.py │ │ │ ├── undevelop.py │ │ │ ├── uninstall.py │ │ │ ├── unit_test.py │ │ │ ├── url.py │ │ │ ├── verify.py │ │ │ ├── versions.py │ │ │ └── view.py │ │ ├── cmd_extensions.py │ │ ├── compilers/ │ │ │ ├── conversion.py │ │ │ └── libraries.py │ │ ├── concretization/ │ │ │ ├── compiler_runtimes.py │ │ │ ├── conditional_dependencies.py │ │ │ ├── core.py │ │ │ ├── errors.py │ │ │ ├── flag_mixing.py │ │ │ ├── preferences.py │ │ │ ├── requirements.py │ │ │ └── splicing.py │ │ ├── config.py │ │ ├── config_values.py │ │ ├── conftest.py │ │ ├── container/ │ │ │ ├── cli.py │ │ │ ├── conftest.py │ │ │ ├── docker.py │ │ │ ├── images.py │ │ │ └── singularity.py │ │ ├── cray_manifest.py │ │ ├── cvs_fetch.py │ │ ├── data/ │ │ │ ├── compiler_verbose_output/ │ │ │ │ ├── cce-8.6.5.txt │ │ │ │ ├── clang-4.0.1.txt │ │ │ │ ├── clang-9.0.0-apple-ld.txt │ │ │ │ ├── collect2-6.3.0-gnu-ld.txt │ │ │ │ ├── gcc-7.3.1.txt │ │ │ │ ├── icc-16.0.3.txt │ │ │ │ ├── nag-6.2-gcc-6.5.0.txt │ │ │ │ ├── obscure-parsing-rules.txt │ │ │ │ └── xl-13.1.5.txt │ │ │ ├── compression/ │ │ │ │ ├── Foo │ │ │ │ ├── Foo.Z │ │ │ │ ├── Foo.bz2 │ │ │ │ ├── Foo.cxx │ │ │ │ ├── Foo.tar.Z │ │ │ │ ├── Foo.tar.bz2 │ │ │ │ ├── Foo.tar.xz │ │ │ │ ├── Foo.tbz │ │ │ │ ├── Foo.tbz2 │ │ │ │ ├── Foo.tgz │ │ │ │ ├── Foo.txz │ │ │ │ └── Foo.xz │ │ │ ├── config/ │ │ │ │ ├── base/ │ │ │ │ │ └── config.yaml │ │ │ │ ├── bootstrap.yaml │ │ │ │ ├── concretizer.yaml │ │ │ │ ├── config.yaml │ │ │ │ ├── include.yaml │ │ │ │ ├── modules.yaml │ │ │ │ ├── packages.yaml │ │ │ │ └── repos.yaml │ │ │ ├── conftest/ │ │ │ │ └── diff-test/ │ │ │ │ ├── package-0.txt │ │ │ │ ├── package-1.txt │ │ │ │ ├── package-2.txt │ │ │ │ └── package-3.txt │ │ │ ├── directory_search/ │ │ │ │ ├── README.txt │ │ │ │ ├── a/ │ │ │ │ │ ├── c.h │ │ │ │ │ ├── foobar.txt │ │ │ │ │ ├── libc.a │ │ │ │ │ └── libc.lib │ │ │ │ ├── b/ │ │ │ │ │ ├── b.h │ │ │ │ │ ├── bar.txp │ │ │ │ │ ├── d.h │ │ │ │ │ ├── liba.a │ │ │ │ │ ├── liba.lib │ │ │ │ │ ├── libd.a │ │ │ │ │ └── libd.lib │ │ │ │ └── c/ │ │ │ │ ├── a.h │ │ │ │ ├── bar.txt │ │ │ │ ├── libb.a │ │ │ │ └── libb.lib │ │ │ ├── filter_file/ │ │ │ │ ├── selfextract.bsx │ │ │ │ ├── start_stop.txt │ │ │ │ └── x86_cpuid_info.c │ │ │ ├── make/ │ │ │ │ ├── affirmative/ │ │ │ │ │ ├── capital_makefile/ │ │ │ │ │ │ └── Makefile │ │ │ │ │ ├── check_test/ │ │ │ │ │ │ └── Makefile │ │ │ │ │ ├── expansion/ │ │ │ │ │ │ └── Makefile │ │ │ │ │ ├── gnu_makefile/ │ │ │ │ │ │ └── GNUmakefile │ │ │ │ │ ├── include/ │ │ │ │ │ │ ├── Makefile │ │ │ │ │ │ └── make.mk │ │ │ │ │ ├── lowercase_makefile/ │ │ │ │ │ │ └── makefile │ │ │ │ │ ├── prerequisites/ │ │ │ │ │ │ └── Makefile │ │ │ │ │ ├── spaces/ │ │ │ │ │ │ └── Makefile │ │ │ │ │ ├── test_check/ │ │ │ │ │ │ └── Makefile │ │ │ │ │ └── three_targets/ │ │ │ │ │ └── Makefile │ │ │ │ └── negative/ │ │ │ │ ├── no_makefile/ │ │ │ │ │ └── readme.txt │ │ │ │ ├── partial_match/ │ │ │ │ │ └── Makefile │ │ │ │ └── variable/ │ │ │ │ └── Makefile │ │ │ ├── microarchitectures/ │ │ │ │ └── microarchitectures.json │ │ │ ├── mirrors/ │ │ │ │ ├── legacy_yaml/ │ │ │ │ │ └── build_cache/ │ │ │ │ │ ├── test-debian6-core2/ │ │ │ │ │ │ └── gcc-4.5.0/ │ │ │ │ │ │ └── zlib-1.2.11/ │ │ │ │ │ │ └── test-debian6-core2-gcc-4.5.0-zlib-1.2.11-t5mczux3tfqpxwmg7egp7axy2jvyulqk.spack │ │ │ │ │ └── test-debian6-core2-gcc-4.5.0-zlib-1.2.11-t5mczux3tfqpxwmg7egp7axy2jvyulqk.spec.yaml │ │ │ │ ├── signed_json/ │ │ │ │ │ └── linux-ubuntu18.04-haswell-gcc-8.4.0-zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb.spec.json.sig │ │ │ │ └── v2_layout/ │ │ │ │ ├── signed/ │ │ │ │ │ └── build_cache/ │ │ │ │ │ ├── _pgp/ │ │ │ │ │ │ ├── CBAB2C1032C6FF5078049EC0FA61D50C12CAD37E.pub │ │ │ │ │ │ └── index.json │ │ │ │ │ ├── index.json │ │ │ │ │ ├── index.json.hash │ │ │ │ │ ├── test-debian6-core2/ │ │ │ │ │ │ └── gcc-10.2.1/ │ │ │ │ │ │ ├── libdwarf-20130729/ │ │ │ │ │ │ │ └── test-debian6-core2-gcc-10.2.1-libdwarf-20130729-sk2gqqz4n5njmvktycnd25wq25jxiqkr.spack │ │ │ │ │ │ └── libelf-0.8.13/ │ │ │ │ │ │ └── test-debian6-core2-gcc-10.2.1-libelf-0.8.13-rqh2vuf6fqwkmipzgi2wjx352mq7y7ez.spack │ │ │ │ │ ├── test-debian6-core2-gcc-10.2.1-libdwarf-20130729-sk2gqqz4n5njmvktycnd25wq25jxiqkr.spec.json.sig │ │ │ │ │ ├── test-debian6-core2-gcc-10.2.1-libelf-0.8.13-rqh2vuf6fqwkmipzgi2wjx352mq7y7ez.spec.json.sig │ │ │ │ │ ├── test-debian6-m1/ │ │ │ │ │ │ ├── gcc-10.2.1/ │ │ │ │ │ │ │ ├── libdwarf-20130729/ │ │ │ │ │ │ │ │ └── test-debian6-m1-gcc-10.2.1-libdwarf-20130729-u5uz3dcch5if4eve4sef67o2rf2lbfgh.spack │ │ │ │ │ │ │ └── libelf-0.8.13/ │ │ │ │ │ │ │ └── test-debian6-m1-gcc-10.2.1-libelf-0.8.13-jr3yipyxyjulcdvckwwwjrrumis7glpa.spack │ │ │ │ │ │ └── none-none/ │ │ │ │ │ │ ├── compiler-wrapper-1.0/ │ │ │ │ │ │ │ └── test-debian6-m1-none-none-compiler-wrapper-1.0-qeehcxyvluwnihsc2qxstmpomtxo3lrc.spack │ │ │ │ │ │ └── gcc-runtime-10.2.1/ │ │ │ │ │ │ └── test-debian6-m1-none-none-gcc-runtime-10.2.1-izgzpzeljwairalfjm3k6fntbb64nt6n.spack │ │ │ │ │ ├── test-debian6-m1-gcc-10.2.1-libdwarf-20130729-u5uz3dcch5if4eve4sef67o2rf2lbfgh.spec.json.sig │ │ │ │ │ ├── test-debian6-m1-gcc-10.2.1-libelf-0.8.13-jr3yipyxyjulcdvckwwwjrrumis7glpa.spec.json.sig │ │ │ │ │ ├── test-debian6-m1-none-none-compiler-wrapper-1.0-qeehcxyvluwnihsc2qxstmpomtxo3lrc.spec.json.sig │ │ │ │ │ └── test-debian6-m1-none-none-gcc-runtime-10.2.1-izgzpzeljwairalfjm3k6fntbb64nt6n.spec.json.sig │ │ │ │ └── unsigned/ │ │ │ │ └── build_cache/ │ │ │ │ ├── index.json │ │ │ │ ├── index.json.hash │ │ │ │ ├── test-debian6-core2/ │ │ │ │ │ └── gcc-10.2.1/ │ │ │ │ │ ├── libdwarf-20130729/ │ │ │ │ │ │ └── test-debian6-core2-gcc-10.2.1-libdwarf-20130729-sk2gqqz4n5njmvktycnd25wq25jxiqkr.spack │ │ │ │ │ └── libelf-0.8.13/ │ │ │ │ │ └── test-debian6-core2-gcc-10.2.1-libelf-0.8.13-rqh2vuf6fqwkmipzgi2wjx352mq7y7ez.spack │ │ │ │ ├── test-debian6-core2-gcc-10.2.1-libdwarf-20130729-sk2gqqz4n5njmvktycnd25wq25jxiqkr.spec.json │ │ │ │ ├── test-debian6-core2-gcc-10.2.1-libelf-0.8.13-rqh2vuf6fqwkmipzgi2wjx352mq7y7ez.spec.json │ │ │ │ ├── test-debian6-m1/ │ │ │ │ │ ├── gcc-10.2.1/ │ │ │ │ │ │ ├── libdwarf-20130729/ │ │ │ │ │ │ │ └── test-debian6-m1-gcc-10.2.1-libdwarf-20130729-u5uz3dcch5if4eve4sef67o2rf2lbfgh.spack │ │ │ │ │ │ └── libelf-0.8.13/ │ │ │ │ │ │ └── test-debian6-m1-gcc-10.2.1-libelf-0.8.13-jr3yipyxyjulcdvckwwwjrrumis7glpa.spack │ │ │ │ │ └── none-none/ │ │ │ │ │ ├── compiler-wrapper-1.0/ │ │ │ │ │ │ └── test-debian6-m1-none-none-compiler-wrapper-1.0-qeehcxyvluwnihsc2qxstmpomtxo3lrc.spack │ │ │ │ │ └── gcc-runtime-10.2.1/ │ │ │ │ │ └── test-debian6-m1-none-none-gcc-runtime-10.2.1-izgzpzeljwairalfjm3k6fntbb64nt6n.spack │ │ │ │ ├── test-debian6-m1-gcc-10.2.1-libdwarf-20130729-u5uz3dcch5if4eve4sef67o2rf2lbfgh.spec.json │ │ │ │ ├── test-debian6-m1-gcc-10.2.1-libelf-0.8.13-jr3yipyxyjulcdvckwwwjrrumis7glpa.spec.json │ │ │ │ ├── test-debian6-m1-none-none-compiler-wrapper-1.0-qeehcxyvluwnihsc2qxstmpomtxo3lrc.spec.json │ │ │ │ └── test-debian6-m1-none-none-gcc-runtime-10.2.1-izgzpzeljwairalfjm3k6fntbb64nt6n.spec.json │ │ │ ├── modules/ │ │ │ │ ├── lmod/ │ │ │ │ │ ├── alter_environment.yaml │ │ │ │ │ ├── autoload_all.yaml │ │ │ │ │ ├── autoload_direct.yaml │ │ │ │ │ ├── complex_hierarchy.yaml │ │ │ │ │ ├── conflicts.yaml │ │ │ │ │ ├── core_compilers.yaml │ │ │ │ │ ├── core_compilers_at_equal.yaml │ │ │ │ │ ├── core_compilers_empty.yaml │ │ │ │ │ ├── exclude.yaml │ │ │ │ │ ├── hide_implicits.yaml │ │ │ │ │ ├── missing_core_compilers.yaml │ │ │ │ │ ├── module_path_separator.yaml │ │ │ │ │ ├── no_arch.yaml │ │ │ │ │ ├── no_hash.yaml │ │ │ │ │ ├── override_template.yaml │ │ │ │ │ ├── projections.yaml │ │ │ │ │ ├── with_view.yaml │ │ │ │ │ └── wrong_conflicts.yaml │ │ │ │ └── tcl/ │ │ │ │ ├── alter_environment.yaml │ │ │ │ ├── autoload_all.yaml │ │ │ │ ├── autoload_direct.yaml │ │ │ │ ├── autoload_with_constraints.yaml │ │ │ │ ├── conflicts.yaml │ │ │ │ ├── exclude.yaml │ │ │ │ ├── exclude_implicits.yaml │ │ │ │ ├── hide_implicits.yaml │ │ │ │ ├── invalid_naming_scheme.yaml │ │ │ │ ├── invalid_token_in_env_var_name.yaml │ │ │ │ ├── module_path_separator.yaml │ │ │ │ ├── naming_scheme.yaml │ │ │ │ ├── no_arch.yaml │ │ │ │ ├── override_config.yaml │ │ │ │ ├── override_template.yaml │ │ │ │ ├── prerequisites_all.yaml │ │ │ │ ├── prerequisites_direct.yaml │ │ │ │ ├── projections.yaml │ │ │ │ ├── suffix-format.yaml │ │ │ │ ├── suffix.yaml │ │ │ │ └── wrong_conflicts.yaml │ │ │ ├── ninja/ │ │ │ │ ├── .gitignore │ │ │ │ ├── affirmative/ │ │ │ │ │ ├── check_test/ │ │ │ │ │ │ └── build.ninja │ │ │ │ │ ├── include/ │ │ │ │ │ │ ├── build.ninja │ │ │ │ │ │ └── include.ninja │ │ │ │ │ ├── simple/ │ │ │ │ │ │ └── build.ninja │ │ │ │ │ ├── spaces/ │ │ │ │ │ │ └── build.ninja │ │ │ │ │ ├── subninja/ │ │ │ │ │ │ ├── build.ninja │ │ │ │ │ │ └── subninja.ninja │ │ │ │ │ ├── test_check/ │ │ │ │ │ │ └── build.ninja │ │ │ │ │ └── three_targets/ │ │ │ │ │ └── build.ninja │ │ │ │ └── negative/ │ │ │ │ ├── no_ninja/ │ │ │ │ │ └── readme.txt │ │ │ │ ├── partial_match/ │ │ │ │ │ └── build.ninja │ │ │ │ ├── rule/ │ │ │ │ │ └── build.ninja │ │ │ │ └── variable/ │ │ │ │ └── build.ninja │ │ │ ├── patch/ │ │ │ │ ├── foo.patch │ │ │ │ └── foo.tgz │ │ │ ├── sourceme_first.bat │ │ │ ├── sourceme_first.sh │ │ │ ├── sourceme_lmod.sh │ │ │ ├── sourceme_modules.bat │ │ │ ├── sourceme_modules.sh │ │ │ ├── sourceme_parameters.bat │ │ │ ├── sourceme_parameters.sh │ │ │ ├── sourceme_second.bat │ │ │ ├── sourceme_second.sh │ │ │ ├── sourceme_unicode.bat │ │ │ ├── sourceme_unicode.sh │ │ │ ├── sourceme_unset.bat │ │ │ ├── sourceme_unset.sh │ │ │ ├── style/ │ │ │ │ ├── broken.dummy │ │ │ │ └── fixed.py │ │ │ ├── templates/ │ │ │ │ ├── a.txt │ │ │ │ ├── extension.tcl │ │ │ │ └── override.txt │ │ │ ├── templates_again/ │ │ │ │ ├── b.txt │ │ │ │ └── override_from_modules.txt │ │ │ ├── test/ │ │ │ │ └── test_stage/ │ │ │ │ └── gavrxt67t7yaiwfek7dds7lgokmoaiin/ │ │ │ │ ├── printing-package-1.0-hzgcoow-test-out.txt │ │ │ │ ├── printing-package-1.0-hzgcoow-tested.txt │ │ │ │ └── results.txt │ │ │ ├── unparse/ │ │ │ │ ├── README.md │ │ │ │ ├── amdfftw.txt │ │ │ │ ├── grads.txt │ │ │ │ ├── legion.txt │ │ │ │ ├── llvm.txt │ │ │ │ ├── mfem.txt │ │ │ │ ├── py-torch.txt │ │ │ │ └── trilinos.txt │ │ │ └── web/ │ │ │ ├── 1.html │ │ │ ├── 2.html │ │ │ ├── 3.html │ │ │ ├── 4.html │ │ │ ├── fragment.html │ │ │ ├── index.html │ │ │ ├── index_with_fragment.html │ │ │ └── index_with_javascript.html │ │ ├── database.py │ │ ├── detection.py │ │ ├── directives.py │ │ ├── directory_layout.py │ │ ├── entry_points.py │ │ ├── env.py │ │ ├── environment/ │ │ │ └── mutate.py │ │ ├── environment_modifications.py │ │ ├── error_messages.py │ │ ├── externals.py │ │ ├── fetch_strategy.py │ │ ├── flag_handlers.py │ │ ├── gcs_fetch.py │ │ ├── git_fetch.py │ │ ├── graph.py │ │ ├── hg_fetch.py │ │ ├── hooks/ │ │ │ └── absolutify_elf_sonames.py │ │ ├── install.py │ │ ├── installer.py │ │ ├── installer_build_graph.py │ │ ├── installer_tui.py │ │ ├── jobserver.py │ │ ├── link_paths.py │ │ ├── llnl/ │ │ │ ├── llnl_string.py │ │ │ ├── url.py │ │ │ └── util/ │ │ │ ├── __init__.py │ │ │ ├── argparsewriter.py │ │ │ ├── file_list.py │ │ │ ├── filesystem.py │ │ │ ├── lang.py │ │ │ ├── link_tree.py │ │ │ ├── lock.py │ │ │ ├── symlink.py │ │ │ └── tty/ │ │ │ ├── __init__.py │ │ │ ├── colify.py │ │ │ ├── color.py │ │ │ ├── log.py │ │ │ └── tty.py │ │ ├── main.py │ │ ├── make_executable.py │ │ ├── mirror.py │ │ ├── module_parsing.py │ │ ├── modules/ │ │ │ ├── __init__.py │ │ │ ├── common.py │ │ │ ├── conftest.py │ │ │ ├── lmod.py │ │ │ └── tcl.py │ │ ├── multimethod.py │ │ ├── namespace_trie.py │ │ ├── new_installer.py │ │ ├── oci/ │ │ │ ├── image.py │ │ │ ├── integration_test.py │ │ │ ├── mock_registry.py │ │ │ └── urlopen.py │ │ ├── optional_deps.py │ │ ├── package_class.py │ │ ├── packages.py │ │ ├── packaging.py │ │ ├── patch.py │ │ ├── permissions.py │ │ ├── projections.py │ │ ├── provider_index.py │ │ ├── relocate.py │ │ ├── relocate_text.py │ │ ├── repo.py │ │ ├── reporters.py │ │ ├── rewiring.py │ │ ├── s3_fetch.py │ │ ├── sbang.py │ │ ├── schema.py │ │ ├── spack_yaml.py │ │ ├── spec_dag.py │ │ ├── spec_list.py │ │ ├── spec_semantics.py │ │ ├── spec_syntax.py │ │ ├── spec_yaml.py │ │ ├── stage.py │ │ ├── svn_fetch.py │ │ ├── tag.py │ │ ├── tengine.py │ │ ├── test_suite.py │ │ ├── traverse.py │ │ ├── url_fetch.py │ │ ├── url_parse.py │ │ ├── url_substitution.py │ │ ├── util/ │ │ │ ├── __init__.py │ │ │ ├── archive.py │ │ │ ├── compression.py │ │ │ ├── editor.py │ │ │ ├── elf.py │ │ │ ├── environment.py │ │ │ ├── executable.py │ │ │ ├── file_cache.py │ │ │ ├── git.py │ │ │ ├── ld_so_conf.py │ │ │ ├── log_parser.py │ │ │ ├── module_cmd.py │ │ │ ├── package_hash.py │ │ │ ├── path.py │ │ │ ├── prefix.py │ │ │ ├── remote_file_cache.py │ │ │ ├── spack_lock_wrapper.py │ │ │ ├── spack_yaml.py │ │ │ ├── timer.py │ │ │ ├── unparse/ │ │ │ │ ├── __init__.py │ │ │ │ └── unparse.py │ │ │ ├── util_gpg.py │ │ │ └── util_url.py │ │ ├── utilities.py │ │ ├── variant.py │ │ ├── verification.py │ │ ├── versions.py │ │ ├── views.py │ │ └── web.py │ ├── tokenize.py │ ├── traverse.py │ ├── url.py │ ├── url_buildcache.py │ ├── user_environment.py │ ├── util/ │ │ ├── __init__.py │ │ ├── archive.py │ │ ├── compression.py │ │ ├── cpus.py │ │ ├── crypto.py │ │ ├── ctest_log_parser.py │ │ ├── editor.py │ │ ├── elf.py │ │ ├── environment.py │ │ ├── executable.py │ │ ├── file_cache.py │ │ ├── file_permissions.py │ │ ├── filesystem.py │ │ ├── format.py │ │ ├── gcs.py │ │ ├── git.py │ │ ├── gpg.py │ │ ├── hash.py │ │ ├── ld_so_conf.py │ │ ├── libc.py │ │ ├── lock.py │ │ ├── log_parse.py │ │ ├── module_cmd.py │ │ ├── naming.py │ │ ├── package_hash.py │ │ ├── parallel.py │ │ ├── path.py │ │ ├── pattern.py │ │ ├── prefix.py │ │ ├── remote_file_cache.py │ │ ├── s3.py │ │ ├── socket.py │ │ ├── spack_json.py │ │ ├── spack_yaml.py │ │ ├── timer.py │ │ ├── typing.py │ │ ├── unparse/ │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ └── unparser.py │ │ ├── url.py │ │ ├── web.py │ │ └── windows_registry.py │ ├── variant.py │ ├── vendor/ │ │ ├── __init__.py │ │ ├── _pyrsistent_version.py │ │ ├── altgraph/ │ │ │ ├── Dot.py │ │ │ ├── Graph.py │ │ │ ├── GraphAlgo.py │ │ │ ├── GraphStat.py │ │ │ ├── GraphUtil.py │ │ │ ├── LICENSE │ │ │ ├── ObjectGraph.py │ │ │ └── __init__.py │ │ ├── archspec/ │ │ │ ├── LICENSE │ │ │ ├── LICENSE-APACHE │ │ │ ├── LICENSE-MIT │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ ├── cli.py │ │ │ ├── cpu/ │ │ │ │ ├── __init__.py │ │ │ │ ├── alias.py │ │ │ │ ├── detect.py │ │ │ │ ├── microarchitecture.py │ │ │ │ └── schema.py │ │ │ ├── json/ │ │ │ │ ├── COPYRIGHT │ │ │ │ ├── LICENSE-APACHE │ │ │ │ ├── LICENSE-MIT │ │ │ │ ├── NOTICE │ │ │ │ ├── README.md │ │ │ │ └── cpu/ │ │ │ │ ├── cpuid.json │ │ │ │ ├── cpuid_schema.json │ │ │ │ ├── microarchitectures.json │ │ │ │ └── microarchitectures_schema.json │ │ │ └── vendor/ │ │ │ └── cpuid/ │ │ │ ├── LICENSE │ │ │ ├── README.md │ │ │ ├── cpuid.py │ │ │ └── example.py │ │ ├── attr/ │ │ │ ├── __init__.py │ │ │ ├── __init__.pyi │ │ │ ├── _cmp.py │ │ │ ├── _cmp.pyi │ │ │ ├── _compat.py │ │ │ ├── _config.py │ │ │ ├── _funcs.py │ │ │ ├── _make.py │ │ │ ├── _next_gen.py │ │ │ ├── _version_info.py │ │ │ ├── _version_info.pyi │ │ │ ├── converters.py │ │ │ ├── converters.pyi │ │ │ ├── exceptions.py │ │ │ ├── exceptions.pyi │ │ │ ├── filters.py │ │ │ ├── filters.pyi │ │ │ ├── py.typed │ │ │ ├── setters.py │ │ │ ├── setters.pyi │ │ │ ├── validators.py │ │ │ └── validators.pyi │ │ ├── attrs/ │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── __init__.pyi │ │ │ ├── converters.py │ │ │ ├── exceptions.py │ │ │ ├── filters.py │ │ │ ├── py.typed │ │ │ ├── setters.py │ │ │ └── validators.py │ │ ├── distro/ │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ ├── distro.py │ │ │ └── py.typed │ │ ├── jinja2/ │ │ │ ├── LICENSE.rst │ │ │ ├── __init__.py │ │ │ ├── _identifier.py │ │ │ ├── async_utils.py │ │ │ ├── bccache.py │ │ │ ├── compiler.py │ │ │ ├── constants.py │ │ │ ├── debug.py │ │ │ ├── defaults.py │ │ │ ├── environment.py │ │ │ ├── exceptions.py │ │ │ ├── ext.py │ │ │ ├── filters.py │ │ │ ├── idtracking.py │ │ │ ├── lexer.py │ │ │ ├── loaders.py │ │ │ ├── meta.py │ │ │ ├── nativetypes.py │ │ │ ├── nodes.py │ │ │ ├── optimizer.py │ │ │ ├── parser.py │ │ │ ├── py.typed │ │ │ ├── runtime.py │ │ │ ├── sandbox.py │ │ │ ├── tests.py │ │ │ ├── utils.py │ │ │ └── visitor.py │ │ ├── jsonschema/ │ │ │ ├── COPYING │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ ├── _format.py │ │ │ ├── _legacy_validators.py │ │ │ ├── _reflect.py │ │ │ ├── _types.py │ │ │ ├── _utils.py │ │ │ ├── _validators.py │ │ │ ├── benchmarks/ │ │ │ │ ├── __init__.py │ │ │ │ ├── issue232.py │ │ │ │ └── json_schema_test_suite.py │ │ │ ├── cli.py │ │ │ ├── compat.py │ │ │ ├── exceptions.py │ │ │ ├── schemas/ │ │ │ │ ├── draft3.json │ │ │ │ ├── draft4.json │ │ │ │ ├── draft6.json │ │ │ │ └── draft7.json │ │ │ └── validators.py │ │ ├── macholib/ │ │ │ ├── LICENSE │ │ │ ├── MachO.py │ │ │ ├── MachOGraph.py │ │ │ ├── MachOStandalone.py │ │ │ ├── SymbolTable.py │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ ├── _cmdline.py │ │ │ ├── dyld.py │ │ │ ├── dylib.py │ │ │ ├── framework.py │ │ │ ├── itergraphreport.py │ │ │ ├── mach_o.py │ │ │ ├── macho_dump.py │ │ │ ├── macho_find.py │ │ │ ├── macho_standalone.py │ │ │ ├── ptypes.py │ │ │ └── util.py │ │ ├── markupsafe/ │ │ │ ├── LICENSE.rst │ │ │ ├── __init__.py │ │ │ ├── _native.py │ │ │ ├── _speedups.c │ │ │ ├── _speedups.pyi │ │ │ └── py.typed │ │ ├── pyrsistent/ │ │ │ ├── LICENSE.mit │ │ │ ├── __init__.py │ │ │ ├── _checked_types.py │ │ │ ├── _field_common.py │ │ │ ├── _helpers.py │ │ │ ├── _immutable.py │ │ │ ├── _pbag.py │ │ │ ├── _pclass.py │ │ │ ├── _pdeque.py │ │ │ ├── _plist.py │ │ │ ├── _pmap.py │ │ │ ├── _precord.py │ │ │ ├── _pset.py │ │ │ ├── _pvector.py │ │ │ ├── _toolz.py │ │ │ ├── _transformations.py │ │ │ ├── py.typed │ │ │ ├── typing.py │ │ │ └── typing.pyi │ │ ├── ruamel/ │ │ │ └── yaml/ │ │ │ ├── __init__.py │ │ │ ├── anchor.py │ │ │ ├── comments.py │ │ │ ├── compat.py │ │ │ ├── composer.py │ │ │ ├── configobjwalker.py │ │ │ ├── constructor.py │ │ │ ├── cyaml.py │ │ │ ├── dumper.py │ │ │ ├── emitter.py │ │ │ ├── error.py │ │ │ ├── events.py │ │ │ ├── loader.py │ │ │ ├── main.py │ │ │ ├── nodes.py │ │ │ ├── parser.py │ │ │ ├── py.typed │ │ │ ├── reader.py │ │ │ ├── representer.py │ │ │ ├── resolver.py │ │ │ ├── scalarbool.py │ │ │ ├── scalarfloat.py │ │ │ ├── scalarint.py │ │ │ ├── scalarstring.py │ │ │ ├── scanner.py │ │ │ ├── serializer.py │ │ │ ├── timestamp.py │ │ │ ├── tokens.py │ │ │ └── util.py │ │ ├── ruamel.yaml.LICENSE │ │ ├── six.LICENSE │ │ ├── six.py │ │ ├── typing_extensions.LICENSE │ │ ├── typing_extensions.py │ │ └── typing_extensions.pyi │ ├── verify.py │ ├── verify_libraries.py │ └── version/ │ ├── __init__.py │ ├── common.py │ ├── git_ref_lookup.py │ ├── lookup.py │ └── version_types.py ├── pyproject.toml ├── pytest.ini ├── share/ │ └── spack/ │ ├── bash/ │ │ └── spack-completion.bash │ ├── bootstrap/ │ │ ├── github-actions-v0.6/ │ │ │ ├── clingo.json │ │ │ ├── gnupg.json │ │ │ ├── metadata.yaml │ │ │ └── patchelf.json │ │ ├── github-actions-v2/ │ │ │ ├── clingo.json │ │ │ ├── gnupg.json │ │ │ ├── metadata.yaml │ │ │ └── patchelf.json │ │ └── spack-install/ │ │ └── metadata.yaml │ ├── csh/ │ │ ├── pathadd.csh │ │ └── spack.csh │ ├── docker/ │ │ ├── entrypoint.bash │ │ └── modules.yaml │ ├── docs/ │ │ └── docker/ │ │ └── module-file-tutorial/ │ │ ├── Dockerfile │ │ ├── packages.yaml │ │ └── spack.sh │ ├── fish/ │ │ └── spack-completion.fish │ ├── keys/ │ │ └── tutorial.pub │ ├── qa/ │ │ ├── bashcov │ │ ├── completion-test.sh │ │ ├── config_state.py │ │ ├── configuration/ │ │ │ ├── config.yaml │ │ │ ├── packages.yaml │ │ │ └── windows_config.yaml │ │ ├── environment_activation.py │ │ ├── flake8_formatter.py │ │ ├── run-build-tests │ │ ├── run-shell-tests │ │ ├── run-style-tests │ │ ├── run-unit-tests │ │ ├── scopes/ │ │ │ ├── false/ │ │ │ │ └── concretizer.yaml │ │ │ ├── true/ │ │ │ │ ├── .spack-env/ │ │ │ │ │ └── transaction_lock │ │ │ │ └── spack.yaml │ │ │ └── wp/ │ │ │ └── concretizer.yaml │ │ ├── setup-env-test.csh │ │ ├── setup-env-test.fish │ │ ├── setup-env-test.sh │ │ ├── setup.sh │ │ ├── setup_spack_installer.ps1 │ │ ├── test-env-cfg.sh │ │ ├── test-framework.sh │ │ ├── validate_last_exit.ps1 │ │ ├── vcvarsall.ps1 │ │ └── windows_test_setup.ps1 │ ├── setup-env.bat │ ├── setup-env.csh │ ├── setup-env.fish │ ├── setup-env.ps1 │ ├── setup-env.sh │ ├── setup-tutorial-env.sh │ ├── spack-completion.bash │ ├── spack-completion.fish │ └── templates/ │ ├── bootstrap/ │ │ └── spack.yaml │ ├── container/ │ │ ├── Dockerfile │ │ ├── almalinux_8.dockerfile │ │ ├── almalinux_9.dockerfile │ │ ├── alpine_3.dockerfile │ │ ├── amazonlinux_2.dockerfile │ │ ├── bootstrap-base.dockerfile │ │ ├── centos_stream9.dockerfile │ │ ├── fedora.dockerfile │ │ ├── leap-15.dockerfile │ │ ├── rockylinux_8.dockerfile │ │ ├── rockylinux_9.dockerfile │ │ ├── singularity.def │ │ ├── ubuntu_2004.dockerfile │ │ └── ubuntu_2404.dockerfile │ ├── depfile/ │ │ └── Makefile │ ├── misc/ │ │ ├── buildcache_index.html │ │ └── graph.dot │ ├── mock-repository/ │ │ ├── build_system.pyt │ │ └── package.pyt │ ├── modules/ │ │ ├── modulefile.lua │ │ └── modulefile.tcl │ └── reports/ │ ├── cdash/ │ │ ├── Build.xml │ │ ├── Configure.xml │ │ ├── Site.xml │ │ ├── Test.xml │ │ ├── Testing.xml │ │ └── Update.xml │ └── junit.xml └── var/ └── spack/ ├── gpg/ │ └── README.md ├── gpg.mock/ │ ├── README.md │ ├── data/ │ │ ├── content.txt │ │ └── content.txt.asc │ └── keys/ │ ├── external.key │ └── package-signing-key ├── test_repos/ │ └── spack_repo/ │ ├── builder_test/ │ │ ├── packages/ │ │ │ ├── builder_and_mixins/ │ │ │ │ └── package.py │ │ │ ├── callbacks/ │ │ │ │ └── package.py │ │ │ ├── custom_phases/ │ │ │ │ └── package.py │ │ │ ├── gmake/ │ │ │ │ └── package.py │ │ │ ├── gnuconfig/ │ │ │ │ └── package.py │ │ │ ├── inheritance/ │ │ │ │ └── package.py │ │ │ ├── inheritance_only_package/ │ │ │ │ └── package.py │ │ │ ├── old_style_autotools/ │ │ │ │ └── package.py │ │ │ ├── old_style_custom_phases/ │ │ │ │ └── package.py │ │ │ └── old_style_derived/ │ │ │ └── package.py │ │ └── repo.yaml │ ├── builtin_mock/ │ │ ├── build_systems/ │ │ │ ├── __init__.py │ │ │ ├── _checks.py │ │ │ ├── autotools.py │ │ │ ├── bundle.py │ │ │ ├── cmake.py │ │ │ ├── compiler.py │ │ │ ├── generic.py │ │ │ ├── gnu.py │ │ │ ├── makefile.py │ │ │ ├── perl.py │ │ │ ├── python.py │ │ │ ├── sourceforge.py │ │ │ └── sourceware.py │ │ ├── packages/ │ │ │ ├── _7zip/ │ │ │ │ └── package.py │ │ │ ├── _7zip_dependent/ │ │ │ │ └── package.py │ │ │ ├── adios2/ │ │ │ │ └── package.py │ │ │ ├── archive_files/ │ │ │ │ └── package.py │ │ │ ├── ascent/ │ │ │ │ └── package.py │ │ │ ├── attributes_foo/ │ │ │ │ └── package.py │ │ │ ├── attributes_foo_app/ │ │ │ │ └── package.py │ │ │ ├── autotools_conditional_variants_test/ │ │ │ │ └── package.py │ │ │ ├── autotools_config_replacement/ │ │ │ │ └── package.py │ │ │ ├── binutils_for_test/ │ │ │ │ └── package.py │ │ │ ├── boost/ │ │ │ │ └── package.py │ │ │ ├── both_link_and_build_dep_a/ │ │ │ │ └── package.py │ │ │ ├── both_link_and_build_dep_b/ │ │ │ │ └── package.py │ │ │ ├── both_link_and_build_dep_c/ │ │ │ │ └── package.py │ │ │ ├── bowtie/ │ │ │ │ └── package.py │ │ │ ├── brillig/ │ │ │ │ └── package.py │ │ │ ├── build_env_compiler_var_a/ │ │ │ │ └── package.py │ │ │ ├── build_env_compiler_var_b/ │ │ │ │ └── package.py │ │ │ ├── build_error/ │ │ │ │ └── package.py │ │ │ ├── build_warnings/ │ │ │ │ └── package.py │ │ │ ├── bzip2/ │ │ │ │ └── package.py │ │ │ ├── callpath/ │ │ │ │ └── package.py │ │ │ ├── canfail/ │ │ │ │ └── package.py │ │ │ ├── client_not_foo/ │ │ │ │ └── package.py │ │ │ ├── cmake/ │ │ │ │ └── package.py │ │ │ ├── cmake_client/ │ │ │ │ └── package.py │ │ │ ├── cmake_client_inheritor/ │ │ │ │ └── package.py │ │ │ ├── cmake_conditional_variants_test/ │ │ │ │ └── package.py │ │ │ ├── compiler_with_deps/ │ │ │ │ └── package.py │ │ │ ├── compiler_wrapper/ │ │ │ │ ├── cc.sh │ │ │ │ └── package.py │ │ │ ├── conditional_constrained_dependencies/ │ │ │ │ └── package.py │ │ │ ├── conditional_languages/ │ │ │ │ └── package.py │ │ │ ├── conditional_provider/ │ │ │ │ └── package.py │ │ │ ├── conditional_values_in_variant/ │ │ │ │ └── package.py │ │ │ ├── conditional_variant_pkg/ │ │ │ │ └── package.py │ │ │ ├── conditional_virtual_dependency/ │ │ │ │ └── package.py │ │ │ ├── conditionally_extends_direct_dep/ │ │ │ │ └── package.py │ │ │ ├── conditionally_extends_transitive_dep/ │ │ │ │ └── package.py │ │ │ ├── conditionally_patch_dependency/ │ │ │ │ ├── package.py │ │ │ │ └── uuid.patch │ │ │ ├── configure_warning/ │ │ │ │ └── package.py │ │ │ ├── conflict/ │ │ │ │ └── package.py │ │ │ ├── conflict_parent/ │ │ │ │ └── package.py │ │ │ ├── conflict_virtual/ │ │ │ │ └── package.py │ │ │ ├── conflicting_dependent/ │ │ │ │ └── package.py │ │ │ ├── corge/ │ │ │ │ └── package.py │ │ │ ├── cumulative_vrange_bottom/ │ │ │ │ └── package.py │ │ │ ├── cumulative_vrange_middle/ │ │ │ │ └── package.py │ │ │ ├── cumulative_vrange_root/ │ │ │ │ └── package.py │ │ │ ├── cvs_test/ │ │ │ │ └── package.py │ │ │ ├── define_cmake_prefix_paths/ │ │ │ │ └── package.py │ │ │ ├── dep_diamond_patch_mid1/ │ │ │ │ ├── mid1.patch │ │ │ │ └── package.py │ │ │ ├── dep_diamond_patch_mid2/ │ │ │ │ └── package.py │ │ │ ├── dep_diamond_patch_top/ │ │ │ │ ├── package.py │ │ │ │ └── top.patch │ │ │ ├── dep_with_variants/ │ │ │ │ └── package.py │ │ │ ├── dep_with_variants_if_develop/ │ │ │ │ └── package.py │ │ │ ├── dep_with_variants_if_develop_root/ │ │ │ │ └── package.py │ │ │ ├── depb/ │ │ │ │ └── package.py │ │ │ ├── dependency_foo_bar/ │ │ │ │ └── package.py │ │ │ ├── dependency_install/ │ │ │ │ └── package.py │ │ │ ├── dependency_mv/ │ │ │ │ └── package.py │ │ │ ├── dependent_install/ │ │ │ │ └── package.py │ │ │ ├── dependent_of_dev_build/ │ │ │ │ └── package.py │ │ │ ├── depends_on_define_cmake_prefix_paths/ │ │ │ │ └── package.py │ │ │ ├── depends_on_develop/ │ │ │ │ └── package.py │ │ │ ├── depends_on_manyvariants/ │ │ │ │ └── package.py │ │ │ ├── depends_on_openmpi/ │ │ │ │ └── package.py │ │ │ ├── depends_on_run_env/ │ │ │ │ └── package.py │ │ │ ├── depends_on_virtual_with_abi/ │ │ │ │ └── package.py │ │ │ ├── deprecated_client/ │ │ │ │ └── package.py │ │ │ ├── deprecated_versions/ │ │ │ │ └── package.py │ │ │ ├── dev_build_test_dependent/ │ │ │ │ └── package.py │ │ │ ├── dev_build_test_install/ │ │ │ │ └── package.py │ │ │ ├── dev_build_test_install_phases/ │ │ │ │ └── package.py │ │ │ ├── develop_branch_version/ │ │ │ │ └── package.py │ │ │ ├── develop_test/ │ │ │ │ └── package.py │ │ │ ├── develop_test2/ │ │ │ │ └── package.py │ │ │ ├── diamond_link_bottom/ │ │ │ │ └── package.py │ │ │ ├── diamond_link_left/ │ │ │ │ └── package.py │ │ │ ├── diamond_link_right/ │ │ │ │ └── package.py │ │ │ ├── diamond_link_top/ │ │ │ │ └── package.py │ │ │ ├── diff_test/ │ │ │ │ └── package.py │ │ │ ├── direct_dep_foo_bar/ │ │ │ │ └── package.py │ │ │ ├── direct_dep_virtuals_one/ │ │ │ │ └── package.py │ │ │ ├── direct_dep_virtuals_two/ │ │ │ │ └── package.py │ │ │ ├── direct_mpich/ │ │ │ │ └── package.py │ │ │ ├── dla_future/ │ │ │ │ └── package.py │ │ │ ├── dt_diamond/ │ │ │ │ └── package.py │ │ │ ├── dt_diamond_bottom/ │ │ │ │ └── package.py │ │ │ ├── dt_diamond_left/ │ │ │ │ └── package.py │ │ │ ├── dt_diamond_right/ │ │ │ │ └── package.py │ │ │ ├── dtbuild1/ │ │ │ │ └── package.py │ │ │ ├── dtbuild2/ │ │ │ │ └── package.py │ │ │ ├── dtbuild3/ │ │ │ │ └── package.py │ │ │ ├── dtlink1/ │ │ │ │ └── package.py │ │ │ ├── dtlink2/ │ │ │ │ └── package.py │ │ │ ├── dtlink3/ │ │ │ │ └── package.py │ │ │ ├── dtlink4/ │ │ │ │ └── package.py │ │ │ ├── dtlink5/ │ │ │ │ └── package.py │ │ │ ├── dtrun1/ │ │ │ │ └── package.py │ │ │ ├── dtrun2/ │ │ │ │ └── package.py │ │ │ ├── dtrun3/ │ │ │ │ └── package.py │ │ │ ├── dttop/ │ │ │ │ └── package.py │ │ │ ├── dtuse/ │ │ │ │ └── package.py │ │ │ ├── dual_cmake_autotools/ │ │ │ │ └── package.py │ │ │ ├── dyninst/ │ │ │ │ └── package.py │ │ │ ├── ecp_viz_sdk/ │ │ │ │ └── package.py │ │ │ ├── emacs/ │ │ │ │ └── package.py │ │ │ ├── extendee/ │ │ │ │ └── package.py │ │ │ ├── extends_spec/ │ │ │ │ └── package.py │ │ │ ├── extension1/ │ │ │ │ └── package.py │ │ │ ├── extension2/ │ │ │ │ └── package.py │ │ │ ├── external_buildable_with_variant/ │ │ │ │ └── package.py │ │ │ ├── external_common_gdbm/ │ │ │ │ └── package.py │ │ │ ├── external_common_openssl/ │ │ │ │ └── package.py │ │ │ ├── external_common_perl/ │ │ │ │ └── package.py │ │ │ ├── external_common_python/ │ │ │ │ └── package.py │ │ │ ├── external_non_default_variant/ │ │ │ │ └── package.py │ │ │ ├── externalmodule/ │ │ │ │ └── package.py │ │ │ ├── externalprereq/ │ │ │ │ └── package.py │ │ │ ├── externaltest/ │ │ │ │ └── package.py │ │ │ ├── externaltool/ │ │ │ │ └── package.py │ │ │ ├── externalvirtual/ │ │ │ │ └── package.py │ │ │ ├── fail_test_audit/ │ │ │ │ └── package.py │ │ │ ├── fail_test_audit_docstring/ │ │ │ │ └── package.py │ │ │ ├── fail_test_audit_impl/ │ │ │ │ └── package.py │ │ │ ├── failing_build/ │ │ │ │ └── package.py │ │ │ ├── failing_empty_install/ │ │ │ │ └── package.py │ │ │ ├── fake/ │ │ │ │ └── package.py │ │ │ ├── fetch_options/ │ │ │ │ └── package.py │ │ │ ├── fftw/ │ │ │ │ └── package.py │ │ │ ├── find_externals1/ │ │ │ │ └── package.py │ │ │ ├── forward_multi_value/ │ │ │ │ └── package.py │ │ │ ├── garply/ │ │ │ │ └── package.py │ │ │ ├── gcc/ │ │ │ │ └── package.py │ │ │ ├── gcc_runtime/ │ │ │ │ └── package.py │ │ │ ├── git_ref_commit_dep/ │ │ │ │ └── package.py │ │ │ ├── git_ref_package/ │ │ │ │ └── package.py │ │ │ ├── git_sparse_a/ │ │ │ │ └── package.py │ │ │ ├── git_sparse_b/ │ │ │ │ └── package.py │ │ │ ├── git_sparsepaths_pkg/ │ │ │ │ └── package.py │ │ │ ├── git_sparsepaths_version/ │ │ │ │ └── package.py │ │ │ ├── git_svn_top_level/ │ │ │ │ └── package.py │ │ │ ├── git_test/ │ │ │ │ └── package.py │ │ │ ├── git_test_commit/ │ │ │ │ └── package.py │ │ │ ├── git_top_level/ │ │ │ │ └── package.py │ │ │ ├── git_url_svn_top_level/ │ │ │ │ └── package.py │ │ │ ├── git_url_top_level/ │ │ │ │ └── package.py │ │ │ ├── glibc/ │ │ │ │ └── package.py │ │ │ ├── gmake/ │ │ │ │ └── package.py │ │ │ ├── gmt/ │ │ │ │ └── package.py │ │ │ ├── gmt_concrete_mv_dependency/ │ │ │ │ └── package.py │ │ │ ├── gnuconfig/ │ │ │ │ └── package.py │ │ │ ├── hash_test1/ │ │ │ │ ├── package.py │ │ │ │ ├── patch1.patch │ │ │ │ └── patch2.patch │ │ │ ├── hash_test2/ │ │ │ │ ├── package.py │ │ │ │ └── patch1.patch │ │ │ ├── hash_test3/ │ │ │ │ └── package.py │ │ │ ├── hash_test4/ │ │ │ │ └── package.py │ │ │ ├── hdf5/ │ │ │ │ └── package.py │ │ │ ├── hg_test/ │ │ │ │ └── package.py │ │ │ ├── hg_top_level/ │ │ │ │ └── package.py │ │ │ ├── hpcviewer/ │ │ │ │ └── package.py │ │ │ ├── hwloc/ │ │ │ │ └── package.py │ │ │ ├── hypre/ │ │ │ │ └── package.py │ │ │ ├── impossible_concretization/ │ │ │ │ └── package.py │ │ │ ├── indirect_mpich/ │ │ │ │ └── package.py │ │ │ ├── installed_deps_a/ │ │ │ │ └── package.py │ │ │ ├── installed_deps_b/ │ │ │ │ └── package.py │ │ │ ├── installed_deps_c/ │ │ │ │ └── package.py │ │ │ ├── installed_deps_d/ │ │ │ │ └── package.py │ │ │ ├── installed_deps_e/ │ │ │ │ └── package.py │ │ │ ├── intel_oneapi_compilers/ │ │ │ │ └── package.py │ │ │ ├── intel_parallel_studio/ │ │ │ │ └── package.py │ │ │ ├── invalid_github_patch_url/ │ │ │ │ └── package.py │ │ │ ├── invalid_github_pull_commits_patch_url/ │ │ │ │ └── package.py │ │ │ ├── invalid_gitlab_patch_url/ │ │ │ │ └── package.py │ │ │ ├── invalid_selfhosted_gitlab_patch_url/ │ │ │ │ └── package.py │ │ │ ├── leaf_adds_virtual/ │ │ │ │ └── package.py │ │ │ ├── libceed/ │ │ │ │ └── package.py │ │ │ ├── libdwarf/ │ │ │ │ └── package.py │ │ │ ├── libelf/ │ │ │ │ ├── local.patch │ │ │ │ └── package.py │ │ │ ├── libtool_deletion/ │ │ │ │ └── package.py │ │ │ ├── libtool_installation/ │ │ │ │ └── package.py │ │ │ ├── libxsmm/ │ │ │ │ └── package.py │ │ │ ├── licenses_1/ │ │ │ │ └── package.py │ │ │ ├── llvm/ │ │ │ │ └── package.py │ │ │ ├── llvm_client/ │ │ │ │ └── package.py │ │ │ ├── long_boost_dependency/ │ │ │ │ └── package.py │ │ │ ├── low_priority_provider/ │ │ │ │ └── package.py │ │ │ ├── maintainers_1/ │ │ │ │ └── package.py │ │ │ ├── maintainers_2/ │ │ │ │ └── package.py │ │ │ ├── maintainers_3/ │ │ │ │ └── package.py │ │ │ ├── many_conditional_deps/ │ │ │ │ └── package.py │ │ │ ├── many_virtual_consumer/ │ │ │ │ └── package.py │ │ │ ├── manyvariants/ │ │ │ │ └── package.py │ │ │ ├── mesa/ │ │ │ │ └── package.py │ │ │ ├── middle_adds_virtual/ │ │ │ │ └── package.py │ │ │ ├── mirror_gnu/ │ │ │ │ └── package.py │ │ │ ├── mirror_gnu_broken/ │ │ │ │ └── package.py │ │ │ ├── mirror_sourceforge/ │ │ │ │ └── package.py │ │ │ ├── mirror_sourceforge_broken/ │ │ │ │ └── package.py │ │ │ ├── mirror_sourceware/ │ │ │ │ └── package.py │ │ │ ├── mirror_sourceware_broken/ │ │ │ │ ├── mirror-gnu-broken/ │ │ │ │ │ └── package.py │ │ │ │ └── package.py │ │ │ ├── missing_dependency/ │ │ │ │ └── package.py │ │ │ ├── mixedversions/ │ │ │ │ └── package.py │ │ │ ├── mixing_parent/ │ │ │ │ └── package.py │ │ │ ├── modifies_run_env/ │ │ │ │ └── package.py │ │ │ ├── module_long_help/ │ │ │ │ └── package.py │ │ │ ├── module_manpath_append/ │ │ │ │ └── package.py │ │ │ ├── module_manpath_prepend/ │ │ │ │ └── package.py │ │ │ ├── module_manpath_setenv/ │ │ │ │ └── package.py │ │ │ ├── module_path_separator/ │ │ │ │ └── package.py │ │ │ ├── module_setenv_raw/ │ │ │ │ └── package.py │ │ │ ├── mpi/ │ │ │ │ └── package.py │ │ │ ├── mpich/ │ │ │ │ └── package.py │ │ │ ├── mpich2/ │ │ │ │ └── package.py │ │ │ ├── mpileaks/ │ │ │ │ └── package.py │ │ │ ├── multi_provider_mpi/ │ │ │ │ └── package.py │ │ │ ├── multimethod/ │ │ │ │ └── package.py │ │ │ ├── multimethod_base/ │ │ │ │ └── package.py │ │ │ ├── multimethod_diamond/ │ │ │ │ └── package.py │ │ │ ├── multimethod_diamond_parent/ │ │ │ │ └── package.py │ │ │ ├── multimethod_inheritor/ │ │ │ │ └── package.py │ │ │ ├── multimodule_inheritance/ │ │ │ │ └── package.py │ │ │ ├── multivalue_variant/ │ │ │ │ └── package.py │ │ │ ├── multivalue_variant_multi_defaults/ │ │ │ │ └── package.py │ │ │ ├── multivalue_variant_multi_defaults_dependent/ │ │ │ │ └── package.py │ │ │ ├── mvapich2/ │ │ │ │ └── package.py │ │ │ ├── mvdefaults/ │ │ │ │ └── package.py │ │ │ ├── needs_relocation/ │ │ │ │ └── package.py │ │ │ ├── needs_text_relocation/ │ │ │ │ └── package.py │ │ │ ├── netlib_blas/ │ │ │ │ └── package.py │ │ │ ├── netlib_lapack/ │ │ │ │ └── package.py │ │ │ ├── netlib_scalapack/ │ │ │ │ └── package.py │ │ │ ├── ninja/ │ │ │ │ └── package.py │ │ │ ├── no_redistribute/ │ │ │ │ └── package.py │ │ │ ├── no_redistribute_dependent/ │ │ │ │ └── package.py │ │ │ ├── no_url_or_version/ │ │ │ │ └── package.py │ │ │ ├── non_existing_conditional_dep/ │ │ │ │ └── package.py │ │ │ ├── nosource/ │ │ │ │ └── package.py │ │ │ ├── nosource_bundle/ │ │ │ │ └── package.py │ │ │ ├── nosource_install/ │ │ │ │ └── package.py │ │ │ ├── noversion/ │ │ │ │ └── package.py │ │ │ ├── noversion_bundle/ │ │ │ │ └── package.py │ │ │ ├── old_external/ │ │ │ │ └── package.py │ │ │ ├── old_sbang/ │ │ │ │ └── package.py │ │ │ ├── openblas/ │ │ │ │ └── package.py │ │ │ ├── openblas_with_lapack/ │ │ │ │ └── package.py │ │ │ ├── openmpi/ │ │ │ │ └── package.py │ │ │ ├── openssl/ │ │ │ │ └── package.py │ │ │ ├── optional_dep_test/ │ │ │ │ └── package.py │ │ │ ├── optional_dep_test_2/ │ │ │ │ └── package.py │ │ │ ├── optional_dep_test_3/ │ │ │ │ └── package.py │ │ │ ├── othervirtual/ │ │ │ │ └── package.py │ │ │ ├── override_context_templates/ │ │ │ │ └── package.py │ │ │ ├── override_module_templates/ │ │ │ │ └── package.py │ │ │ ├── package_base_extendee/ │ │ │ │ └── package.py │ │ │ ├── parallel_package_a/ │ │ │ │ └── package.py │ │ │ ├── parallel_package_b/ │ │ │ │ └── package.py │ │ │ ├── parallel_package_c/ │ │ │ │ └── package.py │ │ │ ├── paraview/ │ │ │ │ └── package.py │ │ │ ├── parent_foo/ │ │ │ │ └── package.py │ │ │ ├── parent_foo_bar/ │ │ │ │ └── package.py │ │ │ ├── parent_foo_bar_fee/ │ │ │ │ └── package.py │ │ │ ├── patch/ │ │ │ │ ├── bar.patch │ │ │ │ ├── baz.patch │ │ │ │ ├── biz.patch │ │ │ │ ├── foo.patch │ │ │ │ └── package.py │ │ │ ├── patch_a_dependency/ │ │ │ │ ├── libelf.patch │ │ │ │ └── package.py │ │ │ ├── patch_inheritance/ │ │ │ │ └── package.py │ │ │ ├── patch_several_dependencies/ │ │ │ │ ├── bar.patch │ │ │ │ ├── baz.patch │ │ │ │ ├── foo.patch │ │ │ │ └── package.py │ │ │ ├── patchelf/ │ │ │ │ └── package.py │ │ │ ├── perl/ │ │ │ │ └── package.py │ │ │ ├── perl_extension/ │ │ │ │ └── package.py │ │ │ ├── pkg_a/ │ │ │ │ └── package.py │ │ │ ├── pkg_b/ │ │ │ │ └── package.py │ │ │ ├── pkg_c/ │ │ │ │ └── package.py │ │ │ ├── pkg_e/ │ │ │ │ └── package.py │ │ │ ├── pkg_with_c_link_dep/ │ │ │ │ └── package.py │ │ │ ├── pkg_with_zlib_dep/ │ │ │ │ └── package.py │ │ │ ├── placeholder/ │ │ │ │ └── package.py │ │ │ ├── preferred_test/ │ │ │ │ └── package.py │ │ │ ├── printing_package/ │ │ │ │ └── package.py │ │ │ ├── py_extension1/ │ │ │ │ └── package.py │ │ │ ├── py_extension2/ │ │ │ │ └── package.py │ │ │ ├── py_extension3/ │ │ │ │ └── package.py │ │ │ ├── py_numpy/ │ │ │ │ └── package.py │ │ │ ├── py_pip/ │ │ │ │ └── package.py │ │ │ ├── py_test_callback/ │ │ │ │ └── package.py │ │ │ ├── py_wheel/ │ │ │ │ └── package.py │ │ │ ├── python/ │ │ │ │ └── package.py │ │ │ ├── python_venv/ │ │ │ │ └── package.py │ │ │ ├── quantum_espresso/ │ │ │ │ └── package.py │ │ │ ├── quux/ │ │ │ │ └── package.py │ │ │ ├── raiser/ │ │ │ │ └── package.py │ │ │ ├── redistribute_x/ │ │ │ │ └── package.py │ │ │ ├── redistribute_y/ │ │ │ │ └── package.py │ │ │ ├── requires_clang/ │ │ │ │ └── package.py │ │ │ ├── requires_clang_or_gcc/ │ │ │ │ └── package.py │ │ │ ├── requires_virtual/ │ │ │ │ └── package.py │ │ │ ├── root/ │ │ │ │ └── package.py │ │ │ ├── root_adds_virtual/ │ │ │ │ └── package.py │ │ │ ├── ruff/ │ │ │ │ └── package.py │ │ │ ├── second_dependency_foo_bar_fee/ │ │ │ │ └── package.py │ │ │ ├── shell_a/ │ │ │ │ └── package.py │ │ │ ├── shell_b/ │ │ │ │ └── package.py │ │ │ ├── simple_inheritance/ │ │ │ │ └── package.py │ │ │ ├── simple_resource/ │ │ │ │ └── package.py │ │ │ ├── simple_standalone_test/ │ │ │ │ └── package.py │ │ │ ├── single_language_virtual/ │ │ │ │ └── package.py │ │ │ ├── singlevalue_variant/ │ │ │ │ └── package.py │ │ │ ├── singlevalue_variant_dependent/ │ │ │ │ └── package.py │ │ │ ├── singlevalue_variant_dependent_type/ │ │ │ │ └── package.py │ │ │ ├── sombrero/ │ │ │ │ └── package.py │ │ │ ├── some_virtual_mv/ │ │ │ │ └── package.py │ │ │ ├── some_virtual_preferred/ │ │ │ │ └── package.py │ │ │ ├── splice_a/ │ │ │ │ └── package.py │ │ │ ├── splice_depends_on_t/ │ │ │ │ └── package.py │ │ │ ├── splice_h/ │ │ │ │ └── package.py │ │ │ ├── splice_t/ │ │ │ │ └── package.py │ │ │ ├── splice_vh/ │ │ │ │ └── package.py │ │ │ ├── splice_vt/ │ │ │ │ └── package.py │ │ │ ├── splice_z/ │ │ │ │ └── package.py │ │ │ ├── sticky_variant/ │ │ │ │ └── package.py │ │ │ ├── sticky_variant_dependent/ │ │ │ │ └── package.py │ │ │ ├── svn_test/ │ │ │ │ └── package.py │ │ │ ├── svn_top_level/ │ │ │ │ └── package.py │ │ │ ├── symly/ │ │ │ │ └── package.py │ │ │ ├── test_build_callbacks/ │ │ │ │ └── package.py │ │ │ ├── test_dep_with_imposed_conditions/ │ │ │ │ └── package.py │ │ │ ├── test_dependency/ │ │ │ │ └── package.py │ │ │ ├── test_error/ │ │ │ │ └── package.py │ │ │ ├── test_fail/ │ │ │ │ └── package.py │ │ │ ├── test_install_callbacks/ │ │ │ │ └── package.py │ │ │ ├── transitive_conditional_virtual_dependency/ │ │ │ │ └── package.py │ │ │ ├── trigger_and_effect_deps/ │ │ │ │ └── package.py │ │ │ ├── trigger_external_non_default_variant/ │ │ │ │ └── package.py │ │ │ ├── trilinos/ │ │ │ │ └── package.py │ │ │ ├── trivial_install_test_dependent/ │ │ │ │ └── package.py │ │ │ ├── trivial_install_test_package/ │ │ │ │ └── package.py │ │ │ ├── trivial_pkg_with_valid_hash/ │ │ │ │ └── package.py │ │ │ ├── trivial_smoke_test/ │ │ │ │ ├── package.py │ │ │ │ └── test/ │ │ │ │ └── test_file.in │ │ │ ├── unconstrainable_conflict/ │ │ │ │ └── package.py │ │ │ ├── unsat_provider/ │ │ │ │ └── package.py │ │ │ ├── unsat_virtual_dependency/ │ │ │ │ └── package.py │ │ │ ├── url_list_test/ │ │ │ │ └── package.py │ │ │ ├── url_only_override/ │ │ │ │ └── package.py │ │ │ ├── url_only_override_with_gaps/ │ │ │ │ └── package.py │ │ │ ├── url_override/ │ │ │ │ └── package.py │ │ │ ├── url_test/ │ │ │ │ └── package.py │ │ │ ├── v1_consumer/ │ │ │ │ └── package.py │ │ │ ├── v1_provider/ │ │ │ │ └── package.py │ │ │ ├── variant_function_validator/ │ │ │ │ └── package.py │ │ │ ├── variant_on_dependency_condition_a/ │ │ │ │ └── package.py │ │ │ ├── variant_on_dependency_condition_b/ │ │ │ │ └── package.py │ │ │ ├── variant_on_dependency_condition_root/ │ │ │ │ └── package.py │ │ │ ├── variant_values/ │ │ │ │ └── package.py │ │ │ ├── variant_values_override/ │ │ │ │ └── package.py │ │ │ ├── vdefault_or_external/ │ │ │ │ └── package.py │ │ │ ├── vdefault_or_external_root/ │ │ │ │ └── package.py │ │ │ ├── vendorsb/ │ │ │ │ └── package.py │ │ │ ├── version_test_dependency_preferred/ │ │ │ │ └── package.py │ │ │ ├── version_test_pkg/ │ │ │ │ └── package.py │ │ │ ├── version_test_root/ │ │ │ │ └── package.py │ │ │ ├── view_dir/ │ │ │ │ └── package.py │ │ │ ├── view_file/ │ │ │ │ └── package.py │ │ │ ├── view_ignore_conflict/ │ │ │ │ └── package.py │ │ │ ├── view_not_ignored/ │ │ │ │ └── package.py │ │ │ ├── view_resolve_conflict_middle/ │ │ │ │ └── package.py │ │ │ ├── view_resolve_conflict_top/ │ │ │ │ └── package.py │ │ │ ├── view_symlinked_dir/ │ │ │ │ └── package.py │ │ │ ├── virtual_abi_1/ │ │ │ │ └── package.py │ │ │ ├── virtual_abi_2/ │ │ │ │ └── package.py │ │ │ ├── virtual_abi_multi/ │ │ │ │ └── package.py │ │ │ ├── virtual_with_abi/ │ │ │ │ └── package.py │ │ │ ├── virtual_with_versions/ │ │ │ │ └── package.py │ │ │ ├── vtk_m/ │ │ │ │ └── package.py │ │ │ ├── when_directives_false/ │ │ │ │ └── package.py │ │ │ ├── when_directives_true/ │ │ │ │ └── package.py │ │ │ ├── with_constraint_met/ │ │ │ │ └── package.py │ │ │ ├── wrong_variant_in_conflicts/ │ │ │ │ └── package.py │ │ │ ├── wrong_variant_in_depends_on/ │ │ │ │ └── package.py │ │ │ ├── zlib/ │ │ │ │ ├── package.py │ │ │ │ └── w_patch.patch │ │ │ └── zmpi/ │ │ │ └── package.py │ │ └── repo.yaml │ ├── compiler_runtime_test/ │ │ ├── packages/ │ │ │ ├── pkg_a/ │ │ │ │ └── package.py │ │ │ └── pkg_b/ │ │ │ └── package.py │ │ └── repo.yaml │ ├── diff/ │ │ ├── packages/ │ │ │ ├── i1/ │ │ │ │ └── package.py │ │ │ ├── i2/ │ │ │ │ └── package.py │ │ │ ├── p1/ │ │ │ │ └── package.py │ │ │ ├── p2/ │ │ │ │ └── package.py │ │ │ ├── p3/ │ │ │ │ └── package.py │ │ │ └── p4/ │ │ │ └── package.py │ │ └── repo.yaml │ ├── duplicates_test/ │ │ ├── packages/ │ │ │ ├── cycle_a/ │ │ │ │ └── package.py │ │ │ ├── cycle_b/ │ │ │ │ └── package.py │ │ │ ├── gmake/ │ │ │ │ └── package.py │ │ │ ├── hdf5/ │ │ │ │ └── package.py │ │ │ ├── pinned_gmake/ │ │ │ │ └── package.py │ │ │ ├── pkg_config/ │ │ │ │ └── package.py │ │ │ ├── py_floating/ │ │ │ │ └── package.py │ │ │ ├── py_numpy/ │ │ │ │ └── package.py │ │ │ ├── py_setuptools/ │ │ │ │ └── package.py │ │ │ ├── py_shapely/ │ │ │ │ └── package.py │ │ │ ├── python/ │ │ │ │ └── package.py │ │ │ ├── unify_build_deps_a/ │ │ │ │ └── package.py │ │ │ ├── unify_build_deps_b/ │ │ │ │ └── package.py │ │ │ ├── unify_build_deps_c/ │ │ │ │ └── package.py │ │ │ └── virtual_build/ │ │ │ └── package.py │ │ └── repo.yaml │ ├── edges_test/ │ │ ├── packages/ │ │ │ ├── blas_only_client/ │ │ │ │ └── package.py │ │ │ ├── conditional_edge/ │ │ │ │ └── package.py │ │ │ ├── openblas/ │ │ │ │ └── package.py │ │ │ └── zlib/ │ │ │ └── package.py │ │ └── repo.yaml │ ├── find/ │ │ ├── packages/ │ │ │ ├── a0/ │ │ │ │ └── package.py │ │ │ ├── b0/ │ │ │ │ └── package.py │ │ │ ├── c0/ │ │ │ │ └── package.py │ │ │ ├── d0/ │ │ │ │ └── package.py │ │ │ └── e0/ │ │ │ └── package.py │ │ └── repo.yaml │ ├── flags_test/ │ │ ├── packages/ │ │ │ ├── t/ │ │ │ │ └── package.py │ │ │ ├── u/ │ │ │ │ └── package.py │ │ │ ├── v/ │ │ │ │ └── package.py │ │ │ ├── w/ │ │ │ │ └── package.py │ │ │ ├── x/ │ │ │ │ └── package.py │ │ │ └── y/ │ │ │ └── package.py │ │ └── repo.yaml │ ├── requirements_test/ │ │ ├── packages/ │ │ │ ├── t/ │ │ │ │ └── package.py │ │ │ ├── u/ │ │ │ │ └── package.py │ │ │ ├── v/ │ │ │ │ └── package.py │ │ │ ├── x/ │ │ │ │ └── package.py │ │ │ └── y/ │ │ │ └── package.py │ │ └── repo.yaml │ └── tutorial/ │ ├── packages/ │ │ ├── armadillo/ │ │ │ ├── package.py │ │ │ └── undef_linux.patch │ │ ├── elpa/ │ │ │ └── package.py │ │ ├── hdf5/ │ │ │ ├── h5f90global-mult-obj-same-equivalence-same-common-block.patch │ │ │ ├── h5public-skip-mpicxx.patch │ │ │ ├── hdf5_1.8_gcc10.patch │ │ │ ├── package.py │ │ │ ├── pre-c99-comments.patch │ │ │ └── test/ │ │ │ ├── dump.out │ │ │ └── spack.h5 │ │ ├── mpich/ │ │ │ ├── mpich32_clang.patch │ │ │ └── package.py │ │ └── netlib_lapack/ │ │ ├── ibm-xl.patch │ │ ├── package.py │ │ ├── testing.patch │ │ └── undefined_declarations.patch │ └── repo.yaml └── vendoring/ ├── patches/ │ ├── altgraph-version.patch │ ├── distro.patch │ ├── jsonschema.attr.patch │ ├── jsonschema.patch │ ├── jsonschema.vendoring.patch │ └── ruamelyaml.patch └── vendor.txt ================================================ FILE CONTENTS ================================================ ================================================ FILE: .ci/gitlab/forward_dotenv_variables.py ================================================ import sys from typing import Dict import yaml def read_dotenv(file_name: str) -> Dict[str, str]: result = [] with open(file_name, "r", encoding="utf-8") as fd: for field in fd: if field.strip()[0] == "#": continue data = field.strip("\n").split("=", 1) try: result.append((data[0], data[1])) except IndexError: print(f"Skipping bad value: {field}") return dict(result) if __name__ == "__main__": dotenv = read_dotenv(sys.argv[1]) if not dotenv: exit(0) with open(sys.argv[2], "r", encoding="utf-8") as fd: conf = yaml.load(fd, Loader=yaml.Loader) if "variables" not in conf: conf["variables"] = {} conf["variables"].update(dotenv) with open(sys.argv[2], "w", encoding="utf-8") as fd: yaml.dump(conf, fd, Dumper=yaml.Dumper) ================================================ FILE: .ci/gitlab-ci.yml ================================================ # Gitlab configuraiton for spack/spack stages: - packages variables: SPACK_PACKAGES_CHECKOUT_VERSION: develop .clone_packages: &clone_packages - mkdir -p ${REPO_DESTINATION} - cd ${REPO_DESTINATION} - git init - git remote add origin https://github.com/spack/spack-packages.git - git fetch --depth 1 origin ${SPACK_PACKAGES_CHECKOUT_VERSION} - git checkout FETCH_HEAD - cd - dotenv: stage: .pre image: ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18 tags: [ spack, service ] script: - export REPO_DESTINATION=etc/spack-packages - *clone_packages - repo_commit=$(git -C ${REPO_DESTINATION} rev-parse FETCH_HEAD) - echo "SPACK_CHECKOUT_VERSION=${repo_commit}" >> ${CI_PROJECT_DIR}/env - echo "SPACK_CHECKOUT_REPO=spack/spack-packages" >> ${CI_PROJECT_DIR}/env - cat ${CI_PROJECT_DIR}/env - python3 ${CI_PROJECT_DIR}/.ci/gitlab/forward_dotenv_variables.py ${CI_PROJECT_DIR}/env ${REPO_DESTINATION}/.ci/gitlab/.gitlab-ci.yml artifacts: paths: - etc/spack-packages/.ci/gitlab/.gitlab-ci.yml spack-packages: stage: packages trigger: strategy: depend include: - artifact: etc/spack-packages/.ci/gitlab/.gitlab-ci.yml job: dotenv ================================================ FILE: .codecov.yml ================================================ coverage: precision: 2 round: nearest range: 60...90 status: project: default: threshold: 2.0% ignore: - lib/spack/spack/test/.* - lib/spack/docs/.* - lib/spack/spack/vendor/.* - share/spack/qa/.* comment: off # Inline codecov annotations make the code hard to read, and they add # annotations in files that seemingly have nothing to do with the PR. github_checks: annotations: false # Attempt to fix "Missing base commit" messages in the codecov UI. # Because we do not run full tests on package PRs, package PRs' merge # commits on `develop` don't have coverage info. It appears that # codecov will give you an error if the pseudo-base's coverage data # doesn't all apply properly to the real PR base. # # See here for docs: # https://docs.codecov.com/docs/comparing-commits#pseudo-comparison # See here for another potential solution: # https://community.codecov.com/t/2480/15 codecov: allow_coverage_offsets: true ================================================ FILE: .devcontainer/postCreateCommand.sh ================================================ #!/bin/bash # Load spack environment at terminal startup cat <> /root/.bashrc . /workspaces/spack/share/spack/setup-env.sh EOF # Load spack environment in this script . /workspaces/spack/share/spack/setup-env.sh # Ensure generic targets for maximum matching with buildcaches spack config --scope site add "packages:all:require:[target=x86_64_v3]" spack config --scope site add "concretizer:targets:granularity:generic" # Find compiler and install gcc-runtime spack compiler find --scope site # Setup buildcaches spack mirror add --scope site develop https://binaries.spack.io/develop spack buildcache keys --install --trust ================================================ FILE: .devcontainer/ubuntu20.04/devcontainer.json ================================================ { "name": "Ubuntu 20.04", "image": "ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01", "postCreateCommand": "./.devcontainer/postCreateCommand.sh" } ================================================ FILE: .devcontainer/ubuntu22.04/devcontainer.json ================================================ { "name": "Ubuntu 22.04", "image": "ghcr.io/spack/ubuntu-22.04:v2024-05-07", "postCreateCommand": "./.devcontainer/postCreateCommand.sh" } ================================================ FILE: .dockerignore ================================================ .git/* opt/spack/* /etc/spack/* !/etc/spack/defaults share/spack/dotkit/* share/spack/lmod/* share/spack/modules/* lib/spack/spack/test/* var/spack/cache/* ================================================ FILE: .flake8 ================================================ # -*- conf -*- # flake8 settings for Spack. # # These exceptions are for Spack core files. We're slightly more lenient # with packages. See .flake8_packages for that. # # This is the only flake8 rule Spack violates somewhat flagrantly # - E731: do not assign a lambda expression, use a def # # This is the only flake8 exception needed when using Black. # - E203: white space around slice operators can be required, ignore : warn # # We still allow these in packages (Would like to get rid of them or rely on mypy # in the future) # - F403: from/import * used; unable to detect undefined names # - F405: undefined name or from * # - F821: undefined name (needed with from/import *) # [flake8] #ignore = E129,,W503,W504,F999,N801,N813,N814,F403,F405,E203 extend-ignore = E731,E203 max-line-length = 99 # F4: Import # - F405: `name` may be undefined, or undefined from star imports: `module` # # F8: Name # - F821: undefined name `name` # per-file-ignores = var/spack/*/package.py:F403,F405,F821 *-ci-package.py:F403,F405,F821 # exclude things we usually do not want linting for. # These still get linted when passed explicitly, as when spack flake8 passes # them on the command line. exclude = .git etc/ opt/ share/ var/spack/cache/ var/spack/gpg*/ var/spack/junit-report/ var/spack/mock-configs/ lib/spack/spack/vendor/ __pycache__ var format = spack [flake8:local-plugins] report = spack = flake8_formatter:SpackFormatter paths = ./share/spack/qa/ ================================================ FILE: .git-blame-ignore-revs ================================================ # .git-blame-ignore-revs # Formatted entire codebase with black 23 603569e321013a1a63a637813c94c2834d0a0023 # Formatted entire codebase with black 22 f52f6e99dbf1131886a80112b8c79dfc414afb7c # Formatted all rst files 1377d42c16c6912faa77259c0a1f665210ccfd85 ================================================ FILE: .gitattributes ================================================ *.bat text eol=crlf *.py diff=python *.py text eol=lf lib/spack/spack/vendor/* linguist-vendored ================================================ FILE: .github/CODE_OF_CONDUCT.md ================================================ # Spack Community Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the Spack project or its community. Examples of representing the project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of the project may be further defined and clarified by Spack maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maintainers@spack.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ ================================================ FILE: .github/CONTRIBUTING.md ================================================ # Contributing to Spack All contributions to Spack must be made under both the Apache License, Version 2.0 (Apache-2.0) and the MIT license (MIT). Before contributing to Spack, you should read the [Contribution Guide](https://spack.readthedocs.io/en/latest/contribution_guide.html), which is maintained as part of Spack's documentation. ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yml ================================================ name: "\U0001F41E Bug report" description: Report a bug in the core of Spack (command not working as expected, etc.) labels: [bug, triage] body: - type: textarea id: reproduce attributes: label: Steps to reproduce description: | Explain, in a clear and concise way, the command you ran and the result you were trying to achieve. Example: "I ran `spack find` to list all the installed packages and ..." placeholder: | ```console $ spack $ spack ... ``` validations: required: true - type: textarea id: error attributes: label: Error message description: | If Spack reported an error, provide the error message. If it did not report an error but the output appears incorrect, provide the incorrect output. If there was no error message and no output but the result is incorrect, describe how it does not match what you expect. placeholder: | ```console $ spack --debug --stacktrace ``` - type: textarea id: information attributes: label: Information on your system description: Please include the output of `spack debug report` validations: required: true - type: markdown attributes: value: | If you have any relevant configuration detail (custom `packages.yaml` or `modules.yaml`, etc.) you can add that here as well. - type: checkboxes id: checks attributes: label: General information options: - label: I have run `spack debug report` and reported the version of Spack/Python/Platform required: true - label: I have searched the issues of this repo and believe this is not a duplicate required: true - label: I have run the failing commands in debug mode and reported the output required: true - type: markdown attributes: value: | We encourage you to try, as much as possible, to reduce your problem to the minimal example that still reproduces the issue. That would help us a lot in fixing it quickly and effectively! If you want to ask a question about the tool (how to use it, what it can currently do, etc.), try the `#general` channel on [our Slack](https://slack.spack.io/) first. We have a welcoming community and chances are you'll get your reply faster and without opening an issue. Other than that, thanks for taking the time to contribute to Spack! ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: true contact_links: - name: "\U0001F4A5 Package build error" url: https://github.com/spack/spack-packages/issues/new?template=build_error.yml about: Report installation issues in the spack/spack-packages repository - name: "\U0001F4A5 Package test error" url: https://github.com/spack/spack-packages/issues/new?template=test_error.yml about: Report standalone package test issues in the spack/spack-packages repository ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.yml ================================================ name: "\U0001F38A Feature request" description: Suggest adding a feature that is not yet in Spack labels: [feature] body: - type: textarea id: summary attributes: label: Summary description: Please add a concise summary of your suggestion here. validations: required: true - type: textarea id: rationale attributes: label: Rationale description: Is your feature request related to a problem? Please describe it! - type: textarea id: description attributes: label: Description description: Describe the solution you'd like and the alternatives you have considered. - type: textarea id: additional_information attributes: label: Additional information description: Add any other context about the feature request here. - type: checkboxes id: checks attributes: label: General information options: - label: I have searched the issues of this repo and believe this is not a duplicate required: true - type: markdown attributes: value: | If you want to ask a question about the tool (how to use it, what it can currently do, etc.), try the `#general` channel on [our Slack](https://slack.spack.io/) first. We have a welcoming community and chances are you'll get your reply faster and without opening an issue. Other than that, thanks for taking the time to contribute to Spack! ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" - package-ecosystem: "pip" directories: - "/.github/workflows/requirements/coverage" - "/.github/workflows/requirements/style" - "/.github/workflows/requirements/unit_tests" - "/lib/spack/docs" schedule: interval: "daily" ================================================ FILE: .github/labeler.yml ================================================ bootstrap: - changed-files: - any-glob-to-any-file: lib/spack/spack/bootstrap/** binary-caches: - changed-files: - any-glob-to-any-file: lib/spack/spack/binary_distribution.py - any-glob-to-any-file: lib/spack/spack/cmd/buildcache.py ci: - changed-files: - any-glob-to-any-file: .ci/** - any-glob-to-any-file: .github/** - any-glob-to-any-file: lib/spack/spack/ci/** commands: - changed-files: - any-glob-to-any-file: lib/spack/spack/cmd/** config: - changed-files: - any-glob-to-any-file: etc/spack/** - any-glob-to-any-file: lib/spack/spack/cmd/config.py - any-glob-to-any-file: lib/spack/spack/config.py - any-glob-to-any-file: lib/spack/spack/schema/** docs: - changed-files: - any-glob-to-any-file: .readthedocs.yml - any-glob-to-any-file: lib/spack/docs/** environments: - changed-files: - any-glob-to-any-file: lib/spack/spack/cmd/env.py - any-glob-to-any-file: lib/spack/spack/environment/** mirrors: - changed-files: - any-glob-to-any-file: lib/spack/spack/cmd/mirror.py - any-glob-to-any-file: lib/spack/spack/mirrors/** modules: - changed-files: - any-glob-to-any-file: lib/spack/spack/cmd/module.py - any-glob-to-any-file: lib/spack/spack/modules/** solver: - changed-files: - any-glob-to-any-file: lib/spack/spack/solver/** style: - changed-files: - any-glob-to-any-file: .flake8 - any-glob-to-any-file: .github/workflows/prechecks.yml - any-glob-to-any-file: .github/workflows/requirements/style/** - any-glob-to-any-file: lib/spack/spack/cmd/style.py - any-glob-to-any-file: pyproject.toml unit-tests: - changed-files: - any-glob-to-any-file: .codecov.yml - any-glob-to-any-file: lib/spack/spack/cmd/unit_test.py - any-glob-to-any-file: lib/spack/spack/test/** - any-glob-to-any-file: pyproject.toml - any-glob-to-any-file: pytest.ini - any-glob-to-any-file: var/spack/test_repos/** vendor: - changed-files: - any-glob-to-any-file: lib/spack/spack/vendor/** versions: - changed-files: - any-glob-to-any-file: lib/spack/spack/version/** ================================================ FILE: .github/pull_request_template.md ================================================ ================================================ FILE: .github/workflows/bin/canonicalize.py ================================================ #!/usr/bin/env python3 # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import ast import os import subprocess import sys from itertools import product from typing import List def run_git_command(*args: str, dir: str) -> None: """Run a git command in the output directory.""" subprocess.run( [ "git", "-c", "user.email=example@example.com", "-c", "user.name=Example", "-c", "init.defaultBranch=main", "-c", "color.ui=always", "-C", dir, *args, ], check=True, stdout=sys.stdout, stderr=sys.stderr, ) def run(root: str, output_dir: str) -> None: """Recurse over a directory and canonicalize all Python files.""" from spack.util.package_hash import RemoveDocstrings, unparse count = 0 stack = [root] while stack: current = stack.pop() for entry in os.scandir(current): if entry.is_dir(follow_symlinks=False): stack.append(entry.path) elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".py"): try: with open(entry.path, "r") as f: src = f.read() except OSError: continue canonical_dir = os.path.join(output_dir, os.path.relpath(current, root)) os.makedirs(canonical_dir, exist_ok=True) with open(os.path.join(canonical_dir, entry.name), "w") as f: f.write( unparse(RemoveDocstrings().visit(ast.parse(src)), py_ver_consistent=True) ) count += 1 assert count > 0, "No Python files found in the specified directory." def compare( input_dir: str, output_dir: str, python_versions: List[str], spack_versions: List[str] ) -> None: """Compare canonicalized files across different Python versions and error if they differ.""" # Create a git repo in output_dir to track changes os.makedirs(output_dir, exist_ok=True) run_git_command("init", dir=output_dir) pairs = list(product(spack_versions, python_versions)) if len(pairs) < 2: raise ValueError("At least two Python or two Spack versions must be given for comparison.") changes_with_previous: List[int] = [] for i, (spack_dir, python_exe) in enumerate(pairs): print(f"\033[1;97mCanonicalizing with {python_exe} and {spack_dir}...\033[0m", flush=True) # Point PYTHONPATH to the given Spack library for the subprocess if not os.path.isdir(spack_dir): raise ValueError(f"Invalid Spack dir: {spack_dir}") env = os.environ.copy() spack_pythonpath = os.path.join(spack_dir, "lib", "spack") if "PYTHONPATH" in env and env["PYTHONPATH"]: env["PYTHONPATH"] = f"{spack_pythonpath}{os.pathsep}{env['PYTHONPATH']}" else: env["PYTHONPATH"] = spack_pythonpath subprocess.run( [python_exe, __file__, "--run", "--input-dir", input_dir, "--output-dir", output_dir], check=True, stdout=sys.stdout, stderr=sys.stderr, env=env, ) if i > 0: try: run_git_command("diff", "--exit-code", "HEAD", dir=output_dir) except subprocess.CalledProcessError: changes_with_previous.append(i) # The first run creates a commit for reference run_git_command("add", ".", dir=output_dir) run_git_command( "commit", "--quiet", "--allow-empty", # makes this idempotent when running locally "-m", f"Canonicalized with {python_exe} and {spack_dir}", dir=output_dir, ) for i in changes_with_previous: previous_spack, previous_python = pairs[i - 1] current_spack, current_python = pairs[i] print( f"\033[1;31mChanges detected between {previous_python} ({previous_spack}) and " f"{current_python} ({current_spack})\033[0m" ) if changes_with_previous: exit(1) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Canonicalize Spack package files.") parser.add_argument("--run", action="store_true", help="Generate canonicalized sources.") parser.add_argument("--spack", nargs="+", help="Specify one or more Spack versions.") parser.add_argument("--python", nargs="+", help="Specify one or more Python versions.") parser.add_argument("--input-dir", type=str, required=True, help="A repo's packages dir.") parser.add_argument( "--output-dir", type=str, required=True, help="The output directory for canonicalized package files.", ) args = parser.parse_args() if args.run: run(args.input_dir, args.output_dir) else: compare(args.input_dir, args.output_dir, args.python, args.spack) ================================================ FILE: .github/workflows/bin/execute_installer.ps1 ================================================ $ proc = Start-Process ${{ env.spack_installer }}\spack.exe "/install /quiet" -Passthru $handle = $proc.Handle # cache proc.Handle $proc.WaitForExit(); if ($proc.ExitCode -ne 0) { Write-Warning "$_ exited with status code $($proc.ExitCode)" } ================================================ FILE: .github/workflows/bin/format-rst.py ================================================ #!/usr/bin/env python3 # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This script formats reStructuredText files to ensure one sentence per line and no trailing whitespace. It exits with a non-zero status if any files were modified.""" import difflib import importlib import io import json import os import re import subprocess import sys from typing import List import black from docutils import nodes from docutils.core import publish_doctree from docutils.parsers.rst import Directive, directives from ruamel.yaml import YAML from spack.vendor import jsonschema import spack.schema #: Map Spack config sections to their corresponding JSON schema SECTION_AND_SCHEMA = [ # The first property's key is the config section name (next(iter(m.schema["properties"])), m.schema) # Dynamically load all modules in spack.schema to be future-proof for m in ( importlib.import_module(f"spack.schema.{f[:-3]}") for f in os.listdir(os.path.dirname(spack.schema.__file__)) if f.endswith(".py") and f != "__init__.py" ) if hasattr(m, "schema") and len(m.schema.get("properties", {})) == 1 ] assert SECTION_AND_SCHEMA, "no schemas found" END_OF_SENTENCE = re.compile( r""" ( (?: (? str: return f"\033[1;33mwarning:\033[0m {msg}" class Warning: def __init__(self, path: str, line: int, message: str) -> None: self.path = path self.line = line self.message = message def __str__(self) -> str: return _warning(f"{self.path}:{self.line}: {self.message}") class CodeBlockWarning(Warning): def __init__(self, path: str, line: int, message: str, diff: str): super().__init__(path, line, f"{message}\n{diff}") def __str__(self) -> str: return _warning(f"{self.path}:{self.line}: {self.message}") class ValidationWarning(Warning): pass class SphinxCodeBlock(Directive): """Defines a code-block directive with the options Sphinx supports.""" has_content = True optional_arguments = 1 # language required_arguments = 0 option_spec = { "force": directives.unchanged, "linenos": directives.unchanged, "dedent": directives.unchanged, "lineno-start": directives.unchanged, "emphasize-lines": directives.unchanged, "caption": directives.unchanged, "class": directives.unchanged, "name": directives.unchanged, } def run(self) -> List[nodes.Node]: # Produce a literal block with block.attributes["language"] set. language = self.arguments[0] if self.arguments else "python" literal = nodes.literal_block("\n".join(self.content), "\n".join(self.content)) literal["language"] = language return [literal] directives.register_directive("code-block", SphinxCodeBlock) class ParagraphInfo: lineno: int end_lineno: int src: str lines: List[str] def __init__(self, line: int, src: str) -> None: self.lineno = line self.src = src self.lines = src.splitlines() self.end_lineno = line + len(self.lines) - 1 def _is_node_in_table(node: nodes.Node) -> bool: """Check if a node is inside a table by walking up the parent chain.""" while node.parent: node = node.parent if isinstance(node, nodes.table): return True return False def _validate_schema(data: object) -> None: if not isinstance(data, dict): return for section, schema in SECTION_AND_SCHEMA: if section in data: jsonschema.validate(data, schema) def _format_code_blocks(document: nodes.document, path: str) -> List[Warning]: """Try to parse and format Python, YAML, and JSON code blocks. This does *not* update the sources, but collects issues for later reporting. Returns a list of warnings.""" issues: List[Warning] = [] for code_block in document.findall(nodes.literal_block): language = code_block.attributes.get("language", "") if language not in ("python", "yaml", "json"): continue original = code_block.astext() line = code_block.line if code_block.line else 0 possible_config_data = None try: if language == "python": formatted = black.format_str(original, mode=black.FileMode(line_length=99)) elif language == "yaml": yaml = YAML(pure=True) yaml.width = 10000 # do not wrap lines yaml.preserve_quotes = True # do not force particular quotes buf = io.BytesIO() possible_config_data = yaml.load(original) yaml.dump(possible_config_data, buf) formatted = buf.getvalue().decode("utf-8") elif language == "json": formatted = json.dumps(json.loads(original), indent=2) else: assert False except Exception as e: issues.append(Warning(path, line, f"formatting failed: {e}: {original!r}")) continue try: _validate_schema(possible_config_data) except jsonschema.ValidationError as e: issues.append(ValidationWarning(path, line, f"schema validation failed: {e.message}")) if formatted == original: continue diff = "\n".join( difflib.unified_diff( original.splitlines(), formatted.splitlines(), lineterm="", fromfile=f"{path}:{line} (original)", tofile=f"{path}:{line} (suggested, NOT required)", ) ) # ignore suggestions to quote double colons like this: # # - build_stage:: # + 'build_stage:': # if diff and not DOUBLE_COLON_WARNING.search(diff): issues.append(CodeBlockWarning(path, line, "formatting suggested:", diff)) return issues def _format_paragraphs(document: nodes.document, path: str, src_lines: List[str]) -> bool: """Format paragraphs in the document. Returns True if ``src_lines`` was modified.""" paragraphs = [ ParagraphInfo(line=p.line, src=p.rawsource) for p in document.findall(nodes.paragraph) if p.line is not None and p.rawsource and not _is_node_in_table(p) ] # Work from bottom to top to avoid messing up line numbers paragraphs.sort(key=lambda p: p.lineno, reverse=True) modified = False for p in paragraphs: # docutils does not give us the column offset, so we'll find it ourselves. col_offset = src_lines[p.lineno - 1].rfind(p.lines[0]) assert col_offset >= 0, f"{path}:{p.lineno}: rst parsing error." prefix = lambda i: " " * col_offset if i > 0 else src_lines[p.lineno - 1][:col_offset] # Defensive check to ensure the source paragraph matches the docutils paragraph for i, line in enumerate(p.lines): line_lhs = f"{prefix(i)}{line}" line_rhs = src_lines[p.lineno - 1 + i].rstrip() # docutils trims trailing whitespace assert line_lhs == line_rhs, f"{path}:{p.lineno + i}: rst parsing error." # Replace current newlines with whitespace, and then split sentences. new_paragraph_src = END_OF_SENTENCE.sub(r"\1\n", p.src.replace("\n", " ")) new_paragraph_lines = [ f"{prefix(i)}{line.lstrip()}" for i, line in enumerate(new_paragraph_src.splitlines()) ] if new_paragraph_lines != src_lines[p.lineno - 1 : p.end_lineno]: modified = True src_lines[p.lineno - 1 : p.end_lineno] = new_paragraph_lines return modified def reformat_rst_file(path: str, warnings: List[Warning]) -> bool: """Reformat a reStructuredText file "in-place". Returns True if modified, False otherwise.""" with open(path, "r", encoding="utf-8") as f: src = f.read() src_lines = src.splitlines() document: nodes.document = publish_doctree(src, settings_overrides=DOCUTILS_SETTING) warnings.extend(_format_code_blocks(document, path)) if not _format_paragraphs(document, path, src_lines): return False with open(f"{path}.tmp", "w", encoding="utf-8") as f: f.write("\n".join(src_lines)) f.write("\n") os.rename(f"{path}.tmp", path) print(f"Fixed reStructuredText formatting: {path}", flush=True) return True def main(*files: str) -> None: modified = False warnings: List[Warning] = [] for f in files: modified |= reformat_rst_file(f, warnings) if modified: subprocess.run(["git", "--no-pager", "diff", "--color=always", "--", *files]) for warning in sorted(warnings, key=lambda w: isinstance(w, ValidationWarning)): print(warning, flush=True, file=sys.stderr) if warnings: print( _warning(f"completed with {len(warnings)} potential issues"), flush=True, file=sys.stderr, ) sys.exit(1 if modified else 0) if __name__ == "__main__": main(*sys.argv[1:]) ================================================ FILE: .github/workflows/bin/generate_spack_yaml_containerize.sh ================================================ #!/bin/bash (echo "spack:" \ && echo " specs: []" \ && echo " container:" \ && echo " format: docker" \ && echo " images:" \ && echo " os: \"${SPACK_YAML_OS}\"" \ && echo " spack:" \ && echo " ref: ${GITHUB_REF}") > spack.yaml ================================================ FILE: .github/workflows/bin/setup_git.ps1 ================================================ git config --global user.email "spack@example.com" git config --global user.name "Test User" git config --global core.longpaths true if ($(git branch --show-current) -ne "develop") { git branch develop origin/develop } ================================================ FILE: .github/workflows/bin/setup_git.sh ================================================ #!/bin/bash -e git config --global user.email "spack@example.com" git config --global user.name "Test User" # create a local pr base branch if [[ -n $GITHUB_BASE_REF ]]; then git fetch origin "${GITHUB_BASE_REF}:${GITHUB_BASE_REF}" fi ================================================ FILE: .github/workflows/bin/system_shortcut_check.ps1 ================================================ param ($systemFolder, $shortcut) $start = [System.Environment]::GetFolderPath("$systemFolder") Invoke-Item "$start\Programs\Spack\$shortcut" ================================================ FILE: .github/workflows/bootstrap.yml ================================================ name: Bootstrapping on: # This Workflow can be triggered manually workflow_dispatch: workflow_call: schedule: # nightly at 2:16 AM - cron: "16 2 * * *" concurrency: group: bootstrap-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: distros-clingo-sources: if: github.repository == 'spack/spack' runs-on: ubuntu-latest container: ${{ matrix.image }} strategy: matrix: image: ["fedora:latest", "opensuse/leap:latest"] steps: - name: Setup Fedora if: ${{ matrix.image == 'fedora:latest' }} run: | dnf install -y \ bzip2 curl file gcc-c++ gcc gcc-gfortran git gzip \ make patch unzip which xz python3 python3-devel tree \ cmake bison bison-devel libstdc++-static gawk - name: Setup OpenSUSE if: ${{ matrix.image == 'opensuse/leap:latest' }} run: | # Harden CI by applying the workaround described here: https://www.suse.com/support/kb/doc/?id=000019505 zypper update -y || zypper update -y zypper install -y \ bzip2 curl file gcc-c++ gcc gcc-fortran tar git gpg2 gzip \ make patch unzip which xz python3 python3-devel tree \ cmake bison - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Bootstrap clingo run: | . share/spack/setup-env.sh spack config add config:installer:new spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack solve zlib tree ~/.spack/bootstrap/store/ clingo-sources: if: github.repository == 'spack/spack' runs-on: ${{ matrix.runner }} strategy: matrix: runner: ["macos-15-intel", "macos-latest", "ubuntu-latest"] steps: - name: Setup macOS if: ${{ matrix.runner != 'ubuntu-latest' }} run: brew install bison tree - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - name: Bootstrap clingo run: | . share/spack/setup-env.sh spack config add config:installer:new spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 export PATH="$(brew --prefix bison)/bin:$(brew --prefix cmake)/bin:$PATH" spack solve zlib tree ~/.spack/bootstrap/store/ gnupg-sources: if: github.repository == 'spack/spack' runs-on: ${{ matrix.runner }} strategy: matrix: runner: ["macos-15-intel", "macos-latest", "ubuntu-latest"] steps: - name: Setup macOS if: ${{ matrix.runner != 'ubuntu-latest' }} run: brew install tree gawk - name: Remove system executables run: | while [ -n "$(command -v gpg gpg2 patchelf)" ]; do sudo rm $(command -v gpg gpg2 patchelf) done - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Bootstrap GnuPG run: | . share/spack/setup-env.sh spack config add config:installer:new spack solve zlib spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack gpg list tree ~/.spack/bootstrap/store/ from-binaries: if: github.repository == 'spack/spack' runs-on: ${{ matrix.runner }} strategy: matrix: runner: ["macos-15-intel", "macos-latest", "ubuntu-latest"] steps: - name: Setup macOS if: ${{ matrix.runner != 'ubuntu-latest' }} run: brew install tree - name: Remove system executables run: | while [ -n "$(command -v gpg gpg2 patchelf)" ]; do sudo rm $(command -v gpg gpg2 patchelf) done - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: | 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - name: Set bootstrap sources run: | . share/spack/setup-env.sh spack bootstrap disable github-actions-v0.6 spack bootstrap disable spack-install - name: Bootstrap clingo run: | . share/spack/setup-env.sh for ver in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do ver_dir="$(find "$RUNNER_TOOL_CACHE/Python" -wholename "*/${ver}.*/*/bin" | grep . || true)" export SPACK_PYTHON="$ver_dir/python3" if [ ! -d "$ver_dir" ] || ! "$SPACK_PYTHON" --version; then echo "Python $ver not found" exit 1 fi spack solve zlib done tree ~/.spack/bootstrap/store - name: Bootstrap GnuPG run: | . share/spack/setup-env.sh spack config add config:installer:new spack gpg list tree ~/.spack/bootstrap/store/ windows: if: github.repository == 'spack/spack' runs-on: "windows-latest" steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - name: Setup Windows run: | Remove-Item -Path (Get-Command gpg).Path Remove-Item -Path (Get-Command file).Path - name: Bootstrap clingo run: | ./share/spack/setup-env.ps1 spack bootstrap disable github-actions-v2 spack bootstrap disable github-actions-v0.6 spack -d solve zlib ./share/spack/qa/validate_last_exit.ps1 tree $env:userprofile/.spack/bootstrap/store/ - name: Bootstrap GnuPG run: | ./share/spack/setup-env.ps1 spack -d gpg list ./share/spack/qa/validate_last_exit.ps1 tree $env:userprofile/.spack/bootstrap/store/ dev-bootstrap: runs-on: ubuntu-latest container: registry.access.redhat.com/ubi8/ubi steps: - name: Install dependencies run: | dnf install -y \ bzip2 curl gcc-c++ gcc gcc-gfortran git gnupg2 gzip \ make patch python3.11 tcl unzip which xz - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup repo and non-root user run: | git --version git config --global --add safe.directory '*' git fetch --unshallow . .github/workflows/bin/setup_git.sh - name: Setup a virtual environment with platform-python run: | python3.11 -m venv ~/platform-spack-311 source ~/platform-spack-311/bin/activate pip install --upgrade pip clingo - name: Bootstrap Spack development environment run: | source ~/platform-spack-311/bin/activate source share/spack/setup-env.sh spack debug report spack -d bootstrap now --dev ================================================ FILE: .github/workflows/build-containers.yml ================================================ name: Containers on: # This Workflow can be triggered manually workflow_dispatch: # Build new Spack develop containers nightly. schedule: - cron: '34 0 * * *' # Run on pull requests that modify this file pull_request: branches: - develop paths: - '.github/workflows/build-containers.yml' - 'share/spack/docker/*' - 'share/spack/templates/container/*' - 'lib/spack/spack/container/*' # Let's also build & tag Spack containers on releases. release: types: [published] concurrency: group: build_containers-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: deploy-images: runs-on: ubuntu-latest permissions: packages: write strategy: # Even if one container fails to build we still want the others # to continue their builds. fail-fast: false # A matrix of Dockerfile paths, associated tags, and which architectures # they support. matrix: # Meaning of the various items in the matrix list # 0: Container name (e.g. ubuntu-bionic) # 1: Platforms to build for # 2: Base image (e.g. ubuntu:22.04) dockerfile: [[amazon-linux, 'linux/amd64,linux/arm64', 'amazonlinux:2'], [centos-stream9, 'linux/amd64,linux/arm64', 'centos:stream9'], [leap15, 'linux/amd64,linux/arm64', 'opensuse/leap:15'], [ubuntu-focal, 'linux/amd64,linux/arm64', 'ubuntu:20.04'], [ubuntu-jammy, 'linux/amd64,linux/arm64', 'ubuntu:22.04'], [ubuntu-noble, 'linux/amd64,linux/arm64', 'ubuntu:24.04'], [almalinux8, 'linux/amd64,linux/arm64', 'almalinux:8'], [almalinux9, 'linux/amd64,linux/arm64', 'almalinux:9'], [rockylinux8, 'linux/amd64,linux/arm64', 'rockylinux:8'], [rockylinux9, 'linux/amd64,linux/arm64', 'rockylinux:9'], [fedora39, 'linux/amd64,linux/arm64', 'fedora:39'], [fedora40, 'linux/amd64,linux/arm64', 'fedora:40']] name: Build ${{ matrix.dockerfile[0] }} if: github.repository == 'spack/spack' steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Determine latest release tag id: latest run: | git fetch --quiet --tags echo "tag=$(git tag --list --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1)" | tee -a $GITHUB_OUTPUT - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 id: docker_meta with: images: | ghcr.io/${{ github.repository_owner }}/${{ matrix.dockerfile[0] }} ${{ github.repository_owner }}/${{ matrix.dockerfile[0] }} tags: | type=schedule,pattern=nightly type=schedule,pattern=develop type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} type=ref,event=branch type=ref,event=pr type=raw,value=latest,enable=${{ github.ref == format('refs/tags/{0}', steps.latest.outputs.tag) }} - name: Generate the Dockerfile env: SPACK_YAML_OS: "${{ matrix.dockerfile[2] }}" run: | .github/workflows/bin/generate_spack_yaml_containerize.sh . share/spack/setup-env.sh mkdir -p dockerfiles/${{ matrix.dockerfile[0] }} spack containerize --last-stage=bootstrap | tee dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile printf "Preparing to build ${{ env.container }} from dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile" if [ ! -f "dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile" ]; then printf "dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile does not exist" exit 1; fi - name: Upload Dockerfile uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dockerfiles_${{ matrix.dockerfile[0] }} path: dockerfiles - name: Set up QEMU uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to GitHub Container Registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to DockerHub if: github.event_name != 'pull_request' uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build & Deploy ${{ matrix.dockerfile[0] }} uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: context: dockerfiles/${{ matrix.dockerfile[0] }} platforms: ${{ matrix.dockerfile[1] }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} merge-dockerfiles: runs-on: ubuntu-latest needs: deploy-images steps: - name: Merge Artifacts uses: actions/upload-artifact/merge@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dockerfiles pattern: dockerfiles_* delete-merged: true ================================================ FILE: .github/workflows/ci.yaml ================================================ name: ci on: push: branches: - develop - releases/** pull_request: branches: - develop - releases/** merge_group: concurrency: group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: # Check which files have been updated by the PR changes: runs-on: ubuntu-latest # Set job outputs to values from filter step outputs: bootstrap: ${{ steps.filter.outputs.bootstrap }} core: ${{ steps.filter.outputs.core }} packages: ${{ steps.filter.outputs.packages }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }} with: fetch-depth: 0 # For pull requests it's not necessary to checkout the code - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: # For merge group events, compare against the target branch (main) base: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_ref || '' }} # For merge group events, use the merge group head ref ref: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.ref }} # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below # Don't run if we only modified packages in the # built-in repository or documentation filters: | bootstrap: - 'lib/spack/**' - 'share/spack/**' - '.github/workflows/bootstrap.yml' - '.github/workflows/ci.yaml' core: - './!(var/**)/**' - 'var/spack/test_repos/**' packages: - 'var/**' # Some links for easier reference: # # "github" context: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context # job outputs: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs # setting environment variables from earlier steps: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable # bootstrap: if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.bootstrap == 'true' }} needs: [ prechecks, changes ] uses: ./.github/workflows/bootstrap.yml secrets: inherit unit-tests: if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }} needs: [ prechecks, changes ] uses: ./.github/workflows/unit_tests.yaml secrets: inherit prechecks: needs: [ changes ] uses: ./.github/workflows/prechecks.yml secrets: inherit with: with_coverage: ${{ needs.changes.outputs.core }} with_packages: ${{ needs.changes.outputs.packages }} import-check: needs: [ changes ] uses: ./.github/workflows/import-check.yaml all-prechecks: needs: [ prechecks ] if: ${{ always() }} runs-on: ubuntu-latest steps: - name: Success run: | [ "${{ needs.prechecks.result }}" = "success" ] && exit 0 [ "${{ needs.prechecks.result }}" = "skipped" ] && exit 0 echo "Unit tests failed." exit 1 coverage: needs: [ unit-tests, prechecks ] if: ${{ needs.changes.outputs.core }} uses: ./.github/workflows/coverage.yml secrets: inherit all: needs: [ unit-tests, coverage, bootstrap ] if: ${{ always() }} runs-on: ubuntu-latest # See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/accessing-contextual-information-about-workflow-runs#needs-context steps: - name: Status summary run: | if [ "${{ needs.unit-tests.result }}" = "success" ] || [ "${{ needs.unit-tests.result }}" = "skipped" ]; then if [ "${{ needs.bootstrap.result }}" = "success" ] || [ "${{ needs.bootstrap.result }}" = "skipped" ]; then exit 0 else echo "Bootstrap tests failed." exit 1 fi else echo "Unit tests failed." exit 1 fi ================================================ FILE: .github/workflows/coverage.yml ================================================ name: coverage on: workflow_call: jobs: # Upload coverage reports to codecov once as a single bundle upload: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.14' - name: Install python dependencies run: pip install -r .github/workflows/requirements/coverage/requirements.txt - name: Download coverage artifact files uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: coverage-* path: coverage merge-multiple: true - run: ls -la coverage - run: coverage combine -a coverage/.coverage* - run: coverage xml - name: "Upload coverage report to CodeCov" uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 with: verbose: true fail_ci_if_error: false token: ${{ secrets.CODECOV_TOKEN }} ================================================ FILE: .github/workflows/import-check.yaml ================================================ name: import-check on: workflow_call: jobs: # Check we don't make the situation with circular imports worse import-check: continue-on-error: true runs-on: ubuntu-latest steps: - uses: julia-actions/setup-julia@4a12c5f801ca5ef0458bba44687563ef276522dd # v3.0.0 with: version: '1.10' - uses: julia-actions/cache@9a93c5fb3e9c1c20b60fc80a478cae53e38618a4 # v3.0.2 # PR: use the base of the PR as the old commit - name: Checkout PR base commit if: github.event_name == 'pull_request' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.pull_request.base.sha }} path: old # not a PR: use the previous commit as the old commit - name: Checkout previous commit if: github.event_name != 'pull_request' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 2 path: old - name: Checkout previous commit if: github.event_name != 'pull_request' run: git -C old reset --hard HEAD^ - name: Checkout new commit uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: new - name: Install circular import checker uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: haampie/circular-import-fighter ref: f1c56367833f3c82f6a85dc58595b2cd7995ad48 path: circular-import-fighter - name: Install dependencies working-directory: circular-import-fighter run: make -j dependencies - name: Circular import check (without inline imports) working-directory: circular-import-fighter run: make -j compare "SPACK_ROOT=../old ../new" - name: Circular import check (with inline imports) working-directory: circular-import-fighter run: make clean-graph && make -j compare "SPACK_ROOT=../old ../new" IMPORTS_FLAGS=--inline ================================================ FILE: .github/workflows/prechecks.yml ================================================ name: prechecks on: workflow_call: inputs: with_coverage: required: true type: string with_packages: required: true type: string concurrency: group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: validate: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.13' - name: Install Python Packages run: | pip install -r .github/workflows/requirements/style/requirements.txt # Validate that the code can be run on all the Python versions supported by Spack - name: vermin run: | vermin --backport importlib \ --backport argparse \ --violations \ --backport typing \ -t=3.6- \ -vvv \ --exclude-regex lib/spack/spack/vendor \ lib/spack/spack/ lib/spack/llnl/ bin/ var/spack/test_repos # Check that __slots__ are used properly - name: slotscheck run: | ./bin/spack python -m slotscheck --exclude-modules="spack.test|spack.vendor" lib/spack/spack/ # Run style checks on the files that have been changed style: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 2 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.13' - name: Install Python packages run: | pip install -r .github/workflows/requirements/style/requirements.txt echo "PYTHONPATH=$PWD/lib/spack" >> $GITHUB_ENV - name: Run style tests (code) run: | bin/spack style --base HEAD^1 bin/spack license verify pylint -j $(nproc) --disable=all --enable=unspecified-encoding --ignore-paths=lib/spack/spack/vendor lib - name: Run style tests (docs) run: .github/workflows/bin/format-rst.py $(git ls-files 'lib/spack/docs/*.rst') ================================================ FILE: .github/workflows/requirements/coverage/requirements.txt ================================================ coverage==7.13.5 ================================================ FILE: .github/workflows/requirements/style/requirements.txt ================================================ black==25.12.0 clingo==5.8.0 flake8==7.3.0 isort==7.0.0 mypy==1.20.1 vermin==1.8.0 pylint==4.0.5 docutils==0.22.4 ruamel.yaml==0.19.1 slotscheck==0.19.1 ruff==0.15.11 ================================================ FILE: .github/workflows/requirements/unit_tests/requirements.txt ================================================ pytest==9.0.3 pytest-cov==7.1.0 pytest-xdist==3.8.0 coverage[toml]<=7.11.0 clingo==5.8.0 ================================================ FILE: .github/workflows/stale.yaml ================================================ name: 'Close stale issues and PRs' on: schedule: # Run every day at 1:14 UTC - cron: '14 1 * * *' workflow_dispatch: jobs: stale: runs-on: ubuntu-latest permissions: actions: write issues: write pull-requests: write steps: - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: # Issues configuration stale-issue-message: > This issue has been automatically marked as stale because it has not had any activity in the last 6 months. It will be closed in 30 days if there is no further activity. If the issue is waiting for a reply from maintainers, feel free to ping them as a reminder. If it is waiting and has no comments yet, feel free to ping `@spack/spack-releasers` or simply leave a comment saying this should not be marked stale. This will also reset the issue's stale state. Thank you for your contributions! close-issue-message: > This issue was closed because it had no activity for 30 days after being marked stale. If you feel this is in error, please feel free to reopen this issue. stale-issue-label: 'stale' any-of-issue-labels: 'build-error,unreproducible,question,documentation,environments' exempt-issue-labels: 'pinned,triage,impact-low,impact-medium,impact-high' # Pull requests configuration stale-pr-message: > This pull request has been automatically marked as stale because it has not had any activity in the last 6 months. It will be closed in 30 days if there is no further activity. If the pull request is waiting for a reply from reviewers, feel free to ping them as a reminder. If it is waiting and has no assigned reviewer, feel free to ping `@spack/spack-releasers` or simply leave a comment saying this should not be marked stale. This will reset the pull request's stale state. To get more eyes on your pull request, you can post a link in the #pull-requests channel of the Spack Slack. Thank you for your contributions! close-pr-message: > This pull request was closed because it had no activity for 30 days after being marked stale. If you feel this is in error, please feel free to reopen this pull request. stale-pr-label: 'stale' any-of-pr-labels: 'new-package,update-package' exempt-pr-labels: 'pinned' # General configuration ascending: true operations-per-run: 1000 remove-stale-when-updated: true enable-statistics: true days-before-stale: 180 days-before-close: 30 ================================================ FILE: .github/workflows/triage.yml ================================================ #----------------------------------------------------------------------- # DO NOT modify unless you really know what you are doing. # # See https://stackoverflow.com/a/74959635 for more info. # Talk to @alecbcs if you have questions/are not sure of a change's # possible impact to security. #----------------------------------------------------------------------- name: triage on: pull_request_target: branches: - develop jobs: pr: runs-on: ubuntu-latest permissions: contents: read pull-requests: write issues: write steps: - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 ================================================ FILE: .github/workflows/unit_tests.yaml ================================================ name: unit tests on: workflow_dispatch: workflow_call: concurrency: group: unit_tests-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: # Run unit tests with different configurations on linux ubuntu: runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] python-version: ['3.8', '3.9', '3.10', '3.11', '3.14'] on_develop: - ${{ github.ref == 'refs/heads/develop' }} include: - python-version: '3.7' os: ubuntu-22.04 on_develop: ${{ github.ref == 'refs/heads/develop' }} exclude: - python-version: '3.8' os: ubuntu-latest on_develop: false - python-version: '3.9' os: ubuntu-latest on_develop: false - python-version: '3.10' os: ubuntu-latest on_develop: false - python-version: '3.11' os: ubuntu-latest on_develop: false steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} - name: Install System packages run: | sudo apt-get -y update # Needed for unit tests sudo apt-get -y install \ coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build \ cmake bison libbison-dev subversion # On ubuntu 24.04, kcov was removed. It may come back in some future Ubuntu - name: Set up Homebrew id: set-up-homebrew uses: Homebrew/actions/setup-homebrew@40e9946c182a64b3db1bf51be0dcb915f7802aa9 - name: Install kcov with brew run: "brew install kcov" - name: Install Python packages run: | # See https://github.com/coveragepy/coveragepy/issues/2082 pip install --upgrade pip pytest pytest-xdist pytest-cov "coverage<=7.11.0" pip install --upgrade "mypy>=0.900" "click" "ruff" - name: Setup git configuration run: | # Need this for the git tests to succeed. git --version . .github/workflows/bin/setup_git.sh - name: Bootstrap clingo if: ${{ matrix.concretizer == 'clingo' }} env: SPACK_PYTHON: python run: | . share/spack/setup-env.sh spack bootstrap disable spack-install spack bootstrap now spack -v solve zlib - name: Run unit tests env: SPACK_PYTHON: python SPACK_TEST_PARALLEL: 4 COVERAGE: true COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }} UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.14' }} run: | share/spack/qa/run-unit-tests - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-${{ matrix.os }}-python${{ matrix.python-version }} path: coverage include-hidden-files: true # Test shell integration shell: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - name: Install System packages run: | sudo apt-get -y update # Needed for shell tests sudo apt-get install -y coreutils csh zsh tcsh fish dash bash subversion # On ubuntu 24.04, kcov was removed. It may come back in some future Ubuntu - name: Set up Homebrew id: set-up-homebrew uses: Homebrew/actions/setup-homebrew@40e9946c182a64b3db1bf51be0dcb915f7802aa9 - name: Install kcov with brew run: "brew install kcov" - name: Install Python packages run: | pip install --upgrade pip -r .github/workflows/requirements/unit_tests/requirements.txt - name: Setup git configuration run: | # Need this for the git tests to succeed. git --version . .github/workflows/bin/setup_git.sh - name: Run shell tests env: COVERAGE: true run: | share/spack/qa/run-shell-tests - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-shell path: coverage include-hidden-files: true # Test RHEL8 UBI with platform Python. This job is run # only on PRs modifying core Spack rhel8-platform-python: runs-on: ubuntu-latest container: registry.access.redhat.com/ubi8/ubi steps: - name: Install dependencies run: | dnf install -y \ bzip2 curl gcc-c++ gcc gcc-gfortran git gnupg2 gzip \ make patch tcl unzip which xz - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup repo and non-root user run: | git --version git config --global --add safe.directory '*' git fetch --unshallow . .github/workflows/bin/setup_git.sh - name: Setup a virtual environment with platform-python run: | /usr/libexec/platform-python -m venv ~/platform-spack source ~/platform-spack/bin/activate pip install --upgrade pip pytest coverage[toml] pytest-xdist - name: Bootstrap Spack development environment and run unit tests run: | source ~/platform-spack/bin/activate source share/spack/setup-env.sh spack debug report spack -d bootstrap now pytest --verbose -x -n3 --dist loadfile -k 'not cvs and not svn and not hg' # Test for the clingo based solver (using clingo-cffi) clingo-cffi: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.13' - name: Install System packages run: | sudo apt-get -y update sudo apt-get -y install coreutils gfortran graphviz gnupg2 - name: Install Python packages run: | pip install --upgrade pip -r .github/workflows/requirements/unit_tests/requirements.txt pip install --upgrade -r .github/workflows/requirements/style/requirements.txt - name: Run unit tests (full suite with coverage) env: COVERAGE: true COVERAGE_FILE: coverage/.coverage-clingo-cffi run: | . share/spack/setup-env.sh spack bootstrap disable spack-install spack bootstrap disable github-actions-v0.6 spack bootstrap disable github-actions-v2 spack bootstrap status spack solve zlib pytest --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml -x -n3 lib/spack/spack/test/concretization/core.py - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-clingo-cffi path: coverage include-hidden-files: true # Run unit tests on MacOS macos: runs-on: ${{ matrix.os }} strategy: matrix: os: [macos-15-intel, macos-latest] python-version: ["3.14"] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} - name: Install Python packages run: | pip install --upgrade pip # See https://github.com/coveragepy/coveragepy/issues/2082 pip install --upgrade -r .github/workflows/requirements/unit_tests/requirements.txt - name: Setup Homebrew packages run: | brew install dash fish gcc gnupg kcov - name: Run unit tests env: COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }} run: | git --version . .github/workflows/bin/setup_git.sh . share/spack/setup-env.sh spack bootstrap disable spack-install spack solve zlib python3 -m pytest --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml --dist loadfile -x -n4 - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-${{ matrix.os }}-python${{ matrix.python-version }} path: coverage include-hidden-files: true # Run unit tests on Windows windows: defaults: run: shell: powershell Invoke-Expression -Command "./share/spack/qa/windows_test_setup.ps1"; {0} runs-on: windows-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.14' - name: Install Python packages run: | python -m pip install --upgrade pip pywin32 -r .github/workflows/requirements/unit_tests/requirements.txt python -m pip install --upgrade pip -r .github/workflows/requirements/style/requirements.txt - name: Create local develop run: | ./.github/workflows/bin/setup_git.ps1 - name: Unit Test env: COVERAGE_FILE: coverage/.coverage-windows run: | python -m pytest -x --verbose --cov --cov-config=pyproject.toml ./share/spack/qa/validate_last_exit.ps1 - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage-windows path: coverage include-hidden-files: true canonicalization: name: package.py canonicalization runs-on: ubuntu-latest container: image: ghcr.io/spack/all-pythons:2025-10-10 steps: - name: Checkout Spack (current) uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: spack-current - name: Checkout Spack (previous) uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: spack-previous ref: ${{ github.event.pull_request.base.sha || github.event.before }} - name: Checkout Spack Packages uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: spack/spack-packages path: spack-packages - name: Test package.py canonicalization run: spack-current/.github/workflows/bin/canonicalize.py --spack $PWD/spack-previous $PWD/spack-current --python python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 python3.14 --input-dir spack-packages/repos/spack_repo/builtin/packages/ --output-dir canonicalized ================================================ FILE: .gitignore ================================================ ########################## # Spack-specific ignores # ########################## /var/spack/stage /var/spack/cache /var/spack/environments /opt /share/spack/modules /share/spack/lmod # Debug logs spack-db.* *.in.log *.out.log CLAUDE.md # Configuration: Ignore everything in /etc/spack, # except defaults and site scopes that ship with spack /etc/spack/* !/etc/spack/defaults !/etc/spack/site/README.md ########################### # Coding agent state ########################### .claude/ .gemini/ .codex/ ########################### # Python-specific ignores # ########################### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ #lib/ #lib64/ parts/ sdist/ #var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ !/lib/spack/env # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ ######################## # Vim-specific ignores # ######################## # Swap [._]*.s[a-v][a-z] !*.svg # comment out if you don't need vector files [._]*.sw[a-p] [._]s[a-rt-v][a-z] [._]ss[a-gi-z] [._]sw[a-p] # Session Session.vim Sessionx.vim # Temporary .netrwhist *~ # Auto-generated tag files tags # Persistent undo [._]*.un~ ########################## # Emacs-specific ignores # ########################## *~ \#*\# /.emacs.desktop /.emacs.desktop.lock *.elc auto-save-list tramp .\#* # Org-mode .org-id-locations # flymake-mode *_flymake.* # eshell files /eshell/history /eshell/lastdir # zsh byte-compiled files *.zwc # elpa packages /elpa/ # reftex files *.rel # AUCTeX auto folder /auto/ # cask packages .cask/ dist/ # Flycheck flycheck_*.el # server auth directory /server/ # projectiles files .projectile # directory configuration .dir-locals.el # network security /network-security.data ############################ # Eclipse-specific ignores # ############################ .metadata #bin/ tmp/ *.tmp *.bak *.swp *~.nib local.properties .settings/ .loadpath .recommenders # External tool builders .externalToolBuilders/ # Locally stored "Eclipse launch configurations" *.launch # PyDev specific (Python IDE for Eclipse) *.pydevproject # CDT-specific (C/C++ Development Tooling) .cproject # CDT- autotools .autotools # Java annotation processor (APT) .factorypath # PDT-specific (PHP Development Tools) .buildpath # sbteclipse plugin .target # Tern plugin .tern-project # TeXlipse plugin .texlipse # STS (Spring Tool Suite) .springBeans # Code Recommenders .recommenders/ # Annotation Processing .apt_generated/ .apt_generated_test/ # Scala IDE specific (Scala & Java development for Eclipse) .cache-main .scala_dependencies .worksheet # Uncomment this line if you wish to ignore the project description file. # Typically, this file would be tracked if it contains build/dependency configurations: #.project ################################## # Visual Studio-specific ignores # ################################## .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json *.code-workspace # Local History for Visual Studio Code .history/ ################################# # Sublime Text-specific ignores # ################################# # Cache files for Sublime Text *.tmlanguage.cache *.tmPreferences.cache *.stTheme.cache # Workspace files are user-specific *.sublime-workspace # Project files should be checked into the repository, unless a significant # proportion of contributors will probably not be using Sublime Text # *.sublime-project # SFTP configuration file sftp-config.json sftp-config-alt*.json # Package control specific files Package Control.last-run Package Control.ca-list Package Control.ca-bundle Package Control.system-ca-bundle Package Control.cache/ Package Control.ca-certs/ Package Control.merged-ca-bundle Package Control.user-ca-bundle oscrypto-ca-bundle.crt bh_unicode_properties.cache # Sublime-github package stores a github token in this file # https://packagecontrol.io/packages/sublime-github GitHub.sublime-settings ############################## # JetBrains-specific ignores # ############################## # Ignore the entire folder since it may conatin more files than # just the ones listed below .idea/ # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff .idea/**/workspace.xml .idea/**/tasks.xml .idea/**/usage.statistics.xml .idea/**/dictionaries .idea/**/shelf # Generated files .idea/**/contentModel.xml # Sensitive or high-churn files .idea/**/dataSources/ .idea/**/dataSources.ids .idea/**/dataSources.local.xml .idea/**/sqlDataSources.xml .idea/**/dynamic.xml .idea/**/uiDesigner.xml .idea/**/dbnavigator.xml # Gradle .idea/**/gradle.xml .idea/**/libraries # Gradle and Maven with auto-import # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. # .idea/artifacts # .idea/compiler.xml # .idea/jarRepositories.xml # .idea/modules.xml # .idea/*.iml # .idea/modules # *.iml # *.ipr # CMake cmake-build-*/ # Mongo Explorer plugin .idea/**/mongoSettings.xml # File-based project format *.iws # IntelliJ out/ # mpeltonen/sbt-idea plugin .idea_modules/ # JIRA plugin atlassian-ide-plugin.xml # Cursive Clojure plugin .idea/replstate.xml # Crashlytics plugin (for Android Studio and IntelliJ) com_crashlytics_export_strings.xml crashlytics.properties crashlytics-build.properties fabric.properties # Editor-based Rest Client .idea/httpRequests # Android studio 3.1+ serialized cache file .idea/caches/build_file_checksums.ser ########################## # macOS-specific ignores # ########################## # General .DS_Store .AppleDouble .LSOverride # Icon must end with two \r Icon # Thumbnails ._* # Files that might appear in the root of a volume .DocumentRevisions-V100 .fseventsd .Spotlight-V100 .TemporaryItems .Trashes .VolumeIcon.icns .com.apple.timemachine.donotpresent # Directories potentially created on remote AFP share .AppleDB .AppleDesktop Network Trash Folder Temporary Items .apdisk ########################## # Linux-specific ignores # ########################## *~ # temporary files which can be created if a process still has a handle open of a deleted file .fuse_hidden* # KDE directory preferences .directory # Linux trash folder which might appear on any partition or disk .Trash-* # .nfs files are created when an open file is removed but is still being accessed .nfs* ############################ # Windows-specific ignores # ############################ # Windows thumbnail cache files Thumbs.db Thumbs.db:encryptable ehthumbs.db ehthumbs_vista.db # Dump file *.stackdump # Folder config file [Dd]esktop.ini # Recycle Bin used on file shares $RECYCLE.BIN/ # Windows Installer files *.cab *.msi *.msix *.msm *.msp # Windows shortcuts *.lnk ================================================ FILE: .mailmap ================================================ Abhinav Bhatele Abhinav Bhatele Adam Moody Adam T. Moody Alfredo Gimenez Alfredo Gimenez Alfredo Gimenez Alfredo Adolfo Gimenez Andrew Williams Andrew Williams Axel Huebl Axel Huebl Axel Huebl Axel Huebl Ben Boeckel Ben Boeckel Ben Boeckel Ben Boeckel Benedikt Hegner Benedikt Hegner Brett Viren Brett Viren David Boehme David Boehme David Boehme David Boehme David Poliakoff David Poliakoff Dhanannjay Deo Dhanannjay 'Djay' Deo Elizabeth Fischer Elizabeth F Elizabeth Fischer Elizabeth F Elizabeth Fischer Elizabeth Fischer Elizabeth Fischer citibeth Geoffrey Oxberry Geoffrey Oxberry Glenn Johnson Glenn Johnson Glenn Johnson Glenn Johnson Gregory Becker Gregory Becker Gregory Becker Gregory Becker Gregory Becker Gregory Becker Gregory L. Lee Greg Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory Lee Harmen Stoppels Harmen Stoppels Ian Lee Ian Lee James Wynne III James Riley Wynne III James Wynne III James Wynne III Joachim Protze jprotze Kathleen Shea kshea21 Kelly (KT) Thompson Kelly (KT) Thompson Kelly Thompson Kevin Brandstatter Kevin Brandstatter Luc Jaulmes Luc Jaulmes Mario Melara Mario Melara Mark Miller miller86 Massimiliano Culpo Massimiliano Culpo Massimiliano Culpo alalazo Mayeul d'Avezac Mayeul d'Avezac Mitchell Devlin Mitchell Devlin Nicolas Richart Nicolas Nicolas Richart Nicolas Richart Peter Scheibel scheibelp Robert D. French Robert D. French Robert D. French Robert.French Robert D. French robertdfrench Saravan Pantham Saravan Pantham Sergey Kosukhin Sergey Kosukhin Stephen Herbein Stephen Herbein Todd Gamblin George Todd Gamblin Todd Gamblin Todd Gamblin Tom Scogland Tom Scogland Tom Scogland Tom Scogland Tzanio Kolev Tzanio ================================================ FILE: .readthedocs.yml ================================================ version: 2 build: os: "ubuntu-24.04" apt_packages: - graphviz - inkscape - xindy tools: python: "3.14" jobs: post_checkout: - git fetch --unshallow || true # get accurate "Last updated on" info sphinx: configuration: lib/spack/docs/conf.py fail_on_warning: true formats: - pdf python: install: - requirements: lib/spack/docs/requirements.txt search: ranking: _modules/*: -10 spack_repo.*.html: -10 spack_repo.html: -10 spack.*.html: -10 spack.html: -10 command_index.html: 4 advanced_topics.html: 5 binary_caches.html: 5 bootstrapping.html: 5 build_settings.html: 5 build_systems.html: 5 build_systems/*.html: 5 chain.html: 5 config_yaml.html: 5 configuring_compilers.html: 5 containers.html: 5 roles_and_responsibilities.html: 5 contribution_guide.html: 5 developer_guide.html: 5 package_review_guide.html: 5 env_vars_yaml.html: 5 environments.html: 5 extensions.html: 5 features.html: 5 getting_help.html: 5 getting_started.html: 5 gpu_configuration.html: 5 include_yaml.html: 5 installing_prerequisites.html: 5 mirrors.html: 5 module_file_support.html: 5 package_api.html: 5 package_fundamentals.html: 5 packages_yaml.html: 5 packaging_guide_advanced.html: 5 packaging_guide_build.html: 5 packaging_guide_creation.html: 5 packaging_guide_testing.html: 5 pipelines.html: 5 replace_conda_homebrew.html: 5 repositories.html: 5 signing.html: 5 spec_syntax.html: 5 windows.html: 5 frequently_asked_questions.html: 6 ================================================ FILE: CHANGELOG.md ================================================ # v1.1.1 (2026-01-14) ## Usability and performance enhancements * solver: do a precheck for non-existing and deprecated versions #51555 * improvements to solver performance (PRs 51591, 51605, 51612, 51625) * python 3.14 support (PRs 51686, 51687, 51688, 51689, 51663) * display when conditions with dependencies in spack info #51588 * spack repo remove: allow removing from unspecified scope #51563 * spack compiler info: show non-external compilers too #51718 ## Improvements to the experimental new installer * support forkserver #51788 (for python 3.14 support) * support --dirty, --keep-stage, and `skip patch` arguments #51558 * implement --use-buildcache, --cache-only, --use-cache and --only arguments #51593 * implement overwrite, keep_prefix #51622 * implement --dont-restage #51623 * fix logging #51787 ## Bugfixes * repo.py: support rhel 7 #51617 * solver: match glibc constraints by hash #51559 * buildache list: list the component prefix not the root #51635 * solver: fix issue with conditional language dependencies #51692 * repo.py: fix checking out commits #51695 * spec parser: ensure toolchains are expanded to different objects #51731 * RHEL7 git 1.8.3.1 fix #51779 * RewireTask.complete: return value from \_process\_binary\_cache\_tarball #51825 ## Documentation * docs: fix default projections setting discrepancy #51640 # v1.1.0 (2025-11-14) `v1.1.0` features major improvements to **compiler handling** and **configuration management**, a significant refactoring of **externals**, and exciting new **experimental features** like a console UI for parallel installations and concretization caching. ## Major new features 1. **Enhanced Compiler Control and Unmixing** * Compiler unmixing (#51135) * Propagated compiler preferences (#51383) In Spack v1.0, support for compilers as nodes made it much easier to mix compilers for the same language on different packages in a Spec. This increased flexibility, but did not offer options to constrain compiler selection when needed. * #51135 introduces the `concretizer:compiler_mixing` config option. When disabled, all specs in the "root unification set" (root specs and their transitive link/run deps) will be assigned a single compiler for each language. You can also specify a list of packages to be excepted from the restriction. * #51383 introduces the `%%` sigil in the spec syntax. While `%` specifies a direct dependency for a single node, `%%` specifies a dependency for that node and a preference for its transitive link/run dependencies (at the same priority as the `prefer` key in `packages.yaml` config). 2. **Customizable configuration** (#51162) All configuration now stems from `$spack/etc/spack` and `$spack/etc/spack/defaults`, so the owner of a Spack instance can have full control over what configuration scopes exist. * Scopes included in configuration can be named, and the builtin `site`, `user`, `system`, etc. scopes are now defined in configuration rather than hard-coded. * `$spack/etc/spack/defaults` is the lowest priority. * `$spack/etc/spack` *includes* the other scopes at lower precedence than itself. * You can override with any scopes *except* the defaults with `include::`. e.g., `include::[]` in an environment allows you to ignore everything but defaults entirely. Here is `$spack/etc/spack/include.yaml`: ```yaml include: # user configuration scope - name: "user" path: "~/.spack" optional: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' # site configuration scope - name: "site" path: "$spack/etc/spack/site" optional: true # system configuration scope - name: "system" path: "/etc/spack" optional: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' ``` NOTE: This change inverts the priority order of configuration in `$spack/etc/spack` and `~/.spack`. See the [configuration docs](https://spack.readthedocs.io/en/latest/configuration.html) and [include docs](https://spack.readthedocs.io/en/latest/include_yaml.html) for more information. 3. **Git includes** (#51191) Configuration files can now be included directly from a **remote Git repository**. This allows for easier sharing and versioning of complex configurations across teams or projects. These entries accept the same syntax as remote repository configuration, and can likewise be conditional with `when:`. ```yaml include: - git: https://github.com/spack/spack-configs branch: main when: os == "centos7" paths: - USC/config/config.yaml - USC/config/packages.yaml ``` See [the docs](https://spack.readthedocs.io/en/latest/include_yaml.html#git-repository-files) for details. 4. **Externals Can Now Have Dependencies** (#51118) Externals are treated as concrete specs, so there is a 1:1 mapping between an entry in `packages.yaml` and any installed external spec (for a fixed repository). Their YAML specification has been extended to allow modeling dependencies of external specs. This might be quite useful to better capture e.g. ROCm installations that are already installed on a given system, or in similar cases. To be backward compatible with external specs specifying a compiler, for instance `mpich %gcc@9`, Spack will match the compiler specification to an existing external. It will fail when the specification is ambiguous, or if it does not match any other externals. ## Experimental Features 5. **New installer UI** (experimental, see #51434) New, experimental console UI for the Spack installer that allows: * Spack to show progress on multiple parallel processes concurrently; * Users to view logs for different installations independently; and * Spack to share a jobserver among multiple parallel builds. Demo: https://asciinema.org/a/755827 Usage: * Run this to enable by default (and persist across runs): ``` spack config add config:installer:new ``` or use: ``` spack -c config:installer:new install ... ``` to try one run with the new UI. * The `-j` flag in spack install `-j ...` is all you need, it will build packages in parallel. There is no need to set `-p`; the installer spawns as many builds as it can and shares work by default. * Use `n` for next logs and `p/N` for previous logs * Use `v` to toggle between logs and overview * Use `q` or `Esc` to go from logs back to overview. * Use `/` to enter search mode: filters the overview as you type; press `Enter` to follow logs or `Esc` to exit search mode. > [!WARNING] > This feature is experimental because it is not feature-complete to match the existing installer. See the issue #51515 for a list of features that are not completed. Particularly note that the new installer locks the entire database, and other spack instances will not install concurrently while it is running. 6. **Concretization Caching** (experimental, see #50905, #51448) Spack can cache concretization outputs for performance. With caching, Spack will still set up the concretization problem, but it can look up the solve result and avoid long solve times. This feature is currently off by default, but you can enable it with: ``` spack config add concretizer:concretization_cache:enable:true ``` > [!WARNING] > Currently there is a bug that the cache will return results that do not properly reflect changes in the `package_hash` (that is, changes in the `package.py` source code). We will enable caching by default in a future release, when this bug is fixed. ## Potentially breaking changes * Configurable configuration changes the precedence of the `site` scope. * The `spack` scope (in `/etc/spack` within the Spack installation) is now the highest precedence scope * The `site` scope is now *lower* precedence than `spack` and `user`. * If you previously had configuration files in in `$spack/etc/spack`, they will take precedence over configuration in `~/.spack`. If you do not want that, move them to `$spack/etc/spack/site`. * See #51162 for details. * Fixed a bug with command-line and environment scope ordering. The environment scope could previously override custom command-line scopes. Now, the active environment is *always* lower precedence than any configuration scopes provided on the command line. (#51461) ## Other notable improvements ### Improved error messages * solver: catch invalid dependencies during concretization (#51176) * improved errors for requirements (#45800) ### Performance Improvements * `spack mirror create --all` now runs in parallel (#50901) * `spack develop`: fast automatic reconcretization (#51140) * Don't spawn a process for `--fake` installs (#51491) * Use `gethostname` instead of `getfqdn` (#51481) * Check for `commit` variant only if not developing (#51507) * Concretization performance improvements (#51160, #51152, #51416) * spack diff: fix performance bug (#51270) ### Concretizer improvements * concretizer: fix direct dep w/ virtuals issue (#51037) * solver: reduce items in edge optimizations (#51503) ### UI and Commands * Managed environments can now be organized into folders (#50994) * `spack info` shows full info about conditional dependencies and can filter by spec. (#51137) * `spack help` is now reorganized and has color sections (#51484) * `spack clean --all` means all (no exception for bootstrap cache) (#50984) * `--variants-by-name` no longer used (#51450) * `spack env create`: allow creation from env or env dir (#51433) ## Notable Bugfixes * mirror: clean up stage when retrying (#43519) * Many smaller concretization fixes (#51361, #51355, #51341, #51347, #51282, #51190, #51226, #51065, #51064, #51074) * Bugfix for failed multi-node parallel installations (#50933) ## Spack community stats * 1,681 commits * 8,611 packages in the 2025.11.0 release, 112 new since 2025.07.0 * 276 people contributed to this release * 265 committers to packages * 31 committers to core See the [2025.11.0 release](https://github.com/spack/spack-packages/releases/tag/v2025.11.0) of [spack-packages](https://github.com/spack/spack-packages/) for more details. # v1.0.4 (2026-02-23) ## Bug fixes * Concretizer bugfixes: * solver: remove a special case for provider weighting #51347 * solver: improve timeout handling and add Ctrl-C interrupt safety #51341 * solver: simplify interrupt/timeout logic #51349 * Repo management bugfixes: * repo.py: support rhel 7 #51617 * repo.py: fix checking out commits #51695 * git: pull_checkout_branch RHEL7 git 1.8.3.1 fix #51779 * git: fix locking issue in pull_checkout_branch #51854 * spack repo remove: allow removing from unspecified scope #51563 * build_environment.py: Prevent deadlock on install process join #51429 * Fix typo in untrack_env #51554 * audit.py: fix re.sub(..., N) positional count arg #51735 ## Enhancements * Support Macos Tahoe (#51373, #51394, #51479) * Support for Python 3.14, except for t-strings (#51686, #51687, #51688, #51697, #51663) * spack info: show conditional dependencies and licenses; allow filtering #51137 * Spack fetch less likely to fail due to AI download protections #51496 * config: relax concurrent_packages to minimum 0 #51840 * This avoids forward-incompatibility with Spack v1.2 * Documentation improvements (#51315, #51640) # v1.0.3 (2026-02-20) Skipped due to a failure in the release process. # v1.0.2 (2025-09-11) ## Bug Fixes * `spack config edit` can now open malformed YAML files. (#51088) * `spack edit -b` supports specifying the repository path or its namespace. (#51084) * `spack repo list` escapes the color code for paths that contain `@g`. (#51178) * Fixed various issues on the solver: * Improved the error message when an invalid dependency is specified in the input. (#51176) * Build the preferred compiler with itself by default. (#51201) * Fixed a performance regression when using `unify:when_possible`. (#51226) * Fixed an issue with strong preferences, when provider details are given. (#51263) * Fixed an issue when specifying flags on a package that appears multiple times in the DAG. (#51218) * Fixed a regression for `zsh` in `spack env activate --prompt`. (#51258) * Fix a few cases where the `when` context manager was not dealing with direct dependencies correctly. (#51259) * Various fixes to string representations of specs. (#51207) ## Enhancements * Various improvements to the documentation (#51145, #51151, #51147, #51181, #51172, #51188, #51195) * Greatly improve the performance of `spack diff`. (#51270) * `spack solve` highlights optimization weights in a more intuitive way. (#51198) # v1.0.1 (2025-08-11) ## Bug Fixes * Ensure forward compatibility of package hashing with the upcoming Python 3.14 release. (#51042) * The `spack diff` command now shows differences in runtime dependencies (e.g., `gcc-runtime`, `glibc`), which were previously hidden. (#51076) * Fix a regression where the solver would mishandle a compiler that was required as both a build and a link dependency. (#51074) * Resolved issues with selecting external packages that have a specific compiler specified. (#51064) * Fix a bug where the concretizer would compute solution scores incorrectly when the package does not depend on a compiler. (#51037) * The solver now correctly evaluates and respects package requirements that specify a hash. (#51065) * Fix an issue where sparse checkouts for different packages could overwrite each other in a source cache or mirror. (#51080) * Prevent `spack repo add` from overwriting the default branch when initially cloning a repository. (#51105) * Add exception handling for bad URLs when fetching git provenance information. (#51022) * Spack no longer conflates git warning messages with command output. (#51045) * Fix an issue with non-path-based package repositories in environments. (#51055) * Spack now validates the terminal size and will fall back to `LINES` and `COLUMNS` environment variables if detection fails. (#51090) * Fix an issue where the package's fetcher was not being set correctly. (#51108) * Ensure `spack tutorial` clones Spack v1.0 instead of v0.23. (#51091) ## Enhancements * Various improvements to the documentation (#51014, #51033, #51039, #51049, #51066, #51073, #51079, #51082, #51083, #51086, #51126, #51131, #51132, #51025) # v1.0.0 (2025-07-20) `v1.0.0` is a major feature release and a significant milestone. It introduces compiler dependencies, a foundational change that has been in development for almost seven years, and the project's first stable package API. If you are interested in more information, you can find more details on the road to v1.0, as well as its features, in talks from the 2025 Spack User Meeting. For example: * [State of the Spack Community](https://www.youtube.com/watch?v=4rInmUfuiZQ&list=PLRKq_yxxHw29-JcpG2CZ-xKK2U8Hw8O1t&index=2) * [Spack v1.0 overview](https://www.youtube.com/watch?v=nFksqSDNwQA&list=PLRKq_yxxHw29-JcpG2CZ-xKK2U8Hw8O1t&index=4) Introducing some of these features required us to make breaking changes. In most cases, we've also provided tools (in the form of Spack commands) that you can use to automatically migrate your packages and configuration. ## Overview - [Overview](#overview) - [Stable Package API](#stable-package-api) - [Separate Package Repository](#separate-package-repository) - [Updating and Pinning Packages](#updating-and-pinning-packages) - [Breaking changes related to package repositories](#breaking-changes-related-to-package-repositories) - [Migrating to the new package API](#migrating-to-the-new-package-api) - [Compiler dependencies](#compiler-dependencies) - [Compiler configuration](#compiler-configuration) - [Languages are virtual dependencies](#languages-are-virtual-dependencies) - [The meaning of % has changed](#the-meaning-of-%25-has-changed) - [Virtual assignment syntax](#virtual-assignment-syntax) - [Toolchains](#toolchains) - [Ordering of variants and compilers now matters](#ordering-of-variants-and-compilers-now-matters) - [Additional Major Features](#additional-major-features) - [Concurrent Package Builds](#concurrent-package-builds) - [Content-addressed build caches](#content-addressed-build-caches) - [Better provenance and mirroring for git](#better-provenance-and-mirroring-for-git) - [Environment variables in environments](#environment-variables-in-environments) - [Better include functionality](#better-include-functionality) - [New commands and options](#new-commands-and-options) - [Notable refactors](#notable-refactors) - [Documentation](#documentation) - [Notable Bugfixes](#notable-bugfixes) - [Additional deprecations, removals, and breaking changes](#additional-deprecations-removals-and-breaking-changes) - [Spack community stats](#spack-community-stats) ## Stable Package API In Spack `v1.0`, the package repository is separate from the Spack tool, giving you more control over the versioning of package recipes. There is also a stable [Package API](https://spack.readthedocs.io/en/latest/package_api.html) that is versioned separately from Spack. This release of Spack supports package API from `v1.0` up to `v2.2`. The older `v1.0` package API is deprecated and may be removed in a future release, but we are guaranteeing that any Spack `v1.x` release will be backward compatible with Package API `v.2.x` -- i.e., it can execute code from the packages in *this* Spack release. See the [Package API Documentation](https://spack.readthedocs.io/en/latest/package_api.html) for full details on package versioning and compatibility. The high level details are: 1. The `spack.package` Python module defines the Package API; 2. The Package API *minor version* is incremented when new functions or classes are exported from `spack.package`; and 3. The major version is incremented when functions or classes are removed or have breaking changes to their signatures (a rare occurrence). This independent versioning allows package authors to utilize new Spack features without waiting for a new Spack release. Older Spack packages (API `v1.0`) may import code from outside of `spack.package`, e.g., from `spack.*` or `llnl.util.*`. This is deprecated and *not* included in the API guarantee. We will remove support for these packages in a future Spack release. ### Separate Package Repository The Spack `builtin` package repository no longer lives in the Spack git repository. You can find it here: * https://github.com/spack/spack-packages Spack clones the package repository automatically when you first run, so you do not have to manage this manually. By default, Spack version `v1.0` uses the `v2025.07` release of `spack-packages`. You can find out more about it by looking at the [package releases](https://github.com/spack/spack-packages/releases). Downloaded package repos are stored by default within `~/.spack`, but the fetch destination can be configured. (#50650). If you want your package repository to live somewhere else, run, e.g.: ``` spack repo set --destination ~/spack-packages builtin ``` You can also configure your *own* package repositories to be fetched automatically from git urls, just as you can with `builtin`. See the [repository configuration docs](https://spack.readthedocs.io/en/latest/repositories.html) for details. ### Updating and Pinning Packages You can tell Spack to update the core package repository from a branch. For example, on `develop` or on a release, you can run commands like: ```shell # pull the latest packages spack repo update ``` or ```shell # check out a specific commit of the spack-packages repo spack repo update --commit 2bf4ab9585c8d483cc8581d65912703d3f020393 builtin ``` which will set up your configuration like this: ```yaml repos: builtin: git: "https://github.com/spack/spack-packages.git" commit: 2bf4ab9585c8d483cc8581d65912703d3f020393 ``` You can use this within an environment to pin a specific version of its package files. See the [repository configuration docs](https://spack.readthedocs.io/en/latest/repositories.html) for more details (#50868, #50997, #51021). ### Breaking changes related to package repositories 1. The builtin repo now lives in `var/spack/repos/spack_repo/builtin` instead of `var/spack/repos/builtin`, and it has a new layout, which you can learn about in the [repo docs](https://spack.readthedocs.io/en/latest/repositories.html). 2. The module `spack.package` no longer exports the following symbols, mostly related to build systems: `AspellDictPackage`, `AutotoolsPackage`, `BundlePackage`, `CachedCMakePackage`, `cmake_cache_filepath`, `cmake_cache_option`, `cmake_cache_path`, `cmake_cache_string`, `CargoPackage`, `CMakePackage`, `generator`, `CompilerPackage`, `CudaPackage`, `Package`, `GNUMirrorPackage`, `GoPackage`, `IntelPackage`, `IntelOneApiLibraryPackageWithSdk`, `IntelOneApiLibraryPackage`, `IntelOneApiStaticLibraryList`, `IntelOneApiPackage`, `INTEL_MATH_LIBRARIES`, `LuaPackage`, `MakefilePackage`, `MavenPackage`, `MesonPackage`, `MSBuildPackage`, `NMakePackage`, `OctavePackage`, `PerlPackage`, `PythonExtension`, `PythonPackage`, `QMakePackage`, `RacketPackage`, `RPackage`, `ROCmPackage`, `RubyPackage`, `SConsPackage`, `SIPPackage`, `SourceforgePackage`, `SourcewarePackage`, `WafPackage`, `XorgPackage` These are now part of the `builtin` package repository, not part of core spack or its package API. When using repositories with package API `v2.0` and higher, *you must explicitly import these package classes* from the appropriate module in `spack_repo.builtin.build_systems` (see #50452 for more). e.g., for `CMakePackage`, you would write: ```python from spack_repo.builtin.build_systems.cmake import CMakePackage ``` Note that `GenericBuilder` and `Package` *are* part of the core package API. They are currently re-exported from `spack_repo.builtin.build_systems.generic` for backward compatibility but may be removed from the package repo. You should prefer to import them from `spack.package`. The original names will still work for old-style (`v1.0`) package repositories but *not* in `v2.0` package repositories. Note that this means that the API stability promise does *not* include old-style package repositories. They are deprecated and will be removed in a future version. So, you should update as soon as you can. 3. Package directory names within `v2.0` repositories are now valid Python modules | Old | New | Description | |-----------------------|-----------------------|-------------------------------------| | `py-numpy/package.py` | `py_numpy/package.py` | hyphen is replaced by underscore. | | `7zip/package.py` | `_7zip/package.py` | leading digits now preceded by _ | | `pass/package.py` | `_pass/package.py` | Python reserved words preceded by _ | 4. Spack has historically injected `import` statements into package recipes, so there was no need to use `from spack.package import *` (though we have included it in `builtin` packages for years. `from spack.package import *` (or more specific imports) will be necessary in packages. The magic we added in the early days of Spack was causing IDEs, code editors, and other tools not to be able to understand Spack packages. Now they use standard Python import semantics and should be compatible with modern Python tooling. This change was also necessary to support Python 3.13. (see #47947 for more details). ### Migrating to the new package API Support will remain in place for the old repository layout for *at least a year*, so that you can continue to use old-style repos in conjunction with earlier versions. If you have custom repositories that need to migrate to the new layout, you can upgrade them to package API `v2.x` by running: ``` spack repo migrate ``` This will make the following changes to your repository: 1. If you used to import from `spack.pkg.builtin` in Python, you now need to import from `spack_repo.builtin` instead: ```python # OLD: no longer supported from spack.pkg.builtin.my_pkg import MyPackage # NEW: spack_repo is a Python namespace package from spack_repo.builtin.packages.my_pkg.package import MyPackage ``` 2. Normalized directory names for packages 3. New-style `spack.package` imports See #50507, #50579, and #50594 for more. ## Compiler dependencies Prior to `v1.0`, compilers in Spack were attributes on nodes in the spec graph, with a name and a version (e.g., `gcc@12.0.0`). In `v1.0` compilers are packages like any other package in Spack (see #45189). This means that they can have variants, targets, and other attributes that regular nodes have. Here, we list the major changes that users should be aware of for this new model. ### Compiler configuration In Spack `v1.0`, `compilers.yaml` is deprecated. `compilers.yaml` is still read by Spack, if present. We will continue to support this for at least a year, but we may remove it after that. Users are encouraged to migrate their configuration to use `packages.yaml` instead. Old style `compilers.yaml` specification: ```yaml compilers: - compiler: spec: gcc@12.3.1 paths: c: /usr/bin/gcc cxx: /usr/bin/g++ fc: /usr/bin/gfortran modules: [...] ``` New style `packages.yaml` compiler specification: ```yaml packages: gcc: externals: - spec: gcc@12.3.1+binutils prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc cxx: /usr/bin/g++ fc: /usr/bin/gfortran modules: [...] ``` See [Configuring Compilers](https://spack.readthedocs.io/en/latest/configuring_compilers.html) for more details. ### Languages are virtual dependencies Packages that need a C, C++, or Fortran compiler now **must** depend on `c`, `cxx`, or `fortran` as a build dependency, e.g.: ```python class MyPackage(Package): depends_on("c", type="build") depends_on("cxx", type="build") depends_on("fortran", type="build") ``` Historically, Spack assumed that *every* package was compiled with C, C++, and Fortran. In Spack `v1.0`, we allow packages to simply not have a compiler if they do not need one. For example, pure Python packages would not depend on any of these, and you should not add these dependencies to packages that do not need them. [Spack `v0.23`](https://github.com/spack/spack/releases/tag/v0.23.0) introduced language virtual dependencies, and we have back-ported them to `0.21.3` and `v0.22.2`. In pre-1.0 Spack releases, these are a no-op. They are present so that language dependencies do not cause an error. This allows you to more easily use older Spack versions together with `v1.0`. See #45217 for more details. ### The meaning of `%` has changed In Spack `v0.x`, `%` specified a compiler with a name and an optional version. In Spack `v1.0`, it simply means "direct dependency". It is similar to the caret `^`, which means "direct *or* transitive dependency". Unlike `^`, which specifies a dependency that needs to be unified for the whole graph, `%` can specify direct dependencies of particular nodes. This means you can use it to mix and match compilers, or `cmake` versions, or any other package for which *multiple* versions of the same build dependency are needed in the same graph. For example, in this spec: ``` foo ^hdf5 %cmake@3.1.2 ^zlib-ng %cmake@3.2.4 ``` `hdf5` and `zlib-ng` are both transitive dependencies of `foo`, but `hdf5` will be built with `cmake@3.1.2` and `zlib-ng` will be built with `%cmake@3.2.4`. This is similar to mixing compilers, but you can now use `%` with other types of build dependencies, as well. You can have multiple versions of packages in the same graph, as long as they are purely build dependencies. ### Virtual assignment syntax You can still specify compilers with `foo %gcc`, in which case `gcc` will be used to satisfy any `c`, `cxx`, and `fortran` dependencies of `foo`, but you can also be specific about the compiler that should be used for each language. To mix, e.g., `clang` and `gfortran`, you can now use *virtual assignment* like so: ```console spack install foo %c,cxx=gcc %fortran=gfortran ``` This says to use `gcc` for `c` and `cxx`, and `gfortran` for `fortran`. It is functionally equivalent to the already supported edge attribute syntax: ``` spack install foo %[virtuals=c,cxx] gcc %[virtuals=fortran] gfortran ``` But, virtual assignment is more legible. We use it as the default formatting for virtual edge attributes, and we print it in the output of `spack spec`, `spack find`, etc. For example: ```console > spack spec zlib - zlib@1.3.1+optimize+pic+shared build_system=makefile arch=darwin-sequoia-m1 %c,cxx=apple-clang@16.0.0 [e] ^apple-clang@16.0.0 build_system=bundle arch=darwin-sequoia-m1 - ^compiler-wrapper@1.0 build_system=generic arch=darwin-sequoia-m1 [+] ^gmake@4.4.1~guile build_system=generic arch=darwin-sequoia-m1 %c=apple-clang@16.0.0 [+] ^compiler-wrapper@1.0 build_system=generic arch=darwin-sequoia-m1 ``` You can see above that only `zlib` and `gmake` are compiled, and `gmake` uses only `c`. The other nodes are either external, and we cannot detect the compiler (`apple-clang`) or they are not compiled (`compiler-wrapper` is a shell script). ### Toolchains Spack now has a concept of a "toolchain", which can be configured in `toolchains.yaml`. A toolchain is an alias for common dependencies, flags, and other spec properties that you can attach to a node in a graph with `%`. Toolchains are versatile and composable as they are simply aliases for regular specs. You can use them to represent mixed compiler combinations, compiler/MPI/numerical library groups, particular runtime libraries, and flags -- all to be applied together. This allows you to do with compiler dependencies what we used to do with `compilers.yaml`, and more. Example mixed clang/gfortran toolchain: ```yaml toolchains: clang_gfortran: - spec: %c=clang when: %c - spec: %cxx=clang when: %cxx - spec: %fortran=gcc when: %fortran - spec: cflags="-O3 -g" - spec: cxxflags="-O3 -g" - spec: fflags="-O3 -g" ``` This enables you to write `spack install foo %clang_gfortran`, and Spack will resolve the `%clang_gfortran` toolchain to include the dependencies and flags listed in `toolchains.yaml`. You could also couple the intel compilers with `mvapich2` like so: ```yaml toolchains: intel_mvapich2: - spec: %c=intel-oneapi-compilers @2025.1.1 when: %c - spec: %cxx=intel-oneapi-compilers @2025.1.1 when: %cxx - spec: %fortran=intel-oneapi-compilers @2025.1.1 when: %fortran - spec: %mpi=mvapich2 @2.3.7-1 +cuda when: %mpi ``` The `when:` conditions here ensure that toolchain constraints are only applied when needed. See the [toolchains documentation](https://spack.readthedocs.io/en/latest/advanced_topics.html#defining-and-using-toolchains) or #50481 for details. ### Ordering of variants and compilers now matters In Spack `v0.x`, these two specs parse the same: ``` pkg %gcc +foo pkg +foo %gcc ``` The `+foo` variant applies to `pkg` in either case. In Spack `v1.0`, there is a breaking change, and `+foo` in `pkg %gcc +foo` now applies to `gcc`, since `gcc` is a normal package. This ensures we have the following symmetry: ``` pkg +foo %dep +bar # `pkg +foo` depends on `dep +bar` directly pkg +foo ^dep +bar # `pkg +foo` depends on `dep +bar` directly or transitively ``` In Spack `v1.0` you may get errors at concretization time if `+foo` is not a variant of `gcc` in specs like`%pkg %gcc +foo`. You can use the `spack style --spec-strings` command to update `package.py` files, `spack.yaml` files: ```shell # dry run spack style --spec-strings $(git ls-files) # if you have a git repo spack style --spec-strings spack.yaml # environments ``` ```shell # use --fix to perform the changes listed by the dry run spack style --fix --spec-strings $(git ls-files) spack style --fix --spec-strings spack.yaml ``` See #49808, #49438, #49439 for details. ## Additional Major Features ### Concurrent Package Builds This release has completely reworked Spack's build scheduler, and it adds a `-p`/ `--concurrent-packages` argument to `spack install`, which can greatly accelerate builds with many packages. You can use it in combination with `spack install -j`. For example, this command: ``` spack install -p 4 -j 16 ``` runs up to 4 package builds at once, each with up to 16 make jobs. The default for `--concurrent-packages` is currently 1, so you must enable this feature yourself, either on the command line or by setting `config:concurrent_packages` (#50856): ```yaml config: concurrent_packages: 1 ``` As before, you can run `spack install` on multiple nodes in a cluster, if the filesystem where Spack's `install_tree` is located supports locking. We will make concurrent package builds the default in `1.1`, when we plan to include support for `gmake`'s jobserver protocol and for line-synced output. Currently, setting `-p` higher than 1 can make Spack's output difficult to read. ### Content-addressed build caches Spack `v1.0` changes the format of build caches to address a number of scaling and consistency issues with our old (aging) buildcache layout. The new buildcache format is content-addressed and enables us to make many operations atomic (and therefore safer). It is also more extensible than the old buildcache format and can enable features like split debug info and different signing methods in the future. See #48713 for more details. Spack `v1.0` can still read, but not write to, the old build caches. The new build cache format is *not* backward compatible with the old format, *but* you can have a new build cache and an old build cache coexist beside each other. If you push to an old build cache, new binaries will start to show up in the new format. You can migrate an old buildcache to the new format using the `spack buildcache migrate` command. It is nondestructive and can migrate an old build cache to a new one in-place. That is, it creates the new buildcache within the same directory, alongside the old buildcache. As with other major changes, the old buildcache format is deprecated in `v1.0`, but will not be removed for at least a year. ### Better provenance and mirroring for git Spack now resolves and preserves the commit of any git-based version at concretization time, storing the precise commit built on the Spec in a reserved `commit` variant. This allows us to better reproduce git builds. See #48702 for details. Historically, Spack has only stored the ref name, e.g. the branch or tag, for git versions that did not already contain full commits. Now we can know exactly what was built regardless of how it was fetched. As a consequence of this change, mirroring git repositories is also more robust. See #50604, #50906 for details. ### Environment variables in environments You can now specify environment variables in your environment that should be set on activation (and unset on deactivation): ```yaml spack: specs: - cmake%gcc env_vars: set: MY_FAVORITE_VARIABLE: "TRUE" ``` The syntax allows the same modifications that are allowed for modules: `set:`, `unset:`, `prepend_path:`, `append_path:`, etc. See [the docs](https://spack.readthedocs.io/en/latest/env_vars_yaml.html or #47587 for more. ### Better include functionality Spack allows you to include local or remote configuration files through `include.yaml`, and includes can be optional (i.e. include them only if they exist) or conditional (only include them under certain conditions: ```yaml spack: include: - /path/to/a/required/config.yaml - path: /path/to/$os/$target/config optional: true - path: /path/to/os-specific/config-dir when: os == "ventura" ``` You can use this in an environment, or in an `include.yaml` in an existing configuration scope. Included configuration files are required *unless* they are explicitly optional or the entry's condition evaluates to `false`. Optional includes are specified with the `optional` clause and conditional ones with the ``when`` clause. Conditionals use the same syntax as [spec list references](https://spack.readthedocs.io/en/latest/environments.html#spec-list-references) The [docs on `include.yaml`](https://spack.readthedocs.io/en/latest/include_yaml.html) have more information. You can also look at #48784. ## New commands and options * `spack repo update` will pull the latest packages (#50868, #50997) * `spack style --spec-strings` fixes old configuration file and packages (#49485) * `spack repo migrate`: migrates old repositories to the new layout (#50507) * `spack ci` no longer has a `--keep-stage` flag (#49467) * The new `spack config scopes` subcommand will list active configuration scopes (#41455, #50703) * `spack cd --repo ` (#50845) * `spack location --repo ` (#50845) * `--force` is now a common argument for all commands that do concretization (#48838) ## Notable refactors * All of Spack is now in one top-level `spack` Python package * The `spack_installable` package is gone as it's no longer needed (#50996) * The top-level `llnl` package has been moved to `spack.llnl` and will likely be refactored more later (#50989) * Vendored dependencies that were previously in `_vendoring` are now in `spack.vendor` (#51005) * Increased determinism when generating inputs for the ASP solver, leading to more consistent concretization results (#49471) * Added fast, stable spec comparison, which also increases determinism of concretizer inputs, and more consistent results (#50625) * Test deps are now part of the DAG hash, so builds with tests enabled will (correctly) have different hashes from builds without tests enabled (#48936) * `spack spec` in an environment or on the command line will show unified output with the specs provided as roots (#47574) * users can now set a timeout in `concretizer.yaml` in case they frequently hit long solves (#47661) * GoPackage: respect `-j`` concurrency (#48421) * We are using static analysis to speed up concretization (#48729) ## Documentation We have overhauled a number of sections of the documentation. * The basics section of the documentation has been reorganized and updated (#50932) * The [packaging guide](https://spack.readthedocs.io/en/latest/packaging_guide_creation.html) has been rewritten and broken into four separate, logically ordered sections (#50884). * As mentioned above the entire [`spack.package` API](https://spack.readthedocs.io/en/latest/package_api.html) has been documented and consolidated to one package (#51010) ## Notable Bugfixes * A race that would cause timeouts in certain parallel builds has been fixed. Every build now stages its own patches and cannot fight over them (causing a timeout) with other builds (#50697) * The `command_line` scope is now *always* the top level. Previously environments could override command line settings (#48255) * `setup-env.csh` is now hardened to avoid conflicts with user aliases (#49670) ## Additional deprecations, removals, and breaking changes 1. `spec["pkg"]` searches only direct dependencies and transitive link/run dependencies, ordered by depth. This avoids situations where we pick up unwanted deps of build/test deps. To reach those, you need to do `spec["build_dep"]["pkg"]` explicitly (#49016). 2. `spec["mpi"]` no longer works to refer to `spec` itself on specs like `openmpi` and `mpich` that could provide `mpi`. We only find `"mpi"` if it is provided by some dependency (see #48984). 3. We have removed some long-standing internal API methods on `spack.spec.Spec` so that we can decouple internal modules in the Spack code. `spack.spec` was including too many different parts of Spack. * `Spec.concretize()` and `Spec.concretized()` have been removed. Use `spack.concretize.concretize_one(spec)` instead (#47971, #47978) * `Spec.is_virtual`` is now spack.repo.PATH.is_virtual (#48986) * `Spec.virtual_dependencies` has been removed (#49079) 4. #50603: Platform config scopes are now opt-in. If you want to use subdirectories like `darwin` or `linux` in your scopes, you'll need to include them explicitly in an `include.yaml` or in your `spack.yaml` file, like so: ```yaml include: - include: "${platform}" optional: true ``` 5. #48488, #48502: buildcache entries created with Spack 0.19 and older using `spack buildcache create --rel` will no longer be relocated upon install. These old binaries should continue to work, except when they are installed with different `config:install_tree:projections` compared to what they were built with. Similarly, buildcache entries created with Spack 0.15 and older that contain long shebang lines wrapped with sbang will no longer be relocated. 6. #50462: the `package.py` globals `std_cmake_args`, `std_pip_args`, `std_meson_args` were removed. They were deprecated in Spack 0.23. Use `CMakeBuilder.std_args(pkg)`, `PythonPipBuilder.std_args(pkg)` and `MesonBuilder.std_args(pkg)` instead. 7. #50605, #50616: If you were using `update_external_dependencies()` in your private packages, note that it is going away in 1.0 to get it out of the package API. It is instead being moved into the concretizer, where it can change in the future, when we have a better way to deal with dependencies of externals, without breaking the package API. We suspect that nobody was doing this, but it's technically a breaking change. 8. #48838: Two breaking command changes: * `spack install` no longer has a `-f` / `--file` option -- write `spack install ./path/to/spec.json` instead. * `spack mirror create` no longer has a short `-f` option -- use `spack mirror create --file` instead. 9. We no longer support the PGI compilers. They have been replaced by `nvhpc` (#47195) 10. Python 3.8 is deprecated in the Python package, as it is EOL (#46913) 11. The `target=fe` / `target=frontend` and `target=be` / `target=backend` targets from Spack's orignal compilation model for cross-compiled Cray and BlueGene systems are now deprecated (#47756) ## Spack community stats * 2,276 commits updated package recipes * 8,499 total packages, 214 new since v0.23.0 * 372 people contributed to this release * 363 committers to packages * 63 committers to core # v0.23.1 (2025-02-19) ## Bugfixes - Fix a correctness issue of `ArchSpec.intersects` (#48741) - Make `extra_attributes` order independent in Spec hashing (#48615, #48854) - Fix issue where system proxy settings were not respected in OCI build caches (#48783) - Fix an issue where the `--test` concretizer flag was not forwarded correctly (#48417) - Fix an issue where `codesign` and `install_name_tool` would not preserve hardlinks on Darwin (#47808) - Fix an issue on Darwin where codesign would run on unmodified binaries (#48568) - Patch configure scripts generated with libtool < 2.5.4, to avoid redundant flags when creating shared libraries on Darwin (#48671) - Fix issue related to mirror URL paths on Windows (#47898) - Esnure proper UTF-8 encoding/decoding in logging (#48005) - Fix issues related to `filter_file` (#48038, #48108) - Fix issue related to creating bootstrap source mirrors (#48235) - Fix issue where command line config arguments were not always top level (#48255) - Fix an incorrect typehint of `concretized()` (#48504) - Improve mention of next Spack version in warning (#47887) - Tests: fix forward compatibility with Python 3.13 (#48209) - Docs: encourage use of `--oci-username-variable` and `--oci-password-variable` (#48189) - Docs: ensure Getting Started has bootstrap list output in correct place (#48281) - CI: allow GitHub actions to run on forks of Spack with different project name (#48041) - CI: make unit tests work on Ubuntu 24.04 (#48151) - CI: re-enable cray pipelines (#47697) ## Package updates - `qt-base`: fix rpath for dependents (#47424) - `gdk-pixbuf`: fix outdated URL (#47825) # v0.23.0 (2024-11-13) `v0.23.0` is a major feature release. We are planning to make this the last major release before Spack `v1.0` in June 2025. Alongside `v0.23`, we will be making pre-releases (alpha, beta, etc.) of `v1.0`, and we encourage users to try them and send us feedback, either on GitHub or on Slack. You can track the road to `v1.0` here: * https://github.com/spack/spack/releases * https://github.com/spack/spack/discussions/30634 ## Features in this Release 1. **Language virtuals** Your packages can now explicitly depend on the languages they require. Historically, Spack has considered C, C++, and Fortran compiler dependencies to be implicit. In `v0.23`, you should ensure that new packages add relevant C, C++, and Fortran dependencies like this: ```python depends_on("c", type="build") depends_on("cxx", type="build") depends_on("fortran", type="build") ``` We encourage you to add these annotations to your packages now, to prepare for Spack `v1.0.0`. In `v1.0.0`, these annotations will be necessary for your package to use C, C++, and Fortran compilers. Note that you should *not* add language dependencies to packages that don't need them, e.g., pure python packages. We have already auto-generated these dependencies for packages in the `builtin` repository (see #45217), based on the types of source files present in each package's source code. We *may* have added too many or too few language dependencies, so please submit pull requests to correct packages if you find that the language dependencies are incorrect. Note that we have also backported support for these dependencies to `v0.21.3` and `v0.22.2`, to make all of them forward-compatible with `v0.23`. This should allow you to move easily between older and newer Spack releases without breaking your packages. 2. **Spec splicing** We are working to make binary installation more seamless in Spack. `v0.23` introduces "splicing", which allows users to deploy binaries using local, optimized versions of a binary interface, even if they were not built with that interface. For example, this would allow you to build binaries in the cloud using `mpich` and install them on a system using a local, optimized version of `mvapich2` *without rebuilding*. Spack preserves full provenance for the installed packages and knows that they were built one way but deployed another. Our intent is to leverage this across many key HPC binary packages, e.g. MPI, CUDA, ROCm, and libfabric. Fundamentally, splicing allows Spack to redeploy an existing spec with different dependencies than how it was built. There are two interfaces to splicing. a. Explicit Splicing #39136 introduced the explicit splicing interface. In the concretizer config, you can specify a target spec and a replacement by hash. ```yaml concretizer: splice: explicit: - target: mpi replacement: mpich/abcdef ``` Here, every installation that would normally use the target spec will instead use its replacement. Above, any spec using *any* `mpi` will be spliced to depend on the specific `mpich` installation requested. This *can* go wrong if you try to replace something built with, e.g., `openmpi` with `mpich`, and it is on the user to ensure ABI compatibility between target and replacement specs. This currently requires some expertise to use, but it will allow users to reuse the binaries they create across more machines and environments. b. Automatic Splicing (experimental) #46729 introduced automatic splicing. In the concretizer config, enable automatic splicing: ```yaml concretizer: splice: automatic: true ``` or run: ```console spack config add concretizer:splice:automatic:true ``` The concretizer will select splices for ABI compatibility to maximize package reuse. Packages can denote ABI compatibility using the `can_splice` directive. No packages in Spack yet use this directive, so if you want to use this feature you will need to add `can_splice` annotations to your packages. We are working on ways to add more ABI compatibility information to the Spack package repository, and this directive may change in the future. See the documentation for more details: * https://spack.readthedocs.io/en/latest/build_settings.html#splicing * https://spack.readthedocs.io/en/latest/packaging_guide.html#specifying-abi-compatibility 3. Broader variant propagation Since #42931, you can specify propagated variants like `hdf5 build_type==RelWithDebInfo` or `trilinos ++openmp` to propagate a variant to all dependencies for which it is relevant. This is valid *even* if the variant does not exist on the package or its dependencies. See https://spack.readthedocs.io/en/latest/basic_usage.html#variants. 4. Query specs by namespace #45416 allows a package's namespace (indicating the repository it came from) to be treated like a variant. You can request packages from particular repos like this: ```console spack find zlib namespace=builtin spack find zlib namespace=myrepo ``` Previously, the spec syntax only allowed namespaces to be prefixes of spec names, e.g. `builtin.zlib`. The previous syntax still works. 5. `spack spec` respects environment settings and `unify:true` `spack spec` did not previously respect environment lockfiles or unification settings, which made it difficult to see exactly how a spec would concretize within an environment. Now it does, so the output you get with `spack spec` will be *the same* as what your environment will concretize to when you run `spack concretize`. Similarly, if you provide multiple specs on the command line with `spack spec`, it will concretize them together if `unify:true` is set. See #47556 and #44843. 6. Less noisy `spack spec` output `spack spec` previously showed output like this: ```console > spack spec /v5fn6xo Input spec -------------------------------- - /v5fn6xo Concretized -------------------------------- [+] openssl@3.3.1%apple-clang@16.0.0~docs+shared arch=darwin-sequoia-m1 ... ``` But the input spec is redundant, and we know we run `spack spec` to concretize the input spec. `spack spec` now *only* shows the concretized spec. See #47574. 7. Better output for `spack find -c` In an environmnet, `spack find -c` lets you search the concretized, but not yet installed, specs, just as you would the installed ones. As with `spack spec`, this should make it easier for you to see what *will* be built before building and installing it. See #44713. 8. `spack -C `: use an environment's configuration without activation Spack environments allow you to associate: 1. a set of (possibly concretized) specs, and 2. configuration When you activate an environment, you're using both of these. Previously, we supported: * `spack -e ` to run spack in the context of a specific environment, and * `spack -C ` to run spack using a directory with configuration files. You can now also pass an environment to `spack -C` to use *only* the environment's configuration, but not the specs or lockfile. See #45046. ## New commands, options, and directives * The new `spack env track` command (#41897) takes a non-managed Spack environment and adds a symlink to Spack's `$environments_root` directory, so that it will be included for reference counting for commands like `spack uninstall` and `spack gc`. If you use free-standing directory environments, this is useful for preventing Spack from removing things required by your environments. You can undo this tracking with the `spack env untrack` command. * Add `-t` short option for `spack --backtrace` (#47227) `spack -d / --debug` enables backtraces on error, but it can be very verbose, and sometimes you just want the backtrace. `spack -t / --backtrace` provides that option. * `gc`: restrict to specific specs (#46790) If you only want to garbage-collect specific packages, you can now provide them on the command line. This gives users finer-grained control over what is uninstalled. * oci buildcaches now support `--only=package`. You can now push *just* a package and not its dependencies to an OCI registry. This allows dependents of non-redistributable specs to be stored in OCI registries without an error. See #45775. ## Notable refactors * Variants are now fully conditional The `variants` dictionary on packages was previously keyed by variant name, and allowed only one definition of any given variant. Spack is now smart enough to understand that variants may have different values and defaults for different versions. For example, `warpx` prior to `23.06` only supported builds for one dimensionality, and newer `warpx` versions could be built with support for many different dimensions: ```python variant( "dims", default="3", values=("1", "2", "3", "rz"), multi=False, description="Number of spatial dimensions", when="@:23.05", ) variant( "dims", default="1,2,rz,3", values=("1", "2", "3", "rz"), multi=True, description="Number of spatial dimensions", when="@23.06:", ) ``` Previously, the default for the old version of `warpx` was not respected and had to be specified manually. Now, Spack will select the right variant definition for each version at concretization time. This allows variants to evolve more smoothly over time. See #44425 for details. ## Highlighted bugfixes 1. Externals no longer override the preferred provider (#45025). External definitions could interfere with package preferences. Now, if `openmpi` is the preferred `mpi`, and an external `mpich` is defined, a new `openmpi` *will* be built if building it is possible. Previously we would prefer `mpich` despite the preference. 2. Composable `cflags` (#41049). This release fixes a longstanding bug that concretization would fail if there were different `cflags` specified in `packages.yaml`, `compilers.yaml`, or on `the` CLI. Flags and their ordering are now tracked in the concretizer and flags from multiple sources will be merged. 3. Fix concretizer Unification for included environments (#45139). ## Deprecations, removals, and syntax changes 1. The old concretizer has been removed from Spack, along with the `config:concretizer` config option. Spack will emit a warning if the option is present in user configuration, since it now has no effect. Spack now uses a simpler bootstrapping mechanism, where a JSON prototype is tweaked slightly to get an initial concrete spec to download. See #45215. 2. Best-effort expansion of spec matrices has been removed. This feature did not work with the "new" ASP-based concretizer, and did not work with `unify: True` or `unify: when_possible`. Use the [exclude key](https://spack.readthedocs.io/en/latest/environments.html#spec-matrices) for the environment to exclude invalid components, or use multiple spec matrices to combine the list of specs for which the constraint is valid and the list of specs for which it is not. See #40792. 3. The old Cray `platform` (based on Cray PE modules) has been removed, and `platform=cray` is no longer supported. Since `v0.19`, Spack has handled Cray machines like Linux clusters with extra packages, and we have encouraged using this option to support Cray. The new approach allows us to correctly handle Cray machines with non-SLES operating systems, and it is much more reliable than making assumptions about Cray modules. See the `v0.19` release notes and #43796 for more details. 4. The `config:install_missing_compilers` config option has been deprecated, and it is a no-op when set in `v0.23`. Our new compiler dependency model will replace it with a much more reliable and robust mechanism in `v1.0`. See #46237. 5. Config options that deprecated in `v0.21` have been removed in `v0.23`. You can now only specify preferences for `compilers`, `targets`, and `providers` globally via the `packages:all:` section. Similarly, you can only specify `versions:` locally for a specific package. See #44061 and #31261 for details. 6. Spack's old test interface has been removed (#45752), having been deprecated in `v0.22.0` (#34236). All `builtin` packages have been updated to use the new interface. See the [stand-alone test documentation]( https://spack.readthedocs.io/en/latest/packaging_guide.html#stand-alone-tests) 7. The `spack versions --safe-only` option, deprecated since `v0.21.0`, has been removed. See #45765. * The `--dependencies` and `--optimize` arguments to `spack ci` have been deprecated. See #45005. ## Binary caches 1. Public binary caches now include an ML stack for Linux/aarch64 (#39666)We now build an ML stack for Linux/aarch64 for all pull requests and on develop. The ML stack includes both CPU-only and CUDA builds for Horovod, Hugging Face, JAX, Keras, PyTorch,scikit-learn, TensorBoard, and TensorFlow, and related packages. The CPU-only stack also includes XGBoost. See https://cache.spack.io/tag/develop/?stack=ml-linux-aarch64-cuda. 2. There is also now an stack of developer tools for macOS (#46910), which is analogous to the Linux devtools stack. You can use this to avoid building many common build dependencies. See https://cache.spack.io/tag/develop/?stack=developer-tools-darwin. ## Architecture support * archspec has been updated to `v0.2.5`, with support for `zen5` * Spack's CUDA package now supports the Grace Hopper `9.0a` compute capability (#45540) ## Windows * Windows bootstrapping: `file` and `gpg` (#41810) * `scripts` directory added to PATH on Windows for python extensions (#45427) * Fix `spack load --list` and `spack unload` on Windows (#35720) ## Other notable changes * Bugfix: `spack find -x` in environments (#46798) * Spec splices are now robust to duplicate nodes with the same name in a spec (#46382) * Cache per-compiler libc calculations for performance (#47213) * Fixed a bug in external detection for openmpi (#47541) * Mirror configuration allows username/password as environment variables (#46549) * Default library search caps maximum depth (#41945) * Unify interface for `spack spec` and `spack solve` commands (#47182) * Spack no longer RPATHs directories in the default library search path (#44686) * Improved performance of Spack database (#46554) * Enable package reuse for packages with versions from git refs (#43859) * Improved handling for `uuid` virtual on macos (#43002) * Improved tracking of task queueing/requeueing in the installer (#46293) ## Spack community stats * Over 2,000 pull requests updated package recipes * 8,307 total packages, 329 new since `v0.22.0` * 140 new Python packages * 14 new R packages * 373 people contributed to this release * 357 committers to packages * 60 committers to core # v0.22.2 (2024-09-21) ## Bugfixes - Forward compatibility with Spack 0.23 packages with language dependencies (#45205, #45191) - Forward compatibility with `urllib` from Python 3.12.6+ (#46453, #46483) - Bump vendored `archspec` for better aarch64 support (#45721, #46445) - Support macOS Sequoia (#45018, #45127) - Fix regression in `{variants.X}` and `{variants.X.value}` format strings (#46206) - Ensure shell escaping of environment variable values in load and activate commands (#42780) - Fix an issue where `spec[pkg]` considers specs outside the current DAG (#45090) - Do not halt concretization on unknown variants in externals (#45326) - Improve validation of `develop` config section (#46485) - Explicitly disable `ccache` if turned off in config, to avoid cache pollution (#45275) - Improve backwards compatibility in `include_concrete` (#45766) - Fix issue where package tags were sometimes repeated (#45160) - Make `setup-env.sh` "sourced only" by dropping execution bits (#45641) - Make certain source/binary fetch errors recoverable instead of a hard error (#45683) - Remove debug statements in package hash computation (#45235) - Remove redundant clingo warnings (#45269) - Remove hard-coded layout version (#45645) - Do not initialize previous store state in `use_store` (#45268) - Docs improvements (#46475) ## Package updates - `chapel` major update (#42197, #44931, #45304) # v0.22.1 (2024-07-04) ## Bugfixes - Fix reuse of externals on Linux (#44316) - Ensure parent gcc-runtime version >= child (#44834, #44870) - Ensure the latest gcc-runtime is rpath'ed when multiple exist among link deps (#44219) - Improve version detection of glibc (#44154) - Improve heuristics for solver (#44893, #44976, #45023) - Make strong preferences override reuse (#44373) - Reduce verbosity when C compiler is missing (#44182) - Make missing ccache executable an error when required (#44740) - Make every environment view containing `python` a `venv` (#44382) - Fix external detection for compilers with os but no target (#44156) - Fix version optimization for roots (#44272) - Handle common implementations of pagination of tags in OCI build caches (#43136) - Apply fetched patches to develop specs (#44950) - Avoid Windows wrappers for filesystem utilities on non-Windows (#44126) - Fix issue with long filenames in build caches on Windows (#43851) - Fix formatting issue in `spack audit` (#45045) - CI fixes (#44582, #43965, #43967, #44279, #44213) ## Package updates - protobuf: fix 3.4:3.21 patch checksum (#44443) - protobuf: update hash for patch needed when="@3.4:3.21" (#44210) - git: bump v2.39 to 2.45; deprecate unsafe versions (#44248) - gcc: use -rpath {rpath_dir} not -rpath={rpath dir} (#44315) - Remove mesa18 and libosmesa (#44264) - Enforce consistency of `gl` providers (#44307) - Require libiconv for iconv (#44335, #45026). Notice that glibc/musl also provide iconv, but are not guaranteed to be complete. Set `packages:iconv:require:[glibc]` to restore the old behavior. - py-matplotlib: qualify when to do a post install (#44191) - rust: fix v1.78.0 instructions (#44127) - suite-sparse: improve setting of the `libs` property (#44214) - netlib-lapack: provide blas and lapack together (#44981) # v0.22.0 (2024-05-12) `v0.22.0` is a major feature release. ## Features in this release 1. **Compiler dependencies** We are in the process of making compilers proper dependencies in Spack, and a number of changes in `v0.22` support that effort. You may notice nodes in your dependency graphs for compiler runtime libraries like `gcc-runtime` or `libgfortran`, and you may notice that Spack graphs now include `libc`. We've also begun moving compiler configuration from `compilers.yaml` to `packages.yaml` to make it consistent with other externals. We are trying to do this with the least disruption possible, so your existing `compilers.yaml` files should still work. We expect to be done with this transition by the `v0.23` release in November. * #41104: Packages compiled with `%gcc` on Linux, macOS and FreeBSD now depend on a new package `gcc-runtime`, which contains a copy of the shared compiler runtime libraries. This enables gcc runtime libraries to be installed and relocated when using a build cache. When building minimal Spack-generated container images it is no longer necessary to install libgfortran, libgomp etc. using the system package manager. * #42062: Packages compiled with `%oneapi` now depend on a new package `intel-oneapi-runtime`. This is similar to `gcc-runtime`, and the runtimes can provide virtuals and compilers can inject dependencies on virtuals into compiled packages. This allows us to model library soname compatibility and allows compilers like `%oneapi` to provide virtuals like `sycl` (which can also be provided by standalone libraries). Note that until we have an agreement in place with intel, Intel packages are marked `redistribute(source=False, binary=False)` and must be downloaded outside of Spack. * #43272: changes to the optimization criteria of the solver improve the hit-rate of buildcaches by a fair amount. The solver more relaxed compatibility rules and will not try to strictly match compilers or targets of reused specs. Users can still enforce the previous strict behavior with `require:` sections in `packages.yaml`. Note that to enforce correct linking, Spack will *not* reuse old `%gcc` and `%oneapi` specs that do not have the runtime libraries as a dependency. * #43539: Spack will reuse specs built with compilers that are *not* explicitly configured in `compilers.yaml`. Because we can now keep runtime libraries in build cache, we do not require you to also have a local configured compiler to *use* the runtime libraries. This improves reuse in buildcaches and avoids conflicts with OS updates that happen underneath Spack. * #43190: binary compatibility on `linux` is now based on the `libc` version, instead of on the `os` tag. Spack builds now detect the host `libc` (`glibc` or `musl`) and add it as an implicit external node in the dependency graph. Binaries with a `libc` with the same name and a version less than or equal to that of the detected `libc` can be reused. This is only on `linux`, not `macos` or `Windows`. * #43464: each package that can provide a compiler is now detectable using `spack external find`. External packages defining compiler paths are effectively used as compilers, and `spack external find -t compiler` can be used as a substitute for `spack compiler find`. More details on this transition are in [the docs](https://spack.readthedocs.io/en/latest/getting_started.html#manual-compiler-configuration) 2. **Improved `spack find` UI for Environments** If you're working in an environment, you likely care about: * What are the roots * Which ones are installed / not installed * What's been added that still needs to be concretized We've tweaked `spack find` in environments to show this information much more clearly. Installation status is shown next to each root, so you can see what is installed. Roots are also shown in bold in the list of installed packages. There is also a new option for `spack find -r` / `--only-roots` that will only show env roots, if you don't want to look at all the installed specs. More details in #42334. 3. **Improved command-line string quoting** We are making some breaking changes to how Spack parses specs on the CLI in order to respect shell quoting instead of trying to fight it. If you (sadly) had to write something like this on the command line: ``` spack install zlib cflags=\"-O2 -g\" ``` That will now result in an error, but you can now write what you probably expected to work in the first place: ``` spack install zlib cflags="-O2 -g" ``` Quoted can also now include special characters, so you can supply flags like: ``` spack install zlib ldflags='-Wl,-rpath=$ORIGIN/_libs' ``` To reduce ambiguity in parsing, we now require that you *not* put spaces around `=` and `==` when for flags or variants. This would not have broken before but will now result in an error: ``` spack install zlib cflags = "-O2 -g" ``` More details and discussion in #30634. 4. **Revert default `spack install` behavior to `--reuse`** We changed the default concretizer behavior from `--reuse` to `--reuse-deps` in #30990 (in `v0.20`), which meant that *every* `spack install` invocation would attempt to build a new version of the requested package / any environment roots. While this is a common ask for *upgrading* and for *developer* workflows, we don't think it should be the default for a package manager. We are going to try to stick to this policy: 1. Prioritize reuse and build as little as possible by default. 2. Only upgrade or install duplicates if they are explicitly asked for, or if there is a known security issue that necessitates an upgrade. With the install command you now have three options: * `--reuse` (default): reuse as many existing installations as possible. * `--reuse-deps` / `--fresh-roots`: upgrade (freshen) roots but reuse dependencies if possible. * `--fresh`: install fresh versions of requested packages (roots) and their dependencies. We've also introduced `--fresh-roots` as an alias for `--reuse-deps` to make it more clear that it may give you fresh versions. More details in #41302 and #43988. 5. **More control over reused specs** You can now control which packages to reuse and how. There is a new `concretizer:reuse` config option, which accepts the following properties: - `roots`: `true` to reuse roots, `false` to reuse just dependencies - `exclude`: list of constraints used to select which specs *not* to reuse - `include`: list of constraints used to select which specs *to* reuse - `from`: list of sources for reused specs (some combination of `local`, `buildcache`, or `external`) For example, to reuse only specs compiled with GCC, you could write: ```yaml concretizer: reuse: roots: true include: - "%gcc" ``` Or, if `openmpi` must be used from externals, and it must be the only external used: ```yaml concretizer: reuse: roots: true from: - type: local exclude: ["openmpi"] - type: buildcache exclude: ["openmpi"] - type: external include: ["openmpi"] ``` 6. **New `redistribute()` directive** Some packages can't be redistributed in source or binary form. We need an explicit way to say that in a package. Now there is a `redistribute()` directive so that package authors can write: ```python class MyPackage(Package): redistribute(source=False, binary=False) ``` Like other directives, this works with `when=`: ```python class MyPackage(Package): # 12.0 and higher are proprietary redistribute(source=False, binary=False, when="@12.0:") # can't redistribute when we depend on some proprietary dependency redistribute(source=False, binary=False, when="^proprietary-dependency") ``` More in #20185. 7. **New `conflict:` and `prefer:` syntax for package preferences** Previously, you could express conflicts and preferences in `packages.yaml` through some contortions with `require:`: ```yaml packages: zlib-ng: require: - one_of: ["%clang", "@:"] # conflict on %clang - any_of: ["+shared", "@:"] # strong preference for +shared ``` You can now use `require:` and `prefer:` for a much more readable configuration: ```yaml packages: zlib-ng: conflict: - "%clang" prefer: - "+shared" ``` See [the documentation](https://spack.readthedocs.io/en/latest/packages_yaml.html#conflicts-and-strong-preferences) and #41832 for more details. 8. **`include_concrete` in environments** You may want to build on the *concrete* contents of another environment without changing that environment. You can now include the concrete specs from another environment's `spack.lock` with `include_concrete`: ```yaml spack: specs: [] concretizer: unify: true include_concrete: - /path/to/environment1 - /path/to/environment2 ``` Now, when *this* environment is concretized, it will bring in the already concrete specs from `environment1` and `environment2`, and build on top of them without changing them. This is useful if you have phased deployments, where old deployments should not be modified but you want to use as many of them as possible. More details in #33768. 9. **`python-venv` isolation** Spack has unique requirements for Python because it: 1. installs every package in its own independent directory, and 2. allows users to register *external* python installations. External installations may contain their own installed packages that can interfere with Spack installations, and some distributions (Debian and Ubuntu) even change the `sysconfig` in ways that alter the installation layout of installed Python packages (e.g., with the addition of a `/local` prefix on Debian or Ubuntu). To isolate Spack from these and other issues, we now insert a small `python-venv` package in between `python` and packages that need to install Python code. This isolates Spack's build environment, isolates Spack from any issues with an external python, and resolves a large number of issues we've had with Python installations. See #40773 for further details. ## New commands, options, and directives * Allow packages to be pushed to build cache after install from source (#42423) * `spack develop`: stage build artifacts in same root as non-dev builds #41373 * Don't delete `spack develop` build artifacts after install (#43424) * `spack find`: add options for local/upstream only (#42999) * `spack logs`: print log files for packages (either partially built or installed) (#42202) * `patch`: support reversing patches (#43040) * `develop`: Add -b/--build-directory option to set build_directory package attribute (#39606) * `spack list`: add `--namespace` / `--repo` option (#41948) * directives: add `checked_by` field to `license()`, add some license checks * `spack gc`: add options for environments and build dependencies (#41731) * Add `--create` to `spack env activate` (#40896) ## Performance improvements * environment.py: fix excessive re-reads (#43746) * ruamel yaml: fix quadratic complexity bug (#43745) * Refactor to improve `spec format` speed (#43712) * Do not acquire a write lock on the env post install if no views (#43505) * asp.py: fewer calls to `spec.copy()` (#43715) * spec.py: early return in `__str__` * avoid `jinja2` import at startup unless needed (#43237) ## Other new features of note * `archspec`: update to `v0.2.4`: support for Windows, bugfixes for `neoverse-v1` and `neoverse-v2` detection. * `spack config get`/`blame`: with no args, show entire config * `spack env create `: dir if dir-like (#44024) * ASP-based solver: update os compatibility for macOS (#43862) * Add handling of custom ssl certs in urllib ops (#42953) * Add ability to rename environments (#43296) * Add config option and compiler support to reuse across OS's (#42693) * Support for prereleases (#43140) * Only reuse externals when configured (#41707) * Environments: Add support for including views (#42250) ## Binary caches * Build cache: make signed/unsigned a mirror property (#41507) * tools stack ## Removals, deprecations, and syntax changes * remove `dpcpp` compiler and package (#43418) * spack load: remove --only argument (#42120) ## Notable Bugfixes * repo.py: drop deleted packages from provider cache (#43779) * Allow `+` in module file names (#41999) * `cmd/python`: use runpy to allow multiprocessing in scripts (#41789) * Show extension commands with spack -h (#41726) * Support environment variable expansion inside module projections (#42917) * Alert user to failed concretizations (#42655) * shell: fix zsh color formatting for PS1 in environments (#39497) * spack mirror create --all: include patches (#41579) ## Spack community stats * 7,994 total packages; 525 since `v0.21.0` * 178 new Python packages, 5 new R packages * 358 people contributed to this release * 344 committers to packages * 45 committers to core # v0.21.3 (2024-10-02) ## Bugfixes - Forward compatibility with Spack 0.23 packages with language dependencies (#45205, #45191) - Forward compatibility with `urllib` from Python 3.12.6+ (#46453, #46483) - Bump `archspec` to 0.2.5-dev for better aarch64 and Windows support (#42854, #44005, #45721, #46445) - Support macOS Sequoia (#45018, #45127, #43862) - CI and test maintenance (#42909, #42728, #46711, #41943, #43363) # v0.21.2 (2024-03-01) ## Bugfixes - Containerize: accommodate nested or pre-existing spack-env paths (#41558) - Fix setup-env script, when going back and forth between instances (#40924) - Fix using fully-qualified namespaces from root specs (#41957) - Fix a bug when a required provider is requested for multiple virtuals (#42088) - OCI buildcaches: - only push in parallel when forking (#42143) - use pickleable errors (#42160) - Fix using sticky variants in externals (#42253) - Fix a rare issue with conditional requirements and multi-valued variants (#42566) ## Package updates - rust: add v1.75, rework a few variants (#41161,#41903) - py-transformers: add v4.35.2 (#41266) - mgard: fix OpenMP on AppleClang (#42933) # v0.21.1 (2024-01-11) ## New features - Add support for reading buildcaches created by Spack v0.22 (#41773) ## Bugfixes - spack graph: fix coloring with environments (#41240) - spack info: sort variants in --variants-by-name (#41389) - Spec.format: error on old style format strings (#41934) - ASP-based solver: - fix infinite recursion when computing concretization errors (#41061) - don't error for type mismatch on preferences (#41138) - don't emit spurious debug output (#41218) - Improve the error message for deprecated preferences (#41075) - Fix MSVC preview version breaking clingo build on Windows (#41185) - Fix multi-word aliases (#41126) - Add a warning for unconfigured compiler (#41213) - environment: fix an issue with deconcretization/reconcretization of specs (#41294) - buildcache: don't error if a patch is missing, when installing from binaries (#41986) - Multiple improvements to unit-tests (#41215,#41369,#41495,#41359,#41361,#41345,#41342,#41308,#41226) ## Package updates - root: add a webgui patch to address security issue (#41404) - BerkeleyGW: update source urls (#38218) # v0.21.0 (2023-11-11) `v0.21.0` is a major feature release. ## Features in this release 1. **Better error messages with condition chaining** In v0.18, we added better error messages that could tell you what problem happened, but they couldn't tell you *why* it happened. `0.21` adds *condition chaining* to the solver, and Spack can now trace back through the conditions that led to an error and build a tree of causes potential causes and where they came from. For example: ```console $ spack solve hdf5 ^cmake@3.0.1 ==> Error: concretization failed for the following reasons: 1. Cannot satisfy 'cmake@3.0.1' 2. Cannot satisfy 'cmake@3.0.1' required because hdf5 ^cmake@3.0.1 requested from CLI 3. Cannot satisfy 'cmake@3.18:' and 'cmake@3.0.1 required because hdf5 ^cmake@3.0.1 requested from CLI required because hdf5 depends on cmake@3.18: when @1.13: required because hdf5 ^cmake@3.0.1 requested from CLI 4. Cannot satisfy 'cmake@3.12:' and 'cmake@3.0.1 required because hdf5 depends on cmake@3.12: required because hdf5 ^cmake@3.0.1 requested from CLI required because hdf5 ^cmake@3.0.1 requested from CLI ``` More details in #40173. 2. **OCI build caches** You can now use an arbitrary [OCI](https://opencontainers.org) registry as a build cache: ```console $ spack mirror add my_registry oci://user/image # Dockerhub $ spack mirror add my_registry oci://ghcr.io/haampie/spack-test # GHCR $ spack mirror set --push --oci-username ... --oci-password ... my_registry # set login creds $ spack buildcache push my_registry [specs...] ``` And you can optionally add a base image to get *runnable* images: ```console $ spack buildcache push --base-image ubuntu:23.04 my_registry python Pushed ... as [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack $ docker run --rm -it [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack ``` This creates a container image from the Spack installations on the host system, without the need to run `spack install` from a `Dockerfile` or `sif` file. It also addresses the inconvenience of losing binaries of dependencies when `RUN spack install` fails inside `docker build`. Further, the container image layers and build cache tarballs are the same files. This means that `spack install` and `docker pull` use the exact same underlying binaries. If you previously used `spack install` inside of `docker build`, this feature helps you save storage by a factor two. More details in #38358. 3. **Multiple versions of build dependencies** Increasingly, complex package builds require multiple versions of some build dependencies. For example, Python packages frequently require very specific versions of `setuptools`, `cython`, and sometimes different physics packages require different versions of Python to build. The concretizer enforced that every solve was *unified*, i.e., that there only be one version of every package. The concretizer now supports "duplicate" nodes for *build dependencies*, but enforces unification through transitive link and run dependencies. This will allow it to better resolve complex dependency graphs in ecosystems like Python, and it also gets us very close to modeling compilers as proper dependencies. This change required a major overhaul of the concretizer, as well as a number of performance optimizations. See #38447, #39621. 4. **Cherry-picking virtual dependencies** You can now select only a subset of virtual dependencies from a spec that may provide more. For example, if you want `mpich` to be your `mpi` provider, you can be explicit by writing: ``` hdf5 ^[virtuals=mpi] mpich ``` Or, if you want to use, e.g., `intel-parallel-studio` for `blas` along with an external `lapack` like `openblas`, you could write: ``` strumpack ^[virtuals=mpi] intel-parallel-studio+mkl ^[virtuals=lapack] openblas ``` The `virtuals=mpi` is an edge attribute, and dependency edges in Spack graphs now track which virtuals they satisfied. More details in #17229 and #35322. Note for packaging: in Spack 0.21 `spec.satisfies("^virtual")` is true if and only if the package specifies `depends_on("virtual")`. This is different from Spack 0.20, where depending on a provider implied depending on the virtual provided. See #41002 for an example where `^mkl` was being used to test for several `mkl` providers in a package that did not depend on `mkl`. 5. **License directive** Spack packages can now have license metadata, with the new `license()` directive: ```python license("Apache-2.0") ``` Licenses use [SPDX identifiers](https://spdx.org/licenses), and you can use SPDX expressions to combine them: ```python license("Apache-2.0 OR MIT") ``` Like other directives in Spack, it's conditional, so you can handle complex cases like Spack itself: ```python license("LGPL-2.1", when="@:0.11") license("Apache-2.0 OR MIT", when="@0.12:") ``` More details in #39346, #40598. 6. **`spack deconcretize` command** We are getting close to having a `spack update` command for environments, but we're not quite there yet. This is the next best thing. `spack deconcretize` gives you control over what you want to update in an already concrete environment. If you have an environment built with, say, `meson`, and you want to update your `meson` version, you can run: ```console spack deconcretize meson ``` and have everything that depends on `meson` rebuilt the next time you run `spack concretize`. In a future Spack version, we'll handle all of this in a single command, but for now you can use this to drop bits of your lockfile and resolve your dependencies again. More in #38803. 7. **UI Improvements** The venerable `spack info` command was looking shabby compared to the rest of Spack's UI, so we reworked it to have a bit more flair. `spack info` now makes much better use of terminal space and shows variants, their values, and their descriptions much more clearly. Conditional variants are grouped separately so you can more easily understand how packages are structured. More in #40998. `spack checksum` now allows you to filter versions from your editor, or by version range. It also notifies you about potential download URL changes. See #40403. 8. **Environments can include definitions** Spack did not previously support using `include:` with The [definitions](https://spack.readthedocs.io/en/latest/environments.html#spec-list-references) section of an environment, but now it does. You can use this to curate lists of specs and more easily reuse them across environments. See #33960. 9. **Aliases** You can now add aliases to Spack commands in `config.yaml`, e.g. this might enshrine your favorite args to `spack find` as `spack f`: ```yaml config: aliases: f: find -lv ``` See #17229. 10. **Improved autoloading of modules** Spack 0.20 was the first release to enable autoloading of direct dependencies in module files. The downside of this was that `module avail` and `module load` tab completion would show users too many modules to choose from, and many users disabled generating modules for dependencies through `exclude_implicits: true`. Further, it was necessary to keep hashes in module names to avoid file name clashes. In this release, you can start using `hide_implicits: true` instead, which exposes only explicitly installed packages to the user, while still autoloading dependencies. On top of that, you can safely use `hash_length: 0`, as this config now only applies to the modules exposed to the user -- you don't have to worry about file name clashes for hidden dependencies. Note: for `tcl` this feature requires Modules 4.7 or higher 11. **Updated container labeling** Nightly Docker images from the `develop` branch will now be tagged as `:develop` and `:nightly`. The `:latest` tag is no longer associated with `:develop`, but with the latest stable release. Releases will be tagged with `:{major}`, `:{major}.{minor}` and `:{major}.{minor}.{patch}`. `ubuntu:18.04` has also been removed from the list of generated Docker images, as it is no longer supported. See #40593. ## Other new commands and directives * `spack env activate` without arguments now loads a `default` environment that you do not have to create (#40756). * `spack find -H` / `--hashes`: a new shortcut for piping `spack find` output to other commands (#38663) * Add `spack checksum --verify`, fix `--add` (#38458) * New `default_args` context manager factors out common args for directives (#39964) * `spack compiler find --[no]-mixed-toolchain` lets you easily mix `clang` and `gfortran` on Linux (#40902) ## Performance improvements * `spack external find` execution is now much faster (#39843) * `spack location -i` now much faster on success (#40898) * Drop redundant rpaths post install (#38976) * ASP-based solver: avoid cycles in clingo using hidden directive (#40720) * Fix multiple quadratic complexity issues in environments (#38771) ## Other new features of note * archspec: update to v0.2.2, support for Sapphire Rapids, Power10, Neoverse V2 (#40917) * Propagate variants across nodes that don't have that variant (#38512) * Implement fish completion (#29549) * Can now distinguish between source/binary mirror; don't ping mirror.spack.io as much (#34523) * Improve status reporting on install (add [n/total] display) (#37903) ## Windows This release has the best Windows support of any Spack release yet, with numerous improvements and much larger swaths of tests passing: * MSVC and SDK improvements (#37711, #37930, #38500, #39823, #39180) * Windows external finding: update default paths; treat .bat as executable on Windows (#39850) * Windows decompression: fix removal of intermediate file (#38958) * Windows: executable/path handling (#37762) * Windows build systems: use ninja and enable tests (#33589) * Windows testing (#36970, #36972, #36973, #36840, #36977, #36792, #36834, #34696, #36971) * Windows PowerShell support (#39118, #37951) * Windows symlinking and libraries (#39933, #38599, #34701, #38578, #34701) ## Notable refactors * User-specified flags take precedence over others in Spack compiler wrappers (#37376) * Improve setup of build, run, and test environments (#35737, #40916) * `make` is no longer a required system dependency of Spack (#40380) * Support Python 3.12 (#40404, #40155, #40153) * docs: Replace package list with packages.spack.io (#40251) * Drop Python 2 constructs in Spack (#38720, #38718, #38703) ## Binary cache and stack updates * e4s arm stack: duplicate and target neoverse v1 (#40369) * Add macOS ML CI stacks (#36586) * E4S Cray CI Stack (#37837) * e4s cray: expand spec list (#38947) * e4s cray sles ci: expand spec list (#39081) ## Removals, deprecations, and syntax changes * ASP: targets, compilers and providers soft-preferences are only global (#31261) * Parser: fix ambiguity with whitespace in version ranges (#40344) * Module file generation is disabled by default; you'll need to enable it to use it (#37258) * Remove deprecated "extra_instructions" option for containers (#40365) * Stand-alone test feature deprecation postponed to v0.22 (#40600) * buildcache push: make `--allow-root` the default and deprecate the option (#38878) ## Notable Bugfixes * Bugfix: propagation of multivalued variants (#39833) * Allow `/` in git versions (#39398) * Fetch & patch: actually acquire stage lock, and many more issues (#38903) * Environment/depfile: better escaping of targets with Git versions (#37560) * Prevent "spack external find" to error out on wrong permissions (#38755) * lmod: allow core compiler to be specified with a version range (#37789) ## Spack community stats * 7,469 total packages, 303 new since `v0.20.0` * 150 new Python packages * 34 new R packages * 353 people contributed to this release * 336 committers to packages * 65 committers to core # v0.20.3 (2023-10-31) ## Bugfixes - Fix a bug where `spack mirror set-url` would drop configured connection info (reverts #34210) - Fix a minor issue with package hash computation for Python 3.12 (#40328) # v0.20.2 (2023-10-03) ## Features in this release Spack now supports Python 3.12 (#40155) ## Bugfixes - Improve escaping in Tcl module files (#38375) - Make repo cache work on repositories with zero mtime (#39214) - Ignore errors for newer, incompatible buildcache version (#40279) - Print an error when git is required, but missing (#40254) - Ensure missing build dependencies get installed when using `spack install --overwrite` (#40252) - Fix an issue where Spack freezes when the build process unexpectedly exits (#39015) - Fix a bug where installation failures cause an unrelated `NameError` to be thrown (#39017) - Fix an issue where Spack package versions would be incorrectly derived from git tags (#39414) - Fix a bug triggered when file locking fails internally (#39188) - Prevent "spack external find" to error out when a directory cannot be accessed (#38755) - Fix multiple performance regressions in environments (#38771) - Add more ignored modules to `pyproject.toml` for `mypy` (#38769) # v0.20.1 (2023-07-10) ## Spack Bugfixes - Spec removed from an environment where not actually removed if `--force` was not given (#37877) - Speed-up module file generation (#37739) - Hotfix for a few recipes that treat CMake as a link dependency (#35816) - Fix re-running stand-alone test a second time, which was getting a trailing spurious failure (#37840) - Fixed reading JSON manifest on Cray, reporting non-concrete specs (#37909) - Fixed a few bugs when generating Dockerfiles from Spack (#37766,#37769) - Fixed a few long-standing bugs when generating module files (#36678,#38347,#38465,#38455) - Fixed issues with building Python extensions using an external Python (#38186) - Fixed compiler removal from command line (#38057) - Show external status as [e] (#33792) - Backported `archspec` fixes (#37793) - Improved a few error messages (#37791) # v0.20.0 (2023-05-21) `v0.20.0` is a major feature release. ## Features in this release 1. **`requires()` directive and enhanced package requirements** We've added some more enhancements to requirements in Spack (#36286). There is a new `requires()` directive for packages. `requires()` is the opposite of `conflicts()`. You can use it to impose constraints on this package when certain conditions are met: ```python requires( "%apple-clang", when="platform=darwin", msg="This package builds only with clang on macOS" ) ``` More on this in [the docs]( https://spack.rtfd.io/en/latest/packaging_guide.html#conflicts-and-requirements). You can also now add a `when:` clause to `requires:` in your `packages.yaml` configuration or in an environment: ```yaml packages: openmpi: require: - any_of: ["%gcc"] when: "@:4.1.4" message: "Only OpenMPI 4.1.5 and up can build with fancy compilers" ``` More details can be found [here]( https://spack.readthedocs.io/en/latest/build_settings.html#package-requirements) 2. **Exact versions** Spack did not previously have a way to distinguish a version if it was a prefix of some other version. For example, `@3.2` would match `3.2`, `3.2.1`, `3.2.2`, etc. You can now match *exactly* `3.2` with `@=3.2`. This is useful, for example, if you need to patch *only* the `3.2` version of a package. The new syntax is described in [the docs]( https://spack.readthedocs.io/en/latest/basic_usage.html#version-specifier). Generally, when writing packages, you should prefer to use ranges like `@3.2` over the specific versions, as this allows the concretizer more leeway when selecting versions of dependencies. More details and recommendations are in the [packaging guide]( https://spack.readthedocs.io/en/latest/packaging_guide.html#ranges-versus-specific-versions). See #36273 for full details on the version refactor. 3. **New testing interface** Writing package tests is now much simpler with a new [test interface]( https://spack.readthedocs.io/en/latest/packaging_guide.html#stand-alone-tests). Writing a test is now as easy as adding a method that starts with `test_`: ```python class MyPackage(Package): ... def test_always_fails(self): """use assert to always fail""" assert False def test_example(self): """run installed example""" example = which(self.prefix.bin.example) example() ``` You can use Python's native `assert` statement to implement your checks -- no more need to fiddle with `run_test` or other test framework methods. Spack will introspect the class and run `test_*` methods when you run `spack test`, 4. **More stable concretization** * Now, `spack concretize` will *only* concretize the new portions of the environment and will not change existing parts of an environment unless you specify `--force`. This has always been true for `unify:false`, but not for `unify:true` and `unify:when_possible` environments. Now it is true for all of them (#37438, #37681). * The concretizer has a new `--reuse-deps` argument that *only* reuses dependencies. That is, it will always treat the *roots* of your environment as it would with `--fresh`. This allows you to upgrade just the roots of your environment while keeping everything else stable (#30990). 5. **Weekly develop snapshot releases** Since last year, we have maintained a buildcache of `develop` at https://binaries.spack.io/develop, but the cache can grow to contain so many builds as to be unwieldy. When we get a stable `develop` build, we snapshot the release and add a corresponding tag the Spack repository. So, you can use a stack from a specific day. There are now tags in the spack repository like: * `develop-2023-05-14` * `develop-2023-05-18` that correspond to build caches like: * https://binaries.spack.io/develop-2023-05-14/e4s * https://binaries.spack.io/develop-2023-05-18/e4s We plan to store these snapshot releases weekly. 6. **Specs in buildcaches can be referenced by hash.** * Previously, you could run `spack buildcache list` and see the hashes in buildcaches, but referring to them by hash would fail. * You can now run commands like `spack spec` and `spack install` and refer to buildcache hashes directly, e.g. `spack install /abc123` (#35042) 7. **New package and buildcache index websites** Our public websites for searching packages have been completely revamped and updated. You can check them out here: * *Package Index*: https://packages.spack.io * *Buildcache Index*: https://cache.spack.io Both are searchable and more interactive than before. Currently major releases are shown; UI for browsing `develop` snapshots is coming soon. 8. **Default CMake and Meson build types are now Release** Spack has historically defaulted to building with optimization and debugging, but packages like `llvm` can be enormous with debug turned on. Our default build type for all Spack packages is now `Release` (#36679, #37436). This has a number of benefits: * much smaller binaries; * higher default optimization level; and * defining `NDEBUG` disables assertions, which may lead to further speedups. You can still get the old behavior back through requirements and package preferences. ## Other new commands and directives * `spack checksum` can automatically add new versions to package (#24532) * new command: `spack pkg grep` to easily search package files (#34388) * New `maintainers` directive (#35083) * Add `spack buildcache push` (alias to `buildcache create`) (#34861) * Allow using `-j` to control the parallelism of concretization (#37608) * Add `--exclude` option to 'spack external find' (#35013) ## Other new features of note * editing: add higher-precedence `SPACK_EDITOR` environment variable * Many YAML formatting improvements from updating `ruamel.yaml` to the latest version supporting Python 3.6. (#31091, #24885, #37008). * Requirements and preferences should not define (non-git) versions (#37687, #37747) * Environments now store spack version/commit in `spack.lock` (#32801) * User can specify the name of the `packages` subdirectory in repositories (#36643) * Add container images supporting RHEL alternatives (#36713) * make version(...) kwargs explicit (#36998) ## Notable refactors * buildcache create: reproducible tarballs (#35623) * Bootstrap most of Spack dependencies using environments (#34029) * Split `satisfies(..., strict=True/False)` into two functions (#35681) * spack install: simplify behavior when inside environments (#35206) ## Binary cache and stack updates * Major simplification of CI boilerplate in stacks (#34272, #36045) * Many improvements to our CI pipeline's reliability ## Removals, Deprecations, and disablements * Module file generation is disabled by default; you'll need to enable it to use it (#37258) * Support for Python 2 was deprecated in `v0.19.0` and has been removed. `v0.20.0` only supports Python 3.6 and higher. * Deprecated target names are no longer recognized by Spack. Use generic names instead: * `graviton` is now `cortex_a72` * `graviton2` is now `neoverse_n1` * `graviton3` is now `neoverse_v1` * `blacklist` and `whitelist` in module configuration were deprecated in `v0.19.0` and are removed in this release. Use `exclude` and `include` instead. * The `ignore=` parameter of the `extends()` directive has been removed. It was not used by any builtin packages and is no longer needed to avoid conflicts in environment views (#35588). * Support for the old YAML buildcache format has been removed. It was deprecated in `v0.19.0` (#34347). * `spack find --bootstrap` has been removed. It was deprecated in `v0.19.0`. Use `spack --bootstrap find` instead (#33964). * `spack bootstrap trust` and `spack bootstrap untrust` are now removed, having been deprecated in `v0.19.0`. Use `spack bootstrap enable` and `spack bootstrap disable`. * The `--mirror-name`, `--mirror-url`, and `--directory` options to buildcache and mirror commands were deprecated in `v0.19.0` and have now been removed. They have been replaced by positional arguments (#37457). * Deprecate `env:` as top level environment key (#37424) * deprecate buildcache create --rel, buildcache install --allow-root (#37285) * Support for very old perl-like spec format strings (e.g., `$_$@$%@+$+$=`) has been removed (#37425). This was deprecated in in `v0.15` (#10556). ## Notable Bugfixes * bugfix: don't fetch package metadata for unknown concrete specs (#36990) * Improve package source code context display on error (#37655) * Relax environment manifest filename requirements and lockfile identification criteria (#37413) * `installer.py`: drop build edges of installed packages by default (#36707) * Bugfix: package requirements with git commits (#35057, #36347) * Package requirements: allow single specs in requirement lists (#36258) * conditional variant values: allow boolean (#33939) * spack uninstall: follow run/link edges on --dependents (#34058) ## Spack community stats * 7,179 total packages, 499 new since `v0.19.0` * 329 new Python packages * 31 new R packages * 336 people contributed to this release * 317 committers to packages * 62 committers to core # v0.19.1 (2023-02-07) ### Spack Bugfixes * `buildcache create`: make "file exists" less verbose (#35019) * `spack mirror create`: don't change paths to urls (#34992) * Improve error message for requirements (#33988) * uninstall: fix accidental cubic complexity (#34005) * scons: fix signature for `install_args` (#34481) * Fix `combine_phase_logs` text encoding issues (#34657) * Use a module-like object to propagate changes in the MRO, when setting build env (#34059) * PackageBase should not define builder legacy attributes (#33942) * Forward lookup of the "run_tests" attribute (#34531) * Bugfix for timers (#33917, #33900) * Fix path handling in prefix inspections (#35318) * Fix libtool filter for Fujitsu compilers (#34916) * Bug fix for duplicate rpath errors on macOS when creating build caches (#34375) * FileCache: delete the new cache file on exception (#34623) * Propagate exceptions from Spack python console (#34547) * Tests: Fix a bug/typo in a `config_values.py` fixture (#33886) * Various CI fixes (#33953, #34560, #34560, #34828) * Docs: remove monitors and analyzers, typos (#34358, #33926) * bump release version for tutorial command (#33859) # v0.19.0 (2022-11-11) `v0.19.0` is a major feature release. ## Major features in this release 1. **Package requirements** Spack's traditional [package preferences]( https://spack.readthedocs.io/en/latest/build_settings.html#package-preferences) are soft, but we've added hard requirements to `packages.yaml` and `spack.yaml` (#32528, #32369). Package requirements use the same syntax as specs: ```yaml packages: libfabric: require: "@1.13.2" mpich: require: - one_of: ["+cuda", "+rocm"] ``` More details in [the docs]( https://spack.readthedocs.io/en/latest/build_settings.html#package-requirements). 2. **Environment UI Improvements** * Fewer surprising modifications to `spack.yaml` (#33711): * `spack install` in an environment will no longer add to the `specs:` list; you'll need to either use `spack add ` or `spack install --add `. * Similarly, `spack uninstall` will not remove from your environment's `specs:` list; you'll need to use `spack remove` or `spack uninstall --remove`. This will make it easier to manage an environment, as there is clear separation between the stack to be installed (`spack.yaml`/`spack.lock`) and which parts of it should be installed (`spack install` / `spack uninstall`). * `concretizer:unify:true` is now the default mode for new environments (#31787) We see more users creating `unify:true` environments now. Users who need `unify:false` can add it to their environment to get the old behavior. This will concretize every spec in the environment independently. * Include environment configuration from URLs (#29026, [docs]( https://spack.readthedocs.io/en/latest/environments.html#included-configurations)) You can now include configuration in your environment directly from a URL: ```yaml spack: include: - https://github.com/path/to/raw/config/compilers.yaml ``` 4. **Multiple Build Systems** An increasing number of packages in the ecosystem need the ability to support multiple build systems (#30738, [docs]( https://spack.readthedocs.io/en/latest/packaging_guide.html#multiple-build-systems)), either across versions, across platforms, or within the same version of the software. This has been hard to support through multiple inheritance, as methods from different build system superclasses would conflict. `package.py` files can now define separate builder classes with installation logic for different build systems, e.g.: ```python class ArpackNg(CMakePackage, AutotoolsPackage): build_system( conditional("cmake", when="@0.64:"), conditional("autotools", when="@:0.63"), default="cmake", ) class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder): def cmake_args(self): pass class Autotoolsbuilder(spack.build_systems.autotools.AutotoolsBuilder): def configure_args(self): pass ``` 5. **Compiler and variant propagation** Currently, compiler flags and variants are inconsistent: compiler flags set for a package are inherited by its dependencies, while variants are not. We should have these be consistent by allowing for inheritance to be enabled or disabled for both variants and compiler flags. Example syntax: - `package ++variant`: enabled variant that will be propagated to dependencies - `package +variant`: enabled variant that will NOT be propagated to dependencies - `package ~~variant`: disabled variant that will be propagated to dependencies - `package ~variant`: disabled variant that will NOT be propagated to dependencies - `package cflags==-g`: `cflags` will be propagated to dependencies - `package cflags=-g`: `cflags` will NOT be propagated to dependencies Syntax for non-boolean variants is similar to compiler flags. More in the docs for [variants]( https://spack.readthedocs.io/en/latest/basic_usage.html#variants) and [compiler flags]( https://spack.readthedocs.io/en/latest/basic_usage.html#compiler-flags). 6. **Enhancements to git version specifiers** * `v0.18.0` added the ability to use git commits as versions. You can now use the `git.` prefix to specify git tags or branches as versions. All of these are valid git versions in `v0.19` (#31200): ```console foo@abcdef1234abcdef1234abcdef1234abcdef1234 # raw commit foo@git.abcdef1234abcdef1234abcdef1234abcdef1234 # commit with git prefix foo@git.develop # the develop branch foo@git.0.19 # use the 0.19 tag ``` * `v0.19` also gives you more control over how Spack interprets git versions, in case Spack cannot detect the version from the git repository. You can suffix a git version with `=` to force Spack to concretize it as a particular version (#30998, #31914, #32257): ```console # use mybranch, but treat it as version 3.2 for version comparison foo@git.mybranch=3.2 # use the given commit, but treat it as develop for version comparison foo@git.abcdef1234abcdef1234abcdef1234abcdef1234=develop ``` More in [the docs]( https://spack.readthedocs.io/en/latest/basic_usage.html#version-specifier) 7. **Changes to Cray EX Support** Cray machines have historically had their own "platform" within Spack, because we needed to go through the module system to leverage compilers and MPI installations on these machines. The Cray EX programming environment now provides standalone `craycc` executables and proper `mpicc` wrappers, so Spack can treat EX machines like Linux with extra packages (#29392). We expect this to greatly reduce bugs, as external packages and compilers can now be used by prefix instead of through modules. We will also no longer be subject to reproducibility issues when modules change from Cray PE release to release and from site to site. This also simplifies dealing with the underlying Linux OS on cray systems, as Spack will properly model the machine's OS as either SuSE or RHEL. 8. **Improvements to tests and testing in CI** * `spack ci generate --tests` will generate a `.gitlab-ci.yml` file that not only does builds but also runs tests for built packages (#27877). Public GitHub pipelines now also run tests in CI. * `spack test run --explicit` will only run tests for packages that are explicitly installed, instead of all packages. 9. **Experimental binding link model** You can add a new option to `config.yaml` to make Spack embed absolute paths to needed shared libraries in ELF executables and shared libraries on Linux (#31948, [docs]( https://spack.readthedocs.io/en/latest/config_yaml.html#shared-linking-bind)): ```yaml config: shared_linking: type: rpath bind: true ``` This can improve launch time at scale for parallel applications, and it can make installations less susceptible to environment variables like `LD_LIBRARY_PATH`, even especially when dealing with external libraries that use `RUNPATH`. You can think of this as a faster, even higher-precedence version of `RPATH`. ## Other new features of note * `spack spec` prints dependencies more legibly. Dependencies in the output now appear at the *earliest* level of indentation possible (#33406) * You can override `package.py` attributes like `url`, directly in `packages.yaml` (#33275, [docs]( https://spack.readthedocs.io/en/latest/build_settings.html#assigning-package-attributes)) * There are a number of new architecture-related format strings you can use in Spack configuration files to specify paths (#29810, [docs]( https://spack.readthedocs.io/en/latest/configuration.html#config-file-variables)) * Spack now supports bootstrapping Clingo on Windows (#33400) * There is now support for an `RPATH`-like library model on Windows (#31930) ## Performance Improvements * Major performance improvements for installation from binary caches (#27610, #33628, #33636, #33608, #33590, #33496) * Test suite can now be parallelized using `xdist` (used in GitHub Actions) (#32361) * Reduce lock contention for parallel builds in environments (#31643) ## New binary caches and stacks * We now build nearly all of E4S with `oneapi` in our buildcache (#31781, #31804, #31804, #31803, #31840, #31991, #32117, #32107, #32239) * Added 3 new machine learning-centric stacks to binary cache: `x86_64_v3`, CUDA, ROCm (#31592, #33463) ## Removals and Deprecations * Support for Python 3.5 is dropped (#31908). Only Python 2.7 and 3.6+ are officially supported. * This is the last Spack release that will support Python 2 (#32615). Spack `v0.19` will emit a deprecation warning if you run it with Python 2, and Python 2 support will soon be removed from the `develop` branch. * `LD_LIBRARY_PATH` is no longer set by default by `spack load` or module loads. Setting `LD_LIBRARY_PATH` in Spack environments/modules can cause binaries from outside of Spack to crash, and Spack's own builds use `RPATH` and do not need `LD_LIBRARY_PATH` set in order to run. If you still want the old behavior, you can run these commands to configure Spack to set `LD_LIBRARY_PATH`: ```console spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH] spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH] ``` * The `spack:concretization:[together|separately]` has been removed after being deprecated in `v0.18`. Use `concretizer:unify:[true|false]`. * `config:module_roots` is no longer supported after being deprecated in `v0.18`. Use configuration in module sets instead (#28659, [docs]( https://spack.readthedocs.io/en/latest/module_file_support.html)). * `spack activate` and `spack deactivate` are no longer supported, having been deprecated in `v0.18`. Use an environment with a view instead of activating/deactivating ([docs]( https://spack.readthedocs.io/en/latest/environments.html#configuration-in-spack-yaml)). * The old YAML format for buildcaches is now deprecated (#33707). If you are using an old buildcache with YAML metadata you will need to regenerate it with JSON metadata. * `spack bootstrap trust` and `spack bootstrap untrust` are deprecated in favor of `spack bootstrap enable` and `spack bootstrap disable` and will be removed in `v0.20`. (#33600) * The `graviton2` architecture has been renamed to `neoverse_n1`, and `graviton3` is now `neoverse_v1`. Buildcaches using the old architecture names will need to be rebuilt. * The terms `blacklist` and `whitelist` have been replaced with `include` and `exclude` in all configuration files (#31569). You can use `spack config update` to automatically fix your configuration files. ## Notable Bugfixes * Permission setting on installation now handles effective uid properly (#19980) * `buildable:true` for an MPI implementation now overrides `buildable:false` for `mpi` (#18269) * Improved error messages when attempting to use an unconfigured compiler (#32084) * Do not punish explicitly requested compiler mismatches in the solver (#30074) * `spack stage`: add missing --fresh and --reuse (#31626) * Fixes for adding build system executables like `cmake` to package scope (#31739) * Bugfix for binary relocation with aliased strings produced by newer `binutils` (#32253) ## Spack community stats * 6,751 total packages, 335 new since `v0.18.0` * 141 new Python packages * 89 new R packages * 303 people contributed to this release * 287 committers to packages * 57 committers to core # v0.18.1 (2022-07-19) ### Spack Bugfixes * Fix several bugs related to bootstrapping (#30834,#31042,#31180) * Fix a regression that was causing spec hashes to differ between Python 2 and Python 3 (#31092) * Fixed compiler flags for oneAPI and DPC++ (#30856) * Fixed several issues related to concretization (#31142,#31153,#31170,#31226) * Improved support for Cray manifest file and `spack external find` (#31144,#31201,#31173,#31186) * Assign a version to openSUSE Tumbleweed according to the GLIBC version in the system (#19895) * Improved Dockerfile generation for `spack containerize` (#29741,#31321) * Fixed a few bugs related to concurrent execution of commands (#31509,#31493,#31477) ### Package updates * WarpX: add v22.06, fixed libs property (#30866,#31102) * openPMD: add v0.14.5, update recipe for @develop (#29484,#31023) # v0.18.0 (2022-05-28) `v0.18.0` is a major feature release. ## Major features in this release 1. **Concretizer now reuses by default** `spack install --reuse` was introduced in `v0.17.0`, and `--reuse` is now the default concretization mode. Spack will try hard to resolve dependencies using installed packages or binaries (#30396). To avoid reuse and to use the latest package configurations, (the old default), you can use `spack install --fresh`, or add configuration like this to your environment or `concretizer.yaml`: ```yaml concretizer: reuse: false ``` 2. **Finer-grained hashes** Spack hashes now include `link`, `run`, *and* `build` dependencies, as well as a canonical hash of package recipes. Previously, hashes only included `link` and `run` dependencies (though `build` dependencies were stored by environments). We coarsened the hash to reduce churn in user installations, but the new default concretizer behavior mitigates this concern and gets us reuse *and* provenance. You will be able to see the build dependencies of new installations with `spack find`. Old installations will not change and their hashes will not be affected. (#28156, #28504, #30717, #30861) 3. **Improved error messages** Error handling with the new concretizer is now done with optimization criteria rather than with unsatisfiable cores, and Spack reports many more details about conflicting constraints. (#30669) 4. **Unify environments when possible** Environments have thus far supported `concretization: together` or `concretization: separately`. These have been replaced by a new preference in `concretizer.yaml`: ```yaml concretizer: unify: [true|false|when_possible] ``` `concretizer:unify:when_possible` will *try* to resolve a fully unified environment, but if it cannot, it will create multiple configurations of some packages where it has to. For large environments that previously had to be concretized separately, this can result in a huge speedup (40-50x). (#28941) 5. **Automatically find externals on Cray machines** Spack can now automatically discover installed packages in the Cray Programming Environment by running `spack external find` (or `spack external read-cray-manifest` to *only* query the PE). Packages from the PE (e.g., `cray-mpich` are added to the database with full dependency information, and compilers from the PE are added to `compilers.yaml`. Available with the June 2022 release of the Cray Programming Environment. (#24894, #30428) 6. **New binary format and hardened signing** Spack now has an updated binary format, with improvements for security. The new format has a detached signature file, and Spack verifies the signature before untarring or decompressing the binary package. The previous format embedded the signature in a `tar` file, which required the client to run `tar` *before* verifying (#30750). Spack can still install from build caches using the old format, but we encourage users to switch to the new format going forward. Production GitLab pipelines have been hardened to securely sign binaries. There is now a separate signing stage so that signing keys are never exposed to build system code, and signing keys are ephemeral and only live as long as the signing pipeline stage. (#30753) 7. **Bootstrap mirror generation** The `spack bootstrap mirror` command can automatically create a mirror for bootstrapping the concretizer and other needed dependencies in an air-gapped environment. (#28556) 8. **Nascent Windows support** Spack now has initial support for Windows. Spack core has been refactored to run in the Windows environment, and a small number of packages can now build for Windows. More details are [in the documentation](https://spack.rtfd.io/en/latest/getting_started.html#spack-on-windows) (#27021, #28385, many more) 9. **Makefile generation** `spack env depfile` can be used to generate a `Makefile` from an environment, which can be used to build packages the environment in parallel on a single node. e.g.: ```console spack -e myenv env depfile > Makefile make ``` Spack propagates `gmake` jobserver information to builds so that their jobs can share cores. (#30039, #30254, #30302, #30526) 10. **New variant features** In addition to being conditional themselves, variants can now have [conditional *values*](https://spack.readthedocs.io/en/latest/packaging_guide.html#conditional-possible-values) that are only possible for certain configurations of a package. (#29530) Variants can be [declared "sticky"](https://spack.readthedocs.io/en/latest/packaging_guide.html#sticky-variants), which prevents them from being enabled or disabled by the concretizer. Sticky variants must be set explicitly by users on the command line or in `packages.yaml`. (#28630) * Allow conditional possible values in variants * Add a "sticky" property to variants ## Other new features of note * Environment views can optionally link only `run` dependencies with `link:run` (#29336) * `spack external find --all` finds library-only packages in addition to build dependencies (#28005) * Customizable `config:license_dir` option (#30135) * `spack external find --path PATH` takes a custom search path (#30479) * `spack spec` has a new `--format` argument like `spack find` (#27908) * `spack concretize --quiet` skips printing concretized specs (#30272) * `spack info` now has cleaner output and displays test info (#22097) * Package-level submodule option for git commit versions (#30085, #30037) * Using `/hash` syntax to refer to concrete specs in an environment now works even if `/hash` is not installed. (#30276) ## Major internal refactors * full hash (see above) * new develop versioning scheme `0.19.0-dev0` * Allow for multiple dependencies/dependents from the same package (#28673) * Splice differing virtual packages (#27919) ## Performance Improvements * Concretization of large environments with `unify: when_possible` is much faster than concretizing separately (#28941, see above) * Single-pass view generation algorithm is 2.6x faster (#29443) ## Archspec improvements * `oneapi` and `dpcpp` flag support (#30783) * better support for `M1` and `a64fx` (#30683) ## Removals and Deprecations * Spack no longer supports Python `2.6` (#27256) * Removed deprecated `--run-tests` option of `spack install`; use `spack test` (#30461) * Removed deprecated `spack flake8`; use `spack style` (#27290) * Deprecate `spack:concretization` config option; use `concretizer:unify` (#30038) * Deprecate top-level module configuration; use module sets (#28659) * `spack activate` and `spack deactivate` are deprecated in favor of environments; will be removed in `0.19.0` (#29430; see also `link:run` in #29336 above) ## Notable Bugfixes * Fix bug that broke locks with many parallel builds (#27846) * Many bugfixes and consistency improvements for the new concretizer and `--reuse` (#30357, #30092, #29835, #29933, #28605, #29694, #28848) ## Packages * `CMakePackage` uses `CMAKE_INSTALL_RPATH_USE_LINK_PATH` (#29703) * Refactored `lua` support: `lua-lang` virtual supports both `lua` and `luajit` via new `LuaPackage` build system(#28854) * PythonPackage: now installs packages with `pip` (#27798) * Python: improve site_packages_dir handling (#28346) * Extends: support spec, not just package name (#27754) * `find_libraries`: search for both .so and .dylib on macOS (#28924) * Use stable URLs and `?full_index=1` for all github patches (#29239) ## Spack community stats * 6,416 total packages, 458 new since `v0.17.0` * 219 new Python packages * 60 new R packages * 377 people contributed to this release * 337 committers to packages * 85 committers to core # v0.17.3 (2022-07-14) ### Spack bugfixes * Fix missing chgrp on symlinks in package installations (#30743) * Allow having non-existing upstreams (#30744, #30746) * Fix `spack stage` with custom paths (#30448) * Fix failing call for `spack buildcache save-specfile` (#30637) * Fix globbing in compiler wrapper (#30699) # v0.17.2 (2022-04-13) ### Spack bugfixes * Fix --reuse with upstreams set in an environment (#29680) * config add: fix parsing of validator error to infer type from oneOf (#29475) * Fix spack -C command_line_scope used in conjunction with other flags (#28418) * Use Spec.constrain to construct spec lists for stacks (#28783) * Fix bug occurring when searching for inherited patches in packages (#29574) * Fixed a few bugs when manipulating symlinks (#28318, #29515, #29636) * Fixed a few minor bugs affecting command prompt, terminal title and argument completion (#28279, #28278, #28939, #29405, #29070, #29402) * Fixed a few bugs affecting the spack ci command (#29518, #29419) * Fix handling of Intel compiler environment (#29439) * Fix a few edge cases when reindexing the DB (#28764) * Remove "Known issues" from documentation (#29664) * Other miscellaneous bugfixes (0b72e070583fc5bcd016f5adc8a84c99f2b7805f, #28403, #29261) # v0.17.1 (2021-12-23) ### Spack Bugfixes * Allow locks to work under high contention (#27846) * Improve errors messages from clingo (#27707 #27970) * Respect package permissions for sbang (#25764) * Fix --enable-locks behavior (#24675) * Fix log-format reporter ignoring install errors (#25961) * Fix overloaded argparse keys (#27379) * Allow style commands to run with targets other than "develop" (#27472) * Log lock messages to debug level, instead of verbose level (#27408) * Handle invalid unicode while logging (#21447) * spack audit: fix API calls to variants (#27713) * Provide meaningful message for empty environment installs (#28031) * Added opensuse leap containers to spack containerize (#27837) * Revert "patches: make re-applied patches idempotent" (#27625) * MANPATH can use system defaults (#21682) * Add "setdefault" subcommand to `spack module tcl` (#14686) * Regenerate views when specs already installed (#28113) ### Package bugfixes * Fix external package detection for OpenMPI (#27255) * Update the UPC++ package to 2021.9.0 (#26996) * Added py-vermin v1.3.2 (#28072) # v0.17.0 (2021-11-05) `v0.17.0` is a major feature release. ## Major features in this release 1. **New concretizer is now default** The new concretizer introduced as an experimental feature in `v0.16.0` is now the default (#25502). The new concretizer is based on the [clingo](https://github.com/potassco/clingo) logic programming system, and it enables us to do much higher quality and faster dependency solving The old concretizer is still available via the `concretizer: original` setting, but it is deprecated and will be removed in `v0.18.0`. 2. **Binary Bootstrapping** To make it easier to use the new concretizer and binary packages, Spack now bootstraps `clingo` and `GnuPG` from public binaries. If it is not able to bootstrap them from binaries, it installs them from source code. With these changes, you should still be able to clone Spack and start using it almost immediately. (#21446, #22354, #22489, #22606, #22720, #22720, #23677, #23946, #24003, #25138, #25607, #25964, #26029, #26399, #26599). 3. **Reuse existing packages (experimental)** The most wanted feature from our [2020 user survey](https://spack.io/spack-user-survey-2020/) and the most wanted Spack feature of all time (#25310). `spack install`, `spack spec`, and `spack concretize` now have a `--reuse` option, which causes Spack to minimize the number of rebuilds it does. The `--reuse` option will try to find existing installations and binary packages locally and in registered mirrors, and will prefer to use them over building new versions. This will allow users to build from source *far* less than in prior versions of Spack. This feature will continue to be improved, with configuration options and better CLI expected in `v0.17.1`. It will become the *default* concretization mode in `v0.18.0`. 4. **Better error messages** We have improved the error messages generated by the new concretizer by using *unsatisfiable cores*. Spack will now print a summary of the types of constraints that were violated to make a spec unsatisfiable (#26719). 5. **Conditional variants** Variants can now have a `when=""` clause, allowing them to be conditional based on the version or other attributes of a package (#24858). 6. **Git commit versions** In an environment and on the command-line, you can now provide a full, 40-character git commit as a version for any package with a top-level `git` URL. e.g., `spack install hdf5@45bb27f58240a8da7ebb4efc821a1a964d7712a8`. Spack will compare the commit to tags in the git repository to understand what versions it is ahead of or behind. 7. **Override local config and cache directories** You can now set `SPACK_DISABLE_LOCAL_CONFIG` to disable the `~/.spack` and `/etc/spack` configuration scopes. `SPACK_USER_CACHE_PATH` allows you to move caches out of `~/.spack`, as well (#27022, #26735). This addresses common problems where users could not isolate CI environments from local configuration. 8. **Improvements to Spack Containerize** For added reproducibility, you can now pin the Spack version used by `spack containerize` (#21910). The container build will only build with the Spack version pinned at build recipe creation instead of the latest Spack version. 9. **New commands for dealing with tags** The `spack tags` command allows you to list tags on packages (#26136), and you can list tests and filter tags with `spack test list` (#26842). ## Other new features of note * Copy and relocate environment views as stand-alone installations (#24832) * `spack diff` command can diff two installed specs (#22283, #25169) * `spack -c ` can set one-off config parameters on CLI (#22251) * `spack load --list` is an alias for `spack find --loaded` (#27184) * `spack gpg` can export private key with `--secret` (#22557) * `spack style` automatically bootstraps dependencies (#24819) * `spack style --fix` automatically invokes `isort` (#24071) * build dependencies can be installed from build caches with `--include-build-deps` (#19955) * `spack audit` command for checking package constraints (#23053) * Spack can now fetch from `CVS` repositories (yep, really) (#23212) * `spack monitor` lets you upload analysis about installations to a [spack monitor server](https://github.com/spack/spack-monitor) (#23804, #24321, #23777, #25928)) * `spack python --path` shows which `python` Spack is using (#22006) * `spack env activate --temp` can create temporary environments (#25388) * `--preferred` and `--latest` options for `spack checksum` (#25830) * `cc` is now pure posix and runs on Alpine (#26259) * `SPACK_PYTHON` environment variable sets which `python` spack uses (#21222) * `SPACK_SKIP_MODULES` lets you source `setup-env.sh` faster if you don't need modules (#24545) ## Major internal refactors * `spec.yaml` files are now `spec.json`, yielding a large speed improvement (#22845) * Splicing allows Spack specs to store mixed build provenance (#20262) * More extensive hooks API for installations (#21930) * New internal API for getting the active environment (#25439) ## Performance Improvements * Parallelize separate concretization in environments; Previously 55 min E4S solve now takes 2.5 min (#26264) * Drastically improve YamlFilesystemView file removal performance via batching (#24355) * Speed up spec comparison (#21618) * Speed up environment activation (#25633) ## Archspec improvements * support for new generic `x86_64_v2`, `x86_64_v3`, `x86_64_v4` targets (see [archspec#31](https://github.com/archspec/archspec-json/pull/31)) * `spack arch --generic` lets you get the best generic architecture for your node (#27061) * added support for aocc (#20124), `arm` compiler on `graviton2` (#24904) and on `a64fx` (#24524), ## Infrastructure, buildcaches, and services * Add support for GCS Bucket Mirrors (#26382) * Add `spackbot` to help package maintainers with notifications. See [spack.github.io/spackbot](https://spack.github.io/spackbot/) * Reproducible pipeline builds with `spack ci rebuild` (#22887) * Removed redundant concretizations from GitLab pipeline generation (#26622) * Spack CI no longer generates jobs for unbuilt specs (#20435) * Every pull request pipeline has its own buildcache (#25529) * `--no-add` installs only specified specs and only if already present in… (#22657) * Add environment-aware `spack buildcache sync` command (#25470) * Binary cache installation speedups and improvements (#19690, #20768) ## Deprecations and Removals * `spack setup` was deprecated in v0.16.0, and has now been removed. Use `spack develop` and `spack dev-build`. * Remove unused `--dependencies` flag from `spack load` (#25731) * Remove stubs for `spack module [refresh|find|rm|loads]`, all of which were deprecated in 2018. ## Notable Bugfixes * Deactivate previous env before activating new one (#25409) * Many fixes to error codes from `spack install` (#21319, #27012, #25314) * config add: infer type based on JSON schema validation errors (#27035) * `spack config edit` now works even if `spack.yaml` is broken (#24689) ## Packages * Allow non-empty version ranges like `1.1.0:1.1` (#26402) * Remove `.99`'s from many version ranges (#26422) * Python: use platform-specific site packages dir (#25998) * `CachedCMakePackage` for using *.cmake initial config files (#19316) * `lua-lang` allows swapping `lua` and `luajit` (#22492) * Better support for `ld.gold` and `ld.lld` (#25626) * build times are now stored as metadata in `$prefix/.spack` (#21179) * post-install tests can be reused in smoke tests (#20298) * Packages can use `pypi` attribute to infer `homepage`/`url`/`list_url` (#17587) * Use gnuconfig package for `config.guess` file replacement (#26035) * patches: make re-applied patches idempotent (#26784) ## Spack community stats * 5969 total packages, 920 new since `v0.16.0` * 358 new Python packages, 175 new R packages * 513 people contributed to this release * 490 committers to packages * 105 committers to core * Lots of GPU updates: * ~77 CUDA-related commits * ~66 AMD-related updates * ~27 OneAPI-related commits * 30 commits from AMD toolchain support * `spack test` usage in packages is increasing * 1669 packages with tests (mostly generic python tests) * 93 packages with their own tests # v0.16.3 (2021-09-21) * clang/llvm: fix version detection (#19978) * Fix use of quotes in Python build system (#22279) * Cray: fix extracting paths from module files (#23472) * Use AWS CloudFront for source mirror (#23978) * Ensure all roots of an installed environment are marked explicit in db (#24277) * Fix fetching for Python 3.8 and 3.9 (#24686) * locks: only open lockfiles once instead of for every lock held (#24794) * Remove the EOL centos:6 docker image # v0.16.2 (2021-05-22) * Major performance improvement for `spack load` and other commands. (#23661) * `spack fetch` is now environment-aware. (#19166) * Numerous fixes for the new, `clingo`-based concretizer. (#23016, #23307, #23090, #22896, #22534, #20644, #20537, #21148) * Support for automatically bootstrapping `clingo` from source. (#20652, #20657 #21364, #21446, #21913, #22354, #22444, #22460, #22489, #22610, #22631) * Python 3.10 support: `collections.abc` (#20441) * Fix import issues by using `__import__` instead of Spack package importe. (#23288, #23290) * Bugfixes and `--source-dir` argument for `spack location`. (#22755, #22348, #22321) * Better support for externals in shared prefixes. (#22653) * `spack build-env` now prefers specs defined in the active environment. (#21642) * Remove erroneous warnings about quotes in `from_sourcing_files`. (#22767) * Fix clearing cache of `InternalConfigScope`. (#22609) * Bugfix for active when pkg is already active error. (#22587) * Make `SingleFileScope` able to repopulate the cache after clearing it. (#22559) * Channelflow: Fix the package. (#22483) * More descriptive error message for bugs in `package.py` (#21811) * Use package-supplied `autogen.sh`. (#20319) * Respect `-k/verify-ssl-false` in `_existing_url` method. (#21864) # v0.16.1 (2021-02-22) This minor release includes a new feature and associated fixes: * intel-oneapi support through new packages (#20411, #20686, #20693, #20717, #20732, #20808, #21377, #21448) This release also contains bug fixes/enhancements for: * HIP/ROCm support (#19715, #20095) * concretization (#19988, #20020, #20082, #20086, #20099, #20102, #20128, #20182, #20193, #20194, #20196, #20203, #20247, #20259, #20307, #20362, #20383, #20423, #20473, #20506, #20507, #20604, #20638, #20649, #20677, #20680, #20790) * environment install reporting fix (#20004) * avoid import in ABI compatibility info (#20236) * restore ability of dev-build to skip patches (#20351) * spack find -d spec grouping (#20028) * spack smoke test support (#19987, #20298) * macOS fixes (#20038, #21662) * abstract spec comparisons (#20341) * continuous integration (#17563) * performance improvements for binary relocation (#19690, #20768) * additional sanity checks for variants in builtin packages (#20373) * do not pollute auto-generated configuration files with empty lists or dicts (#20526) plus assorted documentation (#20021, #20174) and package bug fixes/enhancements (#19617, #19933, #19986, #20006, #20097, #20198, #20794, #20906, #21411). # v0.16.0 (2020-11-18) `v0.16.0` is a major feature release. ## Major features in this release 1. **New concretizer (experimental)** Our new backtracking concretizer is now in Spack as an experimental feature. You will need to install `clingo@master+python` and set `concretizer: clingo` in `config.yaml` to use it. The original concretizer is not exhaustive and is not guaranteed to find a solution if one exists. We encourage you to use the new concretizer and to report any bugs you find with it. We anticipate making the new concretizer the default and including all required dependencies for it in Spack `v0.17`. For more details, see #19501. 2. **spack test (experimental)** Users can add `test()` methods to their packages to run smoke tests on installations with the new `spack test` command (the old `spack test` is now `spack unit-test`). `spack test` is environment-aware, so you can `spack install` an environment and `spack test run` smoke tests on all of its packages. Historical test logs can be perused with `spack test results`. Generic smoke tests for MPI implementations, C, C++, and Fortran compilers as well as specific smoke tests for 18 packages. This is marked experimental because the test API (`self.run_test()`) is likely to be change, but we encourage users to upstream tests, and we will maintain and refactor any that are added to mainline packages (#15702). 3. **spack develop** New `spack develop` command allows you to develop several packages at once within a Spack environment. Running `spack develop foo@v1` and `spack develop bar@v2` will check out specific versions of `foo` and `bar` into subdirectories, which you can then build incrementally with `spack install ` (#15256). 4. **More parallelism** Spack previously installed the dependencies of a _single_ spec in parallel. Entire environments can now be installed in parallel, greatly accelerating builds of large environments. get parallelism from individual specs. Spack now parallelizes entire environment builds (#18131). 5. **Customizable base images for spack containerize** `spack containerize` previously only output a `Dockerfile` based on `ubuntu`. You may now specify any base image of your choosing (#15028). 6. **more external finding** `spack external find` was added in `v0.15`, but only `cmake` had support. `spack external find` can now find `bison`, `cuda`, `findutils`, `flex`, `git`, `lustre` `m4`, `mpich`, `mvapich2`, `ncurses`, `openmpi`, `perl`, `spectrum-mpi`, `tar`, and `texinfo` on your system and add them automatically to `packages.yaml`. 7. **Support aocc, nvhpc, and oneapi compilers** We are aggressively pursuing support for the newest vendor compilers, especially those for the U.S. exascale and pre-exascale systems. Compiler classes and auto-detection for `aocc`, `nvhpc`, `oneapi` are now in Spack (#19345, #19294, #19330). ## Additional new features of note * New `spack mark` command can be used to designate packages as explicitly installed, so that `spack gc` will not garbage-collect them (#16662). * `install_tree` can be customized with Spack's projection format (#18341) * `sbang` now lives in the `install_tree` so that all users can access it (#11598) * `csh` and `tcsh` users no longer need to set `SPACK_ROOT` before sourcing `setup-env.csh` (#18225) * Spec syntax now supports `variant=*` syntax for finding any package that has a particular variant (#19381). * Spack respects `SPACK_GNUPGHOME` variable for custom GPG directories (#17139) * Spack now recognizes Graviton chips ## Major refactors * Use spawn instead of fork on Python >= 3.8 on macOS (#18205) * Use indexes for public build caches (#19101, #19117, #19132, #19141, #19209) * `sbang` is an external package now (https://github.com/spack/sbang, #19582) * `archspec` is an external package now (https://github.com/archspec/archspec, #19600) ## Deprecations and Removals * `spack bootstrap` was deprecated in v0.14.0, and has now been removed. * `spack setup` is deprecated as of v0.16.0. * What was `spack test` is now called `spack unit-test`. `spack test` is now the smoke testing feature in (2) above. ## Bugfixes Some of the most notable bugfixes in this release include: * Better warning messages for deprecated syntax in `packages.yaml` (#18013) * `buildcache list --allarch` now works properly (#17827) * Many fixes and tests for buildcaches and binary relcoation (#15687, *#17455, #17418, #17455, #15687, #18110) ## Package Improvements Spack now has 5050 total packages, 720 of which were added since `v0.15`. * ROCm packages (`hip`, `aomp`, more) added by AMD (#19957, #19832, others) * Many improvements for ARM support * `llvm-flang`, `flang`, and `f18` removed, as `llvm` has real `flang` support since Flang was merged to LLVM mainline * Emerging support for `spack external find` and `spack test` in packages. ## Infrastructure * Major infrastructure improvements to pipelines on `gitlab.spack.io` * Support for testing PRs from forks (#19248) is being enabled for all forks to enable rolling, up-to-date binary builds on `develop` # v0.15.4 (2020-08-12) This release contains one feature addition: * Users can set `SPACK_GNUPGHOME` to override Spack's GPG path (#17139) Several bugfixes for CUDA, binary packaging, and `spack -V`: * CUDA package's `.libs` method searches for `libcudart` instead of `libcuda` (#18000) * Don't set `CUDAHOSTCXX` in environments that contain CUDA (#17826) * `buildcache create`: `NoOverwriteException` is a warning, not an error (#17832) * Fix `spack buildcache list --allarch` (#17884) * `spack -V` works with `releases/latest` tag and shallow clones (#17884) And fixes for GitHub Actions and tests to ensure that CI passes on the release branch (#15687, #17279, #17328, #17377, #17732). # v0.15.3 (2020-07-28) This release contains the following bugfixes: * Fix handling of relative view paths (#17721) * Fixes for binary relocation (#17418, #17455) * Fix redundant printing of error messages in build environment (#17709) It also adds a support script for Spack tutorials: * Add a tutorial setup script to share/spack (#17705, #17722) # v0.15.2 (2020-07-23) This minor release includes two new features: * Spack install verbosity is decreased, and more debug levels are added (#17546) * The $spack/share/spack/keys directory contains public keys that may be optionally trusted for public binary mirrors (#17684) This release also includes several important fixes: * MPICC and related variables are now cleand in the build environment (#17450) * LLVM flang only builds CUDA offload components when +cuda (#17466) * CI pipelines no longer upload user environments that can contain secrets to the internet (#17545) * CI pipelines add bootstrapped compilers to the compiler config (#17536) * `spack buildcache list` does not exit on first failure and lists later mirrors (#17565) * Apple's "gcc" executable that is an apple-clang compiler does not generate a gcc compiler config (#17589) * Mixed compiler toolchains are merged more naturally across different compiler suffixes (#17590) * Cray Shasta platforms detect the OS properly (#17467) * Additional more minor fixes. # v0.15.1 (2020-07-10) This minor release includes several important fixes: * Fix shell support on Cray (#17386) * Fix use of externals installed with other Spack instances (#16954) * Fix gcc+binutils build (#9024) * Fixes for usage of intel-mpi (#17378 and #17382) * Fixes to Autotools config.guess detection (#17333 and #17356) * Update `spack install` message to prompt user when an environment is not explicitly activated (#17454) This release also adds a mirror for all sources that are fetched in Spack (#17077). It is expected to be useful when the official website for a Spack package is unavailable. # v0.15.0 (2020-06-28) `v0.15.0` is a major feature release. ## Major Features in this release 1. **Cray support** Spack will now work properly on Cray "Cluster" systems (non XC systems) and after a `module purge` command on Cray systems. See #12989 2. **Virtual package configuration** Virtual packages are allowed in packages.yaml configuration. This allows users to specify a virtual package as non-buildable without needing to specify for each implementation. See #14934 3. **New config subcommands** This release adds `spack config add` and `spack config remove` commands to add to and remove from yaml configuration files from the CLI. See #13920 4. **Environment activation** Anonymous environments are **no longer** automatically activated in the current working directory. To activate an environment from a `spack.yaml` file in the current directory, use the `spack env activate .` command. This removes a concern that users were too easily polluting their anonymous environments with accidental installations. See #17258 5. **Apple clang compiler** The clang compiler and the apple-clang compiler are now separate compilers in Spack. This allows Spack to improve support for the apple-clang compiler. See #17110 6. **Finding external packages** Spack packages can now support an API for finding external installations. This allows the `spack external find` command to automatically add installations of those packages to the user's configuration. See #15158 ## Additional new features of note * support for using Spack with the fish shell (#9279) * `spack load --first` option to load first match (instead of prompting user) (#15622) * support the Cray cce compiler both new and classic versions (#17256, #12989) * `spack dev-build` command: * supports stopping before a specified phase (#14699) * supports automatically launching a shell in the build environment (#14887) * `spack install --fail-fast` allows builds to fail at the first error (rather than best-effort) (#15295) * environments: SpecList references can be dereferenced as compiler or dependency constraints (#15245) * `spack view` command: new support for a copy/relocate view type (#16480) * ci pipelines: see documentation for several improvements * `spack mirror -a` command now supports excluding packages (#14154) * `spack buildcache create` is now environment-aware (#16580) * module generation: more flexible format for specifying naming schemes (#16629) * lmod module generation: packages can be configured as core specs for lmod hierarchy (#16517) ## Deprecations and Removals The following commands were deprecated in v0.13.0, and have now been removed: * `spack configure` * `spack build` * `spack diy` The following commands were deprecated in v0.14.0, and will be removed in the next major release: * `spack bootstrap` ## Bugfixes Some of the most notable bugfixes in this release include: * Spack environments can now contain the string `-h` (#15429) * The `spack install` gracefully handles being backgrounded (#15723, #14682) * Spack uses `-isystem` instead of `-I` in cases that the underlying build system does as well (#16077) * Spack no longer prints any specs that cannot be safely copied into a Spack command (#16462) * Incomplete Spack environments containing python no longer cause problems (#16473) * Several improvements to binary package relocation ## Package Improvements The Spack project is constantly engaged in routine maintenance, bugfixes, and improvements for the package ecosystem. Of particular note in this release are the following: * Spack now contains 4339 packages. There are 430 newly supported packages in v0.15.0 * GCC now builds properly on ARM architectures (#17280) * Python: patched to support compiling mixed C/C++ python modules through distutils (#16856) * improvements to pytorch and py-tensorflow packages * improvements to major MPI implementations: mvapich2, mpich, openmpi, and others ## Spack Project Management: * Much of the Spack CI infrastructure has moved from Travis to GitHub Actions (#16610, #14220, #16345) * All merges to the `develop` branch run E4S CI pipeline (#16338) * New `spack debug report` command makes reporting bugs easier (#15834) # v0.14.2 (2020-04-15) This is a minor release on the `0.14` series. It includes performance improvements and bug fixes: * Improvements to how `spack install` handles foreground/background (#15723) * Major performance improvements for reading the package DB (#14693, #15777) * No longer check for the old `index.yaml` database file (#15298) * Properly activate environments with '-h' in the name (#15429) * External packages have correct `.prefix` in environments/views (#15475) * Improvements to computing env modifications from sourcing files (#15791) * Bugfix on Cray machines when getting `TERM` env variable (#15630) * Avoid adding spurious `LMOD` env vars to Intel modules (#15778) * Don't output [+] for mock installs run during tests (#15609) # v0.14.1 (2020-03-20) This is a bugfix release on top of `v0.14.0`. Specific fixes include: * several bugfixes for parallel installation (#15339, #15341, #15220, #15197) * `spack load` now works with packages that have been renamed (#14348) * bugfix for `suite-sparse` installation (#15326) * deduplicate identical suffixes added to module names (#14920) * fix issues with `configure_args` during module refresh (#11084) * increased test coverage and test fixes (#15237, #15354, #15346) * remove some unused code (#15431) # v0.14.0 (2020-02-23) `v0.14.0` is a major feature release, with 3 highlighted features: 1. **Distributed builds.** Multiple Spack instances will now coordinate properly with each other through locks. This works on a single node (where you've called `spack` several times) or across multiple nodes with a shared filesystem. For example, with SLURM, you could build `trilinos` and its dependencies on 2 24-core nodes, with 3 Spack instances per node and 8 build jobs per instance, with `srun -N 2 -n 6 spack install -j 8 trilinos`. This requires a filesystem with locking enabled, but not MPI or any other library for parallelism. 2. **Build pipelines.** You can also build in parallel through Gitlab CI. Simply create a Spack environment and push it to Gitlab to build on Gitlab runners. Pipeline support is now integrated into a single `spack ci` command, so setting it up is easier than ever. See the [Pipelines section](https://spack.readthedocs.io/en/v0.14.0/pipelines.html) in the docs. 3. **Container builds.** The new `spack containerize` command allows you to create a Docker or Singularity recipe from any Spack environment. There are options to customize the build if you need them. See the [Container Images section](https://spack.readthedocs.io/en/latest/containers.html) in the docs. In addition, there are several other new commands, many bugfixes and improvements, and `spack load` no longer requires modules, so you can use it the same way on your laptop or on your supercomputer. Spack grew by over 300 packages since our last release in November 2019, and the project grew to over 500 contributors. Thanks to all of you for making yet another great release possible. Detailed notes below. ## Major new core features * Distributed builds: spack instances coordinate and build in parallel (#13100) * New `spack ci` command to manage CI pipelines (#12854) * Generate container recipes from environments: `spack containerize` (#14202) * `spack load` now works without using modules (#14062, #14628) * Garbage collect old/unused installations with `spack gc` (#13534) * Configuration files all set environment modifications the same way (#14372, [docs](https://spack.readthedocs.io/en/v0.14.0/configuration.html#environment-modifications)) * `spack commands --format=bash` auto-generates completion (#14393, #14607) * Packages can specify alternate fetch URLs in case one fails (#13881) ## Improvements * Improved locking for concurrency with environments (#14676, #14621, #14692) * `spack test` sends args to `pytest`, supports better listing (#14319) * Better support for aarch64 and cascadelake microarch (#13825, #13780, #13820) * Archspec is now a separate library (see https://github.com/archspec/archspec) * Many improvements to the `spack buildcache` command (#14237, #14346, #14466, #14467, #14639, #14642, #14659, #14696, #14698, #14714, #14732, #14929, #15003, #15086, #15134) ## Selected Bugfixes * Compilers now require an exact match on version (#8735, #14730, #14752) * Bugfix for patches that specified specific versions (#13989) * `spack find -p` now works in environments (#10019, #13972) * Dependency queries work correctly in `spack find` (#14757) * Bugfixes for locking upstream Spack instances chains (#13364) * Fixes for PowerPC clang optimization flags (#14196) * Fix for issue with compilers and specific microarchitectures (#13733, #14798) ## New commands and options * `spack ci` (#12854) * `spack containerize` (#14202) * `spack gc` (#13534) * `spack load` accepts `--only package`, `--only dependencies` (#14062, #14628) * `spack commands --format=bash` (#14393) * `spack commands --update-completion` (#14607) * `spack install --with-cache` has new option: `--no-check-signature` (#11107) * `spack test` now has `--list`, `--list-long`, and `--list-names` (#14319) * `spack install --help-cdash` moves CDash help out of the main help (#13704) ## Deprecations * `spack release-jobs` has been rolled into `spack ci` * `spack bootstrap` will be removed in a future version, as it is no longer needed to set up modules (see `spack load` improvements above) ## Documentation * New section on building container images with Spack (see [docs](https://spack.readthedocs.io/en/latest/containers.html)) * New section on using `spack ci` command to build pipelines (see [docs](https://spack.readthedocs.io/en/latest/pipelines.html)) * Document how to add conditional dependencies (#14694) * Document how to use Spack to replace Homebrew/Conda (#13083, see [docs](https://spack.readthedocs.io/en/latest/workflows.html#using-spack-to-replace-homebrew-conda)) ## Important package changes * 3,908 total packages (345 added since 0.13.0) * Added first cut at a TensorFlow package (#13112) * We now build R without "recommended" packages, manage them w/Spack (#12015) * Elpa and OpenBLAS now leverage microarchitecture support (#13655, #14380) * Fix `octave` compiler wrapper usage (#14726) * Enforce that packages in `builtin` aren't missing dependencies (#13949) # v0.13.4 (2020-02-07) This release contains several bugfixes: * bugfixes for invoking python in various environments (#14349, #14496, #14569) * brought tab completion up to date (#14392) * bugfix for removing extensions from views in order (#12961) * bugfix for nondeterministic hashing for specs with externals (#14390) # v0.13.3 (2019-12-23) This release contains more major performance improvements for Spack environments, as well as bugfixes for mirrors and a `python` issue with RHEL8. * mirror bugfixes: symlinks, duplicate patches, and exception handling (#13789) * don't try to fetch `BundlePackages` (#13908) * avoid re-fetching patches already added to a mirror (#13908) * avoid re-fetching already added patches (#13908) * avoid re-fetching already added patches (#13908) * allow repeated invocations of `spack mirror create` on the same dir (#13908) * bugfix for RHEL8 when `python` is unavailable (#14252) * improve concretization performance in environments (#14190) * improve installation performance in environments (#14263) # v0.13.2 (2019-12-04) This release contains major performance improvements for Spack environments, as well as some bugfixes and minor changes. * allow missing modules if they are blacklisted (#13540) * speed up environment activation (#13557) * mirror path works for unknown versions (#13626) * environments: don't try to modify run-env if a spec is not installed (#13589) * use semicolons instead of newlines in module/python command (#13904) * verify.py: os.path.exists exception handling (#13656) * Document use of the maintainers field (#13479) * bugfix with config caching (#13755) * hwloc: added 'master' version pointing at the HEAD of the master branch (#13734) * config option to allow gpg warning suppression (#13744) * fix for relative symlinks when relocating binary packages (#13727) * allow binary relocation of strings in relative binaries (#13724) # v0.13.1 (2019-11-05) This is a bugfix release on top of `v0.13.0`. Specific fixes include: * `spack find` now displays variants and other spec constraints * bugfix: uninstall should find concrete specs by DAG hash (#13598) * environments: make shell modifications partially unconditional (#13523) * binary distribution: relocate text files properly in relative binaries (#13578) * bugfix: fetch prefers to fetch local mirrors over remote resources (#13545) * environments: only write when necessary (#13546) * bugfix: spack.util.url.join() now handles absolute paths correctly (#13488) * sbang: use utf-8 for encoding when patching (#13490) * Specs with quoted flags containing spaces are parsed correctly (#13521) * targets: print a warning message before downgrading (#13513) * Travis CI: Test Python 3.8 (#13347) * Documentation: Database.query methods share docstrings (#13515) * cuda: fix conflict statements for x86-64 targets (#13472) * cpu: fix clang flags for generic x86_64 (#13491) * syaml_int type should use int.__repr__ rather than str.__repr__ (#13487) * elpa: prefer 2016.05.004 until sse/avx/avx2 issues are resolved (#13530) * trilinos: temporarily constrain netcdf@:4.7.1 (#13526) # v0.13.0 (2019-10-25) `v0.13.0` is our biggest Spack release yet, with *many* new major features. From facility deployment to improved environments, microarchitecture support, and auto-generated build farms, this release has features for all of our users. Spack grew by over 700 packages in the past year, and the project now has over 450 contributors. Thanks to all of you for making this release possible. ## Major new core features - Chaining: use dependencies from external "upstream" Spack instances - Environments now behave more like virtualenv/conda - Each env has a *view*: a directory with all packages symlinked in - Activating an environment sets `PATH`, `LD_LIBRARY_PATH`, `CPATH`, `CMAKE_PREFIX_PATH`, `PKG_CONFIG_PATH`, etc. to point to this view. - Spack detects and builds specifically for your microarchitecture - named, understandable targets like `skylake`, `broadwell`, `power9`, `zen2` - Spack knows which compilers can build for which architectures - Packages can easily query support for features like `avx512` and `sse3` - You can pick a target with, e.g. `spack install foo target=icelake` - Spack stacks: combinatorial environments for facility deployment - Environments can now build cartesian products of specs (with `matrix:`) - Conditional syntax support to exclude certain builds from the stack - Projections: ability to build easily navigable symlink trees environments - Support no-source packages (BundlePackage) to aggregate related packages - Extensions: users can write custom commands that live outside of Spack repo - Support ARM and Fujitsu compilers ## CI/build farm support - `spack release-jobs` can detect `package.py` changes and generate `.gitlab-ci.yml` to create binaries for an environment or stack in parallel (initial support -- will change in future release). - Results of build pipelines can be uploaded to a CDash server. - Spack can now upload/fetch from package mirrors in Amazon S3 ## New commands/options - `spack mirror create --all` downloads *all* package sources/resources/patches - `spack dev-build` runs phases of the install pipeline on the working directory - `spack deprecate` permanently symlinks an old, unwanted package to a new one - `spack verify` chcecks that packages' files match what was originally installed - `spack find --json` prints `JSON` that is easy to parse with, e.g. `jq` - `spack find --format FORMAT` allows you to flexibly print package metadata - `spack spec --json` prints JSON version of `spec.yaml` ## Selected improvements - Auto-build requested compilers if they do not exist - Spack automatically adds `RPATHs` needed to make executables find compiler runtime libraries (e.g., path to newer `libstdc++` in `icpc` or `g++`) - setup-env.sh is now compatible with Bash, Dash, and Zsh - Spack now caps build jobs at min(16, ncores) by default - `spack compiler find` now also throttles number of spawned processes - Spack now writes stage directories directly to `$TMPDIR` instead of symlinking stages within `$spack/var/spack/cache`. - Improved and more powerful `spec` format strings - You can pass a `spec.yaml` file anywhere in the CLI you can type a spec. - Many improvements to binary caching - Gradually supporting new features from Environment Modules v4 - `spack edit` respects `VISUAL` environment variable - Simplified package syntax for specifying build/run environment modifications - Numerous improvements to support for environments across Spack commands - Concretization improvements ## Documentation - Multi-lingual documentation (Started a Japanese translation) - Tutorial now has its own site at spack-tutorial.readthedocs.io - This enables us to keep multiple versions of the tutorial around ## Deprecations - Spack no longer supports dotkit (LLNL's homegrown, now deprecated module tool) - `spack build`, `spack configure`, `spack diy` deprecated in favor of `spack dev-build` and `spack install` ## Important package changes - 3,563 total packages (718 added since 0.12.1) - Spack now defaults to Python 3 (previously preferred 2.7 by default) - Much improved ARM support thanks to Fugaku (RIKEN) and SNL teams - Support new special versions: master, trunk, and head (in addition to develop) - Better finding logic for libraries and headers # v0.12.1 (2018-11-13) This is a minor bugfix release, with a minor fix in the tutorial and a `flake8` fix. Bugfixes * Add `r` back to regex strings in binary distribution * Fix gcc install version in the tutorial # v0.12.0 (2018-11-13) ## Major new features - Spack environments - `spack.yaml` and `spack.lock` files for tracking dependencies - Custom configurations via command line - Better support for linking Python packages into view directories - Packages have more control over compiler flags via flag handlers - Better support for module file generation - Better support for Intel compilers, Intel MPI, etc. - Many performance improvements, improved startup time ## License - As of this release, all of Spack is permissively licensed under Apache-2.0 or MIT, at the user's option. - Consents from over 300 contributors were obtained to make this relicense possible. - Previous versions were distributed under the LGPL license, version 2.1. ## New packages Over 2,900 packages (800 added since last year) Spack would not be possible without our community. Thanks to all of our [contributors](https://github.com/spack/spack/graphs/contributors) for the new features and packages in this release! # v0.11.2 (2018-02-07) This release contains the following fixes: * Fixes for `gfortran` 7 compiler detection (#7017) * Fixes for exceptions thrown during module generation (#7173) # v0.11.1 (2018-01-19) This release contains bugfixes for compiler flag handling. There were issues in `v0.11.0` that caused some packages to be built without proper optimization. Fixes: * Issue #6999: FFTW installed with Spack 0.11.0 gets built without optimisations Includes: * PR #6415: Fixes for flag handling behavior * PR #6960: Fix type issues with setting flag handlers * 880e319: Upstream fixes to `list_url` in various R packages # v0.11.0 (2018-01-17) Spack v0.11.0 contains many improvements since v0.10.0. Below is a summary of the major features, broken down by category. ## New packages - Spack now has 2,178 packages (from 1,114 in v0.10.0) - Many more Python packages (356) and R packages (471) - 48 Exascale Proxy Apps (try `spack list -t proxy-app`) ## Core features for users - Relocatable binary packages (`spack buildcache`, #4854) - Spack now fully supports Python 3 (#3395) - Packages can be tagged and searched by tags (#4786) - Custom module file templates using Jinja (#3183) - `spack bootstrap` command now sets up a basic module environment (#3057) - Simplified and better organized help output (#3033) - Improved, less redundant `spack install` output (#5714, #5950) - Reworked `spack dependents` and `spack dependencies` commands (#4478) ## Major new features for packagers - Multi-valued variants (#2386) - New `conflicts()` directive (#3125) - New dependency type: `test` dependencies (#5132) - Packages can require their own patches on dependencies (#5476) - `depends_on(..., patches=)` - Build interface for passing linker information through Specs (#1875) - Major packages that use blas/lapack now use this interface - Flag handlers allow packages more control over compiler flags (#6415) - Package subclasses support many more build systems: - autotools, perl, qmake, scons, cmake, makefile, python, R, WAF - package-level support for installing Intel HPC products (#4300) - `spack blame` command shows contributors to packages (#5522) - `spack create` now guesses many more build systems (#2707) - Better URL parsing to guess package version URLs (#2972) - Much improved `PythonPackage` support (#3367) ## Core - Much faster concretization (#5716, #5783) - Improved output redirection (redirecting build output works properly #5084) - Numerous improvements to internal structure and APIs ## Tutorials & Documentation - Many updates to documentation - [New tutorial material from SC17](https://spack.readthedocs.io/en/latest/tutorial.html) - configuration - build systems - build interface - working with module generation - Documentation on docker workflows and best practices ## Selected improvements and bug fixes - No longer build Python eggs -- installations are plain directories (#3587) - Improved filtering of system paths from build PATHs and RPATHs (#2083, #3910) - Git submodules are properly handled on fetch (#3956) - Can now set default number of parallel build jobs in `config.yaml` - Improvements to `setup-env.csh` (#4044) - Better default compiler discovery on Mac OS X (#3427) - clang will automatically mix with gfortran - Improved compiler detection on Cray machines (#3075) - Better support for IBM XL compilers - Better tab completion - Resume gracefully after prematurely terminated partial installs (#4331) - Better mesa support (#5170) Spack would not be possible without our community. Thanks to all of our [contributors](https://github.com/spack/spack/graphs/contributors) for the new features and packages in this release! # v0.10.0 (2017-01-17) This is Spack `v0.10.0`. With this release, we will start to push Spack releases more regularly. This is the last Spack release without automated package testing. With the next release, we will begin to run package tests in addition to unit tests. Spack has grown rapidly from 422 to [1,114 packages](https://spack.readthedocs.io/en/v0.10.0/package_list.html), thanks to the hard work of over 100 contributors. Below is a condensed version of all the changes since `v0.9.1`. ### Packages - Grew from 422 to 1,114 packages - Includes major updates like X11, Qt - Expanded HPC, R, and Python ecosystems ### Core - Major speed improvements for spack find and concretization - Completely reworked architecture support - Platforms can have front-end and back-end OS/target combinations - Much better support for Cray and BG/Q cross-compiled environments - Downloads are now cached locally - Support installations in deeply nested directories: patch long shebangs using `sbang` ### Basic usage - Easier global configuration via config.yaml - customize install, stage, and cache locations - Hierarchical configuration scopes: default, site, user - Platform-specific scopes allow better per-platform defaults - Ability to set `cflags`, `cxxflags`, `fflags` on the command line - YAML-configurable support for both Lmod and tcl modules in mainline - `spack install` supports --dirty option for emergencies ### For developers - Support multiple dependency types: `build`, `link`, and `run` - Added `Package` base classes for custom build systems - `AutotoolsPackage`, `CMakePackage`, `PythonPackage`, etc. - `spack create` now guesses many more build systems - Development environment integration with `spack setup` - New interface to pass linking information via `spec` objects - Currently used for `BLAS`/`LAPACK`/`SCALAPACK` libraries - Polymorphic virtual dependency attributes: `spec['blas'].blas_libs` ### Testing & Documentation - Unit tests run continuously on Travis CI for Mac and Linux - Switched from `nose` to `pytest` for unit tests. - Unit tests take 1 minute now instead of 8 - Massively expanded documentation - Docs are now hosted on [spack.readthedocs.io](https://spack.readthedocs.io) ================================================ FILE: CITATION.cff ================================================ # If you are referencing Spack in a publication, please cite the SC'15 paper # described here. # # Here's the raw citation: # # Todd Gamblin, Matthew P. LeGendre, Michael R. Collette, Gregory L. Lee, # Adam Moody, Bronis R. de Supinski, and W. Scott Futral. # The Spack Package Manager: Bringing Order to HPC Software Chaos. # In Supercomputing 2015 (SC’15), Austin, Texas, November 15-20 2015. LLNL-CONF-669890. # # Or, in BibTeX: # # @inproceedings{Gamblin_The_Spack_Package_2015, # address = {Austin, Texas, USA}, # author = {Gamblin, Todd and LeGendre, Matthew and # Collette, Michael R. and Lee, Gregory L. and # Moody, Adam and de Supinski, Bronis R. and Futral, Scott}, # doi = {10.1145/2807591.2807623}, # month = {November 15-20}, # note = {LLNL-CONF-669890}, # series = {Supercomputing 2015 (SC’15)}, # title = {{The Spack Package Manager: Bringing Order to HPC Software Chaos}}, # url = {https://github.com/spack/spack}, # year = {2015} # } # # And here's the CITATION.cff format: # cff-version: 1.2.0 type: software message: "If you are referencing Spack in a publication, please cite the paper below." title: "The Spack Package Manager: Bringing Order to HPC Software Chaos" abstract: >- Large HPC centers spend considerable time supporting software for thousands of users, but the complexity of HPC software is quickly outpacing the capabilities of existing software management tools. Scientific applications require specific versions of compilers, MPI, and other dependency libraries, so using a single, standard software stack is infeasible. However, managing many configurations is difficult because the configuration space is combinatorial in size. We introduce Spack, a tool used at Lawrence Livermore National Laboratory to manage this complexity. Spack provides a novel, re- cursive specification syntax to invoke parametric builds of packages and dependencies. It allows any number of builds to coexist on the same system, and it ensures that installed packages can find their dependencies, regardless of the environment. We show through real-world use cases that Spack supports diverse and demanding applications, bringing order to HPC software chaos. preferred-citation: title: "The Spack Package Manager: Bringing Order to HPC Software Chaos" type: conference-paper url: "https://tgamblin.github.io/pubs/spack-sc15.pdf" authors: - family-names: "Gamblin" given-names: "Todd" - family-names: "LeGendre" given-names: "Matthew" - family-names: "Collette" given-names: "Michael R." - family-names: "Lee" given-names: "Gregory L." - family-names: "Moody" given-names: "Adam" - family-names: "de Supinski" given-names: "Bronis R." - family-names: "Futral" given-names: "Scott" conference: name: "Supercomputing 2015 (SC’15)" city: "Austin" region: "Texas" country: "US" date-start: 2015-11-15 date-end: 2015-11-20 month: 11 year: 2015 identifiers: - description: "The concept DOI of the work." type: doi value: 10.1145/2807591.2807623 - description: "The DOE Document Release Number of the work" type: other value: "LLNL-CONF-669890" authors: - family-names: "Gamblin" given-names: "Todd" - family-names: "LeGendre" given-names: "Matthew" - family-names: "Collette" given-names: "Michael R." - family-names: "Lee" given-names: "Gregory L." - family-names: "Moody" given-names: "Adam" - family-names: "de Supinski" given-names: "Bronis R." - family-names: "Futral" given-names: "Scott" ================================================ FILE: COPYRIGHT ================================================ Intellectual Property Notice ------------------------------ Spack is licensed under the Apache License, Version 2.0 (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or the MIT license, (LICENSE-MIT or http://opensource.org/licenses/MIT), at your option. Copyrights and patents in the Spack project are retained by contributors. No copyright assignment is required to contribute to Spack. Spack was originally developed in 2013 by Lawrence Livermore National Security, LLC. It was originally distributed under the LGPL-2.1 license. Consent from contributors to relicense to Apache-2.0/MIT is documented at https://github.com/spack/spack/issues/9137. SPDX usage ------------ Individual files contain SPDX tags instead of the full license text. This enables machine processing of license information based on the SPDX License Identifiers that are available here: https://spdx.org/licenses/ Files that are dual-licensed as Apache-2.0 OR MIT contain the following text in the license header: SPDX-License-Identifier: (Apache-2.0 OR MIT) External Packages ------------------- Spack bundles most external dependencies in `lib/spack/spack/vendor/`. This directory is automatically maintained using the `vendoring` tool. Spack also includes other vendored components like `ast.unparse` (in `lib/spack/spack/util/unparse/`), `sbang` (in `lib/spack/spack/hooks/` and `bin/`), and `spack.util.ctest_log_parser`. These packages are covered by various permissive licenses. A summary listing follows. See the license included with each for full details. PackageName: altgraph PackageHomePage: https://altgraph.readthedocs.io/en/latest/index.html PackageLicenseDeclared: MIT PackageName: archspec PackageHomePage: https://github.com/archspec/archspec PackageLicenseDeclared: Apache-2.0 OR MIT PackageName: ast.unparse PackageHomePage: https://www.python.org/ PackageLicenseDeclared: PSF-2.0 PackageName: attr PackageHomePage: https://www.attrs.org/ PackageLicenseDeclared: MIT PackageName: attrs PackageHomePage: https://www.attrs.org/ PackageLicenseDeclared: MIT PackageName: ctest_log_parser PackageHomePage: https://github.com/Kitware/CMake PackageLicenseDeclared: BSD-3-Clause PackageName: distro PackageHomePage: https://pypi.python.org/pypi/distro PackageLicenseDeclared: Apache-2.0 PackageName: jinja2 PackageHomePage: https://pypi.python.org/pypi/Jinja2 PackageLicenseDeclared: BSD-3-Clause PackageName: jsonschema PackageHomePage: https://pypi.python.org/pypi/jsonschema PackageLicenseDeclared: MIT PackageName: macholib PackageHomePage: https://macholib.readthedocs.io/en/latest/index.html PackageLicenseDeclared: MIT PackageName: markupsafe PackageHomePage: https://pypi.python.org/pypi/MarkupSafe PackageLicenseDeclared: BSD-3-Clause PackageName: pyrsistent PackageHomePage: http://github.com/tobgu/pyrsistent PackageLicenseDeclared: MIT PackageName: ruamel.yaml PackageHomePage: https://yaml.readthedocs.io/ PackageLicenseDeclared: MIT PackageName: sbang PackageHomePage: https://github.com/spack/sbang PackageLicenseDeclared: Apache-2.0 OR MIT PackageName: six PackageHomePage: https://pypi.org/project/six/ PackageLicenseDeclared: MIT PackageName: typing_extensions PackageHomePage: https://pypi.org/project/typing-extensions/ PackageLicenseDeclared: Python-2.0 ================================================ FILE: LICENSE-APACHE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: LICENSE-MIT ================================================ MIT License Copyright (c) Spack Project Developers. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: NEWS.md ================================================ ## Package API v2.4 - Added the `%%` sigil to spec syntax, to propagate compiler preferences. ## Spack v1.0.0 Deprecated the implicit attributes: - `PackageBase.legacy_buildsystem` - `Builder.legacy_methods` - `Builder.legacy_attributes` - `Builder.legacy_long_methods` ## Package API v2.3 - `spack.package.version` directive: added `git_sparse_paths` parameter. ## Package API v2.2 Added to `spack.package`: - `BuilderWithDefaults` - `ClassProperty` - `CompilerPropertyDetector` - `GenericBuilder` - `HKEY` - `LC_ID_DYLIB` - `LinkTree` - `MachO` - `ModuleChangePropagator` - `Package` - `WindowsRegistryView` - `apply_macos_rpath_fixups` - `classproperty` - `compare_output_file` - `compare_output` - `compile_c_and_execute` - `compiler_spec` - `create_builder` - `dedupe` - `delete_needed_from_elf` - `delete_rpath` - `environment_modifications_for_specs` - `execute_install_time_tests` - `filter_shebang` - `filter_system_paths` - `find_all_libraries` - `find_compilers` - `get_cmake_prefix_path` - `get_effective_jobs` - `get_elf_compat` - `get_path_args_from_module_line` - `get_user` - `has_shebang` - `host_platform` - `is_system_path` - `join_url` - `kernel_version` - `libc_from_dynamic_linker` - `macos_version` - `make_package_test_rpath` - `memoized` - `microarchitecture_flags_from_target` - `microarchitecture_flags` - `module_command` - `parse_dynamic_linker` - `parse_elf` - `path_contains_subdirectory` - `readlink` - `safe_remove` - `sbang_install_path` - `sbang_shebang_line` - `set_env` - `shared_library_suffix` - `spack_script` - `static_library_suffix` - `substitute_version_in_url` - `windows_sfn` ## Package API v2.1 Added to `spack.package`: - `CompilerError` - `SpackError` ================================================ FILE: NOTICE ================================================ This work was produced under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344. This work was prepared as an account of work sponsored by an agency of the United States Government. Neither the United States Government nor Lawrence Livermore National Security, LLC, nor any of their employees makes any warranty, expressed or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness of any information, apparatus, product, or process disclosed, or represents that its use would not infringe privately owned rights. Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or Lawrence Livermore National Security, LLC. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or Lawrence Livermore National Security, LLC, and shall not be used for advertising or product endorsement purposes. ================================================ FILE: README.md ================================================

Spack

CI Status Bootstrap Status Containers Status Documentation Status Code coverage Slack Matrix

**[Getting Started]   •   [Config]   •   [Community]   •   [Contributing]   •   [Packaging Guide]   •   [Packages]** [Getting Started]: https://spack.readthedocs.io/en/latest/getting_started.html [Config]: https://spack.readthedocs.io/en/latest/configuration.html [Community]: #community [Contributing]: https://spack.readthedocs.io/en/latest/contribution_guide.html [Packaging Guide]: https://spack.readthedocs.io/en/latest/packaging_guide_creation.html [Packages]: https://github.com/spack/spack-packages
Spack is a multi-platform package manager that builds and installs multiple versions and configurations of software. It works on Linux, macOS, Windows, and many supercomputers. Spack is non-destructive: installing a new version of a package does not break existing installations, so many configurations of the same package can coexist. Spack offers a simple "spec" syntax that allows users to specify versions and configuration options. Package files are written in pure Python, and specs allow package authors to write a single script for many different builds of the same package. With Spack, you can build your software *all* the ways you want to. See the [Feature Overview](https://spack.readthedocs.io/en/latest/features.html) for examples and highlights. Installation ---------------- To install spack, first make sure you have Python & Git. Then: ```bash git clone --depth=2 https://github.com/spack/spack.git ``` ```bash # For bash/zsh/sh . spack/share/spack/setup-env.sh # For tcsh/csh source spack/share/spack/setup-env.csh # For fish . spack/share/spack/setup-env.fish ``` ```bash # Now you're ready to install a package! spack install zlib-ng ``` Documentation ---------------- [**Full documentation**](https://spack.readthedocs.io/) is available, or run `spack help` or `spack help --all`. For a cheat sheet on Spack syntax, run `spack help --spec`. Tutorial ---------------- We maintain a [**hands-on tutorial**](https://spack-tutorial.readthedocs.io/). It covers basic to advanced usage, packaging, developer features, and large HPC deployments. You can do all of the exercises on your own laptop using a Docker container. Feel free to use these materials to teach users at your organization about Spack. Community ------------------------ Spack is an open source project. Questions, discussion, and contributions are welcome. Contributions can be anything from new packages to bugfixes, documentation, or even new core features. Resources: * **Slack workspace**: [spackpm.slack.com](https://spackpm.slack.com). To get an invitation, visit [slack.spack.io](https://slack.spack.io). * **Matrix space**: [#spack-space:matrix.org](https://matrix.to/#/#spack-space:matrix.org): [bridged](https://github.com/matrix-org/matrix-appservice-slack#matrix-appservice-slack) to Slack. * [**Github Discussions**](https://github.com/spack/spack/discussions): for Q&A and discussions. Note the pinned discussions for announcements. * **X**: [@spackpm](https://twitter.com/spackpm). Be sure to `@mention` us! * **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack): only for announcements. Please use other venues for discussions. Contributing ------------------------ Contributing to Spack is relatively easy. Just send us a [pull request](https://help.github.com/articles/using-pull-requests/). Most contributors will want to contribute to Spack's community package recipes. To do that, you should visit the **[spack-packages repository][Packages]**. If you want to contribute to Spack itself, you can submit a pull request to the [spack repository](https://github.com/spack/spack) (this repository). Your PR must: 1. Make ``develop`` the destination branch; 2. Pass Spack's unit tests, documentation tests, and package build tests; 3. Be [PEP 8](https://www.python.org/dev/peps/pep-0008/) compliant; 4. Sign off all commits with `git commit --signoff`. Signoff says that you agree to the [Developer Certificate of Origin](https://developercertificate.org). Note that this is different from [signing commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits), which you may also do, but it's not required. We enforce these guidelines with our continuous integration (CI) process. To run tests locally, and for helpful tips on git, see our [Contribution Guide](https://spack.readthedocs.io/en/latest/contribution_guide.html). Releases -------- For multi-user site deployments or other use cases that need very stable software installations, we recommend using Spack's [stable releases](https://github.com/spack/spack/releases). Each Spack release series also has a corresponding branch, e.g. `releases/v0.14` has `0.14.x` versions of Spack, and `releases/v0.13` has `0.13.x` versions. We backport important bug fixes to these branches but we do not advance the package versions or make other changes that would change the way Spack concretizes dependencies within a release branch. So, you can base your Spack deployment on a release branch and `git pull` to get fixes, without the package churn that comes with `develop`. The latest release is always available with the `releases/latest` tag. See the [docs on releases](https://spack.readthedocs.io/en/latest/developer_guide.html#releases) for more details. Code of Conduct ------------------------ Please note that Spack has a [**Code of Conduct**](.github/CODE_OF_CONDUCT.md). By participating in the Spack community, you agree to abide by its rules. Authors ---------------- Many thanks go to Spack's [contributors](https://github.com/spack/spack/graphs/contributors). Spack was created by Todd Gamblin, tgamblin@llnl.gov. ### Citing Spack If you are referencing Spack in a publication, please cite the following paper: * Todd Gamblin, Matthew P. LeGendre, Michael R. Collette, Gregory L. Lee, Adam Moody, Bronis R. de Supinski, and W. Scott Futral. [**The Spack Package Manager: Bringing Order to HPC Software Chaos**](https://www.computer.org/csdl/proceedings/sc/2015/3723/00/2807623.pdf). In *Supercomputing 2015 (SC’15)*, Austin, Texas, November 15-20 2015. LLNL-CONF-669890. On GitHub, you can copy this citation in APA or BibTeX format via the "Cite this repository" button. Or, see the comments in `CITATION.cff` for the raw BibTeX. License ---------------- Spack is distributed under the terms of both the MIT license and the Apache License (Version 2.0). Users may choose either license, at their option. All new contributions must be made under both the MIT and Apache-2.0 licenses. See [LICENSE-MIT](https://github.com/spack/spack/blob/develop/LICENSE-MIT), [LICENSE-APACHE](https://github.com/spack/spack/blob/develop/LICENSE-APACHE), [COPYRIGHT](https://github.com/spack/spack/blob/develop/COPYRIGHT), and [NOTICE](https://github.com/spack/spack/blob/develop/NOTICE) for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) LLNL-CODE-811652 ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions We provide security updates for `develop` and for the last two stable (`0.x`) release series of Spack. Security updates will be made available as patch (`0.x.1`, `0.x.2`, etc.) releases. For more on Spack's release structure, see [`README.md`](https://github.com/spack/spack#releases). ## Reporting a Vulnerability You can report a vulnerability using GitHub's private reporting feature: 1. Go to [github.com/spack/spack/security](https://github.com/spack/spack/security). 2. Click "Report a vulnerability" in the upper right corner of that page. 3. Fill out the form and submit your draft security advisory. More details are available in [GitHub's docs](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability). You can expect to hear back about security issues within two days. If your security issue is accepted, we will do our best to release a fix within a week. If fixing the issue will take longer than this, we will discuss timeline options with you. ================================================ FILE: bin/haspywin.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import subprocess import sys def getpywin(): try: import win32con # noqa: F401 except ImportError: print("pyWin32 not installed but is required...\nInstalling via pip:") subprocess.check_call([sys.executable, "-m", "pip", "-q", "install", "--upgrade", "pip"]) subprocess.check_call([sys.executable, "-m", "pip", "-q", "install", "pywin32"]) if __name__ == "__main__": getpywin() ================================================ FILE: bin/sbang ================================================ #!/bin/sh # # Copyright sbang project developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # # `sbang`: Run scripts with long shebang lines. # # Many operating systems limit the length and number of possible # arguments in shebang lines, making it hard to use interpreters that are # deep in the directory hierarchy or require special arguments. # # To use, put the long shebang on the second line of your script, and # make sbang the interpreter, like this: # # #!/bin/sh /path/to/sbang # #!/long/path/to/real/interpreter with arguments # # `sbang` will run the real interpreter with the script as its argument. # # See https://github.com/spack/sbang for more details. # # Generic error handling die() { echo "$@" 1>&2; exit 1 } # set SBANG_DEBUG to make the script print what would normally be executed. exec="exec" if [ -n "${SBANG_DEBUG}" ]; then exec="echo " fi # First argument is the script we want to actually run. script="$1" # ensure that the script actually exists if [ -z "$script" ]; then die "error: sbang requires exactly one argument" elif [ ! -f "$script" ]; then die "$script: no such file or directory" fi # Search the first two lines of script for interpreters. lines=0 while read -r line && [ $lines -ne 2 ]; do if [ "${line#\#!}" != "$line" ]; then shebang_line="${line#\#!}" elif [ "${line#//!}" != "$line" ]; then # // comments shebang_line="${line#//!}" elif [ "${line#--!}" != "$line" ]; then # -- lua comments shebang_line="${line#--!}" elif [ "${line#}" fi lines=$((lines+1)) done < "$script" # error if we did not find any interpreter if [ -z "$shebang_line" ]; then die "error: sbang found no interpreter in $script" fi # parse out the interpreter and first argument IFS=' ' read -r interpreter arg1 rest < /dev/null "$cmd"; then export SPACK_PYTHON="$(command -v "$cmd")" exec "${SPACK_PYTHON}" "$0" "$@" fi done echo "==> Error: spack could not find a python interpreter!" >&2 exit 1 ":""" # Line above is a shell no-op, and ends a python multi-line comment. # The code above runs this file with our preferred python interpreter. import os import sys min_python3 = (3, 6) if sys.version_info[:2] < min_python3: v_info = sys.version_info[:3] msg = "Spack requires Python %d.%d or higher " % min_python3 msg += "You are running spack with Python %d.%d.%d." % v_info sys.exit(msg) # Find spack's location and its prefix. spack_file = os.path.realpath(os.path.expanduser(__file__)) spack_prefix = os.path.dirname(os.path.dirname(spack_file)) # Allow spack libs to be imported in our scripts sys.path.insert(0, os.path.join(spack_prefix, "lib", "spack")) from spack.main import main # noqa: E402 # Once we've set up the system path, run the spack main method if __name__ == "__main__": sys.exit(main()) ================================================ FILE: bin/spack-python ================================================ #!/bin/sh # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # # spack-python # # If you want to write your own executable Python script that uses Spack # modules, on Mac OS or maybe some others, you may be able to do it like # this: # # #!/usr/bin/env spack python # # Mac OS supports the above syntax, but it's not standard and most Linuxes # don't support more than one argument after the shebang command. This # script is a workaround. Do this in your Python script instead: # # #!/usr/bin/env spack-python # # This is compatible across platforms. # exec spack python "$@" ================================================ FILE: bin/spack-tmpconfig ================================================ #!/bin/bash set -euo pipefail [[ -n "${TMPCONFIG_DEBUG:=}" ]] && set -x DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" mkdir -p "${XDG_RUNTIME_DIR:=/tmp}/spack-tests" export TMPDIR="${XDG_RUNTIME_DIR}" export TMP_DIR="$(mktemp -d -t spack-test-XXXXX)" clean_up() { [[ -n "$TMPCONFIG_DEBUG" ]] && printf "cleaning up: $TMP_DIR\n" rm -rf "$TMP_DIR" } trap clean_up EXIT trap clean_up ERR [[ -n "$TMPCONFIG_DEBUG" ]] && printf "Redirecting TMP_DIR and spack directories to $TMP_DIR\n" export BOOTSTRAP="${SPACK_USER_CACHE_PATH:=$HOME/.spack}/bootstrap" export SPACK_USER_CACHE_PATH="$TMP_DIR/user_cache" mkdir -p "$SPACK_USER_CACHE_PATH" private_bootstrap="$SPACK_USER_CACHE_PATH/bootstrap" use_spack='' use_bwrap='' # argument handling while (($# >= 1)) ; do case "$1" in -b) # privatize bootstrap too, useful for CI but not always cheap shift export BOOTSTRAP="$private_bootstrap" ;; -B) # use specified bootstrap dir export BOOTSTRAP="$2" shift 2 ;; -s) # run spack directly with remaining args shift use_spack=1 ;; --contain=bwrap) if bwrap --help 2>&1 > /dev/null ; then use_bwrap=1 else echo Bubblewrap containment requested, but no bwrap command found exit 1 fi shift ;; --) shift break ;; *) break ;; esac done typeset -a CMD if [[ -n "$use_spack" ]] ; then CMD=("$DIR/spack" "$@") else CMD=("$@") fi mkdir -p "$BOOTSTRAP" export SPACK_SYSTEM_CONFIG_PATH="$TMP_DIR/sys_conf" export SPACK_USER_CONFIG_PATH="$TMP_DIR/user_conf" mkdir -p "$SPACK_USER_CONFIG_PATH" cat >"$SPACK_USER_CONFIG_PATH/config.yaml" <"$SPACK_USER_CONFIG_PATH/bootstrap.yaml" < 1)) && echo "Running: ${CMD[@]}" "${CMD[@]}" ================================================ FILE: bin/spack.bat ================================================ :: Copyright Spack Project Developers. See COPYRIGHT file for details. :: :: SPDX-License-Identifier: (Apache-2.0 OR MIT) ::####################################################################### :: :: This file is part of Spack and sets up the spack environment for batch, :: This includes environment modules and lmod support, :: and it also puts spack in your path. The script also checks that at least :: module support exists, and provides suggestions if it doesn't. Source :: it like this: :: :: . /path/to/spack/install/spack_cmd.bat :: @echo off set spack="%SPACK_ROOT%\bin\spack" ::####################################################################### :: This is a wrapper around the spack command that forwards calls to :: 'spack load' and 'spack unload' to shell functions. This in turn :: allows them to be used to invoke environment modules functions. :: :: 'spack load' is smarter than just 'load' because it converts its :: arguments into a unique Spack spec that is then passed to module :: commands. This allows the user to use packages without knowing all :: their installation details. :: :: e.g., rather than requiring a full spec for libelf, the user can type: :: :: spack load libelf :: :: This will first find the available libelf module file and use a :: matching one. If there are two versions of libelf, the user would :: need to be more specific, e.g.: :: :: spack load libelf@0.8.13 :: :: This is very similar to how regular spack commands work and it :: avoids the need to come up with a user-friendly naming scheme for :: spack module files. ::####################################################################### :_sp_shell_wrapper set "_sp_flags=" set "_sp_args=" set "_sp_subcommand=" setlocal enabledelayedexpansion :: commands have the form '[flags] [subcommand] [args]' :: flags will always start with '-', e.g. --help or -V :: subcommands will never start with '-' :: everything after the subcommand is an arg :process_cl_args rem Set first cl argument (denoted by %1) to be processed set t=%1 rem shift moves all cl positional arguments left by one rem meaning %2 is now %1, this allows us to iterate over each rem argument shift rem assign next "first" cl argument to cl_args, will be null when rem there are now further arguments to process set cl_args=%1 if "!t:~0,1!" == "-" ( if defined _sp_subcommand ( rem We already have a subcommand, processing args now if not defined _sp_args ( set "_sp_args=!t!" ) else ( set "_sp_args=!_sp_args! !t!" ) ) else ( if not defined _sp_flags ( set "_sp_flags=!t!" ) else ( set "_sp_flags=!_sp_flags! !t!" ) ) ) else if not defined _sp_subcommand ( set "_sp_subcommand=!t!" ) else ( if not defined _sp_args ( set "_sp_args=!t!" ) else ( set "_sp_args=!_sp_args! !t!" ) ) rem if this is not nu;ll, we have more tokens to process rem start above process again with remaining unprocessed cl args if defined cl_args goto :process_cl_args :: --help, -h and -V flags don't require further output parsing. :: If we encounter, execute and exit if defined _sp_flags ( if NOT "%_sp_flags%"=="%_sp_flags:-h=%" ( python %spack% %_sp_flags% exit /B 0 ) else if NOT "%_sp_flags%"=="%_sp_flags:--help=%" ( python %spack% %_sp_flags% exit /B 0 ) else if NOT "%_sp_flags%"=="%_sp_flags:-V=%" ( python %spack% %_sp_flags% exit /B 0 ) ) if not defined _sp_subcommand ( if not defined _sp_args ( if not defined _sp_flags ( python %spack% --help exit /B 0 ) ) ) :: pass parsed variables outside of local scope. Need to do :: this because delayedexpansion can only be set by setlocal endlocal & ( set "_sp_flags=%_sp_flags%" set "_sp_args=%_sp_args%" set "_sp_subcommand=%_sp_subcommand%" ) :: Filter out some commands. For any others, just run the command. if "%_sp_subcommand%" == "cd" ( goto :case_cd ) else if "%_sp_subcommand%" == "env" ( goto :case_env ) else if "%_sp_subcommand%" == "load" ( goto :case_load ) else if "%_sp_subcommand%" == "unload" ( goto :case_load ) else ( goto :default_case ) ::####################################################################### :case_cd :: Check for --help or -h :: TODO: This is not exactly the same as setup-env. :: In setup-env, '--help' or '-h' must follow the cd :: Here, they may be anywhere in the args if defined _sp_args ( if NOT "%_sp_args%"=="%_sp_args:--help=%" ( python %spack% cd -h goto :end_switch ) else if NOT "%_sp_args%"=="%_sp_args:-h=%" ( python %spack% cd -h goto :end_switch ) ) for /F "tokens=* USEBACKQ" %%F in ( `python %spack% location %_sp_args%`) do ( set "LOC=%%F" ) for %%Z in ("%LOC%") do if EXIST %%~sZ\NUL (cd /d "%LOC%") goto :end_switch :case_env :: If no args or args contain --bat or -h/--help: just execute. if NOT defined _sp_args ( goto :default_case ) if NOT "%_sp_args%"=="%_sp_args:--help=%" ( goto :default_case ) else if NOT "%_sp_args%"=="%_sp_args: -h=%" ( goto :default_case ) else if NOT "%_sp_args%"=="%_sp_args:--bat=%" ( goto :default_case ) else if NOT "%_sp_args%"=="%_sp_args:deactivate=%" ( for /f "tokens=* USEBACKQ" %%I in ( `call python %spack% %_sp_flags% env deactivate --bat %_sp_args:deactivate=%` ) do %%I ) else if NOT "%_sp_args%"=="%_sp_args:activate=%" ( for /f "tokens=* USEBACKQ" %%I in ( `python %spack% %_sp_flags% env activate --bat %_sp_args:activate=%` ) do %%I ) else ( goto :default_case ) goto :end_switch :case_load if NOT defined _sp_args ( exit /B 0 ) :: If args contain --bat, or -h/--help: just execute. if NOT "%_sp_args%"=="%_sp_args:--help=%" ( goto :default_case ) else if NOT "%_sp_args%"=="%_sp_args:-h=%" ( goto :default_case ) else if NOT "%_sp_args%"=="%_sp_args:--bat=%" ( goto :default_case ) else if NOT "%_sp_args%"=="%_sp_args:--list=%" ( goto :default_case ) for /f "tokens=* USEBACKQ" %%I in ( `python %spack% %_sp_flags% %_sp_subcommand% --bat %_sp_args%` ) do %%I goto :end_switch :default_case python %spack% %_sp_flags% %_sp_subcommand% %_sp_args% goto :end_switch :end_switch exit /B %ERRORLEVEL% ::######################################################################## :: Prepends directories to path, if they exist. :: pathadd /path/to/dir # add to PATH :: or pathadd OTHERPATH /path/to/dir # add to OTHERPATH ::######################################################################## :_spack_pathadd set "_pa_varname=PATH" set "_pa_new_path=%~1" if NOT "%~2" == "" ( set "_pa_varname=%~1" set "_pa_new_path=%~2" ) set "_pa_oldvalue=%_pa_varname%" for %%Z in ("%_pa_new_path%") do if EXIST %%~sZ\NUL ( if defined %_pa_oldvalue% ( set "_pa_varname=%_pa_new_path%:%_pa_oldvalue%" ) else ( set "_pa_varname=%_pa_new_path%" ) ) exit /b 0 :: set module system roots :_sp_multi_pathadd for %%I in (%~2) do ( for %%Z in (%_sp_compatible_sys_types%) do ( :pathadd "%~1" "%%I\%%Z" ) ) exit /B %ERRORLEVEL% ================================================ FILE: bin/spack.ps1 ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # SPDX-License-Identifier: (Apache-2.0 OR MIT) # ####################################################################### function Compare-CommonArgs { $CMDArgs = $args[0] # These arguments take precedence and call for no further parsing of arguments # invoke actual Spack entrypoint with that context and exit after "--help", "-h", "--version", "-V" | ForEach-Object { $arg_opt = $_ if(($CMDArgs) -and ([bool]($CMDArgs.Where({$_ -eq $arg_opt})))) { return $true } } return $false } function Read-SpackArgs { $SpackCMD_params = @() $SpackSubCommand = $NULL $SpackSubCommandArgs = @() $args_ = $args[0] $args_ | ForEach-Object { if (!$SpackSubCommand) { if($_.SubString(0,1) -eq "-") { $SpackCMD_params += $_ } else{ $SpackSubCommand = $_ } } else{ $SpackSubCommandArgs += $_ } } return $SpackCMD_params, $SpackSubCommand, $SpackSubCommandArgs } function Set-SpackEnv { # This method is responsible # for processing the return from $(spack ) # which are returned as System.Object[]'s containing # a list of env commands # Invoke-Expression can only handle one command at a time # so we iterate over the list to invoke the env modification # expressions one at a time foreach($envop in $args[0]){ Invoke-Expression $envop } } function Invoke-SpackCD { if (Compare-CommonArgs $SpackSubCommandArgs) { python "$Env:SPACK_ROOT/bin/spack" cd -h } else { $LOC = $(python "$Env:SPACK_ROOT/bin/spack" location $SpackSubCommandArgs) if (($NULL -ne $LOC)){ if ( Test-Path -Path $LOC){ Set-Location $LOC } else{ exit 1 } } else { exit 1 } } } function Invoke-SpackEnv { if (Compare-CommonArgs $SpackSubCommandArgs[0]) { python "$Env:SPACK_ROOT/bin/spack" env -h } else { $SubCommandSubCommand = $SpackSubCommandArgs[0] $SubCommandSubCommandArgs = $SpackSubCommandArgs[1..$SpackSubCommandArgs.Count] switch ($SubCommandSubCommand) { "activate" { if (Compare-CommonArgs $SubCommandSubCommandArgs) { python "$Env:SPACK_ROOT/bin/spack" env activate $SubCommandSubCommandArgs } elseif ([bool]($SubCommandSubCommandArgs.Where({$_ -eq "--pwsh"}))) { python "$Env:SPACK_ROOT/bin/spack" env activate $SubCommandSubCommandArgs } elseif (!$SubCommandSubCommandArgs) { python "$Env:SPACK_ROOT/bin/spack" env activate $SubCommandSubCommandArgs } else { $SpackEnv = $(python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params env activate "--pwsh" $SubCommandSubCommandArgs) Set-SpackEnv $SpackEnv } } "deactivate" { if ([bool]($SubCommandSubCommandArgs.Where({$_ -eq "--pwsh"}))) { python"$Env:SPACK_ROOT/bin/spack" env deactivate $SubCommandSubCommandArgs } elseif($SubCommandSubCommandArgs) { python "$Env:SPACK_ROOT/bin/spack" env deactivate -h } else { $SpackEnv = $(python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params env deactivate "--pwsh") Set-SpackEnv $SpackEnv } } default {python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs} } } } function Invoke-SpackLoad { if (Compare-CommonArgs $SpackSubCommandArgs) { python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs } elseif ([bool]($SpackSubCommandArgs.Where({($_ -eq "--pwsh") -or ($_ -eq "--list")}))) { python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs } else { $SpackEnv = $(python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand "--pwsh" $SpackSubCommandArgs) Set-SpackEnv $SpackEnv } } $SpackCMD_params, $SpackSubCommand, $SpackSubCommandArgs = Read-SpackArgs $args if (Compare-CommonArgs $SpackCMD_params) { python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs exit $LASTEXITCODE } # Process Spack commands with special conditions # all other commands are piped directly to Spack switch($SpackSubCommand) { "cd" {Invoke-SpackCD} "env" {Invoke-SpackEnv} "load" {Invoke-SpackLoad} "unload" {Invoke-SpackLoad} default {python "$Env:SPACK_ROOT/bin/spack" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs} } exit $LASTEXITCODE ================================================ FILE: bin/spack_cmd.bat ================================================ @ECHO OFF :: (c) 2021 Lawrence Livermore National Laboratory :: To use this file independently of Spack's installer, execute this script in its directory, or add the :: associated bin directory to your PATH. Invoke to launch Spack Shell. :: :: source_dir/spack/bin/spack_cmd.bat :: call "%~dp0..\share\spack\setup-env.bat" pushd %SPACK_ROOT% %comspec% /K ================================================ FILE: bin/spack_pwsh.ps1 ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) $Env:SPACK_PS1_PATH="$PSScriptRoot\..\share\spack\setup-env.ps1" & (Get-Process -Id $pid).Path -NoExit { . $Env:SPACK_PS1_PATH ; Push-Location $ENV:SPACK_ROOT } ================================================ FILE: etc/spack/defaults/base/concretizer.yaml ================================================ # ------------------------------------------------------------------------- # This is the default spack configuration file. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing # `$SPACK_ROOT/etc/spack/concretizer.yaml`, `~/.spack/concretizer.yaml`, # or by adding a `concretizer:` section to an environment. # ------------------------------------------------------------------------- concretizer: # Whether to consider installed packages or packages from buildcaches when # concretizing specs. If `true`, we'll try to use as many installs/binaries # as possible, rather than building. If `false`, we'll always give you a fresh # concretization. If `dependencies`, we'll only reuse dependencies but # give you a fresh concretization for your root specs. reuse: true # Options that tune which targets are considered for concretization. The # concretization process is very sensitive to the number targets, and the time # needed to reach a solution increases noticeably with the number of targets # considered. targets: # Determine whether we want to target specific or generic # microarchitectures. Valid values are: "microarchitectures" or "generic". # An example of "microarchitectures" would be "skylake" or "bulldozer", # while an example of "generic" would be "aarch64" or "x86_64_v4". granularity: microarchitectures # If "false" allow targets that are incompatible with the current host (for # instance concretize with target "icelake" while running on "haswell"). # If "true" only allow targets that are compatible with the host. host_compatible: true # When "true" concretize root specs of environments together, so that each unique # package in an environment corresponds to one concrete spec. This ensures # environments can always be activated. When "false" perform concretization separately # on each root spec, allowing different versions and variants of the same package in # an environment. unify: true # Option to deal with possible duplicate nodes (i.e. different nodes from the same package) in the DAG. duplicates: # "none": allows a single node for any package in the DAG. # "minimal": allows the duplication of 'build-tools' nodes only # (e.g. py-setuptools, cmake etc.) # "full" (experimental): allows separation of the entire build-tool stack (e.g. the entire "cmake" subDAG) strategy: minimal # Maximum number of duplicates in a DAG, when using a strategy that allows duplicates. "default" is the # number used if there isn't a more specific alternative max_dupes: default: 1 # Virtuals c: 2 cxx: 2 fortran: 1 # Regular packages cmake: 2 gmake: 2 python: 2 python-venv: 2 py-cython: 2 py-flit-core: 2 py-pip: 2 py-setuptools: 2 py-versioneer: 2 py-wheel: 2 xcb-proto: 2 # Compilers gcc: 2 llvm: 2 # Option to specify compatibility between operating systems for reuse of compilers and packages # Specified as a key: [list] where the key is the os that is being targeted, and the list contains the OS's # it can reuse. Note this is a directional compatibility so mutual compatibility between two OS's # requires two entries i.e. os_compatible: {sonoma: [monterey], monterey: [sonoma]} os_compatible: {} # If false, force all link/run dependencies of root to match c/c++/Fortran compiler. If this is # a list, then the listed packages are allowed to use a different compiler, but all others must # match. compiler_mixing: true # Option to specify whether to support splicing. Splicing allows for # the relinking of concrete package dependencies in order to better # reuse already built packages with ABI compatible dependencies splice: explicit: [] automatic: false # Maximum time, in seconds, allowed for the 'solve' phase. If set to 0, there is no time limit. timeout: 0 # If set to true, exceeding the timeout will always result in a concretization error. If false, # the best (suboptimal) model computed before the timeout is used. # # Setting this to false yields unreproducible results, so we advise to use that value only # for debugging purposes (e.g. check which constraints can help Spack concretize faster). error_on_timeout: true # Static analysis may reduce the concretization time by generating smaller ASP problems, in # cases where there are requirements that prevent part of the search space to be explored. static_analysis: false # If enabled, concretizations are cached in the misc_cache. The cache is keyed by the hash # of solver inputs, so we only need to run setup (not solve) if there is a cache hit. # Feature is experimental: enabling may result in potentially invalid concretizations # if package recipes are changed, see: https://github.com/spack/spack/issues/51553 concretization_cache: enable: false # Options to control the behavior of the concretizer with external specs externals: # Either 'architecture_only', to complete external specs with just the architecture of the # current host, or 'default_variants' to complete external specs also with missing variants, # using their default value. completion: default_variants ================================================ FILE: etc/spack/defaults/base/config.yaml ================================================ # ------------------------------------------------------------------------- # This is the default spack configuration file. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing files in: # # * $SPACK_ROOT/etc/spack/ - Spack instance settings # * ~/.spack/ - user settings # * $SPACK_ROOT/etc/spack/site - Spack "site" settings. Like instance settings # but lower priority then user settings. # # ------------------------------------------------------------------------- config: # This is the path to the root of the Spack install tree. # You can use $spack here to refer to the root of the spack instance. install_tree: root: $spack/opt/spack projections: all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" # install_tree can include an optional padded length (int or boolean) # default is False (do not pad) # if padded_length is True, Spack will pad as close to the system max path # length as possible # if padded_length is an integer, Spack will pad to that many characters, # assuming it is higher than the length of the install_tree root. # padded_length: 128 # Locations where templates should be found template_dirs: - $spack/share/spack/templates # Directory where licenses should be located license_dir: $spack/etc/spack/licenses # Temporary locations Spack can try to use for builds. # # Recommended options are given below. # # Builds can be faster in temporary directories on some (e.g., HPC) systems. # Specifying `$tempdir` will ensure use of the default temporary directory # (i.e., ``$TMP` or ``$TMPDIR``). # # Another option that prevents conflicts and potential permission issues is # to specify `$user_cache_path/stage`, which ensures each user builds in their # home directory. # # A more traditional path uses the value of `$spack/var/spack/stage`, which # builds directly inside Spack's instance without staging them in a # temporary space. Problems with specifying a path inside a Spack instance # are that it precludes its use as a system package and its ability to be # pip installable. # # In Spack environment files, chaining onto existing system Spack # installations, the $env variable can be used to download, cache and build # into user-writable paths that are relative to the currently active # environment. # # In any case, if the username is not already in the path, Spack will append # the value of `$user` in an attempt to avoid potential conflicts between # users in shared temporary spaces. # # The build stage can be purged with `spack clean --stage` and # `spack clean -a`, so it is important that the specified directory uniquely # identifies Spack staging to avoid accidentally wiping out non-Spack work. build_stage: - $tempdir/$user/spack-stage - $user_cache_path/stage # - $spack/var/spack/stage # Naming format for individual stage directories stage_name: "spack-stage-{name}-{version}-{hash}" # Directory in which to run tests and store test results. # Tests will be stored in directories named by date/time and package # name/hash. test_stage: $user_cache_path/test # Cache directory for already downloaded source tarballs and archived # repositories. This can be purged with `spack clean --downloads`. source_cache: $spack/var/spack/cache ## Directory where spack managed environments are created and stored # environments_root: $spack/var/spack/environments # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` misc_cache: $user_cache_path/cache # Abort downloads after this many seconds if not data is received. # Setting this to 0 will disable the timeout. connect_timeout: 30 # If this is false, tools like curl that use SSL will not verify # certificates. (e.g., curl will use use the -k option) verify_ssl: true # This is where custom certs for proxy/firewall are stored. # It can be a path or environment variable. To match ssl env configuration # the default is the environment variable SSL_CERT_FILE ssl_certs: $SSL_CERT_FILE # Suppress gpg warnings from binary package verification # Only suppresses warnings, gpg failure will still fail the install # Potential rationale to set True: users have already explicitly trusted the # gpg key they are using, and may not want to see repeated warnings that it # is self-signed or something of the sort. suppress_gpg_warnings: false # If set to true, Spack will always check checksums after downloading # archives. If false, Spack skips the checksum step. checksum: true # If set to true, Spack will fetch deprecated versions without warning. # If false, Spack will raise an error when trying to install a deprecated version. deprecated: false # If set to true, `spack install` and friends will NOT clean # potentially harmful variables from the build environment. Use wisely. dirty: false # The language the build environment will use. This will produce English # compiler messages by default, so the log parser can highlight errors. # If set to C, it will use English (see man locale). # If set to the empty string (''), it will use the language from the # user's environment. build_language: C # When set to true, concurrent instances of Spack will use locks to # avoid modifying the install tree, database file, etc. If false, Spack # will disable all locking, but you must NOT run concurrent instances # of Spack. For filesystems that don't support locking, you should set # this to false and run one Spack at a time, but otherwise we recommend # enabling locks. locks: true # The default url fetch method to use. # If set to 'curl', Spack will require curl on the user's system # If set to 'urllib', Spack will use python built-in libs to fetch url_fetch_method: urllib # The maximum number of jobs to use for the build. When using the old # installer, this is the number of jobs per package. In the new installer, # this is the global maximum number of jobs across all packages. When fewer # cores are available, Spack will use fewer jobs. The `-j` command line # argument overrides this option. build_jobs: 16 # The maximum number of concurrent package builds a single Spack process # will perform. The default value of 0 means no package parallelism when using # the old installer, and unlimited package parallelism (other than the limit # set by build_jobs) when using the new installer. Setting this to 1 will # disable package parallelism in both installers. This option is ignored on # Windows. concurrent_packages: 0 # Which installer to use: "old" or "new". installer: new # If set to true, Spack will use ccache to cache C compiles. ccache: false # How long to wait to lock the Spack installation database. This lock is used # when Spack needs to manage its own package metadata and all operations are # expected to complete within the default time limit. The timeout should # therefore generally be left untouched. db_lock_timeout: 60 # How long to wait when attempting to modify a package (e.g. to install it). # This value should typically be 'null' (never time out) unless the Spack # instance only ever has a single user at a time, and only if the user # anticipates that a significant delay indicates that the lock attempt will # never succeed. package_lock_timeout: null # Control how shared libraries are located at runtime on Linux. See the # the Spack documentation for details. shared_linking: # Spack automatically embeds runtime search paths in ELF binaries for their # dependencies. Their type can either be "rpath" or "runpath". For glibc, rpath is # inherited and has precedence over LD_LIBRARY_PATH; runpath is not inherited # and of lower precedence. DO NOT MIX these within the same install tree. type: rpath # (Experimental) Embed absolute paths of dependent libraries directly in ELF # binaries to avoid runtime search. This can improve startup time of # executables with many dependencies, in particular on slow filesystems. bind: false # Controls the handling of missing dynamic libraries after installation. # Options are ignore (default), warn, or error. If set to error, the # installation fails if installed binaries reference dynamic libraries that # are not found in their specified rpaths. missing_library_policy: ignore # Set to 'false' to allow installation on filesystems that doesn't allow setgid bit # manipulation by unprivileged user (e.g. AFS) allow_sgid: true # Whether to show status information during building and installing packages. # This gives information about Spack's current progress as well as the current # and total number of packages. Information is shown both in the terminal # title and inline. install_status: true # Number of seconds a buildcache's index.json is cached locally before probing # for updates, within a single Spack invocation. Defaults to 10 minutes. binary_index_ttl: 600 flags: # Whether to keep -Werror flags active in package builds. keep_werror: 'none' # A mapping of aliases that can be used to define new commands. For instance, # `sp: spec -I` will define a new command `sp` that will execute `spec` with # the `-I` argument. Aliases cannot override existing commands. aliases: concretise: concretize containerise: containerize rm: remove ================================================ FILE: etc/spack/defaults/base/mirrors.yaml ================================================ mirrors: spack-public: binary: false url: https://mirror.spack.io ================================================ FILE: etc/spack/defaults/base/modules.yaml ================================================ # ------------------------------------------------------------------------- # This is the default configuration for Spack's module file generation. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing the following files. # # Per-spack-instance settings (overrides defaults): # $SPACK_ROOT/etc/spack/modules.yaml # # Per-user settings (overrides default and site settings): # ~/.spack/modules.yaml # ------------------------------------------------------------------------- modules: # This maps paths in the package install prefix to environment variables # they should be added to. For example, /bin should be in PATH. prefix_inspections: ./bin: - PATH ./man: - MANPATH ./share/man: - MANPATH ./share/aclocal: - ACLOCAL_PATH ./lib/pkgconfig: - PKG_CONFIG_PATH ./lib64/pkgconfig: - PKG_CONFIG_PATH ./share/pkgconfig: - PKG_CONFIG_PATH ./: - CMAKE_PREFIX_PATH # These are configurations for the module set named "default" default: # Where to install modules roots: tcl: $spack/share/spack/modules lmod: $spack/share/spack/lmod # What type of modules to use ("tcl" and/or "lmod") enable: [] tcl: all: autoload: direct # Default configurations if lmod is enabled lmod: all: autoload: direct hierarchy: - mpi ================================================ FILE: etc/spack/defaults/base/packages.yaml ================================================ # ------------------------------------------------------------------------- # This file controls default concretization preferences for Spack. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing the following files. # # Per-spack-instance settings (overrides defaults): # $SPACK_ROOT/etc/spack/packages.yaml # # Per-user settings (overrides default and site settings): # ~/.spack/packages.yaml # ------------------------------------------------------------------------- packages: all: providers: awk: [gawk] armci: [armcimpi] blas: [openblas] c: [gcc, llvm, intel-oneapi-compilers] cuda-lang: [cuda] cxx: [gcc, llvm, intel-oneapi-compilers] daal: [intel-oneapi-daal] elf: [elfutils] fftw-api: [fftw, amdfftw] flame: [libflame, amdlibflame] fortran: [gcc, llvm, intel-oneapi-compilers] fortran-rt: [gcc-runtime, intel-oneapi-runtime] fuse: [libfuse] gl: [glx, osmesa] glu: [mesa-glu, openglu] golang: [go, gcc] go-or-gccgo-bootstrap: [go-bootstrap, gcc] hip-lang: [llvm-amdgpu] iconv: [libiconv] ipp: [intel-oneapi-ipp] java: [openjdk, jdk] jpeg: [libjpeg-turbo, libjpeg] lapack: [openblas] libc: [glibc, musl] libgfortran: [gcc-runtime] libglx: [mesa+glx] libifcore: [intel-oneapi-runtime] libllvm: [llvm] lua-lang: [lua, lua-luajit-openresty, lua-luajit] luajit: [lua-luajit-openresty, lua-luajit] mariadb-client: [mariadb-c-client, mariadb] mkl: [intel-oneapi-mkl] mpe: [mpe2] mpi: [openmpi, mpich] mysql-client: [mysql, mariadb-c-client] opencl: [pocl] onedal: [intel-oneapi-dal] pbs: [openpbs, torque] pil: [py-pillow] pkgconfig: [pkgconf, pkg-config] qmake: [qt-base, qt] rpc: [libtirpc] scalapack: [netlib-scalapack, amdscalapack] sycl: [hipsycl] szip: [libaec, libszip] tbb: [intel-tbb] unwind: [libunwind] uuid: [util-linux-uuid, libuuid] wasi-sdk: [wasi-sdk-prebuilt] xkbdata-api: [xkeyboard-config, xkbdata] xxd: [xxd-standalone, vim] yacc: [bison, byacc] ziglang: [zig] zlib-api: [zlib-ng+compat, zlib] permissions: read: world write: user cce: buildable: false cray-fftw: buildable: false cray-libsci: buildable: false cray-mpich: buildable: false cray-mvapich2: buildable: false cray-pmi: buildable: false egl: buildable: false essl: buildable: false fj: buildable: false fujitsu-mpi: buildable: false fujitsu-ssl2: buildable: false glibc: buildable: false hpcx-mpi: buildable: false iconv: prefer: [libiconv] mpt: buildable: false musl: buildable: false opengl: buildable: false spectrum-mpi: buildable: false xl: buildable: false ================================================ FILE: etc/spack/defaults/base/repos.yaml ================================================ # ------------------------------------------------------------------------- # This is the default spack repository configuration. It includes the # builtin spack package repository. # # Users can override these settings by editing the following files. # # Per-spack-instance settings (overrides defaults): # $SPACK_ROOT/etc/spack/repos.yaml # # Per-user settings (overrides default and site settings): # ~/.spack/repos.yaml # ------------------------------------------------------------------------- repos: builtin: git: https://github.com/spack/spack-packages.git ================================================ FILE: etc/spack/defaults/bootstrap.yaml ================================================ bootstrap: # If set to false Spack will not bootstrap missing software, # but will instead raise an error. enable: true # Root directory for bootstrapping work. The software bootstrapped # by Spack is installed in a "store" subfolder of this root directory root: $user_cache_path/bootstrap # Methods that can be used to bootstrap software. Each method may or # may not be able to bootstrap all the software that Spack needs, # depending on its type. sources: - name: github-actions-v2 metadata: $spack/share/spack/bootstrap/github-actions-v2 - name: github-actions-v0.6 metadata: $spack/share/spack/bootstrap/github-actions-v0.6 - name: spack-install metadata: $spack/share/spack/bootstrap/spack-install trusted: # By default we trust bootstrapping from sources and from binaries # produced on Github via the workflow github-actions-v2: true github-actions-v0.6: true spack-install: true ================================================ FILE: etc/spack/defaults/darwin/modules.yaml ================================================ # ------------------------------------------------------------------------- # This is the default configuration for Spack's module file generation. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing the following files. # # Per-spack-instance settings (overrides defaults): # $SPACK_ROOT/etc/spack/modules.yaml # # Per-user settings (overrides default and site settings): # ~/.spack/modules.yaml # ------------------------------------------------------------------------- modules: prefix_inspections: ./lib: - DYLD_FALLBACK_LIBRARY_PATH ./lib64: - DYLD_FALLBACK_LIBRARY_PATH ================================================ FILE: etc/spack/defaults/darwin/packages.yaml ================================================ # ------------------------------------------------------------------------- # This file controls default concretization preferences for Spack. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing the following files. # # Per-spack-instance settings (overrides defaults): # $SPACK_ROOT/etc/spack/packages.yaml # # Per-user settings (overrides default and site settings): # ~/.spack/packages.yaml # ------------------------------------------------------------------------- packages: all: providers: c: [apple-clang, llvm, gcc] cxx: [apple-clang, llvm, gcc] elf: [libelf] fortran: [gcc] fuse: [macfuse] gl: [apple-gl] glu: [apple-glu] unwind: [apple-libunwind] uuid: [apple-libuuid] apple-clang: buildable: false apple-gl: buildable: false externals: - spec: apple-gl@4.1.0 prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk apple-glu: buildable: false externals: - spec: apple-glu@1.3.0 prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk apple-libunwind: buildable: false externals: # Apple bundles libunwind version 35.3 with macOS 10.9 and later, # although the version number used here isn't critical - spec: apple-libunwind@35.3 prefix: /usr apple-libuuid: buildable: false externals: # Apple bundles libuuid in libsystem_c version 1353.100.2, # although the version number used here isn't critical - spec: apple-libuuid@1353.100.2 prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk c: prefer: - apple-clang cxx: prefer: - apple-clang fortran: prefer: - gcc ================================================ FILE: etc/spack/defaults/include.yaml ================================================ include: # default platform-specific configuration - path: "${platform}" optional: true # base packages.yaml overridable by platform-specific settings - path: base ================================================ FILE: etc/spack/defaults/linux/modules.yaml ================================================ # ------------------------------------------------------------------------- # This is the default configuration for Spack's module file generation. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing the following files. # # Per-spack-instance settings (overrides defaults): # $SPACK_ROOT/etc/spack/modules.yaml # # Per-user settings (overrides default and site settings): # ~/.spack/modules.yaml # ------------------------------------------------------------------------- modules: {} ================================================ FILE: etc/spack/defaults/windows/config.yaml ================================================ config: locks: false build_stage:: - '$user_cache_path/stage' stage_name: '{name}-{version}-{hash:7}' installer: old ================================================ FILE: etc/spack/defaults/windows/packages.yaml ================================================ # ------------------------------------------------------------------------- # This file controls default concretization preferences for Spack. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing the following files. # # Per-spack-instance settings (overrides defaults): # $SPACK_ROOT/etc/spack/packages.yaml # # Per-user settings (overrides default and site settings): # ~/.spack/packages.yaml # ------------------------------------------------------------------------- packages: all: providers: c : [msvc] cxx: [msvc] mpi: [msmpi] gl: [wgl] mpi: require: - one_of: [msmpi] msvc: buildable: false ================================================ FILE: lib/spack/_vendoring/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import warnings import spack.vendor import spack.error warnings.warn( "The `_vendoring` module will be removed in Spack v1.1", category=spack.error.SpackAPIWarning, stacklevel=2, ) __path__ = spack.vendor.__path__ ================================================ FILE: lib/spack/docs/.gitignore ================================================ .spack/spack-packages .spack/packages.yaml .spack-env _build _spack_root command_index.rst spack*.rst spack.lock ================================================ FILE: lib/spack/docs/.spack/repos.yaml ================================================ repos: builtin: destination: ./spack-packages ================================================ FILE: lib/spack/docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = -W --keep-going SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build export PYTHONPATH := ../../spack:$(PYTHONPATH) APIDOC_FILES = spack*.rst # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext apidoc dashdoc all: html # # This creates a git repository and commits generated html docs. # It them pushes the new branch into THIS repository as gh-pages. # # github for some reason runs jekyll automatically on gh-pages # files, but we don't want that. 'touch .nojekyll' takes care # of it. # gh-pages: _build/html root="$$(git rev-parse --show-toplevel)" && \ cd _build/html && \ rm -rf .git && \ touch .nojekyll && \ git init && \ git add . && \ git commit -m "Spack Documentation" && \ git push -f $$root master:gh-pages && \ rm -rf .git # This version makes gh-pages into a single page that redirects # to spack.readthedocs.io gh-pages-redirect: root="$$(git rev-parse --show-toplevel)" && \ cd _gh_pages_redirect && \ rm -rf .git && \ git init && \ git add . && \ git commit -m "Spack Documentation" && \ git push -f $$root master:gh-pages && \ rm -rf .git upload: rsync -avz --rsh=ssh --delete _build/html/ cab:/usr/global/web-pages/lc/www/adept/docs/spack git push -f github gh-pages apidoc: sphinx-apidoc -f -T -o . ../spack sphinx-apidoc -f -T -o . ../llnl help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -f command_index.rst -rm -rf $(BUILDDIR)/* $(APIDOC_FILES) -rm -rf .spack/spack-packages html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Spack.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Spack.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Spack" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Spack" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." dashdoc: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/dashdoc doc2dash -A -v -n spack -d $(BUILDDIR)/ -f -I index.html -j $(BUILDDIR)/dashdoc @echo @echo "Build finished. The Docset is in $(BUILDDIR)/dashdoc." ================================================ FILE: lib/spack/docs/_gh_pages_redirect/.nojekyll ================================================ ================================================ FILE: lib/spack/docs/_gh_pages_redirect/index.html ================================================

This page has moved to https://spack.readthedocs.io/

================================================ FILE: lib/spack/docs/_static/css/custom.css ================================================ div.versionadded { border-left: 3px solid #0c731f; color: #0c731f; padding-left: 1rem; } .py.property { display: block !important; } div.version-switch { text-align: center; min-height: 2em; } div.version-switch>select { display: inline-block; text-align-last: center; background: none; border: none; border-radius: 0.5em; box-shadow: none; color: var(--color-foreground-primary); cursor: pointer; appearance: none; padding: 0.2em; -webkit-appearance: none; -moz-appearance: none; } div.version-switch select:active, div.version-switch select:focus, div.version-switch select:hover { color: var(--color-foreground-secondary); background: var(--color-background-hover); } .toc-tree li.scroll-current>.reference { font-weight: normal; } .search-results span { background-color: #fff3cd; padding: 0.1rem 0.2rem; border-radius: 2px; } @media (prefers-color-scheme: dark) { body:not([data-theme="light"]) .search-results span { background-color: #664d03; color: #fff3cd; } } .highlight .go { color: #333; } ================================================ FILE: lib/spack/docs/_static/js/versions.js ================================================ // based on https://github.com/readthedocs/sphinx_rtd_theme/blob/3.0.2/sphinx_rtd_theme/static/js/versions.js_t function onSelectorSwitch(event) { const option = event.target.selectedIndex; const item = event.target.options[option]; window.location.href = item.dataset.url; } function initVersionSelector(config) { const versionSwitch = document.querySelector(".version-switch"); if (!versionSwitch) { return; } let versions = config.versions.active; if (config.versions.current.hidden || config.versions.current.type === "external") { versions.unshift(config.versions.current); } const versionSelect = ` `; versionSwitch.innerHTML = versionSelect; versionSwitch.firstElementChild.addEventListener("change", onSelectorSwitch); } function initSearch(currentVersion) { // Numeric versions are PRs which have no search results; use latest instead. const searchVersion = /^\d+$/.test(currentVersion) ? 'latest' : currentVersion; let searchTimeout; let originalContent; const searchInput = document.querySelector(".sidebar-search"); const mainContent = document.getElementById("furo-main-content"); const searchForm = document.querySelector(".sidebar-search-container"); if (!searchInput || !mainContent || !searchForm) { return; } // Store original content originalContent = mainContent.innerHTML; searchInput.addEventListener("input", handleSearchInput); searchInput.addEventListener("keydown", handleTabNavigation); searchForm.addEventListener("submit", handleFormSubmit); function handleSearchInput(e) { const query = e.target.value.trim(); clearTimeout(searchTimeout); if (query.length === 0) { mainContent.innerHTML = originalContent; return; } searchTimeout = setTimeout(function () { performSearch(query); }, 300); } function handleFormSubmit(e) { e.preventDefault(); const query = searchInput.value.trim(); if (query) { performSearch(query); } } function handleTabNavigation(e) { // Check if we're tabbing throught search results if (e.key !== 'Tab' || e.shiftKey) { return; } const searchResults = document.querySelector(".search-results"); if (!searchResults) { return; } // Focus on the first link in search results instead of default behavior e.preventDefault(); const firstLink = searchResults.querySelector("a"); if (firstLink) { firstLink.focus(); } } function performSearch(query) { const fullQuery = `project:spack/${searchVersion} ${query}`; const searchUrl = `/_/api/v3/search/?q=${encodeURIComponent(fullQuery)}`; fetch(searchUrl) .then(function (response) { if (!response.ok) { throw new Error("HTTP error! status: " + response.status); } return response.json(); }) .then(function (data) { displaySearchResults(data, query); }) .catch(function (error) { mainContent.innerHTML = "

Error performing search.

"; }); } function displaySearchResults(data, query) { if (!data.results?.length) { mainContent.innerHTML = `

No Results Found

No results found for "${query}".

`; return; } let html = '

Search Results

'; data.results.forEach((result, index) => { const title = result.highlights?.title?.[0] ?? result.title; html += `

${title}

`; result.blocks?.forEach(block => { const blockTitle = block.highlights?.title?.[0] ?? block.title; html += `

${blockTitle}

`; html += block.highlights?.content?.map(content => `

${content}

`).join('') ?? ''; }); if (index < data.results.length - 1) { html += `
`; } }); html += "
"; mainContent.innerHTML = html; } } document.addEventListener("readthedocs-addons-data-ready", function (event) { const config = event.detail.data(); initVersionSelector(config); initSearch(config.versions.current.slug); }); ================================================ FILE: lib/spack/docs/_templates/base.html ================================================ {% extends "!base.html" %} {%- block extrahead %} {%- if READTHEDOCS %} {%- endif %} {% endblock %} ================================================ FILE: lib/spack/docs/_templates/sidebar/brand.html ================================================ {%- if READTHEDOCS %}
{%- endif %} ================================================ FILE: lib/spack/docs/advanced_topics.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Explore advanced topics in Spack, including auditing packages and configuration, and verifying installations. .. _cmd-spack-audit: Auditing Packages and Configuration =================================== The ``spack audit`` command detects potential issues with configuration and packages: .. command-output:: spack audit -h For instance, it can detect duplicate external specs in ``packages.yaml``, or the use of non-existing variants in directives. A detailed list of the checks currently implemented for each subcommand can be printed with: .. command-output:: spack -v audit list Depending on the use case, users can run the appropriate subcommands to obtain diagnostics. If issues are found, they are reported to stdout: .. code-block:: console % spack audit packages lammps PKG-DIRECTIVES: 1 issue found 1. lammps: wrong variant in "conflicts" directive the variant 'adios' does not exist in spack_repo/builtin/packages/lammps/package.py .. _cmd-spack-verify: Verifying Installations ======================= The ``spack verify`` command can be used to verify the validity of Spack-installed packages any time after installation. ``spack verify manifest`` ------------------------- At installation time, Spack creates a manifest of every file in the installation prefix. For links, Spack tracks the mode, ownership, and destination. For directories, Spack tracks the mode and ownership. For files, Spack tracks the mode, ownership, modification time, hash, and size. The ``spack verify manifest`` command will check, for every file in each package, whether any of those attributes have changed. It will also check for newly added files or deleted files from the installation prefix. Spack can either check all installed packages using the ``-a,--all`` option or accept specs listed on the command line to verify. The ``spack verify manifest`` command can also verify that individual files haven't been altered since installation time. If the given file is not in a Spack installation prefix, Spack will report that it is not owned by any package. To check individual files instead of specs, use the ``-f,--files`` option. Spack installation manifests are included in the tarball signed by Spack for binary package distribution. When installed from a binary package, Spack uses the packaged installation manifest instead of creating one at install time. The ``spack verify`` command also accepts the ``-l,--local`` option to check only local packages (as opposed to those used transparently from ``upstream`` Spack instances) and the ``-j,--json`` option to output machine-readable JSON data for any errors. ``spack verify libraries`` -------------------------- The ``spack verify libraries`` command can be used to verify that packages do not have accidental system dependencies. This command scans the install prefixes of packages for executables and shared libraries, and resolves their needed libraries in their RPATHs. When needed libraries cannot be located, an error is reported. This typically indicates that a package was linked against a system library instead of a library provided by a Spack package. This verification can also be enabled as a post-install hook by setting ``config:shared_linking:missing_library_policy`` to ``error`` or ``warn`` in :ref:`config.yaml `. .. _filesystem-requirements: Filesystem Requirements ======================= By default, Spack needs to be run from a filesystem that supports ``flock`` locking semantics. Nearly all local filesystems and recent versions of NFS support this, but parallel filesystems or NFS volumes may be configured without ``flock`` support enabled. You can determine how your filesystems are mounted with ``mount``. The output for a Lustre filesystem might look like this: .. code-block:: console $ mount | grep lscratch mds1-lnet0@o2ib100:/lsd on /p/lscratchd type lustre (rw,nosuid,lazystatfs,flock) mds2-lnet0@o2ib100:/lse on /p/lscratche type lustre (rw,nosuid,lazystatfs,flock) Note the ``flock`` option on both Lustre mounts. If you do not see this or a similar option for your filesystem, you have a few options. First, you can move your Spack installation to a filesystem that supports locking. Second, you could ask your system administrator to enable ``flock`` for your filesystem. If none of those work, you can disable locking in one of two ways: 1. Run Spack with the ``-L`` or ``--disable-locks`` option to disable locks on a call-by-call basis. 2. Edit :ref:`config.yaml ` and set the ``locks`` option to ``false`` to always disable locking. .. warning:: If you disable locking, concurrent instances of Spack will have no way to avoid stepping on each other. You must ensure that there is only **one** instance of Spack running at a time. Otherwise, Spack may end up with a corrupted database, or you may not be able to see all installed packages when running commands like ``spack find``. If you are unfortunate enough to run into this situation, you may be able to fix it by running ``spack reindex``. This issue typically manifests with the error below: .. code-block:: console $ ./spack find Traceback (most recent call last): File "./spack", line 176, in main() File "./spack", line 154, in main return_val = command(parser, args) File "./spack/lib/spack/spack/cmd/find.py", line 170, in find specs = set(spack.installed_db.query(\**q_args)) File "./spack/lib/spack/spack/database.py", line 551, in query with self.read_transaction(): File "./spack/lib/spack/spack/database.py", line 598, in __enter__ if self._enter() and self._acquire_fn: File "./spack/lib/spack/spack/database.py", line 608, in _enter return self._db.lock.acquire_read(self._timeout) File "./spack/lib/spack/llnl/util/lock.py", line 103, in acquire_read self._lock(fcntl.LOCK_SH, timeout) # can raise LockError. File "./spack/lib/spack/llnl/util/lock.py", line 64, in _lock fcntl.lockf(self._fd, op | fcntl.LOCK_NB) IOError: [Errno 38] Function not implemented ================================================ FILE: lib/spack/docs/binary_caches.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Discover how to create, use, and manage build caches in Spack to share pre-built binary packages and speed up installations. .. _binary_caches: Build Caches ============ To avoid recompilation of Spack packages, installed packages can be pushed to a build cache, and then downloaded and installed by others. Whenever a mirror provides prebuilt packages, Spack will take these packages into account during concretization and installation, making ``spack install`` significantly faster. .. note:: We use the terms "build cache" and "mirror" often interchangeably. Mirrors are used during installation for both sources and prebuilt packages. Build caches refer to mirrors that provide prebuilt packages. Creating a Build Cache ---------------------- Build caches are created via: .. code-block:: console $ spack buildcache push This command takes the locally installed spec and its dependencies, and creates tarballs of their install prefixes. It also generates metadata files, signed with GPG. These tarballs and metadata files are then pushed to the provided build cache, which can be a local directory or a remote URL. Here is an example where a build cache is created in a local directory named "spack-cache", to which we push the "ninja" spec: .. code-block:: console $ spack buildcache push ./spack-cache ninja ==> Selected 30 specs to push to file:///home/spackuser/spack/spack-cache ... ==> [30/30] Pushed ninja@1.12.1/ngldn2k Note that ``ninja`` must be installed locally for this to work. Once you have a build cache, you can add it as a mirror, as discussed next. Finding or Installing Build Cache Files --------------------------------------- To find or install build cache files, a Spack mirror must be configured with: .. code-block:: console $ spack mirror add Both URLs and local paths on the filesystem can be specified. In the previous example, you might add the directory "spack-cache" and call it ``mymirror``: .. code-block:: console $ spack mirror add mymirror ./spack-cache You can see that the mirror is added with ``spack mirror list`` as follows: .. code-block:: console $ spack mirror list mymirror file:///home/spackuser/spack/spack-cache spack-public https://spack-llnl-mirror.s3-us-west-2.amazonaws.com/ At this point, you've created a build cache, but Spack hasn't indexed it, so if you run ``spack buildcache list``, you won't see any results. You need to index this new build cache as follows: .. code-block:: console $ spack buildcache update-index ./spack-cache Now you can use ``list``: .. code-block:: console $ spack buildcache list ==> 24 cached builds. -- linux-ubuntu22.04-sapphirerapids / gcc@12.3.0 ---------------- [ ... ] ninja@1.12.1 With ``mymirror`` configured and an index available, Spack will automatically use it during concretization and installation. That means that you can expect ``spack install ninja`` to fetch prebuilt packages from the mirror. Let's verify by reinstalling ninja: .. code-block:: spec $ spack uninstall ninja $ spack install ninja [ ... ] ==> Installing ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh [24/24] gpg: Signature made Thu 06 Mar 2025 10:03:38 AM MST gpg: using RSA key 75BC0528114909C076E2607418010FFAD73C9B07 gpg: Good signature from "example (GPG created for Spack) " [ultimate] ==> Fetching file:///home/spackuser/spack/spack-cache/blobs/sha256/f0/f08eb62661ad159d2d258890127fc6053f5302a2f490c1c7f7bd677721010ee0 ==> Fetching file:///home/spackuser/spack/spack-cache/blobs/sha256/c7/c79ac6e40dfdd01ac499b020e52e57aa91151febaea3ad183f90c0f78b64a31a ==> Extracting ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh from binary cache ==> ninja: Successfully installed ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh Search: 0.00s. Fetch: 0.11s. Install: 0.11s. Extract: 0.10s. Relocate: 0.00s. Total: 0.22s [+] /home/spackuser/spack/opt/spack/linux-ubuntu22.04-sapphirerapids/gcc-12.3.0/ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh It worked! You've just completed a full example of creating a build cache with a spec of interest, adding it as a mirror, updating its index, listing the contents, and finally, installing from it. By default, Spack falls back to building from sources when the mirror is not available or when the package is simply not already available. To force Spack to install only prebuilt packages, you can use: .. code-block:: console $ spack install --use-buildcache only For example, to combine all of the commands above to add the E4S build cache and then install from it exclusively, you would do: .. code-block:: console $ spack mirror add E4S https://cache.e4s.io $ spack buildcache keys --install --trust $ spack install --use-buildcache only The ``--install`` and ``--trust`` flags install keys to the keyring and trust all downloaded keys. Build Cache Index Views ^^^^^^^^^^^^^^^^^^^^^^^ .. note:: Introduced in Spack v1.2. The addition of this feature does not increment the build cache version (v3). .. note:: Build cache index views are not supported in OCI build caches. Build caches can quickly become large and inefficient to search as binaries are added over time. A common work around to this problem is to break the build cache into stacks that target specific applications or workflows. This allows for curation of binaries as smaller collections of packages that push to their own mirrors that each maintain a smaller search area. However, this approach comes with the trade off of requiring much larger storage and computational footprints due to duplication of common dependencies between stacks. Splitting build caches can also reduce direct fetch hits by reducing the breadth of binaries available in a single mirror. To better address the issues with large search areas, build cache index views (or just "views" in this section) were introduced. A view is a named index which provides a curated view into a larger build cache. This allows build cache maintainers to provide the same granularity of build caches split by stacks without having to pay for the extra storage and compute required for the duplicated dependencies. Views can be created or updated using an active environment, or a list of environment names or paths. The ``spack buildcache`` commands for views are alias of the command ``spack buildcache update-index``. View indices are stored similarly to the top level build cache index, but use an additional prefix of the view name ``/v3/manifests/index/my-stack/index.manifest.json``. .. _cmd-spack-buildcache-create-view: Creating a Build Cache Index View """"""""""""""""""""""""""""""""" Here is an example of creating a view using an active environment. .. code-block:: console $ spack env activate my-stack $ spack install $ spack buildcache push my-mirror $ spack buildcache update-index --name my-view my-mirror It is also possible to create a view from a list of one or more environments by passing the environment names or paths. If a list of environments is passed while inside of an active environment, the active environment is ignored and only the passed environments are considered. .. code-block:: console $ spack buildcache update-index --name my-view my-mirror my-stack /path/to/environment/my-other-stack .. _cmd-spack-buildcache-update-view: Updating a Build Cache Index View """"""""""""""""""""""""""""""""" To prevent accidentally overwriting an existing view, it is required to specify how a view should be updated. It is possible to use one of two options for updating a view index: ``--force`` or ``--append``. Using the ``--force`` option will replace the index as if the previous one did not exist. The ``--append`` option will first read the existing index, and then add the new specs to it. .. code-block:: console $ spack buildcache push my-mirror $ spack buildcache update-index --append --name my-view my-mirror my-stack .. warning:: Using the ``--append`` option with build cache index views is a non-atomic operation. In the case where multiple writers are appending to the same view, the result will only include the state of the last to write. When using ``--append`` for build cache workflows it is up to the user to correctly serialize the update operations. List of Popular Build Caches ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * `Spack Public Build Cache `_: `spack build cache `_ * `Extreme-scale Scientific Software Stack (E4S) `_: `e4s build cache `_ Creating and Trusting GPG keys ------------------------------ .. _cmd-spack-gpg: ``spack gpg`` ^^^^^^^^^^^^^ Spack has support for signing and verifying packages using GPG keys. A separate keyring is used for Spack, so any keys available in the user's home directory are not used. ``spack gpg init`` ^^^^^^^^^^^^^^^^^^ When Spack is first installed, its keyring is empty. Keys stored in :file:`var/spack/gpg` are the default keys for a Spack installation. These keys may be imported by running ``spack gpg init``. This will import the default keys into the keyring as trusted keys. Trusting keys ^^^^^^^^^^^^^ Additional keys may be added to the keyring using: .. code-block:: console $ spack gpg trust Once a key is trusted, packages signed by the owner of the key may be installed. To remove keys from your keyring, use: .. code-block:: console $ spack gpg untrust Key IDs can be email addresses, names, or (preferably) fingerprints. Creating keys ^^^^^^^^^^^^^ You may also create your own key so that you may sign your own packages using .. code-block:: console $ spack gpg create By default, the key has no expiration, but it may be set with the ``--expires `` flag. It is also recommended to add a comment as to the use of the key using the ``--comment `` flag. The public half of the key can also be exported for sharing with others so that they may use packages you have signed using the ``--export `` flag. Secret keys may also be later exported using the ``spack gpg export [...]`` command. .. admonition:: Key creation speed :class: tip The creation of a new GPG key requires generating a lot of random numbers. Depending on the entropy produced on your system, the entire process may take a long time (and may even appear to hang). Virtual machines and cloud instances are particularly likely to display this behavior. To speed it up, you may install tools like ``rngd``, which is usually available as a package in the host OS. Another alternative is ``haveged``, which can be installed on RHEL/CentOS machines. `This Digital Ocean tutorial `_ provides a good overview of sources of randomness. Build Cache Signing ------------------- By default, Spack will add a cryptographic signature to each package pushed to a build cache and verify the signature when installing from a build cache. Keys for signing can be managed with the :ref:`spack gpg ` command, as well as ``spack buildcache keys``, as mentioned above. You can disable signing when pushing with ``spack buildcache push --unsigned`` and disable verification when installing from any build cache with ``spack install --no-check-signature``. Alternatively, signing and verification can be enabled or disabled on a per-build-cache basis: .. code-block:: console $ spack mirror add --signed # enable signing and verification $ spack mirror add --unsigned # disable signing and verification $ spack mirror set --signed # enable signing and verification for an existing mirror $ spack mirror set --unsigned # disable signing and verification for an existing mirror Alternatively, you can edit the ``mirrors.yaml`` configuration file directly: .. code-block:: yaml mirrors: : url: signed: false # disable signing and verification See also :ref:`mirrors`. Relocation ---------- When using build caches across different machines, it is likely that the install root is different from the one used to build the binaries. To address this issue, Spack automatically relocates all paths encoded in binaries and scripts to their new location upon installation. Note that there are some cases where this is not possible: if binaries are built in a relatively short path and then installed to a longer path, there may not be enough space in the binary to encode the new path. In this case, Spack will fail to install the package from the build cache, and a source build is required. To reduce the likelihood of this happening, it is highly recommended to add padding to the install root during the build, as specified in the :ref:`config ` section of the configuration: .. code-block:: yaml config: install_tree: root: /opt/spack padded_length: 128 Automatic Push to a Build Cache --------------------------------- Sometimes it is convenient to push packages to a build cache immediately after they are installed. Spack can do this by setting the ``--autopush`` flag when adding a mirror: .. code-block:: console $ spack mirror add --autopush Or the ``--autopush`` flag can be set for an existing mirror: .. code-block:: console $ spack mirror set --autopush # enable automatic push for an existing mirror $ spack mirror set --no-autopush # disable automatic push for an existing mirror Then, after installing a package, it is automatically pushed to all mirrors with ``autopush: true``. The command .. code-block:: console $ spack install will have the same effect as .. code-block:: console $ spack install $ spack buildcache push # for all caches with autopush: true .. note:: Packages are automatically pushed to a build cache only if they are built from source. .. _binary_caches_oci: OCI / Docker V2 Registries as Build Cache ----------------------------------------- Spack can also use OCI or Docker V2 registries such as Docker Hub, Quay.io, Amazon ECR, GitHub Packages, GitLab Container Registry, JFrog Artifactory, and others as build caches. This is a convenient way to share binaries using public infrastructure or to cache Spack-built binaries in GitHub Actions and GitLab CI. These registries can be used not only to share Spack binaries but also to create and distribute runnable container images. To get started, configure an OCI mirror using ``oci://`` as the scheme and optionally specify variables that hold the username and password (or personal access token) for the registry: .. code-block:: console $ spack mirror add --oci-username-variable REGISTRY_USER \ --oci-password-variable REGISTRY_TOKEN \ my_registry oci://example.com/my_image This registers a mirror in your ``mirrors.yaml`` configuration file that looks as follows: .. code-block:: yaml mirrors: my_registry: url: oci://example.com/my_image access_pair: id_variable: REGISTRY_USER secret_variable: REGISTRY_TOKEN Spack follows the naming conventions of Docker, with Docker Hub as the default registry. To use Docker Hub, you can omit the registry domain: .. code-block:: console $ spack mirror add ... my_registry oci://username/my_image From here, you can use the mirror as any other build cache: .. code-block:: console $ export REGISTRY_USER=... $ export REGISTRY_TOKEN=... $ spack buildcache push my_registry # push to the registry $ spack install # or install from the registry .. note:: Spack defaults to ``https`` for OCI registries, and does not fall back to ``http`` in case of failure. For local registries which use ``http`` instead of ``https``, you can specify ``oci+http://localhost:5000/my_image``. .. _oci-authentication: Authentication with popular Container Registries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Below are instructions for authenticating with some of the most popular container registries. In all cases, you need to generate a (temporary) token to use as the password -- this is not the same as your account password. GHCR """""" To authenticate with GitHub Container Registry (GHCR), you can use your GitHub username as the username. For the password, you can use either: #. A personal access token (PAT) with ``write:packages`` scope. #. A GitHub Actions token (``GITHUB_TOKEN``) with ``packages:write`` permission. See also `GitHub's documentation `_ and :ref:`github-actions-build-cache` below. Docker Hub """""""""" To authenticate with Docker Hub, you can use your Docker Hub username as the username. For the password, you need to generate a personal access token (PAT) on the Docker Hub website. See `Docker's documentation `_ for more information. Amazon ECR """""""""" To authenticate with Amazon ECR, you can use the AWS CLI to generate a temporary password. The username is always ``AWS``. .. code-block:: console $ export AWS_ECR_PASSWORD=$(aws ecr get-login-password --region ) $ spack mirror add \ --oci-username AWS \ --oci-password-variable AWS_ECR_PASSWORD \ my_registry \ oci://XXX.dkr.ecr..amazonaws.com/my/image See also `AWS's documentation `_. Azure Container Registry """""""""""""""""""""""" To authenticate with an Azure Container Registry that has RBAC enabled, you can use the Azure CLI to generate a temporary password for your managed identity. The username is always ``00000000-0000-0000-0000-000000000000``. .. code-block:: console $ export AZURE_ACR_PASSWORD=$(az acr login --name --expose-token --output tsv --query accessToken) $ spack mirror add \ --oci-username 00000000-0000-0000-0000-000000000000 \ --oci-password-variable AZURE_ACR_PASSWORD \ my_registry \ oci://.azurecr.io/my/image See also `Azure's documentation `_. Build Cache and Container Images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A unique feature of build caches on top of OCI registries is that it's incredibly easy to generate a runnable container image with the binaries installed. This is a great way to make applications available to users without requiring them to install Spack -- all you need is Docker, Podman, or any other OCI-compatible container runtime. To produce container images, all you need to do is add the ``--base-image`` flag when pushing to the build cache: .. code-block:: console $ spack buildcache push --base-image ubuntu:20.04 my_registry ninja Pushed to example.com/my_image:ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack $ docker run -it example.com/my_image:ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack root@e4c2b6f6b3f4:/# ninja --version 1.11.1 If ``--base-image`` is not specified, Spack produces distroless images. In practice, you won't be able to run these as containers because they don't come with libc and other system dependencies. However, they are still compatible with tools like ``skopeo``, ``podman``, and ``docker`` for pulling and pushing. See the section :ref:`exporting-images` for more details on how to create container images with Spack. .. _github-actions-build-cache: Spack Build Cache for GitHub Actions ------------------------------------ To significantly speed up Spack in GitHub Actions, binaries can be cached in GitHub Packages. This service is an OCI registry that can be linked to a GitHub repository. Spack offers a public build cache for GitHub Actions with a set of common packages, which lets you get started quickly. See the following resources for more information: * `spack/setup-spack `_ for setting up Spack in GitHub Actions * `spack/github-actions-buildcache `_ for more details on the public build cache .. _cmd-spack-buildcache: ``spack buildcache`` -------------------- ``spack buildcache push`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Create a tarball of an installed Spack package and all its dependencies. Tarballs and specfiles are compressed and checksummed; manifests are signed if GPG2 is available. Commands like ``spack buildcache install`` will search Spack mirrors to get the list of build caches. ============== ======================================================================================================================== Arguments Description ============== ======================================================================================================================== ```` list of partial specs or hashes with a leading ``/`` to match from installed packages and used for creating build caches ``-d `` directory in which ``v3`` and ``blobs`` directories are created, defaults to ``.`` ``-f`` overwrite compressed tarball and spec metadata files if they already exist ``-k `` the key to sign package with. In the case where multiple keys exist, the package will be unsigned unless ``-k`` is used. ``-r`` make paths in binaries relative before creating tarball ``-y`` answer yes to all questions about creating unsigned build caches ============== ======================================================================================================================== ``spack buildcache list`` ^^^^^^^^^^^^^^^^^^^^^^^^^ Retrieves all specs for build caches available on a Spack mirror. ============== ===================================================================================== Arguments Description ============== ===================================================================================== ```` list of partial package specs to be matched against specs downloaded for build caches ============== ===================================================================================== E.g., ``spack buildcache list gcc`` will print only commands to install ``gcc`` package(s). ``spack buildcache install`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Retrieves all specs for build caches available on a Spack mirror and installs build caches with specs matching the input specs. ============== ============================================================================================== Arguments Description ============== ============================================================================================== ```` list of partial package specs or hashes with a leading ``/`` to be installed from build caches ``-f`` remove install directory if it exists before unpacking tarball ``-y`` answer yes to all to don't verify package with gpg questions ============== ============================================================================================== ``spack buildcache keys`` ^^^^^^^^^^^^^^^^^^^^^^^^^ List public keys available on a Spack mirror. ========= ============================================== Arguments Description ========= ============================================== ``-it`` trust the keys downloaded with prompt for each ``-y`` answer yes to all trust all keys downloaded ========= ============================================== .. _build_cache_layout: Build Cache Layout ------------------ This section describes the structure and content of URL-style build caches, as distinguished from OCI-style build caches. The entry point for a binary package is a manifest JSON file that references at least two other files stored as content-addressed blobs. These files include a spec metadata file, as well as the installation directory of the package stored as a compressed archive file. Binary package manifest files are named to indicate the package name and version, as well as the hash of the concrete spec. For example: .. code-block:: text gcc-runtime-12.3.0-qyu2lvgt3nxh7izxycugdbgf5gsdpkjt.spec.manifest.json would contain the manifest for a binary package of ``gcc-runtime@12.3.0``. The ID of the built package is defined to be the DAG hash of the concrete spec and exists in the name of the file as well. The ID distinguishes a particular binary package from all other binary packages with the same package name and version. Below is an example binary package manifest file. Such a file would live in the versioned spec manifests directory of a binary mirror, for example, ``v3/manifests/spec/``: .. code-block:: json { "version": 3, "data": [ { "contentLength": 10731083, "mediaType": "application/vnd.spack.install.v2.tar+gzip", "compression": "gzip", "checksumAlgorithm": "sha256", "checksum": "0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210" }, { "contentLength": 1000, "mediaType": "application/vnd.spack.spec.v5+json", "compression": "gzip", "checksumAlgorithm": "sha256", "checksum": "fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041" } ] } The manifest references both the compressed tar file as well as the compressed spec metadata file, and contains the checksum of each. This checksum is also used as the address of the associated file and, hence, must be known in order to locate the tarball or spec file within the mirror. Once the tarball or spec metadata file is downloaded, the checksum should be computed locally and compared to the checksum in the manifest to ensure the contents have not changed since the binary package was pushed. Spack stores all data files (including compressed tar files, spec metadata, indices, public keys, etc.) within a ``blobs//`` directory, using the first two characters of the checksum as a subdirectory to reduce the number of files in a single folder. Here is a depiction of the organization of binary mirror contents: .. code-block:: text mirror_directory/ v3/ layout.json manifests/ spec/ gcc-runtime/ gcc-runtime-12.3.0-s2nqujezsce4x6uhtvxscu7jhewqzztx.spec.manifest.json gmake/ gmake-4.4.1-lpr4j77rcgkg5536tmiuzwzlcjsiomph.spec.manifest.json compiler-wrapper/ compiler-wrapper-1.0-s7ieuyievp57vwhthczhaq2ogowf3ohe.spec.manifest.json index/ index.manifest.json key/ 75BC0528114909C076E2607418010FFAD73C9B07.key.manifest.json keys.manifest.json blobs/ sha256/ 0f/ 0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210 fb/ fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041 2a/ 2a21836d206ccf0df780ab0be63fdf76d24501375306a35daa6683c409b7922f ... Files within the ``manifests`` directory are organized into subdirectories by the type of entity they represent. Binary package manifests live in the ``spec/`` directory, build cache index manifests live in the ``index/`` directory, and manifests for public keys and their indices live in the ``key/`` subdirectory. Regardless of the type of entity they represent, all manifest files are named with an extension ``.manifest.json``. Every manifest contains a ``data`` array, each element of which refers to an associated file stored as a content-addressed blob. Considering the example spec manifest shown above, the compressed installation archive can be found by picking out the data blob with the appropriate ``mediaType``, which in this case would be ``application/vnd.spack.install.v2.tar+gzip``. The associated file is found by looking in the blobs directory under ``blobs/sha256/fb/`` for the file named with the complete checksum value. As mentioned above, every entity in a build cache is stored as a content-addressed blob pointed to by a manifest. While an example spec manifest (i.e., a manifest for a binary package) is shown above, here is what the manifest of a build cache index looks like: .. code-block:: json { "version": 3, "data": [ { "contentLength": 6411, "mediaType": "application/vnd.spack.db.v8+json", "compression": "none", "checksumAlgorithm": "sha256", "checksum": "225a3e9da24d201fdf9d8247d66217f5b3f4d0fc160db1498afd998bfd115234" } ] } Some things to note about this manifest are that it points to a blob that is not compressed (``compression: "none"``) and that the ``mediaType`` is one we have not seen yet, ``application/vnd.spack.db.v8+json``. The decision not to compress build cache indices stems from the fact that Spack does not yet sign build cache index manifests. Once that changes, you may start to see these indices stored as compressed blobs. For completeness, here are examples of manifests for the other two types of entities you might find in a Spack build cache. First, a public key manifest: .. code-block:: json { "version": 3, "data": [ { "contentLength": 2472, "mediaType": "application/pgp-keys", "compression": "none", "checksumAlgorithm": "sha256", "checksum": "9fc18374aebc84deb2f27898da77d4d4410e5fb44c60c6238cb57fb36147e5c7" } ] } Note the ``mediaType`` of ``application/pgp-keys``. Finally, a public key index manifest: .. code-block:: json { "version": 3, "data": [ { "contentLength": 56, "mediaType": "application/vnd.spack.keyindex.v1+json", "compression": "none", "checksumAlgorithm": "sha256", "checksum": "29b3a0eb6064fd588543bc43ac7d42d708a69058dafe4be0859e3200091a9a1c" } ] } Again, note the ``mediaType`` of ``application/vnd.spack.keyindex.v1+json``. Also, note that both the above manifest examples refer to uncompressed blobs; this is for the same reason Spack does not yet compress build cache index blobs. ================================================ FILE: lib/spack/docs/bootstrapping.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how Spack's bootstrapping feature automatically fetches and installs essential build tools when they are not available on the host system. .. _bootstrapping: .. _cmd-spack-bootstrap: .. _cmd-spack-bootstrap-status: .. _cmd-spack-bootstrap-now: Bootstrapping ============= In the :ref:`Getting started ` section, we already mentioned that Spack can bootstrap some of its dependencies, including ``clingo``. In fact, there is an entire command dedicated to the management of every aspect of bootstrapping: .. command-output:: spack bootstrap --help Spack bootstraps its dependencies automatically the first time they are needed. You can readily check if any prerequisite for using Spack is missing by running: .. code-block:: console % spack bootstrap status Spack v0.19.0 - python@3.8 [FAIL] Core Functionalities [B] MISSING "clingo": required to concretize specs [FAIL] Binary packages [B] MISSING "gpg2": required to sign/verify buildcaches Spack will take care of bootstrapping any missing dependency marked as [B]. Dependencies marked as [-] are instead required to be found on the system. % echo $? 1 In the case of the output shown above, Spack detected that both ``clingo`` and ``gnupg`` are missing, and it's giving detailed information on why they are needed and whether they can be bootstrapped. The return code of this command summarizes the results; if any dependencies are missing, the return code is ``1``, otherwise ``0``. Running a command that concretizes a spec, like: .. code-block:: console % spack solve zlib ==> Installing "clingo-bootstrap@spack%apple-clang@12.0.0~docs~ipo+python build_type=Release arch=darwin-catalina-x86_64" from a buildcache [ ... ] automatically triggers the bootstrapping of clingo from pre-built binaries as expected. Users can also bootstrap all Spack's dependencies in a single command, which is useful to set up containers or other similar environments: .. code-block:: console $ spack bootstrap now ==> Installing "clingo-bootstrap@spack%gcc@10.2.1~docs~ipo+python+static_libstdcpp build_type=Release arch=linux-centos7-x86_64" from a buildcache ==> Installing "patchelf@0.15.0%gcc@10.2.1 ldflags="-static-libstdc++ -static-libgcc" arch=linux-centos7-x86_64" from a buildcache .. _cmd-spack-bootstrap-root: The Bootstrapping Store ----------------------- The software installed for bootstrapping purposes is deployed in a separate store. You can check its location with the following command: .. code-block:: console % spack bootstrap root You can also change it by specifying the desired path: .. code-block:: console % spack bootstrap root /opt/spack/bootstrap You can check what is installed in the bootstrapping store at any time using: .. code-block:: console % spack -b find ==> Showing internal bootstrap store at "/Users/spack/.spack/bootstrap/store" ==> 11 installed packages -- darwin-catalina-x86_64 / apple-clang@12.0.0 ------------------ clingo-bootstrap@spack libassuan@2.5.5 libgpg-error@1.42 libksba@1.5.1 pinentry@1.1.1 zlib@1.2.11 gnupg@2.3.1 libgcrypt@1.9.3 libiconv@1.16 npth@1.6 python@3.8 If needed, you can remove all the software in the current bootstrapping store with: .. code-block:: console % spack clean -b ==> Removing bootstrapped software and configuration in "/Users/spack/.spack/bootstrap" % spack -b find ==> Showing internal bootstrap store at "/Users/spack/.spack/bootstrap/store" ==> 0 installed packages .. _cmd-spack-bootstrap-list: .. _cmd-spack-bootstrap-disable: .. _cmd-spack-bootstrap-enable: .. _cmd-spack-bootstrap-reset: Enabling and Disabling Bootstrapping Methods -------------------------------------------- Bootstrapping is performed by trying the methods listed by: .. command-output:: spack bootstrap list in the order they appear, from top to bottom. By default, Spack is configured to try bootstrapping from pre-built binaries first and to fall back to bootstrapping from sources if that fails. If needed, you can disable bootstrapping altogether by running: .. code-block:: console % spack bootstrap disable in which case, it's your responsibility to ensure Spack runs in an environment where all its prerequisites are installed. You can also configure Spack to skip certain bootstrapping methods by disabling them specifically: .. code-block:: console % spack bootstrap disable github-actions ==> "github-actions" is now disabled and will not be used for bootstrapping tells Spack to skip trying to bootstrap from binaries. To add the "github-actions" method back, you can: .. code-block:: console % spack bootstrap enable github-actions You can also reset the bootstrapping configuration to Spack's defaults: .. code-block:: console % spack bootstrap reset ==> Bootstrapping configuration is being reset to Spack's defaults. Current configuration will be lost. Do you want to continue? [Y/n] % .. _cmd-spack-bootstrap-mirror: .. _cmd-spack-bootstrap-add: Creating a Mirror for Air-Gapped Systems ---------------------------------------- Spack's default bootstrapping configuration requires internet connection to fetch precompiled binaries or source tarballs. Sometimes, though, Spack is deployed on air-gapped systems where such access is denied. To help in these situations, Spack provides a command to create a local mirror containing the source tarballs and/or binary packages needed for bootstrapping. .. code-block:: console % spack bootstrap mirror --binary-packages /opt/bootstrap ==> Adding "clingo-bootstrap@spack+python %apple-clang target=x86_64" and dependencies to the mirror at /opt/bootstrap/local-mirror ==> Adding "gnupg@2.3: %apple-clang target=x86_64" and dependencies to the mirror at /opt/bootstrap/local-mirror ==> Adding "patchelf@0.13.1:0.13.99 %apple-clang target=x86_64" and dependencies to the mirror at /opt/bootstrap/local-mirror ==> Adding binary packages from "https://github.com/alalazo/spack-bootstrap-mirrors/releases/download/v0.1-rc.2/bootstrap-buildcache.tar.gz" to the mirror at /opt/bootstrap/local-mirror To register the mirror on the platform where it's supposed to be used run the following command(s): % spack bootstrap add --trust local-sources /opt/bootstrap/metadata/sources % spack bootstrap add --trust local-binaries /opt/bootstrap/metadata/binaries % spack buildcache update-index /opt/bootstrap/bootstrap_cache Run this command on a machine with internet access, then move the resulting folder to the air-gapped system. Once the local sources are added using the commands suggested at the prompt, they can be used to bootstrap Spack. ================================================ FILE: lib/spack/docs/build_settings.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Understand how to control the build process in Spack by customizing package-specific build settings and environment variables. .. _concretizer-options: Concretization Settings (concretizer.yaml) ========================================== The ``concretizer.yaml`` configuration file allows users to customize aspects of the algorithm used to select the dependencies they install. The default configuration is the following: .. literalinclude:: _spack_root/etc/spack/defaults/base/concretizer.yaml :language: yaml Completion of external nodes ---------------------------- :ref:`The external packages ` available from the ``packages.yaml`` configuration file are usually reporting only a few of the variants defined in the corresponding recipe. Users can configure how Spack deals with missing information for externals via the ``concretizer:externals:completion`` attribute: .. code-block:: yaml concretizer: externals: completion: default_variants This attribute currently allows two possible values: - ``architecture_only``: only the mandatory architectural information is completed on externals - ``default_variants``: external specs are also completed with missing variants, using their default values Reuse Already Installed Packages -------------------------------- The ``reuse`` attribute controls how aggressively Spack reuses binary packages during concretization. The attribute can be either a single value or an object for more complex configurations. In the former case ("single value"), it allows Spack to: 1. Reuse installed packages and build caches for all the specs to be concretized, when ``true``. 2. Reuse installed packages and build caches only for the dependencies of the root specs, when ``dependencies``. 3. Disregard reusing installed packages and build caches, when ``false``. In case finer control over which specs are reused is needed, the value of this attribute can be an object with the following keys: 1. ``roots``: if ``true`` root specs are reused, if ``false`` only dependencies of root specs are reused 2. ``from``: list of sources from which reused specs are taken Each source in ``from`` is itself an object with the following attributes: .. list-table:: Attributes for a source or reusable specs :header-rows: 1 * - Attribute name - Description * - type (mandatory, string) - Can be ``local``, ``buildcache``, or ``external``. * - include (optional, list of specs) - If present, reusable specs must match at least one of the constraints in the list. * - exclude (optional, list of specs) - If present, reusable specs must not match any of the constraints in the list. For instance, the following configuration: .. code-block:: yaml concretizer: reuse: roots: true from: - type: local include: - "%gcc" - "%clang" tells the concretizer to reuse all specs compiled with either ``gcc`` or ``clang`` that are installed in the local store. Any spec from remote build caches is disregarded. To reduce the boilerplate in configuration files, default values for the ``include`` and ``exclude`` options can be pushed up one level: .. code-block:: yaml concretizer: reuse: roots: true include: - "%gcc" from: - type: local - type: buildcache - type: local include: - "foo %oneapi" In the example above, we reuse all specs compiled with ``gcc`` from the local store and remote build caches, and we also reuse ``foo %oneapi``. Note that the last source of specs overrides the default ``include`` attribute. For one-off concretizations, there are command-line arguments for each of the simple "single value" configurations. This means a user can: .. code-block:: console % spack install --reuse to enable reuse for a single installation, or: .. code-block:: console $ spack install --fresh to do a fresh install if ``reuse`` is enabled by default. .. seealso:: FAQ: :ref:`Why does Spack pick particular versions and variants? ` Selection of Target Microarchitectures ------------------------------------------ The options under the ``targets`` attribute control which targets are considered during a solve. Currently, the options in this section are only configurable from the ``concretizer.yaml`` file, and there are no corresponding command-line arguments to enable them for a single solve. The ``granularity`` option can take two possible values: ``microarchitectures`` and ``generic``. If set to: .. code-block:: yaml concretizer: targets: granularity: microarchitectures Spack will consider all the microarchitectures known to ``archspec`` to label nodes for compatibility. If instead the option is set to: .. code-block:: yaml concretizer: targets: granularity: generic Spack will consider only generic microarchitectures. For instance, when running on a Haswell machine, Spack will consider ``haswell`` as the best target in the former case and ``x86_64_v3`` as the best target in the latter case. The ``host_compatible`` option is a Boolean option that determines whether or not the microarchitectures considered during the solve are constrained to be compatible with the host Spack is currently running on. For instance, if this option is set to ``true``, a user cannot concretize for ``target=icelake`` while running on a Haswell machine. Duplicate Nodes --------------- The ``duplicates`` attribute controls whether the DAG can contain multiple configurations of the same package. This is mainly relevant for build dependencies, which may have their version pinned by some nodes and thus be required at different versions by different nodes in the same DAG. The ``strategy`` option controls how the solver deals with duplicates. If the value is ``none``, then a single configuration per package is allowed in the DAG. This means, for instance, that only a single ``cmake`` or a single ``py-setuptools`` version is allowed. The result would be a slightly faster concretization at the expense of making a few specs unsolvable. If the value is ``minimal``, Spack will allow packages tagged as ``build-tools`` to have duplicates. This allows, for instance, to concretize specs whose nodes require different and incompatible ranges of some build tool. For instance, in the figure below, the latest `py-shapely` requires a newer `py-setuptools`, while `py-numpy` still needs an older version: .. figure:: images/shapely_duplicates.svg :width: 5580 :height: 1842 Up to Spack v0.20, ``duplicates:strategy:none`` was the default (and only) behavior. From Spack v0.21, the default behavior is ``duplicates:strategy:minimal``. Splicing -------- The ``splice`` key covers configuration attributes for splicing specs in the solver. "Splicing" is a method for replacing a dependency with another spec that provides the same package or virtual. There are two types of splices, referring to different behaviors for shared dependencies between the root spec and the new spec replacing a dependency: "transitive" and "intransitive". A "transitive" splice is one that resolves all conflicts by taking the dependency from the new node. An "intransitive" splice is one that resolves all conflicts by taking the dependency from the original root. From a theory perspective, hybrid splices are possible but are not modeled by Spack. All spliced specs retain a ``build_spec`` attribute that points to the original spec before any splice occurred. The ``build_spec`` for a non-spliced spec is itself. The figure below shows examples of transitive and intransitive splices: .. figure:: images/splices.png :width: 2308 :height: 1248 The concretizer can be configured to explicitly splice particular replacements for a target spec. Splicing will allow the user to make use of generically built public binary caches while swapping in highly optimized local builds for performance-critical components and/or components that interact closely with the specific hardware details of the system. The most prominent candidate for splicing is MPI providers. MPI packages have relatively well-understood ABI characteristics, and most High Performance Computing facilities deploy highly optimized MPI packages tailored to their particular hardware. The following configuration block configures Spack to replace whatever MPI provider each spec was concretized to use with the particular package of ``mpich`` with the hash that begins ``abcdef``. .. code-block:: yaml concretizer: splice: explicit: - target: mpi replacement: mpich/abcdef transitive: false .. warning:: When configuring an explicit splice, you as the user take on the responsibility for ensuring ABI compatibility between the specs matched by the target and the replacement you provide. If they are not compatible, Spack will not warn you, and your application will fail to run. The ``target`` field of an explicit splice can be any abstract spec. The ``replacement`` field must be a spec that includes the hash of a concrete spec, and the replacement must either be the same package as the target, provide the virtual that is the target, or provide a virtual that the target provides. The ``transitive`` field is optional -- by default, splices will be transitive. .. note:: With explicit splices configured, it is possible for Spack to concretize to a spec that does not satisfy the input. For example, with the configuration above, ``hdf5 ^mvapich2`` will concretize to use ``mpich/abcdef`` instead of ``mvapich2`` as the MPI provider. Spack will warn the user in this case, but will not fail the concretization. .. _automatic_splicing: Automatic Splicing ^^^^^^^^^^^^^^^^^^ The Spack solver can be configured to do automatic splicing for ABI-compatible packages. Automatic splices are enabled in the concretizer configuration section: .. code-block:: yaml concretizer: splice: automatic: true Packages can include ABI-compatibility information using the ``can_splice`` directive. See :ref:`the packaging guide ` for instructions on specifying ABI compatibility using the ``can_splice`` directive. .. note:: The ``can_splice`` directive is experimental and may be changed in future versions. When automatic splicing is enabled, the concretizer will combine any number of ABI-compatible specs if possible to reuse installed packages and packages available from binary caches. The end result of these specs is equivalent to a series of transitive/intransitive splices, but the series may be non-obvious. ================================================ FILE: lib/spack/docs/build_systems/autotoolspackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: An overview of the Autotools build system in Spack for packages that use GNU Autotools. .. _autotoolspackage: Autotools --------- Autotools is a GNU build system that provides a build-script generator. By running the platform-independent ``./configure`` script that comes with the package, you can generate a platform-dependent Makefile. Phases ^^^^^^ The ``AutotoolsBuilder`` and ``AutotoolsPackage`` base classes come with the following phases: #. ``autoreconf`` - generate the configure script #. ``configure`` - generate the Makefiles #. ``build`` - build the package #. ``install`` - install the package Most of the time, the ``autoreconf`` phase will do nothing, but if the package is missing a ``configure`` script, ``autoreconf`` will generate one for you. The other phases run: .. code-block:: console $ ./configure --prefix=/path/to/installation/prefix $ make $ make check # optional $ make install $ make installcheck # optional Of course, you may need to add a few arguments to the ``./configure`` line. Important files ^^^^^^^^^^^^^^^ The most important file for an Autotools-based package is the ``configure`` script. This script is automatically generated by Autotools and generates the appropriate Makefile when run. .. warning:: Watch out for fake Autotools packages! Autotools is a very popular build system, and many people are used to the classic steps to install a package: .. code-block:: console $ ./configure $ make $ make install For this reason, some developers will write their own ``configure`` scripts that have nothing to do with Autotools. These packages may not accept the same flags as other Autotools packages, so it is better to use the ``Package`` base class and create a :ref:`custom build system `. You can tell if a package uses Autotools by running ``./configure --help`` and comparing the output to other known Autotools packages. You should also look for files like: * ``configure.ac`` * ``configure.in`` * ``Makefile.am`` Packages that don't use Autotools aren't likely to have these files. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ Whether or not your package requires Autotools to install depends on how the source code is distributed. Most of the time, when developers distribute tarballs, they will already contain the ``configure`` script necessary for installation. If this is the case, your package does not require any Autotools dependencies. However, a basic rule of version control systems is to never commit code that can be generated. The source code repository itself likely does not have a ``configure`` script. Developers typically write (or auto-generate) a ``configure.ac`` script that contains configuration preferences and a ``Makefile.am`` script that contains build instructions. Then, ``autoconf`` is used to convert ``configure.ac`` into ``configure``, while ``automake`` is used to convert ``Makefile.am`` into ``Makefile.in``. ``Makefile.in`` is used by ``configure`` to generate a platform-dependent ``Makefile`` for you. The following diagram provides a high-level overview of the process: .. figure:: Autoconf-automake-process.* :target: https://commons.wikimedia.org/w/index.php?curid=15581407 `GNU autoconf and automake process for generating makefiles `_ by `Jdthood` under `CC BY-SA 3.0 `_ If a ``configure`` script is not present in your tarball, you will need to generate one yourself. Luckily, Spack already has an ``autoreconf`` phase to do most of the work for you. By default, the ``autoreconf`` phase runs: .. code-block:: console $ autoreconf --install --verbose --force -I /share/aclocal In case you need to add more arguments, override ``autoreconf_extra_args`` in your ``package.py`` on class scope like this: .. code-block:: python autoreconf_extra_args = ["-Im4"] All you need to do is add a few Autotools dependencies to the package. Most stable releases will come with a ``configure`` script, but if you check out a commit from the ``master`` branch, you would want to add: .. code-block:: python depends_on("autoconf", type="build", when="@master") depends_on("automake", type="build", when="@master") depends_on("libtool", type="build", when="@master") It is typically redundant to list the ``m4`` macro processor package as a dependency, since ``autoconf`` already depends on it. Using a custom autoreconf phase """"""""""""""""""""""""""""""" In some cases, it might be needed to replace the default implementation of the autoreconf phase with one running a script interpreter. In this example, the ``bash`` shell is used to run the ``autogen.sh`` script. .. code-block:: python def autoreconf(self, spec, prefix): which("bash")("autogen.sh") If the ``package.py`` has build instructions in a separate :ref:`builder class `, the signature for a phase changes slightly: .. code-block:: python class AutotoolsBuilder(AutotoolsBuilder): def autoreconf(self, pkg, spec, prefix): which("bash")("autogen.sh") patching configure or Makefile.in files """"""""""""""""""""""""""""""""""""""" In some cases, developers might need to distribute a patch that modifies one of the files used to generate ``configure`` or ``Makefile.in``. In this case, these scripts will need to be regenerated. It is preferable to regenerate these manually using the patch, and then create a new patch that directly modifies ``configure``. That way, Spack can use the secondary patch and additional build system dependencies aren't necessary. Old Autotools helper scripts """""""""""""""""""""""""""" Autotools based tarballs come with helper scripts such as ``config.sub`` and ``config.guess``. It is the responsibility of the developers to keep these files up to date so that they run on every platform, but for very old software releases this is impossible. In these cases Spack can help to replace these files with newer ones, without having to add the heavy dependency on ``automake``. Automatic helper script replacement is currently enabled by default on ``ppc64le`` and ``aarch64``, as these are the known cases where old scripts fail. On these targets, ``AutotoolsPackage`` adds a build dependency on ``gnuconfig``, which is a very lightweight package with newer versions of the helper files. Spack then tries to run all the helper scripts it can find in the release, and replaces them on failure with the helper scripts from ``gnuconfig``. To opt out of this feature, use the following setting: .. code-block:: python patch_config_files = False To enable it conditionally on different architectures, define a property and make the package depend on ``gnuconfig`` as a build dependency: .. code-block:: python depends_on("gnuconfig", when="@1.0:") @property def patch_config_files(self): return self.spec.satisfies("@1.0:") .. note:: On some exotic architectures it is necessary to use system provided ``config.sub`` and ``config.guess`` files. In this case, the most transparent solution is to mark the ``gnuconfig`` package as external and non-buildable, with a prefix set to the directory containing the files: .. code-block:: yaml gnuconfig: buildable: false externals: - spec: gnuconfig@master prefix: /usr/share/configure_files/ force_autoreconf """""""""""""""" If for whatever reason you really want to add the original patch and tell Spack to regenerate ``configure``, you can do so using the following setting: .. code-block:: python force_autoreconf = True This line tells Spack to wipe away the existing ``configure`` script and generate a new one. If you only need to do this for a single version, this can be done like so: .. code-block:: python @property def force_autoreconf(self): return self.version == Version("1.2.3") Finding configure flags ^^^^^^^^^^^^^^^^^^^^^^^ Once you have a ``configure`` script present, the next step is to determine what option flags are available. These flags can be found by running: .. code-block:: console $ ./configure --help ``configure`` will display a list of valid flags separated into some or all of the following sections: * Configuration * Installation directories * Fine tuning of the installation directories * Program names * X features * System types * **Optional Features** * **Optional Packages** * **Some influential environment variables** For the most part, you can ignore all but the last 3 sections. The "Optional Features" section lists flags that enable/disable features you may be interested in. The "Optional Packages" section often lists dependencies and the flags needed to locate them. The "environment variables" section lists environment variables that the build system uses to pass flags to the compiler and linker. Adding flags to configure ^^^^^^^^^^^^^^^^^^^^^^^^^ For most of the flags you encounter, you will want a variant to optionally enable/disable them. You can then optionally pass these flags to the ``configure`` call by overriding the ``configure_args`` function like so: .. code-block:: python def configure_args(self): args = [] ... if self.spec.satisfies("+mpi"): args.append("--enable-mpi") else: args.append("--disable-mpi") return args Alternatively, you can use the :ref:`enable_or_disable ` helper: .. code-block:: python def configure_args(self): args = [] ... args.extend(self.enable_or_disable("mpi")) return args Note that we are explicitly disabling MPI support if it is not requested. This is important, as many Autotools packages will enable options by default if the dependencies are found, and disable them otherwise. We want Spack installations to be as deterministic as possible. If two users install a package with the same variants, the goal is that both installations work the same way. See `here `__ and `here `__ for a rationale as to why these so-called "automagic" dependencies are a problem. .. note:: By default, Autotools installs packages to ``/usr``. We don't want this, so Spack automatically adds ``--prefix=/path/to/installation/prefix`` to your list of ``configure_args``. You don't need to add this yourself. .. _autotools_helper_functions: Helper functions ^^^^^^^^^^^^^^^^ You may have noticed that most of the Autotools flags are of the form ``--enable-foo``, ``--disable-bar``, ``--with-baz=``, or ``--without-baz``. Since these flags are so common, Spack provides a couple of helper functions to make your life easier. .. _autotools_enable_or_disable: ``enable_or_disable`` """"""""""""""""""""" Autotools flags for simple boolean variants can be automatically generated by calling the ``enable_or_disable`` method. This is typically used to enable or disable some feature within the package. .. code-block:: python variant( "memchecker", default=False, description="Memchecker support for debugging [degrades performance]", ) ... def configure_args(self): args = [] ... args.extend(self.enable_or_disable("memchecker")) return args In this example, specifying the variant ``+memchecker`` will generate the following configuration options: .. code-block:: console --enable-memchecker ``with_or_without`` """"""""""""""""""" Autotools flags for more complex variants, including boolean variants and multi-valued variants, can be automatically generated by calling the ``with_or_without`` method. .. code-block:: python variant( "schedulers", values=disjoint_sets( ("auto",), ("alps", "lsf", "tm", "slurm", "sge", "loadleveler") ).with_non_feature_values("auto", "none"), description="List of schedulers for which support is enabled; " "'auto' lets openmpi determine", ) if not spec.satisfies("schedulers=auto"): config_args.extend(self.with_or_without("schedulers")) In this example, specifying the variant ``schedulers=slurm,sge`` will generate the following configuration options: .. code-block:: console --with-slurm --with-sge ``enable_or_disable`` is actually functionally equivalent to ``with_or_without``, and accepts the same arguments and variant types; but idiomatic Autotools packages often follow these naming conventions. ``activation_value`` """""""""""""""""""" Autotools parameters that require an option can still be automatically generated, using the ``activation_value`` argument to ``with_or_without`` (or, rarely, ``enable_or_disable``). .. code-block:: python variant( "fabrics", values=disjoint_sets( ("auto",), ("psm", "psm2", "verbs", "mxm", "ucx", "libfabric") ).with_non_feature_values("auto", "none"), description="List of fabrics that are enabled; 'auto' lets openmpi determine", ) if not spec.satisfies("fabrics=auto"): config_args.extend(self.with_or_without("fabrics", activation_value="prefix")) ``activation_value`` accepts a callable that generates the configure parameter value given the variant value; but the special value ``prefix`` tells Spack to automatically use the dependency's installation prefix, which is the most common use for such parameters. In this example, specifying the variant ``fabrics=libfabric`` will generate the following configuration options: .. code-block:: console --with-libfabric= The ``variant`` keyword """"""""""""""""""""""" When Spack variants and configure flags do not correspond one-to-one, the ``variant`` keyword can be passed to ``with_or_without`` and ``enable_or_disable``. For example: .. code-block:: python variant("debug_tools", default=False) config_args += self.enable_or_disable("debug-tools", variant="debug_tools") Or when one variant controls multiple flags: .. code-block:: python variant("debug_tools", default=False) config_args += self.with_or_without("memchecker", variant="debug_tools") config_args += self.with_or_without("profiler", variant="debug_tools") Conditional variants """""""""""""""""""" When a variant is conditional and its condition is not met on the concrete spec, the ``with_or_without`` and ``enable_or_disable`` methods will simply return an empty list. For example: .. code-block:: python variant("profiler", when="@2.0:") config_args += self.with_or_without("profiler") will neither add ``--with-profiler`` nor ``--without-profiler`` when the version is below ``2.0``. Activation overrides """""""""""""""""""" Finally, the behavior of either ``with_or_without`` or ``enable_or_disable`` can be overridden for specific variant values. This is most useful for multi-value variants where some of the variant values require atypical behavior. .. code-block:: python def with_or_without_verbs(self, activated): # Up through version 1.6, this option was named --with-openib. # In version 1.7, it was renamed to be --with-verbs. opt = "verbs" if self.spec.satisfies("@1.7:") else "openib" if not activated: return f"--without-{opt}" return f"--with-{opt}={self.spec['rdma-core'].prefix}" Defining ``with_or_without_verbs`` overrides the behavior of a ``fabrics=verbs`` variant, changing the configure-time option to ``--with-openib`` for older versions of the package and specifying an alternative dependency name: .. code-block:: text --with-openib= Configure script in a sub-directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Occasionally, developers will hide their source code and ``configure`` script in a subdirectory like ``src``. If this happens, Spack won't be able to automatically detect the build system properly when running ``spack create``. You will have to manually change the package base class and tell Spack where the ``configure`` script resides. You can do this like so: .. code-block:: python configure_directory = "src" Building out of source ^^^^^^^^^^^^^^^^^^^^^^ Some packages like ``gcc`` recommend building their software in a different directory than the source code to prevent build pollution. This can be done using the ``build_directory`` variable: .. code-block:: python build_directory = "spack-build" By default, Spack will build the package in the same directory that contains the ``configure`` script. Build and install targets ^^^^^^^^^^^^^^^^^^^^^^^^^ For most Autotools packages, the usual: .. code-block:: console $ configure $ make $ make install is sufficient to install the package. However, if you need to run make with any other targets, for example, to build an optional library or build the documentation, you can add these like so: .. code-block:: python build_targets = ["all", "docs"] install_targets = ["install", "docs"] Testing ^^^^^^^ Autotools-based packages typically provide unit testing via the ``check`` and ``installcheck`` targets. If you build your software with ``spack install --test=root``, Spack will check for the presence of a ``check`` or ``test`` target in the Makefile and run ``make check`` for you. After installation, it will check for an ``installcheck`` target and run ``make installcheck`` if it finds one. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the Autotools build system, see: https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html ================================================ FILE: lib/spack/docs/build_systems/bundlepackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Discover how to create meta-packages known as "bundles" in Spack to group multiple packages together for a single installation. .. _bundlepackage: Bundle ------ ``BundlePackage`` represents a set of packages that are expected to work well together, such as a collection of commonly used software libraries. The associated software is specified as dependencies. If it makes sense, variants, conflicts, and requirements can be added to the package. :ref:`Variants ` ensure that common build options are consistent across the packages supporting them. :ref:`Conflicts ` prevent attempts to build with known bugs and limitations. :ref:`Requirements ` prevent attempts to build without critical options. For example, if ``MyBundlePackage`` is known to only build on ``linux``, it could use the ``require`` directive as follows: .. code-block:: python require("platform=linux", msg="MyBundlePackage only builds on linux") Spack has a number of built-in bundle packages, such as: * `AmdAocl `_ * `EcpProxyApps `_ * `Libc `_ * `Xsdk `_ where ``Xsdk`` also inherits from ``CudaPackage`` and ``RocmPackage`` and ``Libc`` is a virtual bundle package for the C standard library. Creation ^^^^^^^^ Be sure to specify the ``bundle`` template if you are using ``spack create`` to generate a package from the template. For example, use the following command to create a bundle package whose class name will be ``Mybundle``: .. code-block:: console $ spack create --template bundle --name mybundle Phases ^^^^^^ The ``BundlePackage`` base class does not provide any phases by default since the bundle does not represent a build system. URL ^^^^^^ The ``url`` property does not have meaning since there is no package-specific code to fetch. Version ^^^^^^^ At least one ``version`` must be specified in order for the package to build. ================================================ FILE: lib/spack/docs/build_systems/cachedcmakepackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn about the CachedCMakePackage build system in Spack for CMake-based projects. .. _cachedcmakepackage: CachedCMake ----------- The CachedCMakePackage base class is used for CMake-based workflows that create a CMake cache file prior to running ``cmake``. This is useful for packages with arguments longer than the system limit, and for reproducibility. The documentation for this class assumes that the user is familiar with the ``CMakePackage`` class from which it inherits. See the documentation for :ref:`CMakePackage `. Phases ^^^^^^ The ``CachedCMakePackage`` base class comes with the following phases: #. ``initconfig`` - generate the CMake cache file #. ``cmake`` - generate the Makefile #. ``build`` - build the package #. ``install`` - install the package By default, these phases run: .. code-block:: console $ mkdir spack-build $ cd spack-build $ cat << EOF > name-arch-compiler@version.cmake # Write information on compilers and dependencies # includes information on mpi and cuda if applicable $ cmake .. -DCMAKE_INSTALL_PREFIX=/path/to/installation/prefix -C name-arch-compiler@version.cmake $ make $ make test # optional $ make install The ``CachedCMakePackage`` class inherits from the ``CMakePackage`` class, and accepts all of the same options and adds all of the same flags to the ``cmake`` command. Similar to the ``CMakePackage`` class, you may need to add a few arguments yourself, and the ``CachedCMakePackage`` provides the same interface to add those flags. Adding entries to the CMake cache ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In addition to adding flags to the ``cmake`` command, you may need to add entries to the CMake cache in the ``initconfig`` phase. This can be done by overriding one of four methods: #. ``CachedCMakePackage.initconfig_compiler_entries`` #. ``CachedCMakePackage.initconfig_mpi_entries`` #. ``CachedCMakePackage.initconfig_hardware_entries`` #. ``CachedCMakePackage.initconfig_package_entries`` Each of these methods returns a list of CMake cache strings. The distinction between these methods is merely to provide a well-structured and legible CMake cache file -- otherwise, entries from each of these methods are handled identically. Spack also provides convenience methods for generating CMake cache entries. These methods are available at module scope in every Spack package. Because CMake parses boolean options, strings, and paths differently, there are three such methods: #. ``cmake_cache_option`` #. ``cmake_cache_string`` #. ``cmake_cache_path`` These methods each accept three parameters -- the name of the CMake variable associated with the entry, the value of the entry, and an optional comment -- and return strings in the appropriate format to be returned from any of the ``initconfig*`` methods. Additionally, these methods may return comments beginning with the ``#`` character. A typical usage of these methods may look something like this: .. code-block:: python def initconfig_mpi_entries(self): # Get existing MPI configurations entries = super(self, Foo).initconfig_mpi_entries() # The existing MPI configurations key on whether ``mpi`` is in the spec # This spec has an MPI variant, and we need to enable MPI when it is on. # This hypothetical package controls MPI with the ``FOO_MPI`` option to # cmake. if self.spec.satisfies("+mpi"): entries.append(cmake_cache_option("FOO_MPI", True, "enable mpi")) else: entries.append(cmake_cache_option("FOO_MPI", False, "disable mpi")) def initconfig_package_entries(self): # Package specific options entries = [] entries.append("#Entries for build options") bar_on = self.spec.satisfies("+bar") entries.append(cmake_cache_option("FOO_BAR", bar_on, "toggle bar")) entries.append("#Entries for dependencies") if self.spec["blas"].name == "baz": # baz is our blas provider entries.append(cmake_cache_string("FOO_BLAS", "baz", "Use baz")) entries.append(cmake_cache_path("BAZ_PREFIX", self.spec["baz"].prefix)) External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on CMake cache files, see: https://cmake.org/cmake/help/latest/manual/cmake.1.html ================================================ FILE: lib/spack/docs/build_systems/cmakepackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to package software that uses the CMake build system with Spack, covering common practices and customization options for CMake-based packages. .. _cmakepackage: CMake ------ Like Autotools, CMake is a widely-used build-script generator. Designed by Kitware, CMake is the most popular build system for new C, C++, and Fortran projects, and many older projects are switching to it as well. Unlike Autotools, CMake can generate build scripts for builders other than Make: Ninja, Visual Studio, etc. It is therefore cross-platform, whereas Autotools is Unix-only. Phases ^^^^^^ The ``CMakeBuilder`` and ``CMakePackage`` base classes come with the following phases: #. ``cmake`` - generate the Makefile #. ``build`` - build the package #. ``install`` - install the package By default, these phases run: .. code-block:: console $ mkdir spack-build $ cd spack-build $ cmake .. -DCMAKE_INSTALL_PREFIX=/path/to/installation/prefix $ make $ make test # optional $ make install A few more flags are passed to ``cmake`` by default, including flags for setting the build type and flags for locating dependencies. Of course, you may need to add a few arguments yourself. Important files ^^^^^^^^^^^^^^^ A CMake-based package can be identified by the presence of a ``CMakeLists.txt`` file. This file defines the build flags that can be passed to the CMake invocation, as well as linking instructions. If you are familiar with CMake, it can prove very useful for determining dependencies and dependency version requirements. One thing to look for is the ``cmake_minimum_required`` function: .. code-block:: cmake cmake_minimum_required(VERSION 2.8.12) This means that CMake 2.8.12 is the earliest release that will work. You should specify this in a ``depends_on`` statement. CMake-based packages may also contain ``CMakeLists.txt`` in subdirectories. This modularization helps to manage complex builds in a hierarchical fashion. Sometimes these nested ``CMakeLists.txt`` require additional dependencies not mentioned in the top-level file. There's also usually a ``cmake`` or ``CMake`` directory containing additional macros, find scripts, etc. These may prove useful in determining dependency version requirements. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ Every package that uses the CMake build system requires a ``cmake`` dependency. Since this is always the case, the ``CMakePackage`` base class already contains: .. code-block:: python depends_on("cmake", type="build") If you need to specify a particular version requirement, you can override this in your package: .. code-block:: python depends_on("cmake@2.8.12:", type="build") Finding cmake flags ^^^^^^^^^^^^^^^^^^^ To get a list of valid flags that can be passed to ``cmake``, run the following command in the directory that contains ``CMakeLists.txt``: .. code-block:: console $ cmake . -LAH CMake will start by checking for compilers and dependencies. Eventually it will begin to list build options. You'll notice that most of the build options at the top are prefixed with ``CMAKE_``. You can safely ignore most of these options as Spack already sets them for you. This includes flags needed to locate dependencies, RPATH libraries, set the installation directory, and set the build type. The rest of the flags are the ones you should consider adding to your package. They often include flags to enable/disable support for certain features and locate specific dependencies. One thing you'll notice that makes CMake different from Autotools is that CMake has an understanding of build flag hierarchy. That is, certain flags will not display unless their parent flag has been selected. For example, flags to specify the ``lib`` and ``include`` directories for a package might not appear unless CMake found the dependency it was looking for. You may need to manually specify certain flags to explore the full depth of supported build flags, or check the ``CMakeLists.txt`` yourself. .. _cmake_args: Adding flags to cmake ^^^^^^^^^^^^^^^^^^^^^ To add additional flags to the ``cmake`` call, simply override the ``cmake_args`` function. The following example defines values for the flags ``WHATEVER``, ``ENABLE_BROKEN_FEATURE``, ``DETECT_HDF5``, and ``THREADS`` with and without the :meth:`~spack_repo.builtin.build_systems.cmake.CMakeBuilder.define` and :meth:`~spack_repo.builtin.build_systems.cmake.CMakeBuilder.define_from_variant` helper functions: .. code-block:: python def cmake_args(self): args = [ "-DWHATEVER:STRING=somevalue", self.define("ENABLE_BROKEN_FEATURE", False), self.define_from_variant("DETECT_HDF5", "hdf5"), self.define_from_variant("THREADS"), # True if +threads ] return args Spack supports CMake defines from conditional variants too. Whenever the condition on the variant is not met, ``define_from_variant()`` will simply return an empty string, and CMake simply ignores the empty command line argument. For example, the following .. code-block:: python variant("example", default=True, when="@2.0:") def cmake_args(self): return [self.define_from_variant("EXAMPLE", "example")] will generate ``'cmake' '-DEXAMPLE=ON' ...`` when `@2.0: +example` is met, but will result in ``'cmake' '' ...`` when the spec version is below ``2.0``. CMake arguments provided by Spack ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The following default arguments are controlled by Spack: ``CMAKE_INSTALL_PREFIX`` """""""""""""""""""""""" Is set to the package's install directory. ``CMAKE_PREFIX_PATH`` """"""""""""""""""""" CMake finds dependencies through calls to ``find_package()``, ``find_program()``, ``find_library()``, ``find_file()``, and ``find_path()``, which use a list of search paths from ``CMAKE_PREFIX_PATH``. Spack sets this variable to a list of prefixes of the spec's transitive dependencies. For troubleshooting cases where CMake fails to find a dependency, add the ``--debug-find`` flag to ``cmake_args``. ``CMAKE_BUILD_TYPE`` """""""""""""""""""" Every CMake-based package accepts a ``-DCMAKE_BUILD_TYPE`` flag to dictate which level of optimization to use. In order to ensure uniformity across packages, the ``CMakePackage`` base class adds a variant to control this: .. code-block:: python variant( "build_type", default="RelWithDebInfo", description="CMake build type", values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel"), ) However, not every CMake package accepts all four of these options. Grep the ``CMakeLists.txt`` file to see if the default values are missing or replaced. For example, the `dealii `_ package overrides the default variant with: .. code-block:: python variant( "build_type", default="DebugRelease", description="The build type to build", values=("Debug", "Release", "DebugRelease"), ) For more information on ``CMAKE_BUILD_TYPE``, see: https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html ``CMAKE_INSTALL_RPATH`` and ``CMAKE_INSTALL_RPATH_USE_LINK_PATH=ON`` """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" CMake uses different RPATHs during the build and after installation, so that executables can locate the libraries they're linked to during the build, and installed executables do not have RPATHs to build directories. In Spack, we have to make sure that RPATHs are set properly after installation. Spack sets ``CMAKE_INSTALL_RPATH`` to a list of ``/lib`` or ``/lib64`` directories of the spec's link-type dependencies. Apart from that, it sets ``-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON``, which should add RPATHs for directories of linked libraries not in the directories covered by ``CMAKE_INSTALL_RPATH``. Usually it's enough to set only ``-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON``, but the reason to provide both options is that packages may dynamically open shared libraries, which CMake cannot detect. In those cases, the RPATHs from ``CMAKE_INSTALL_RPATH`` are used as search paths. .. note:: Some packages provide stub libraries, which contain an interface for linking without an implementation. When using such libraries, it's best to override the option ``-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF`` in ``cmake_args``, so that stub libraries are not used at runtime. Generators ^^^^^^^^^^ CMake and Autotools are build-script generation tools; they "generate" the Makefiles that are used to build a software package. CMake actually supports multiple generators, not just Makefiles. Another common generator is Ninja. To switch to the Ninja generator, simply add: .. code-block:: python generator("ninja") ``CMakePackage`` defaults to "Unix Makefiles". If you switch to the Ninja generator, make sure to add: .. code-block:: python depends_on("ninja", type="build") to the package as well. Aside from that, you shouldn't need to do anything else. Spack will automatically detect that you are using Ninja and run: .. code-block:: console $ cmake .. -G Ninja $ ninja $ ninja install Spack currently only supports "Unix Makefiles" and "Ninja" as valid generators, but it should be simple to add support for alternative generators. For more information on CMake generators, see: https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html CMakeLists.txt in a sub-directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Occasionally, developers will hide their source code and ``CMakeLists.txt`` in a subdirectory like ``src``. If this happens, Spack won't be able to automatically detect the build system properly when running ``spack create``. You will have to manually change the package base class and tell Spack where ``CMakeLists.txt`` resides. You can do this like so: .. code-block:: python root_cmakelists_dir = "src" Note that this path is relative to the root of the extracted tarball, not to the ``build_directory``. It defaults to the current directory. Building out of source ^^^^^^^^^^^^^^^^^^^^^^ By default, Spack builds every ``CMakePackage`` in a ``spack-build`` sub-directory. If, for whatever reason, you would like to build in a different sub-directory, simply override ``build_directory`` like so: .. code-block:: python build_directory = "my-build" Build and install targets ^^^^^^^^^^^^^^^^^^^^^^^^^ For most CMake packages, the usual: .. code-block:: console $ cmake $ make $ make install is sufficient to install the package. However, if you need to run make with any other targets, for example, to build an optional library or build the documentation, you can add these like so: .. code-block:: python build_targets = ["all", "docs"] install_targets = ["install", "docs"] Testing ^^^^^^^ CMake-based packages typically provide unit testing via the ``test`` target. If you build your software with ``--test=root``, Spack will check for the presence of a ``test`` target in the Makefile and run ``make test`` for you. If you want to run a different test instead, simply override the ``check`` method. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the CMake build system, see: https://cmake.org/cmake/help/latest/ ================================================ FILE: lib/spack/docs/build_systems/cudapackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to packaging CUDA applications with Spack, including helpers for managing CUDA dependencies and architecture-specific builds. .. _cudapackage: Cuda ------ Different from other packages, ``CudaPackage`` does not represent a build system. Instead its goal is to simplify and unify usage of ``CUDA`` in other packages by providing a `mixin-class `_. You can find source for the package at ``__. Variants ^^^^^^^^ This package provides the following variants: * **cuda** This variant is used to enable/disable building with ``CUDA``. The default is disabled (or ``False``). * **cuda_arch** This variant supports the optional specification of one or more architectures. Valid values are maintained in the ``cuda_arch_values`` property and are the numeric character equivalent of the compute capability version (e.g., '10' for version 1.0). Each provided value affects associated ``CUDA`` dependencies and compiler conflicts. The variant builds both PTX code for the *virtual* architecture (e.g. ``compute_10``) and binary code for the *real* architecture (e.g. ``sm_10``). GPUs and their compute capability versions are listed at https://developer.nvidia.com/cuda-gpus. Conflicts ^^^^^^^^^ Conflicts are used to prevent builds with known bugs or issues. While base ``CUDA`` conflicts have been included with this package, you may want to add more for your software. For example, if your package requires ``cuda_arch`` to be specified when ``cuda`` is enabled, you can add the following conflict to your package to terminate such build attempts with a suitable message: .. code-block:: python conflicts("cuda_arch=none", when="+cuda", msg="CUDA architecture is required") Similarly, if your software does not support all versions of the property, you could add ``conflicts`` to your package for those versions. For example, suppose your software does not work with CUDA compute capability versions prior to SM 5.0 (``50``). You can add the following code to display a custom message should a user attempt such a build: .. code-block:: python unsupported_cuda_archs = ["10", "11", "12", "13", "20", "21", "30", "32", "35", "37"] for value in unsupported_cuda_archs: conflicts( f"cuda_arch={value}", when="+cuda", msg=f"CUDA architecture {value} is not supported" ) Methods ^^^^^^^ This package provides one custom helper method, which is used to build standard CUDA compiler flags. **cuda_flags** This built-in static method returns a list of command line flags for the chosen ``cuda_arch`` value(s). The flags are intended to be passed to the CUDA compiler driver (i.e., ``nvcc``). This method must be explicitly called when you are creating the arguments for your build in order to use the values. Usage ^^^^^^ This helper package can be added to your package by adding it as a base class of your package. For example, you can add it to your :ref:`CMakePackage `-based package as follows: .. code-block:: python :emphasize-lines: 1,8-17 class MyCudaPackage(CMakePackage, CudaPackage): ... def cmake_args(self): spec = self.spec args = [] ... if spec.satisfies("+cuda"): # Set up the CUDA macros needed by the build args.append("-DWITH_CUDA=ON") cuda_arch_list = spec.variants["cuda_arch"].value cuda_arch = cuda_arch_list[0] if cuda_arch != "none": args.append(f"-DCUDA_FLAGS=-arch=sm_{cuda_arch}") else: # Ensure build with CUDA is disabled args.append("-DWITH_CUDA=OFF") ... return args assuming only the ``WITH_CUDA`` and ``CUDA_FLAGS`` flags are required. You will need to customize options as needed for your build. This example also illustrates how to check for the ``cuda`` variant using ``self.spec`` and how to retrieve the ``cuda_arch`` variant's value, which is a list, using ``self.spec.variants["cuda_arch"].value``. With over 70 packages using ``CudaPackage`` as of January 2021 there are lots of examples to choose from to get more ideas for using this package. ================================================ FILE: lib/spack/docs/build_systems/custompackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to creating custom build systems in Spack for packaging software with its own build scripts or adding support for new build systems. .. _custompackage: Custom Build Systems -------------------- While the built-in build systems should meet your needs for the vast majority of packages, some packages provide custom build scripts. This guide is intended for the following use cases: * Packaging software with its own custom build system * Adding support for new build systems If you want to add support for a new build system, a good place to start is to look at the definitions of other build systems. This guide focuses mostly on how Spack's build systems work. In this guide, we will be using the `perl `_ and `cmake `_ packages as examples. ``perl``'s build system is a hand-written ``Configure`` shell script, while ``cmake`` bootstraps itself during installation. Both of these packages require custom build systems. Base class ^^^^^^^^^^ If your package does not belong to any of the built-in build systems that Spack already supports, you should inherit from the ``Package`` base class. ``Package`` is a simple base class with a single phase: ``install``. If your package is simple, you may be able to simply write an ``install`` method that gets the job done. However, if your package is more complex and installation involves multiple steps, you should add separate phases as mentioned in the next section. If you are creating a new build system base class, you should inherit from ``PackageBase``. This is the superclass for all build systems in Spack. Phases ^^^^^^ The most important concept in Spack's build system support is the idea of phases. Each build system defines a set of phases that are necessary to install the package. They usually follow some sort of "configure", "build", "install" guideline, but any of those phases may be missing or combined with another phase. If you look at the ``perl`` package, you'll see: .. code-block:: python phases = ("configure", "build", "install") Similarly, ``cmake`` defines: .. code-block:: python phases = ("bootstrap", "build", "install") If we look at the ``cmake`` example, this tells Spack's ``PackageBase`` class to run the ``bootstrap``, ``build``, and ``install`` functions in that order. It is now up to you to define these methods. Phase and phase_args functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If we look at ``perl``, we see that it defines a ``configure`` method: .. code-block:: python def configure(self, spec, prefix): configure = Executable("./Configure") configure(*self.configure_args()) There is also a corresponding ``configure_args`` function that handles all of the arguments to pass to ``Configure``, just like in ``AutotoolsPackage``. Comparatively, the ``build`` and ``install`` phases are pretty simple: .. code-block:: python def build(self, spec, prefix): make() def install(self, spec, prefix): make("install") The ``cmake`` package looks very similar, but with a ``bootstrap`` function instead of ``configure``: .. code-block:: python def bootstrap(self, spec, prefix): bootstrap = Executable("./bootstrap") bootstrap(*self.bootstrap_args()) def build(self, spec, prefix): make() def install(self, spec, prefix): make("install") Again, there is a ``bootstrap_args`` function that determines the correct bootstrap flags to use. ``run_before`` / ``run_after`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Occasionally, you may want to run extra steps either before or after a given phase. This applies not just to custom build systems, but to existing build systems as well. You may need to patch a file that is generated by configure, or install extra files in addition to what ``make install`` copies to the installation prefix. This is where ``@run_before`` and ``@run_after`` come in. These Python decorators allow you to write functions that are called before or after a particular phase. For example, in ``perl``, we see: .. code-block:: python @run_after("install") def install_cpanm(self): spec = self.spec maker = make cpan_dir = join_path("cpanm", "cpanm") if sys.platform == "win32": maker = nmake cpan_dir = join_path(self.stage.source_path, cpan_dir) cpan_dir = windows_sfn(cpan_dir) if "+cpanm" in spec: with working_dir(cpan_dir): perl = spec["perl"].command perl("Makefile.PL") maker() maker("install") This extra step automatically installs ``cpanm`` in addition to the base Perl installation. ``on_package_attributes`` ^^^^^^^^^^^^^^^^^^^^^^^^^ The ``run_before`` / ``run_after`` logic discussed above becomes particularly powerful when combined with the ``@on_package_attributes`` decorator. This decorator allows you to conditionally run certain functions depending on the attributes of that package. The most common example is conditional testing. Many unit tests are prone to failure, even when there is nothing wrong with the installation. Unfortunately, non-portable unit tests and tests that are "supposed to fail" are more common than we would like. Instead of always running unit tests on installation, Spack lets users conditionally run tests with the ``--test=root`` flag. If we wanted to define a function that would conditionally run if and only if this flag is set, we would use the following: .. code-block:: python @on_package_attributes(run_tests=True) def my_test_function(self): ... Testing ^^^^^^^ Let's put everything together and add unit tests to be optionally run during the installation of our package. In the ``perl`` package, we can see: .. code-block:: python @run_after("build") @on_package_attributes(run_tests=True) def build_test(self): if sys.platform == "win32": win32_dir = os.path.join(self.stage.source_path, "win32") win32_dir = windows_sfn(win32_dir) with working_dir(win32_dir): nmake("test", ignore_quotes=True) else: make("test") As you can guess, this runs ``make test`` *after* building the package, if and only if testing is requested. Again, this is not specific to custom build systems, it can be added to existing build systems as well. .. warning:: The order of decorators matters. The following ordering: .. code-block:: python @run_after("install") @on_package_attributes(run_tests=True) def my_test_function(self): ... works as expected. However, if you reverse the ordering: .. code-block:: python @on_package_attributes(run_tests=True) @run_after("install") def my_test_function(self): ... the tests will always be run regardless of whether or not ``--test=root`` is requested. See https://github.com/spack/spack/issues/3833 for more information Ideally, every package in Spack will have some sort of test to ensure that it was built correctly. It is up to the package authors to make sure this happens. If you are adding a package for some software and the developers list commands to test the installation, please add these tests to your ``package.py``. For more information on other forms of package testing, refer to :ref:`Checking an installation `. ================================================ FILE: lib/spack/docs/build_systems/inteloneapipackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to using the Intel oneAPI packages in Spack, including how to build with icx, use the oneAPI Spack environment, and configure externally installed oneAPI tools. .. _inteloneapipackage: IntelOneapi =========== Spack can install and use the Intel oneAPI products. You may either use Spack to install the oneAPI tools or use the `Intel installers`_. After installation, you may use the tools directly, or use Spack to build packages with the tools. The Spack Python class ``IntelOneapiPackage`` is a base class that is used by ``IntelOneapiCompilers``, ``IntelOneapiMkl``, ``IntelOneapiTbb`` and other classes to implement the oneAPI packages. Search for ``oneAPI`` at `packages.spack.io `_ for the full list of available oneAPI packages, or use: .. code-block:: console $ spack list -d oneAPI For more information on a specific package, do: .. code-block:: console $ spack info --all Building a Package With icx --------------------------- In this example, we build patchelf with ``icc`` and ``icx``. The compilers are installed with Spack. Install the oneAPI compilers: .. code-block:: spec $ spack install intel-oneapi-compilers To build the ``patchelf`` Spack package with ``icx``, do: .. code-block:: spec $ spack install patchelf%oneapi Using oneAPI Spack environment ------------------------------- In this example, we build LAMMPS with ``icx`` using Spack environment for oneAPI packages created by Intel. The compilers are installed with Spack like in example above. Install the oneAPI compilers: .. code-block:: spec $ spack install intel-oneapi-compilers Clone `spack-configs `_ repo and activate Intel oneAPI CPU environment: .. code-block:: console $ git clone https://github.com/spack/spack-configs $ spack env activate spack-configs/INTEL/CPU $ spack concretize -f `Intel oneAPI CPU environment `_ contains applications tested and validated by Intel. This list is constantly extended. Currently, it supports: - `Devito `_ - `GROMACS `_ - `HPCG `_ - `HPL `_ - `LAMMPS `_ - `OpenFOAM `_ - `Quantum Espresso `_ - `STREAM `_ - `WRF `_ To build LAMMPS with oneAPI compiler from this environment just run: .. code-block:: spec $ spack install lammps Compiled binaries can be found using: .. code-block:: console $ spack cd -i lammps You can do the same for all other applications from this environment. Using oneAPI MPI to Satisfy a Virtual Dependence ------------------------------------------------ The ``hdf5`` package works with any compatible MPI implementation. To build ``hdf5`` with Intel oneAPI MPI do: .. code-block:: spec $ spack install hdf5 +mpi ^intel-oneapi-mpi Using Externally Installed oneAPI Tools --------------------------------------- Spack can also use oneAPI tools that are manually installed with `Intel Installers`_. The procedures for configuring Spack to use external compilers and libraries are different. Compilers ^^^^^^^^^ To use the compilers, add some information about the installation to ``packages.yaml``. For most users, it is sufficient to do: .. code-block:: console $ spack compiler add /opt/intel/oneapi/compiler/latest/bin Adapt the paths above if you did not install the tools in the default location. After adding the compilers, using them is the same as if you had installed the ``intel-oneapi-compilers`` package. Another option is to manually add the configuration to ``packages.yaml`` as described in :ref:`Compiler configuration `. Before 2024, the directory structure was different: .. code-block:: console $ spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64 $ spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin Libraries ^^^^^^^^^ If you want Spack to use oneMKL that you have installed without Spack in the default location, then add the following to ``~/.spack/packages.yaml``, adjusting the version as appropriate: .. code-block:: yaml intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2021.1.1 prefix: /opt/intel/oneapi/ Using oneAPI Tools Installed by Spack ------------------------------------- Spack can be a convenient way to install and configure compilers and libraries, even if you do not intend to build a Spack package. If you want to build a Makefile project using Spack-installed oneAPI compilers, then use Spack to configure your environment: .. code-block:: spec $ spack load intel-oneapi-compilers And then you can build with: .. code-block:: console $ CXX=icpx make You can also use Spack-installed libraries. For example: .. code-block:: spec $ spack load intel-oneapi-mkl This updates your environment CPATH, LIBRARY_PATH, and other environment variables for building an application with oneMKL. .. _`Intel installers`: https://software.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top.html ================================================ FILE: lib/spack/docs/build_systems/luapackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn about the Lua build system in Spack for packages using rockspec files. .. _luapackage: Lua ------ The ``Lua`` build system is a helper for the common case of Lua packages that provide a rockspec file. This is not meant to take a rock archive, but to build a source archive or repository that provides a rockspec, which should cover most Lua packages. In the case a Lua package builds by Make rather than LuaRocks, prefer MakefilePackage. Phases ^^^^^^ The ``LuaBuilder`` and ``LuaPackage`` base classes come with the following phases: #. ``unpack`` - if using a rock, unpacks the rock and moves into the source directory #. ``preprocess`` - adjust sources or rockspec to fix build #. ``install`` - install the project By default, these phases run: .. code-block:: console # If the archive is a source rock $ luarocks unpack .src.rock $ # preprocess is a no-op by default $ luarocks make .rockspec Any of these phases can be overridden in your package as necessary. Important files ^^^^^^^^^^^^^^^ Packages that use the Lua/LuaRocks build system can be identified by the presence of a ``*.rockspec`` file in their source tree, or can be fetched as a source rock archive (``.src.rock``). This file declares things like build instructions and dependencies. The ``.src.rock`` also contains all code. It is common for the rockspec file to list the Lua version required in a dependency. The LuaPackage class adds appropriate dependencies on a Lua implementation, but it is a good idea to specify the version required with a ``depends_on`` statement. The block normally will be a table definition like this: .. code-block:: lua dependencies = { "lua >= 5.1", } The LuaPackage class supports source repositories and archives containing a rockspec and directly downloading source rock files. It *does not* support downloading dependencies listed inside a rockspec, and thus does not support directly downloading a rockspec as an archive. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ All base dependencies are added by the build system, but LuaRocks is run to avoid downloading extra Lua dependencies during build. If the package needs Lua libraries outside the standard set, they should be added as dependencies. To specify a Lua version constraint but allow all Lua implementations, prefer to use ``depends_on("lua-lang@5.1:5.1.99")`` to express any 5.1 compatible version. If the package requires LuaJit rather than Lua, a ``depends_on("luajit")`` should be used to ensure a LuaJit distribution is used instead of the Lua interpreter. Alternately, if only interpreted Lua will work, ``depends_on("lua")`` will express that. Passing arguments to luarocks make ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you need to pass any arguments to the ``luarocks make`` call, you can override the ``luarocks_args`` method like so: .. code-block:: python def luarocks_args(self): return ["flag1", "flag2"] One common use of this is to override warnings or flags for newer compilers, as in: .. code-block:: python def luarocks_args(self): return ["CFLAGS='-Wno-error=implicit-function-declaration'"] External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the LuaRocks build system, see: https://luarocks.org/ ================================================ FILE: lib/spack/docs/build_systems/makefilepackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to using the Makefile build system in Spack for packages that use plain Makefiles. .. _makefilepackage: Makefile -------- The most primitive build system a package can use is a plain Makefile. Makefiles are simple to write for small projects, but they usually require you to edit the Makefile to set platform and compiler-specific variables. Phases ^^^^^^ The ``MakefileBuilder`` and ``MakefilePackage`` base classes come with 3 phases: #. ``edit`` - edit the Makefile #. ``build`` - build the project #. ``install`` - install the project By default, ``edit`` does nothing, but you can override it to replace hardcoded Makefile variables. The ``build`` and ``install`` phases run: .. code-block:: console $ make $ make install Important files ^^^^^^^^^^^^^^^ The main file that matters for a ``MakefilePackage`` is the Makefile. This file will be named one of the following ways: * GNUmakefile (only works with GNU Make) * Makefile (most common) * makefile Some Makefiles also *include* other configuration files. Check for an ``include`` directive in the Makefile. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ Spack assumes that the operating system will have a valid ``make`` utility installed already, so you don't need to add a dependency on ``make``. However, if the package uses a ``GNUmakefile`` or the developers recommend using GNU Make, you should add a dependency on ``gmake``: .. code-block:: python depends_on("gmake", type="build") Types of Makefile packages ^^^^^^^^^^^^^^^^^^^^^^^^^^ Most of the work involved in packaging software that uses Makefiles involves overriding or replacing hard-coded variables. Many packages make the mistake of hard-coding compilers, usually for GCC or Intel. This is fine if you happen to be using that particular compiler, but Spack is designed to work with *any* compiler, and you need to ensure that this is the case. Depending on how the Makefile is designed, there are 4 common strategies that can be used to set or override the appropriate variables: Environment variables """"""""""""""""""""" Make has multiple types of `assignment operators `_. Some Makefiles use ``=`` to assign variables. The only way to override these variables is to edit the Makefile or override them on the command-line. However, Makefiles that use ``?=`` for assignment honor environment variables. Since Spack already sets ``CC``, ``CXX``, ``F77``, and ``FC``, you won't need to worry about setting these variables. If there are any other variables you need to set, you can do this in the ``setup_build_environment`` method: .. code-block:: python def setup_build_environment(self, env: EnvironmentModifications) -> None: env.set("PREFIX", prefix) env.set("BLASLIB", spec["blas"].libs.ld_flags) `cbench `_ is a good example of a simple package that does this, while `esmf `_ is a good example of a more complex package. Command-line arguments """""""""""""""""""""" If the Makefile ignores environment variables, the next thing to try is command-line arguments. You can do this by overriding the ``build_targets`` attribute. If you don't need access to the spec, you can do this like so: .. code-block:: python build_targets = ["CC=cc"] If you do need access to the spec, you can create a property like so: .. code-block:: python @property def build_targets(self): spec = self.spec return [ "CC=cc", f"BLASLIB={spec['blas'].libs.ld_flags}", ] `cloverleaf `_ is a good example of a package that uses this strategy. Edit Makefile """"""""""""" Some Makefiles are just plain stubborn and will ignore command-line variables. The only way to ensure that these packages build correctly is to directly edit the Makefile. Spack provides a ``FileFilter`` class and a ``filter`` method to help with this. For example: .. code-block:: python def edit(self, spec, prefix): makefile = FileFilter("Makefile") makefile.filter(r"^\s*CC\s*=.*", f"CC = {spack_cc}") makefile.filter(r"^\s*CXX\s*=.*", f"CXX = {spack_cxx}") makefile.filter(r"^\s*F77\s*=.*", f"F77 = {spack_f77}") makefile.filter(r"^\s*FC\s*=.*", f"FC = {spack_fc}") `stream `_ is a good example of a package that involves editing a Makefile to set the appropriate variables. Config file """"""""""" More complex packages often involve Makefiles that *include* a configuration file. These configuration files are primarily composed of variables relating to the compiler, platform, and the location of dependencies or names of libraries. Since these config files are dependent on the compiler and platform, you will often see entire directories of examples for common compilers and architectures. Use these examples to help determine what possible values to use. If the config file is long and only contains one or two variables that need to be modified, you can use the technique above to edit the config file. However, if you end up needing to modify most of the variables, it may be easier to write a new file from scratch. If each variable is independent of each other, a dictionary works well for storing variables: .. code-block:: python def edit(self, spec, prefix): config = { "CC": "cc", "MAKE": "make", } if spec.satisfies("+blas"): config["BLAS_LIBS"] = spec["blas"].libs.joined() with open("make.inc", "w") as inc: for key in config: inc.write(f"{key} = {config[key]}\n") `elk `_ is a good example of a package that uses a dictionary to store configuration variables. If the order of variables is important, it may be easier to store them in a list: .. code-block:: python def edit(self, spec, prefix): config = [ f"INSTALL_DIR = {prefix}", "INCLUDE_DIR = $(INSTALL_DIR)/include", "LIBRARY_DIR = $(INSTALL_DIR)/lib", ] with open("make.inc", "w") as inc: for var in config: inc.write(f"{var}\n") `hpl `_ is a good example of a package that uses a list to store configuration variables. Variables to watch out for ^^^^^^^^^^^^^^^^^^^^^^^^^^ The following is a list of common variables to watch out for. The first two sections are `implicit variables `_ defined by Make and will always use the same name, while the rest are user-defined variables and may vary from package to package. * **Compilers** This includes variables such as ``CC``, ``CXX``, ``F77``, ``F90``, and ``FC``, as well as variables related to MPI compiler wrappers, like ``MPICC`` and friends. * **Compiler flags** This includes variables for compiler flags, such as ``CFLAGS``, ``CXXFLAGS``, ``F77FLAGS``, ``F90FLAGS``, ``FCFLAGS``, and ``CPPFLAGS``. These variables are often hard-coded to contain flags specific to a certain compiler. If these flags don't work for every compiler, you may want to consider filtering them. * **Variables that enable or disable features** This includes variables like ``MPI``, ``OPENMP``, ``PIC``, and ``DEBUG``. These flags often require you to create a variant so that you can either build with or without MPI support, for example. These flags are often compiler-dependent. You should replace them with the appropriate compiler flags, such as ``self.compiler.openmp_flag`` or ``self.compiler.pic_flag``. * **Platform flags** These flags control the type of architecture that the executable is compiled for. Watch out for variables like ``PLAT`` or ``ARCH``. * **Dependencies** Look out for variables that sound like they could be used to locate dependencies, such as ``JAVA_HOME``, ``JPEG_ROOT``, or ``ZLIBDIR``. Also watch out for variables that control linking, such as ``LIBS``, ``LDFLAGS``, and ``INCLUDES``. These variables need to be set to the installation prefix of a dependency, or to the correct linker flags to link to that dependency. * **Installation prefix** If your Makefile has an ``install`` target, it needs some way of knowing where to install. By default, many packages install to ``/usr`` or ``/usr/local``. Since many Spack users won't have sudo privileges, it is imperative that each package is installed to the proper prefix. Look for variables like ``PREFIX`` or ``INSTALL``. Makefiles in a sub-directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Not every package places their Makefile in the root of the package tarball. If the Makefile is in a sub-directory like ``src``, you can tell Spack where to locate it like so: .. code-block:: python build_directory = "src" Manual installation ^^^^^^^^^^^^^^^^^^^ Not every Makefile includes an ``install`` target. If this is the case, you can override the default ``install`` method to manually install the package: .. code-block:: python def install(self, spec, prefix): mkdir(prefix.bin) install("foo", prefix.bin) install_tree("lib", prefix.lib) External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on reading and writing Makefiles, see: https://www.gnu.org/software/make/manual/make.html ================================================ FILE: lib/spack/docs/build_systems/mavenpackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn about the Maven build system in Spack for building and managing Java-based projects. .. _mavenpackage: Maven ------ Apache Maven is a general-purpose build system that does not rely on Makefiles to build software. It is designed for building and managing Java-based projects. Phases ^^^^^^ The ``MavenBuilder`` and ``MavenPackage`` base classes come with the following phases: #. ``build`` - compile code and package into a JAR file #. ``install`` - copy to installation prefix By default, these phases run: .. code-block:: console $ mvn package $ install . Important files ^^^^^^^^^^^^^^^ Maven packages can be identified by the presence of a ``pom.xml`` file. This file lists dependencies and other metadata about the project. There may also be configuration files in the ``.mvn`` directory. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ Maven requires the ``mvn`` executable to build the project. It also requires Java at both build- and run-time. Because of this, the base class automatically adds the following dependencies: .. code-block:: python depends_on("java", type=("build", "run")) depends_on("maven", type="build") In the ``pom.xml`` file, you may see sections like: .. code-block:: xml [1.7,) [3.5.4,) This specifies the versions of Java and Maven that are required to build the package. See https://docs.oracle.com/middleware/1212/core/MAVEN/maven_version.htm#MAVEN402 for a description of this version range syntax. In this case, you should add: .. code-block:: python depends_on("java@7:", type="build") depends_on("maven@3.5.4:", type="build") Passing arguments to the build phase ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The default build and install phases should be sufficient to install most packages. However, you may want to pass additional flags to the build phase. For example: .. code-block:: python def build_args(self): return ["-Pdist,native", "-Dtar", "-Dmaven.javadoc.skip=true"] External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the Maven build system, see: https://maven.apache.org/index.html ================================================ FILE: lib/spack/docs/build_systems/mesonpackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to use the Meson build system in Spack for projects that use Meson for their build process. .. _mesonpackage: Meson ------ Much like Autotools and CMake, Meson is a build system. But it is meant to be both fast and as user friendly as possible. GNOME's goal is to port modules to use the Meson build system. Phases ^^^^^^ The ``MesonBuilder`` and ``MesonPackage`` base classes come with the following phases: #. ``meson`` - generate ninja files #. ``build`` - build the project #. ``install`` - install the project By default, these phases run: .. code-block:: console $ mkdir spack-build $ cd spack-build $ meson .. --prefix=/path/to/installation/prefix $ ninja $ ninja test # optional $ ninja install Any of these phases can be overridden in your package as necessary. There is also a ``check`` method that looks for a ``test`` target in the build file. If a ``test`` target exists and the user runs: .. code-block:: console $ spack install --test=root Spack will run ``ninja test`` after the build phase. Important files ^^^^^^^^^^^^^^^ Packages that use the Meson build system can be identified by the presence of a ``meson.build`` file. This file declares things like build instructions and dependencies. One thing to look for is the ``meson_version`` key that gets passed to the ``project`` function: .. code-block:: none :emphasize-lines: 10 project('gtk+', 'c', version: '3.94.0', default_options: [ 'buildtype=debugoptimized', 'warning_level=1', # We only need c99, but glib needs GNU-specific features # https://github.com/mesonbuild/meson/issues/2289 'c_std=gnu99', ], meson_version: '>= 0.43.0', license: 'LGPLv2.1+') This means that Meson 0.43.0 is the earliest release that will work. You should specify this in a ``depends_on`` statement. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ At the bare minimum, packages that use the Meson build system need ``meson`` and ``ninja`` dependencies. Since this is always the case, the ``MesonPackage`` base class already contains: .. code-block:: python depends_on("meson", type="build") depends_on("ninja", type="build") If you need to specify a particular version requirement, you can override this in your package: .. code-block:: python depends_on("meson@0.43.0:", type="build") depends_on("ninja", type="build") Finding meson flags ^^^^^^^^^^^^^^^^^^^ To get a list of valid flags that can be passed to ``meson``, run the following command in the directory that contains ``meson.build``: .. code-block:: console $ meson setup --help Passing arguments to meson ^^^^^^^^^^^^^^^^^^^^^^^^^^ If you need to pass any arguments to the ``meson`` call, you can override the ``meson_args`` method like so: .. code-block:: python def meson_args(self): return ["--warnlevel=3"] This method can be used to pass flags as well as variables. Note that the ``MesonPackage`` base class already defines variants for ``buildtype``, ``default_library`` and ``strip``, which are mapped to default Meson arguments, meaning that you don't have to specify these. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the Meson build system, see: https://mesonbuild.com/index.html ================================================ FILE: lib/spack/docs/build_systems/octavepackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn about the Octave build system in Spack for installing Octave packages. .. _octavepackage: Octave ------ Octave has its own build system for installing packages. Phases ^^^^^^ The ``OctaveBuilder`` and ``OctavePackage`` base classes have a single phase: #. ``install`` - install the package By default, this phase runs the following command: .. code-block:: console $ octave '--eval' 'pkg prefix ; pkg install ' Beware that uninstallation is not implemented at the moment. After uninstalling a package via Spack, you also need to manually uninstall it from Octave via ``pkg uninstall ``. Finding Octave packages ^^^^^^^^^^^^^^^^^^^^^^^ Most Octave packages are listed at https://octave.sourceforge.io/packages.php. Dependencies ^^^^^^^^^^^^ Usually, the homepage of a package will list dependencies, i.e., ``Dependencies: Octave >= 3.6.0 struct >= 1.0.12``. The same information should be available in the ``DESCRIPTION`` file in the root of each archive. External Documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the Octave build system, see: https://octave.org/doc/v4.4.0/Installing-and-Removing-Packages.html ================================================ FILE: lib/spack/docs/build_systems/perlpackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to packaging Perl modules with Spack, covering when to add a package and build system integration. .. _perlpackage: Perl ------ Much like Octave, Perl has its own language-specific build system. This documentation includes information on when **not** to add a Spack package for a Perl module. .. _suitable_perl_modules: Suitable Modules ^^^^^^^^^^^^^^^^ In general, modules that are part of the standard Perl installation should not be added to Spack. A possible exception is if the module was not part of the standard installation for earlier versions of ``perl`` that are still listed in the package, which you can check by running ``spack info perl``. How do you know if the module is in the standard Perl installation? You check if it is included in the ``CORE`` by entering the following on the command line: .. code-block:: console $ corelist where is case sensitive. Examples of outputs for modules that are and are not in the ``CORE`` using perl v5.42.0 are: .. code-block:: console $ corelist Carp Data for 2025-07-02 Carp was first released with perl 5 $ corelist XML::Writer Data for 2025-07-02 XML::Writer was not in CORE (or so I think) Phases ^^^^^^ The ``PerlBuilder`` and ``PerlPackage`` base classes come with three phases that can be overridden: #. ``configure`` - configure the package #. ``build`` - build the package #. ``install`` - install the package Perl packages have two common modules used for module installation: ``ExtUtils::MakeMaker`` """"""""""""""""""""""" The ``ExtUtils::MakeMaker`` module is just what it sounds like, a module designed to generate Makefiles. It can be identified by the presence of a ``Makefile.PL`` file, and has the following installation steps: .. code-block:: console $ perl Makefile.PL INSTALL_BASE=/path/to/installation/prefix $ make $ make test # optional $ make install ``Module::Build`` """"""""""""""""" The ``Module::Build`` module is a pure-Perl build system, and can be identified by the presence of a ``Build.PL`` file. It has the following installation steps: .. code-block:: console $ perl Build.PL --install_base /path/to/installation/prefix $ ./Build $ ./Build test # optional $ ./Build install If both ``Makefile.PL`` and ``Build.PL`` files exist in the package, Spack will use ``Makefile.PL`` by default. If your package uses a different module, ``PerlPackage`` will need to be extended to support it. ``PerlPackage`` automatically detects which build steps to use, so there shouldn't be much work on the package developer's side to get things working. Finding Perl packages ^^^^^^^^^^^^^^^^^^^^^ Most Perl modules are hosted on CPAN, the Comprehensive Perl Archive Network. If you need to find a package for ``XML::Parser``, for example, you should search for "CPAN XML::Parser". Just make sure that the module is not included in the ``CORE`` (see :ref:`suitable_perl_modules`). Some CPAN pages are versioned. Check for a link to the "Latest Release" to make sure you have the latest version. Package name ^^^^^^^^^^^^ When you use ``spack create`` to create a new Perl package, Spack will automatically prepend ``perl-`` to the front of the package name. This helps to keep Perl modules separate from other packages. The same naming scheme is used for other language extensions, like Python and R. See :ref:`creating-and-editing-packages` for more information on the command. Description ^^^^^^^^^^^ Most CPAN pages have a short description under "NAME" and a longer description under "DESCRIPTION". Use whichever you think is more useful while still being succinct. Homepage ^^^^^^^^ In the top-right corner of the CPAN page, you'll find a "permalink" for the package. This should be used instead of the current URL, as it doesn't contain the version number and will always link to the latest release. URL ^^^^^^ If you haven't found it already, the download URL is on the right side of the page below the permalink. Search for "Download". Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ Every ``PerlPackage`` obviously depends on Perl at build and run-time, so ``PerlPackage`` contains: .. code-block:: python extends("perl") If your package requires a specific version of Perl, you should specify this. Although newer versions of Perl include ``ExtUtils::MakeMaker`` and ``Module::Build`` as "core" modules, you may want to add dependencies on ``perl-extutils-makemaker`` and ``perl-module-build`` anyway. Many people add Perl as an external package, and we want the build to work properly. If your package uses ``Makefile.PL`` to build, add: .. code-block:: python depends_on("perl-extutils-makemaker", type="build") If your package uses ``Build.PL`` to build, add: .. code-block:: python depends_on("perl-module-build", type="build") Perl dependencies ^^^^^^^^^^^^^^^^^ Below the download URL, you will find a "Dependencies" link, which takes you to a page listing all of the dependencies of the package. Packages listed as "Core module" don't need to be added as dependencies, but all direct dependencies should be added. Don't add dependencies of dependencies. These should be added as dependencies to the dependency, not to your package. Passing arguments to configure ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Packages that have non-Perl dependencies often use command-line variables to specify their installation directory. You can pass arguments to ``Makefile.PL`` or ``Build.PL`` by overriding ``configure_args`` like so: .. code-block:: python def configure_args(self): expat = self.spec["expat"].prefix return [ "EXPATLIBPATH={0}".format(expat.lib), "EXPATINCPATH={0}".format(expat.include), ] Testing ^^^^^^^ ``PerlPackage`` provides a simple stand-alone test of the successfully installed package to confirm that installed Perl module(s) can be used. These tests can be performed any time after the installation using ``spack -v test run``. (For more information on the command, see :ref:`cmd-spack-test-run`.) The base class automatically detects Perl modules based on the presence of ``*.pm`` files under the package's library directory. For example, the files under ``perl-bignum``'s Perl library are: .. code-block:: console $ find . -name "*.pm" ./bigfloat.pm ./bigrat.pm ./Math/BigFloat/Trace.pm ./Math/BigInt/Trace.pm ./Math/BigRat/Trace.pm ./bigint.pm ./bignum.pm which results in the package having the ``use_modules`` property containing: .. code-block:: python use_modules = [ "bigfloat", "bigrat", "Math::BigFloat::Trace", "Math::BigInt::Trace", "Math::BigRat::Trace", "bigint", "bignum", ] .. note:: This list can often be used to catch missing dependencies. If the list is somehow wrong, you can provide the names of the modules yourself by overriding ``use_modules`` like so: .. code-block:: python use_modules = ["bigfloat", "bigrat", "bigint", "bignum"] If you only want a subset of the automatically detected modules to be tested, you could instead define the ``skip_modules`` property on the package. So, instead of overriding ``use_modules`` as shown above, you could define the following: .. code-block:: python skip_modules = [ "Math::BigFloat::Trace", "Math::BigInt::Trace", "Math::BigRat::Trace", ] for the same use tests. Alternatives to Spack ^^^^^^^^^^^^^^^^^^^^^ If you need to maintain a stack of Perl modules for a user and don't want to add all of them to Spack, a good alternative is ``cpanm``. If Perl is already installed on your system, it should come with a ``cpan`` executable. To install ``cpanm``, run the following command: .. code-block:: console $ cpan App::cpanminus Now, you can install any Perl module you want by running: .. code-block:: console $ cpanm Module::Name Obviously, these commands can only be run if you have root privileges. Furthermore, ``cpanm`` is not capable of installing non-Perl dependencies. If you need to install to your home directory or need to install a module with non-Perl dependencies, Spack is a better option. External documentation ^^^^^^^^^^^^^^^^^^^^^^ You can find more information on installing Perl modules from source at: http://www.perlmonks.org/?node_id=128077 More generic Perl module installation instructions can be found at: http://www.cpan.org/modules/INSTALL.html ================================================ FILE: lib/spack/docs/build_systems/pythonpackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to packaging Python libraries with Spack, covering PyPI downloads, dependency management, and build system integration. .. _pythonpackage: Python ------ Python packages and modules have their own special build system. This documentation covers everything you'll need to know in order to write a Spack build recipe for a Python library. Terminology ^^^^^^^^^^^ In the Python ecosystem, there are a number of terms that are important to understand. **PyPI** The `Python Package Index `_, where most Python libraries are hosted. **sdist** Source distributions, distributed as tarballs (.tar.gz) and zip files (.zip). Contain the source code of the package. **bdist** Built distributions, distributed as wheels (.whl). Contain the pre-built library. **wheel** A binary distribution format common in the Python ecosystem. This file is actually just a zip file containing specific metadata and code. See the `documentation `_ for more details. **build frontend** Command-line tools used to build and install wheels. Examples include `pip `_, `build `_, and `installer `_. **build backend** Libraries used to define how to build a wheel. Examples include `setuptools `__, `flit `_, `poetry `_, `hatchling `_, `meson `_, and `pdm `_. Downloading ^^^^^^^^^^^ The first step in packaging a Python library is to figure out where to download it from. The vast majority of Python packages are hosted on `PyPI `_, which is :ref:`preferred over GitHub ` for downloading packages. Search for the package name on PyPI to find the project page. The project page is usually located at: .. code-block:: text https://pypi.org/project/ On the project page, there is a "Download files" tab containing download URLs. Whenever possible, we prefer to build Spack packages from source. If PyPI only has wheels, check to see if the project is hosted on GitHub and see if GitHub has source distributions. The project page usually has a "Homepage" and/or "Source code" link for this. If the project is closed-source, it may only have wheels available. For example, ``py-azureml-sdk`` is closed-source and can be downloaded from: .. code-block:: text https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl Once you've found a URL to download the package from, run: .. code-block:: console $ spack create to create a new package template. .. _pypi-vs-github: PyPI vs. GitHub """"""""""""""" Many packages are hosted on PyPI, but are developed on GitHub or another version control system hosting service. The source code can be downloaded from either location, but PyPI is preferred for the following reasons: #. PyPI contains the bare minimum number of files needed to install the package. You may notice that the tarball you download from PyPI does not have the same checksum as the tarball you download from GitHub. When a developer uploads a new release to PyPI, it doesn't contain every file in the repository, only the files necessary to install the package. PyPI tarballs are therefore smaller. #. PyPI is the official source for package managers like ``pip``. Let's be honest, ``pip`` is much more popular than Spack. If the GitHub tarball contains a file not present in the PyPI tarball that causes a bug, the developers may not realize this for quite some time. If the bug was in a file contained in the PyPI tarball, users would notice the bug much more quickly. #. GitHub release may be a beta version. When a developer releases a new version of a package on GitHub, it may not be intended for most users. Until that release also makes its way to PyPI, it should be assumed that the release is not yet ready for general use. #. The checksum for a GitHub release may change. Unfortunately, some developers have a habit of patching releases without incrementing the version number. This results in a change in tarball checksum. Package managers like Spack that use checksums to verify the integrity of a download tarball grind to a halt when the checksum for a known version changes. Most of the time, the change is intentional, and contains a needed bug fix. However, sometimes the change indicates a download source that has been compromised, and a tarball that contains a virus. If this happens, you must contact the developers to determine which is the case. PyPI is nice because it makes it physically impossible to re-release the same version of a package with a different checksum. The only reason to use GitHub instead of PyPI is if PyPI only has wheels or if the PyPI sdist is missing a file needed to build the package. If this is the case, please add a comment above the ``url`` explaining this. PyPI ^^^^^^ Since PyPI is so commonly used to host Python libraries, the ``PythonPackage`` base class has a ``pypi`` attribute that can be set. Once set, ``pypi`` will be used to define the ``homepage``, ``url``, and ``list_url``. For example, the following: .. code-block:: python homepage = "https://pypi.org/project/setuptools/" url = "https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip" list_url = "https://pypi.org/simple/setuptools/" is equivalent to: .. code-block:: python pypi = "setuptools/setuptools-49.2.0.zip" If a package has a different homepage listed on PyPI, you can override it by setting your own ``homepage``. Description ^^^^^^^^^^^ The top of the PyPI project page contains a short description of the package. The "Project description" tab may also contain a longer description of the package. Either of these can be used to populate the package docstring. Dependencies ^^^^^^^^^^^^ Once you've determined the basic metadata for a package, the next step is to determine the build backend. ``PythonPackage`` uses `pip `_ to install the package, but pip requires a backend to actually build the package. To determine the build backend, look for a ``pyproject.toml`` file. If there is no ``pyproject.toml`` file and only a ``setup.py`` or ``setup.cfg`` file, you can assume that the project uses :ref:`setuptools`. If there is a ``pyproject.toml`` file, see if it contains a ``[build-system]`` section. For example: .. code-block:: toml [build-system] requires = [ "setuptools>=42", "wheel", ] build-backend = "setuptools.build_meta" This section does two things: the ``requires`` key lists build dependencies of the project, and the ``build-backend`` key defines the build backend. All of these build dependencies should be added as dependencies to your package: .. code-block:: python depends_on("py-setuptools@42:", type="build") Note that ``py-wheel`` is already listed as a build dependency in the ``PythonPackage`` base class, so you don't need to add it unless you need to specify a specific version requirement or change the dependency type. See `PEP 517 `__ and `PEP 518 `_ for more information on the design of ``pyproject.toml``. Depending on which build backend a project uses, there are various places that run-time dependencies can be listed. Most modern build backends support listing dependencies directly in ``pyproject.toml``. Look for dependencies under the following keys: * ``requires-python`` under ``[project]`` This specifies the version of Python that is required. * ``dependencies`` under ``[project]`` These packages are required for building and installation. You can add them with ``type=("build", "run")``. * ``[project.optional-dependencies]`` This section includes keys with lists of optional dependencies needed to enable those features. You should add a variant that optionally adds these dependencies. This variant should be ``False`` by default. Some build backends may have additional locations where dependencies can be found. distutils """"""""" Before the introduction of setuptools and other build backends, Python packages had to rely on the built-in distutils library. Distutils is missing many of the features that setuptools and other build backends offer, and users are encouraged to use setuptools instead. In fact, distutils was deprecated in Python 3.10 and will be removed in Python 3.12. Because of this, pip actually replaces all imports of distutils with setuptools. If a package uses distutils, you should instead add a build dependency on setuptools. Check for a ``requirements.txt`` file that may list dependencies of the project. .. _setuptools: setuptools """""""""" If the ``pyproject.toml`` lists ``setuptools.build_meta`` as a ``build-backend``, or if the package has a ``setup.py`` that imports ``setuptools``, or if the package has a ``setup.cfg`` file, then it uses setuptools to build. Setuptools is a replacement for the distutils library, and has almost the exact same API. In addition to ``pyproject.toml``, dependencies can be listed in the ``setup.py`` or ``setup.cfg`` file. Look for the following arguments: * ``python_requires`` This specifies the version of Python that is required. * ``setup_requires`` These packages are usually only needed at build-time, so you can add them with ``type="build"``. * ``install_requires`` These packages are required for building and installation. You can add them with ``type=("build", "run")``. * ``extras_require`` These packages are optional dependencies that enable additional functionality. You should add a variant that optionally adds these dependencies. This variant should be ``False`` by default. * ``tests_require`` These are packages that are required to run the unit tests for the package. These dependencies can be specified using the ``type="test"`` dependency type. However, the PyPI tarballs rarely contain unit tests, so there is usually no reason to add these. See https://setuptools.pypa.io/en/latest/userguide/dependency_management.html for more information on how setuptools handles dependency management. See `PEP 440 `_ for documentation on version specifiers in setuptools. flit """""" There are actually two possible build backends for flit, ``flit`` and ``flit_core``. If you see these in the ``pyproject.toml``, add a build dependency to your package. With flit, all dependencies are listed directly in the ``pyproject.toml`` file. Older versions of flit used to store this info in a ``flit.ini`` file, so check for this too. In addition to the default ``pyproject.toml`` keys listed above, older versions of flit may use the following keys: * ``requires`` under ``[tool.flit.metadata]`` These packages are required for building and installation. You can add them with ``type=("build", "run")``. * ``[tool.flit.metadata.requires-extra]`` This section includes keys with lists of optional dependencies needed to enable those features. You should add a variant that optionally adds these dependencies. This variant should be ``False`` by default. See https://flit.pypa.io/en/latest/pyproject_toml.html for more information. poetry """""" Like flit, poetry also has two possible build backends, ``poetry`` and ``poetry_core``. If you see these in the ``pyproject.toml``, add a build dependency to your package. With poetry, all dependencies are listed directly in the ``pyproject.toml`` file. Dependencies are listed in a ``[tool.poetry.dependencies]`` section, and use a `custom syntax `_ for specifying the version requirements. Note that ``~=`` works differently in poetry than in setuptools and flit for versions that start with a zero. hatchling """"""""" If the ``pyproject.toml`` lists ``hatchling.build`` as the ``build-backend``, it uses the hatchling build system. Hatchling uses the default ``pyproject.toml`` keys to list dependencies. See https://hatch.pypa.io/latest/config/dependency/ for more information. meson """""" If the ``pyproject.toml`` lists ``mesonpy`` as the ``build-backend``, it uses the meson build system. Meson uses the default ``pyproject.toml`` keys to list dependencies. See https://meson-python.readthedocs.io/en/latest/tutorials/introduction.html for more information. pdm """""" If the ``pyproject.toml`` lists ``pdm.pep517.api`` as the ``build-backend``, it uses the PDM build system. PDM uses the default ``pyproject.toml`` keys to list dependencies. See https://pdm.fming.dev/latest/ for more information. wheels """""" Some Python packages are closed-source and are distributed as Python wheels. For example, ``py-azureml-sdk`` downloads a ``.whl`` file. This file is simply a zip file, and can be extracted using: .. code-block:: console $ unzip *.whl The zip file will not contain a ``setup.py``, but it will contain a ``METADATA`` file which contains all the information you need to write a ``package.py`` build recipe. Check for lines like: .. code-block:: text Requires-Python: >=3.5,<4 Requires-Dist: azureml-core (~=1.11.0) Requires-Dist: azureml-dataset-runtime[fuse] (~=1.11.0) Requires-Dist: azureml-train (~=1.11.0) Requires-Dist: azureml-train-automl-client (~=1.11.0) Requires-Dist: azureml-pipeline (~=1.11.0) Provides-Extra: accel-models Requires-Dist: azureml-accel-models (~=1.11.0); extra == 'accel-models' Provides-Extra: automl Requires-Dist: azureml-train-automl (~=1.11.0); extra == 'automl' ``Requires-Python`` is equivalent to ``python_requires`` and ``Requires-Dist`` is equivalent to ``install_requires``. ``Provides-Extra`` is used to name optional features (variants) and a ``Requires-Dist`` with ``extra == 'foo'`` will list any dependencies needed for that feature. Passing arguments to setup.py ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The default install phase should be sufficient to install most packages. However, the installation instructions for a package may suggest passing certain flags to the ``setup.py`` call. The ``PythonPackage`` class has two techniques for doing this. Config settings """"""""""""""" These settings are passed to `PEP 517 `__ build backends. For example, ``py-scipy`` package allows you to specify the name of the BLAS/LAPACK library you want pkg-config to search for: .. code-block:: python depends_on("py-pip@22.1:", type="build") def config_settings(self, spec, prefix): return { "blas": spec["blas"].libs.names[0], "lapack": spec["lapack"].libs.names[0], } .. note:: This flag only works for packages that define a ``build-backend`` in ``pyproject.toml``. Also, it is only supported by pip 22.1+, which requires Python 3.7+. For packages that still support Python 3.6 and older, ``install_options`` should be used instead. Global options """""""""""""" These flags are added directly after ``setup.py`` when pip runs ``python setup.py install``. For example, the ``py-pyyaml`` package has an optional dependency on ``libyaml`` that can be enabled like so: .. code-block:: python def global_options(self, spec, prefix): options = [] if spec.satisfies("+libyaml"): options.append("--with-libyaml") else: options.append("--without-libyaml") return options .. note:: Direct invocation of ``setup.py`` is `deprecated `_. This flag forces pip to use a deprecated installation procedure. It should only be used in packages that don't define a ``build-backend`` in ``pyproject.toml`` or packages that still support Python 3.6 and older. Install options """"""""""""""" These flags are added directly after ``install`` when pip runs ``python setup.py install``. For example, the ``py-pyyaml`` package allows you to specify the directories to search for ``libyaml``: .. code-block:: python def install_options(self, spec, prefix): options = [] if spec.satisfies("+libyaml"): options.extend( [ spec["libyaml"].libs.search_flags, spec["libyaml"].headers.include_flags, ] ) return options .. note:: Direct invocation of ``setup.py`` is `deprecated `_. This flag forces pip to use a deprecated installation procedure. It should only be used in packages that don't define a ``build-backend`` in ``pyproject.toml`` or packages that still support Python 3.6 and older. Testing ^^^^^^^ ``PythonPackage`` provides a couple of options for testing packages both during and after the installation process. Import tests """""""""""" Just because a package successfully built does not mean that it built correctly. The most reliable test of whether or not the package was correctly installed is to attempt to import all of the modules that get installed. To get a list of modules, run the following command in the source directory: .. code-block:: pycon >>> import setuptools >>> setuptools.find_packages() ['numpy', 'numpy._build_utils', 'numpy.compat', 'numpy.core', 'numpy.distutils', 'numpy.doc', 'numpy.f2py', 'numpy.fft', 'numpy.lib', 'numpy.linalg', 'numpy.ma', 'numpy.matrixlib', 'numpy.polynomial', 'numpy.random', 'numpy.testing', 'numpy.core.code_generators', 'numpy.distutils.command', 'numpy.distutils.fcompiler'] Large, complex packages like ``numpy`` will return a long list of packages, while other packages like ``six`` will return an empty list. ``py-six`` installs a single ``six.py`` file. In Python packaging lingo, a "package" is a directory containing files like: .. code-block:: none foo/__init__.py foo/bar.py foo/baz.py whereas a "module" is a single Python file. The ``PythonPackage`` base class automatically detects these package and module names for you. If, for whatever reason, the module names detected are wrong, you can provide the names yourself by overriding ``import_modules`` like so: .. code-block:: python import_modules = ["six"] Sometimes the list of module names to import depends on how the package was built. For example, the ``py-pyyaml`` package has a ``+libyaml`` variant that enables the build of a faster optimized version of the library. If the user chooses ``~libyaml``, only the ``yaml`` library will be importable. If the user chooses ``+libyaml``, both the ``yaml`` and ``yaml.cyaml`` libraries will be available. This can be expressed like so: .. code-block:: python @property def import_modules(self): modules = ["yaml"] if self.spec.satisfies("+libyaml"): modules.append("yaml.cyaml") return modules These tests often catch missing dependencies and non-RPATHed libraries. Make sure not to add modules/packages containing the word "test", as these likely won't end up in the installation directory, or may require test dependencies like pytest to be installed. Instead of defining the ``import_modules`` explicitly, only the subset of module names to be skipped can be defined by using ``skip_modules``. If a defined module has submodules, they are skipped as well, e.g., in case the ``plotting`` modules should be excluded from the automatically detected ``import_modules`` ``["nilearn", "nilearn.surface", "nilearn.plotting", "nilearn.plotting.data"]`` set: .. code-block:: python skip_modules = ["nilearn.plotting"] This will set ``import_modules`` to ``["nilearn", "nilearn.surface"]``. Import tests can be run during the installation using ``spack install --test=root`` or at any time after the installation using ``spack test run``. Unit tests """""""""" The package may have its own unit or regression tests. Spack can run these tests during the installation by adding test methods after installation. For example, ``py-numpy`` adds the following as a check to run after the ``install`` phase: .. code-block:: python @run_after("install") @on_package_attributes(run_tests=True) def install_test(self): with working_dir("spack-test", create=True): python("-c", "import numpy; numpy.test('full', verbose=2)") when testing is enabled during the installation (i.e., ``spack install --test=root``). .. note:: Additional information is available on :ref:`install phase tests `. Setup file in a sub-directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Many C/C++ libraries provide optional Python bindings in a subdirectory. To tell pip which directory to build from, you can override the ``build_directory`` attribute. For example, if a package provides Python bindings in a ``python`` directory, you can use: .. code-block:: python build_directory = "python" PythonPackage vs. packages that use Python ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are many packages that make use of Python, but packages that depend on Python are not necessarily ``PythonPackage``'s. Choosing a build system """"""""""""""""""""""" First of all, you need to select a build system. ``spack create`` usually does this for you, but if for whatever reason you need to do this manually, choose ``PythonPackage`` if and only if the package contains one of the following files: * ``pyproject.toml`` * ``setup.py`` * ``setup.cfg`` Choosing a package name """"""""""""""""""""""" Selecting the appropriate package name is a little more complicated than choosing the build system. By default, ``spack create`` will prepend ``py-`` to the beginning of the package name if it detects that the package uses the ``PythonPackage`` build system. However, there are occasionally packages that use ``PythonPackage`` that shouldn't start with ``py-``. For example: * awscli * aws-parallelcluster * busco * easybuild * httpie * mercurial * scons * snakemake The thing these packages have in common is that they are command-line tools that just so happen to be written in Python. Someone who wants to install ``mercurial`` with Spack isn't going to realize that it is written in Python, and they certainly aren't going to assume the package is called ``py-mercurial``. For this reason, we manually renamed the package to ``mercurial``. Likewise, there are occasionally packages that don't use the ``PythonPackage`` build system but should still be prepended with ``py-``. For example: * py-genders * py-py2cairo * py-pygobject * py-pygtk * py-pyqt * py-pyserial * py-sip * py-xpyb These packages are primarily used as Python libraries, not as command-line tools. You may see C/C++ packages that have optional Python language bindings, such as: * antlr * cantera * conduit * pagmo * vtk Don't prepend these kinds of packages with ``py-``. When in doubt, think about how this package will be used. Is it primarily a Python library that will be imported in other Python scripts? Or is it a command-line tool, or C/C++/Fortran program with optional Python modules? The former should be prepended with ``py-``, while the latter should not. ``extends`` vs. ``depends_on`` """""""""""""""""""""""""""""" As mentioned in the :ref:`Packaging Guide `, ``extends`` and ``depends_on`` are very similar, but ``extends`` ensures that the extension and extendee share the same prefix in views. This allows the user to import a Python module without having to add that module to ``PYTHONPATH``. Additionally, ``extends("python")`` adds a dependency on the package ``python-venv``. This improves isolation from the system, whether it's during the build or at runtime: user and system site packages cannot accidentally be used by any package that ``extends("python")``. As a rule of thumb: if a package does not install any Python modules of its own, and merely puts a Python script in the ``bin`` directory, then there is no need for ``extends``. If the package installs modules in the ``site-packages`` directory, it requires ``extends``. Executing ``python`` during the build """"""""""""""""""""""""""""""""""""" Whenever you need to execute a Python command or pass the path of the Python interpreter to the build system, it is best to use the global variable ``python`` directly. For example: .. code-block:: python @run_before("install") def recythonize(self): python("setup.py", "clean") # use the `python` global As mentioned in the previous section, ``extends("python")`` adds an automatic dependency on ``python-venv``, which is a virtual environment that guarantees build isolation. The ``python`` global always refers to the correct Python interpreter, whether the package uses ``extends("python")`` or ``depends_on("python")``. Alternatives to Spack ^^^^^^^^^^^^^^^^^^^^^ PyPI has hundreds of thousands of packages that are not yet in Spack, and ``pip`` may be a perfectly valid alternative to using Spack. The main advantage of Spack over ``pip`` is its ability to compile non-Python dependencies. It can also build cythonized versions of a package or link to an optimized BLAS/LAPACK library like MKL, resulting in calculations that run orders of magnitude faster. Spack does not offer a significant advantage over other Python-management systems for installing and using tools like flake8 and sphinx. But if you need packages with non-Python dependencies like numpy and scipy, Spack will be very valuable to you. Anaconda is another great alternative to Spack, and comes with its own ``conda`` package manager. Like Spack, Anaconda is capable of compiling non-Python dependencies. Anaconda contains many Python packages that are not yet in Spack, and Spack contains many Python packages that are not yet in Anaconda. The main advantage of Spack over Anaconda is its ability to choose a specific compiler and BLAS/LAPACK or MPI library. Spack also has better platform support for supercomputers, and can build optimized binaries for your specific microarchitecture. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on Python packaging, see: * https://packaging.python.org/ For more information on build and installation frontend tools, see: * pip: https://pip.pypa.io/ * build: https://pypa-build.readthedocs.io/ * installer: https://installer.readthedocs.io/ For more information on build backend tools, see: * setuptools: https://setuptools.pypa.io/ * flit: https://flit.pypa.io/ * poetry: https://python-poetry.org/ * hatchling: https://hatch.pypa.io/latest/ * meson: https://meson-python.readthedocs.io/ * pdm: https://pdm.fming.dev/latest/ ================================================ FILE: lib/spack/docs/build_systems/qmakepackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn about the QMake build system in Spack, a script generator for Qt-based projects. .. _qmakepackage: QMake ------ Much like Autotools and CMake, QMake is a build-script generator designed by the developers of Qt. In its simplest form, Spack's ``QMakePackage`` runs the following steps: .. code-block:: console $ qmake $ make $ make check # optional $ make install QMake does not appear to have a standardized way of specifying the installation directory, so you may have to set environment variables or edit ``*.pro`` files to get things working properly. QMake packages will depend on the virtual ``qmake`` package which is provided by multiple versions of Qt: ``qt`` provides Qt up to Qt5, and ``qt-base`` provides Qt from version Qt6 onwards. This split was motivated by the desire to split the single Qt package into its components to allow for more fine-grained installation. To depend on a specific version, refer to the documentation on :ref:`virtual-dependencies`. Phases ^^^^^^ The ``QMakeBuilder`` and ``QMakePackage`` base classes come with the following phases: #. ``qmake`` - generate Makefiles #. ``build`` - build the project #. ``install`` - install the project By default, these phases run: .. code-block:: console $ qmake $ make $ make install Any of these phases can be overridden in your package as necessary. There is also a ``check`` method that looks for a ``check`` target in the Makefile. If a ``check`` target exists and the user runs: .. code-block:: console $ spack install --test=root Spack will run ``make check`` after the build phase. Important files ^^^^^^^^^^^^^^^ Packages that use the QMake build system can be identified by the presence of a ``.pro`` file. This file declares things like build instructions and dependencies. One thing to look for is the ``minQtVersion`` function: .. code-block:: none minQtVersion(5, 6, 0) This means that Qt 5.6.0 is the earliest release that will work. You should specify this in a ``depends_on`` statement. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ At the bare minimum, packages that use the QMake build system need a ``qt`` dependency. Since this is always the case, the ``QMakePackage`` base class already contains: .. code-block:: python depends_on("qt", type="build") If you want to specify a particular version requirement, or need to link to the ``qt`` libraries, you can override this in your package: .. code-block:: python depends_on("qt@5.6.0:") Passing arguments to qmake ^^^^^^^^^^^^^^^^^^^^^^^^^^ If you need to pass any arguments to the ``qmake`` call, you can override the ``qmake_args`` method like so: .. code-block:: python def qmake_args(self): return ["-recursive"] This method can be used to pass flags as well as variables. ``*.pro`` file in a sub-directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If the ``*.pro`` file used to tell QMake how to build the package is found in a sub-directory, you can tell Spack to run all phases in this sub-directory by adding the following to the package: .. code-block:: python build_directory = "src" External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the QMake build system, see: http://doc.qt.io/qt-5/qmake-manual.html ================================================ FILE: lib/spack/docs/build_systems/racketpackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn about the Racket build system in Spack for installing Racket packages and modules. .. _racketpackage: Racket ------ Much like Python, Racket packages and modules have their own special build system. To learn more about the specifics of the Racket package system, please refer to the `Racket Docs `_. Phases ^^^^^^ The ``RacketBuilder`` and ``RacketPackage`` base classes provide an ``install`` phase that can be overridden, corresponding to the use of: .. code-block:: console $ raco pkg install Caveats ^^^^^^^ In principle, ``raco`` supports a second, ``setup`` phase; however, we have not implemented this separately, as in normal circumstances, ``install`` also handles running ``setup`` automatically. Unlike Python, Racket currently only supports two installation scopes for packages, user or system, and keeps a registry of installed packages at each scope in its configuration files. This means we can't simply compose a "``RACKET_PATH``" environment variable listing all of the places packages are installed, and update this at will. Unfortunately, this means that all currently installed packages which extend Racket via ``raco pkg install`` are accessible whenever Racket is accessible. Additionally, because Spack does not implement uninstall hooks, uninstalling a Spack ``rkt-`` package will have no effect on the ``raco`` installed packages visible to your Racket installation. Instead, you must manually run ``raco pkg remove`` to keep the two package managers in a mutually consistent state. ================================================ FILE: lib/spack/docs/build_systems/rocmpackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn about the ROCmPackage helper in Spack, which provides standard variants, dependencies, and conflicts for building packages that target AMD GPUs. .. _rocmpackage: ROCm ------ The ``ROCmPackage`` is not a build system but a helper package. Like ``CudaPackage``, it provides standard variants, dependencies, and conflicts to facilitate building packages targeting AMD GPUs. You can find the source for this package (and suggestions for setting up your ``packages.yaml`` file) at ``__. Variants ^^^^^^^^ This package provides the following variants: * **rocm** This variant is used to enable/disable building with ``rocm``. The default is disabled (or ``False``). * **amdgpu_target** This variant supports the optional specification of the AMD GPU architecture. Valid values are the names of the GPUs (e.g., ``gfx701``), which are maintained in the ``amdgpu_targets`` property. Dependencies ^^^^^^^^^^^^ This package defines basic ROCm dependencies, including ``llvm`` and ``hip``. Conflicts ^^^^^^^^^ Conflicts are used to prevent builds with known bugs or issues. This package already requires that the ``amdgpu_target`` always be specified for ROCm builds. It also defines a conflict that prevents builds with an ``amdgpu_target`` when ``rocm`` is disabled. Refer to :ref:`packaging_conflicts` for more information on package conflicts. Methods ^^^^^^^ This package provides one custom helper method, which is used to build standard AMD HIP compiler flags. **hip_flags** This built-in static method returns the appropriately formatted ``--amdgpu-target`` build option for ``hipcc``. This method must be explicitly called when you are creating the arguments for your build in order to use the values. Usage ^^^^^^ This helper package can be added to your package by adding it as a base class of your package. For example, you can add it to your :ref:`CMakePackage `-based package as follows: .. code-block:: python :emphasize-lines: 1,3-6,13-21 class MyRocmPackage(CMakePackage, ROCmPackage): ... # Ensure +rocm and amdgpu_targets are passed to dependencies depends_on("mydeppackage", when="+rocm") for val in ROCmPackage.amdgpu_targets: depends_on(f"mydeppackage amdgpu_target={val}", when=f"amdgpu_target={val}") ... def cmake_args(self): spec = self.spec args = [] ... if spec.satisfies("+rocm"): # Set up the HIP macros needed by the build args.extend(["-DENABLE_HIP=ON", f"-DHIP_ROOT_DIR={spec['hip'].prefix}"]) rocm_archs = spec.variants["amdgpu_target"].value if "none" not in rocm_archs: args.append(f"-DHIP_HIPCC_FLAGS=--amdgpu-target={','.join(rocm_archs)}") else: # Ensure build with HIP is disabled args.append("-DENABLE_HIP=OFF") ... return args ... assuming only the ``ENABLE_HIP``, ``HIP_ROOT_DIR``, and ``HIP_HIPCC_FLAGS`` macros are required to be set and the only dependency needing ROCm options is ``mydeppackage``. You will need to customize the flags as needed for your build. This example also illustrates how to check for the ``rocm`` variant using ``self.spec`` and how to retrieve the ``amdgpu_target`` variant's value using ``self.spec.variants["amdgpu_target"].value``. All five packages using ``ROCmPackage`` as of January 2021 also use the :ref:`CudaPackage `. So, it is worth looking at those packages to get ideas for creating a package that can support both ``cuda`` and ``rocm``. ================================================ FILE: lib/spack/docs/build_systems/rpackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to packaging R libraries and applications with Spack, including support for CRAN, Bioconductor, and GitHub sources. .. _rpackage: R ------ Like Python, R has its own built-in build system. The R build system is remarkably uniform and well-tested. This makes it one of the easiest build systems to create new Spack packages for. Phases ^^^^^^ The ``RBuilder`` and ``RPackage`` base classes have a single phase: #. ``install`` - install the package By default, this phase runs the following command: .. code-block:: console $ R CMD INSTALL --library=/path/to/installation/prefix/rlib/R/library . Finding R packages ^^^^^^^^^^^^^^^^^^ The vast majority of R packages are hosted on CRAN - The Comprehensive R Archive Network. If you are looking for a particular R package, search for "CRAN " and you should quickly find what you want. If it isn't on CRAN, try Bioconductor, another common R repository. For the purposes of this tutorial, we will be walking through `r-caret `_ as an example. If you search for "CRAN caret", you will quickly find what you are looking for at https://cran.r-project.org/package=caret. https://cran.r-project.org is the main CRAN website. However, CRAN also has a https://cloud.r-project.org site that automatically redirects to `mirrors around the world `_. For stability and performance reasons, we will use https://cloud.r-project.org/package=caret. If you search for "Package source", you will find the download URL for the latest release. Use this URL with ``spack create`` to create a new package. Package name ^^^^^^^^^^^^ The first thing you'll notice is that Spack prepends ``r-`` to the front of the package name. This is how Spack separates R extensions from the rest of the packages in Spack. Without this, we would end up with package name collisions more frequently than we would like. For instance, there are already packages for both: * ``ape`` and ``r-ape`` * ``curl`` and ``r-curl`` * ``gmp`` and ``r-gmp`` * ``jpeg`` and ``r-jpeg`` * ``openssl`` and ``r-openssl`` * ``uuid`` and ``r-uuid`` * ``xts`` and ``r-xts`` Many popular programs written in C/C++ are later ported to R as a separate project. Description ^^^^^^^^^^^ The first thing you'll need to add to your new package is a description. The top of the homepage for ``caret`` lists the following description: Classification and Regression Training Misc functions for training and plotting classification and regression models. The first line is a short description (title) and the second line is a long description. In this case the description is only one line but often the description is several lines. Spack makes use of both short and long descriptions and convention is to use both when creating an R package. Homepage ^^^^^^^^ If you look at the bottom of the page, you'll see: Linking: Please use the canonical form https://CRAN.R-project.org/package=caret to link to this page. Please uphold the wishes of the CRAN admins and use https://cloud.r-project.org/package=caret as the homepage instead of https://cloud.r-project.org/web/packages/caret/index.html. The latter may change without notice. URL ^^^^^^ As previously mentioned, the download URL for the latest release can be found by searching "Package source" on the homepage. List URL ^^^^^^^^ CRAN maintains a single webpage containing the latest release of every single package: https://cloud.r-project.org/src/contrib/ Of course, as soon as a new release comes out, the version you were using in your package is no longer available at that URL. It is moved to an archive directory. If you search for "Old sources", you will find: https://cloud.r-project.org/src/contrib/Archive/caret If you only specify the URL for the latest release, your package will no longer be able to fetch that version as soon as a new release comes out. To get around this, add the archive directory as a ``list_url``. Bioconductor packages ^^^^^^^^^^^^^^^^^^^^^ Bioconductor packages are set up in a similar way to CRAN packages, but there are some very important distinctions. Bioconductor packages can be found at: https://bioconductor.org/. Bioconductor packages are R packages and so follow the same packaging scheme as CRAN packages. What is different is that Bioconductor itself is versioned and released. This scheme, using the Bioconductor package installer, allows further specification of the minimum version of R as well as further restrictions on the dependencies between packages than what is possible with the native R packaging system. Spack cannot replicate these extra features and thus Bioconductor packages in Spack need to be managed as a group during updates in order to maintain package consistency with Bioconductor itself. Another key difference is that, while previous versions of packages are available, they are not available from a site that can be programmatically set, thus a ``list_url`` attribute cannot be used. However, each package is also available in a git repository, with branches corresponding to each Bioconductor release. Thus, it is always possible to retrieve the version of any package corresponding to a Bioconductor release simply by fetching the branch that corresponds to the Bioconductor release of the package repository. For this reason, Spack Bioconductor R packages use the git repository, with the commit of the respective branch used in the ``version()`` attribute of the package. cran and bioc attributes ^^^^^^^^^^^^^^^^^^^^^^^^ Much like the ``pypi`` attribute for Python packages, due to the fact that R packages are obtained from specific repositories, it is possible to set up shortcut attributes that can be used to set ``homepage``, ``url``, ``list_url``, and ``git``. For example, the following ``cran`` attribute: .. code-block:: python cran = "caret" is equivalent to: .. code-block:: python homepage = "https://cloud.r-project.org/package=caret" url = "https://cloud.r-project.org/src/contrib/caret_6.0-86.tar.gz" list_url = "https://cloud.r-project.org/src/contrib/Archive/caret" Likewise, the following ``bioc`` attribute: .. code-block:: python bioc = "BiocVersion" is equivalent to: .. code-block:: python homepage = "https://bioconductor.org/packages/BiocVersion/" git = "https://git.bioconductor.org/packages/BiocVersion" Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ As an extension of the R ecosystem, your package will obviously depend on R to build and run. Normally, we would use ``depends_on`` to express this, but for R packages, we use ``extends``. This implies a special dependency on R, which is used to set environment variables such as ``R_LIBS`` uniformly. Since every R package needs this, the ``RPackage`` base class contains: .. code-block:: python extends("r") Take a close look at the homepage for ``caret``. If you look at the "Depends" section, you'll notice that ``caret`` depends on "R (≥ 3.2.0)". You should add this to your package like so: .. code-block:: python depends_on("r@3.2.0:", type=("build", "run")) R dependencies ^^^^^^^^^^^^^^ R packages are often small and follow the classic Unix philosophy of doing one thing well. They are modular and usually depend on several other packages. You may find a single package with over a hundred dependencies. Luckily, R packages are well-documented and list all of their dependencies in the following sections: * Depends * Imports * LinkingTo As far as Spack is concerned, all three of these dependency types correspond to ``type=("build", "run")``, so you don't have to worry about the details. If you are curious what they mean, https://github.com/spack/spack/issues/2951 has a pretty good summary: ``Depends`` is required and will cause those R packages to be *attached*, that is, their APIs are exposed to the user. ``Imports`` *loads* packages so that *the package* importing these packages can access their APIs, while *not* being exposed to the user. When a user calls ``library(foo)`` s/he *attaches* package ``foo`` and all of the packages under ``Depends``. Any function in one of these packages can be called directly as ``bar()``. If there are conflicts, a user can also specify ``pkgA::bar()`` and ``pkgB::bar()`` to distinguish between them. Historically, there was only ``Depends`` and ``Suggests``, hence the confusing names. Today, maybe ``Depends`` would have been named ``Attaches``. The ``LinkingTo`` is not perfect and there was recently an extensive discussion about API/ABI among other things on the R-devel mailing list among very skilled R developers: * https://stat.ethz.ch/pipermail/r-devel/2016-December/073505.html * https://stat.ethz.ch/pipermail/r-devel/2017-January/073647.html Some packages also have a fourth section: * Suggests These are optional, rarely-used dependencies that a user might find useful. You should **NOT** add these dependencies to your package. R packages already have enough dependencies as it is, and adding optional dependencies can really slow down the concretization process. They can also introduce circular dependencies. A fifth rarely used section is: * Enhances This means that the package can be used as an optional dependency for another package. Again, these packages should **NOT** be listed as dependencies. Core, recommended, and non-core packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you look at "Depends", "Imports", and "LinkingTo", you will notice 3 different types of packages: Core packages """"""""""""" If you look at the ``caret`` homepage, you'll notice a few dependencies that don't have a link to the package, like ``methods``, ``stats``, and ``utils``. These packages are part of the core R distribution and are tied to the R version installed. You can basically consider these to be "R itself". These are so essential to R that it would not make sense for them to be updated via CRAN. If you did, you would basically get a different version of R. Thus, they're updated when R is updated. You can find a list of these core libraries at: https://github.com/wch/r-source/tree/trunk/src/library Recommended packages """""""""""""""""""" When you install R, there is an option called ``--with-recommended-packages``. This flag causes the R installation to include a few "Recommended" packages (legacy term). They are for historical reasons quite tied to the core R distribution, developed by the R core team or people closely related to it. The R core distribution "knows" about these packages, but they are indeed distributed via CRAN. Because they're distributed via CRAN, they can also be updated between R version releases. Spack explicitly adds the ``--without-recommended-packages`` flag to prevent the installation of these packages. Due to the way Spack handles package activation (symlinking packages to the R installation directory), pre-existing recommended packages will cause conflicts for already-existing files. We could either not include these recommended packages in Spack and require them to be installed through ``--with-recommended-packages``, or we could not install them with R and let users choose the version of the package they want to install. We chose the latter. Since these packages are so commonly distributed with the R system, many developers may assume these packages exist and fail to list them as dependencies. Watch out for this. You can find a list of these recommended packages at: https://github.com/wch/r-source/blob/trunk/share/make/vars.mk Non-core packages """"""""""""""""" These are packages that are neither "core" nor "recommended". There are more than 10,000 of these packages hosted on CRAN alone. For each of these package types, if you see that a specific version is required, for example, "lattice (≥ 0.20)", please add this information to the dependency: .. code-block:: python depends_on("r-lattice@0.20:", type=("build", "run")) Non-R dependencies ^^^^^^^^^^^^^^^^^^ Some packages depend on non-R libraries for linking. Check out the `r-stringi `_ package for an example: https://cloud.r-project.org/package=stringi. If you search for the text "SystemRequirements", you will see: ICU4C (>= 52, optional) This is how non-R dependencies are listed. Make sure to add these dependencies. The default dependency type should suffice. Passing arguments to the installation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Some R packages provide additional flags that can be passed to ``R CMD INSTALL``, often to locate non-R dependencies. `r-rmpi `_ is an example of this, as it uses flags for linking to an MPI library. To pass these to the installation command, you can override ``configure_args`` like so: .. code-block:: python def configure_args(self): mpi_name = self.spec["mpi"].name # The type of MPI. Supported values are: # OPENMPI, LAM, MPICH, MPICH2, or CRAY if mpi_name == "openmpi": Rmpi_type = "OPENMPI" elif mpi_name == "mpich": Rmpi_type = "MPICH2" else: raise InstallError("Unsupported MPI type") return [ "--with-Rmpi-type={0}".format(Rmpi_type), "--with-mpi={0}".format(spec["mpi"].prefix), ] There is a similar ``configure_vars`` function that can be overridden to pass variables to the build. Alternatives to Spack ^^^^^^^^^^^^^^^^^^^^^ CRAN hosts over 10,000 R packages, most of which are not in Spack. Many users may not need the advanced features of Spack, and may prefer to install R packages the normal way: .. code-block:: console $ R > install.packages("ggplot2") R will search CRAN for the ``ggplot2`` package and install all necessary dependencies for you. If you want to update all installed R packages to the latest release, you can use: .. code-block:: console > update.packages(ask = FALSE) This works great for users who have internet access, but those on an air-gapped cluster will find it easier to let Spack build a download mirror and install these packages for you. Where Spack really shines is its ability to install non-R dependencies and link to them properly, something the R installation mechanism cannot handle. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on installing R packages, see: https://stat.ethz.ch/R-manual/R-devel/library/utils/html/INSTALL.html For more information on writing R packages, see: https://cloud.r-project.org/doc/manuals/r-release/R-exts.html In particular, https://cloud.r-project.org/doc/manuals/r-release/R-exts.html#Package-Dependencies has a great explanation of the difference between Depends, Imports, and LinkingTo. ================================================ FILE: lib/spack/docs/build_systems/rubypackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Discover the Ruby build system in Spack for installing Ruby gems, with support for gemspec, Rakefile, and pre-packaged .gem files. .. _rubypackage: Ruby ------ Like Perl, Python, and R, Ruby has its own build system for installing Ruby gems. Phases ^^^^^^ The ``RubyBuilder`` and ``RubyPackage`` base classes provide the following phases that can be overridden: #. ``build`` - build everything needed to install #. ``install`` - install everything from build directory For packages that come with a ``*.gemspec`` file, these phases run: .. code-block:: console $ gem build *.gemspec $ gem install *.gem For packages that come with a ``Rakefile`` file, these phases run: .. code-block:: console $ rake package $ gem install *.gem For packages that come pre-packaged as a ``*.gem`` file, the build phase is skipped and the install phase runs: .. code-block:: console $ gem install *.gem These are all standard ``gem`` commands and can be found by running: .. code-block:: console $ gem help commands For packages that only distribute ``*.gem`` files, these files can be downloaded with the ``expand=False`` option in the ``version`` directive. The build phase will be automatically skipped. Important files ^^^^^^^^^^^^^^^ When building from source, Ruby packages can be identified by the presence of any of the following files: * ``*.gemspec`` * ``Rakefile`` * ``setup.rb`` (not yet supported) However, not all Ruby packages are released as source code. Some are only released as ``*.gem`` files. These files can be extracted using: .. code-block:: console $ gem unpack *.gem Description ^^^^^^^^^^^ The ``*.gemspec`` file may contain something like: .. code-block:: ruby summary = "An implementation of the AsciiDoc text processor and publishing toolchain" description = "A fast, open source text processor and publishing toolchain for converting AsciiDoc content to HTML 5, DocBook 5, and other formats." Either of these can be used for the description of the Spack package. Homepage ^^^^^^^^ The ``*.gemspec`` file may contain something like: .. code-block:: ruby homepage = "https://asciidoctor.org" This should be used as the official homepage of the Spack package. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ All Ruby packages require Ruby at build and run-time. For this reason, the base class contains: .. code-block:: python extends("ruby") The ``*.gemspec`` file may contain something like: .. code-block:: ruby required_ruby_version = ">= 2.3.0" This can be added to the Spack package using: .. code-block:: python depends_on("ruby@2.3.0:", type=("build", "run")) Ruby dependencies ^^^^^^^^^^^^^^^^^ When you install a package with ``gem``, it reads the ``*.gemspec`` file in order to determine the dependencies of the package. If the dependencies are not yet installed, ``gem`` downloads them and installs them for you. This may sound convenient, but Spack cannot rely on this behavior for two reasons: #. Spack needs to be able to install packages on air-gapped networks. If there is no internet connection, ``gem`` can't download the package dependencies. By explicitly listing every dependency in the ``package.py``, Spack knows what to download ahead of time. #. Duplicate installations of the same dependency may occur. Spack supports *activation* of Ruby extensions, which involves symlinking the package installation prefix to the Ruby installation prefix. If your package is missing a dependency, that dependency will be installed to the installation directory of the same package. If you try to activate the package + dependency, it may cause a problem if that package has already been activated. For these reasons, you must always explicitly list all dependencies. Although the documentation may list the package's dependencies, often the developers assume people will use ``gem`` and won't have to worry about it. Always check the ``*.gemspec`` file to find the true dependencies. Check for the following clues in the ``*.gemspec`` file: * ``add_runtime_dependency`` These packages are required for installation. * ``add_dependency`` This is an alias for ``add_runtime_dependency`` * ``add_development_dependency`` These packages are optional dependencies used for development. They should not be added as dependencies of the package. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on Ruby packaging, see: https://guides.rubygems.org/ ================================================ FILE: lib/spack/docs/build_systems/sconspackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn about the SCons build system in Spack, a Python-based tool that handles building and linking without relying on Makefiles. .. _sconspackage: SCons ------ SCons is a general-purpose build system that does not rely on Makefiles to build software. SCons is written in Python, and handles all building and linking itself. As far as build systems go, SCons is very non-uniform. It provides a common framework for developers to write build scripts, but the build scripts themselves can vary drastically. Some developers add subcommands like: .. code-block:: console $ scons clean $ scons build $ scons test $ scons install Others don't add any subcommands. Some have configuration options that can be specified through variables on the command line. Others don't. Phases ^^^^^^ As previously mentioned, SCons allows developers to add subcommands like ``build`` and ``install``, but by default, installation usually looks like: .. code-block:: console $ scons $ scons install To facilitate this, the ``SConsBuilder`` and ``SConsPackage`` base classes provide the following phases: #. ``build`` - build the package #. ``install`` - install the package Package developers often add unit tests that can be invoked with ``scons test`` or ``scons check``. Spack provides a ``build_test`` method to handle this. Since we don't know which one the package developer chose, the ``build_test`` method does nothing by default, but can be easily overridden like so: .. code-block:: python def build_test(self): scons("check") Important files ^^^^^^^^^^^^^^^ SCons packages can be identified by their ``SConstruct`` files. These files handle everything from setting up subcommands and command-line options to linking and compiling. One thing to look for is the ``EnsureSConsVersion`` function: .. code-block:: none EnsureSConsVersion(2, 3, 0) This means that SCons 2.3.0 is the earliest release that will work. You should specify this in a ``depends_on`` statement. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ At the bare minimum, packages that use the SCons build system need a ``scons`` dependency. Since this is always the case, the ``SConsPackage`` base class already contains: .. code-block:: python depends_on("scons", type="build") If you want to specify a particular version requirement, you can override this in your package: .. code-block:: python depends_on("scons@2.3.0:", type="build") Finding available options ^^^^^^^^^^^^^^^^^^^^^^^^^ The first place to start when looking for a list of valid options to build a package is ``scons --help``. Some packages like `kahip `_ don't bother overwriting the default SCons help message, so this isn't very useful, but other packages like `serf `_ print a list of valid command-line variables: .. code-block:: console $ scons --help scons: Reading SConscript files ... Checking for GNU-compatible C compiler...yes scons: done reading SConscript files. PREFIX: Directory to install under ( /path/to/PREFIX ) default: /usr/local actual: /usr/local LIBDIR: Directory to install architecture dependent libraries under ( /path/to/LIBDIR ) default: $PREFIX/lib actual: /usr/local/lib APR: Path to apr-1-config, or to APR's install area ( /path/to/APR ) default: /usr actual: /usr APU: Path to apu-1-config, or to APR's install area ( /path/to/APU ) default: /usr actual: /usr OPENSSL: Path to OpenSSL's install area ( /path/to/OPENSSL ) default: /usr actual: /usr ZLIB: Path to zlib's install area ( /path/to/ZLIB ) default: /usr actual: /usr GSSAPI: Path to GSSAPI's install area ( /path/to/GSSAPI ) default: None actual: None DEBUG: Enable debugging info and strict compile warnings (yes|no) default: False actual: False APR_STATIC: Enable using a static compiled APR (yes|no) default: False actual: False CC: Command name or path of the C compiler default: None actual: gcc CFLAGS: Extra flags for the C compiler (space-separated) default: None actual: LIBS: Extra libraries passed to the linker, e.g. "-l -l" (space separated) default: None actual: None LINKFLAGS: Extra flags for the linker (space-separated) default: None actual: CPPFLAGS: Extra flags for the C preprocessor (space separated) default: None actual: None Use scons -H for help about command-line options. More advanced packages like `cantera `_ use ``scons --help`` to print a list of subcommands: .. code-block:: console $ scons --help scons: Reading SConscript files ... SCons build script for Cantera Basic usage: 'scons help' - print a description of user-specifiable options. 'scons build' - Compile Cantera and the language interfaces using default options. 'scons clean' - Delete files created while building Cantera. '[sudo] scons install' - Install Cantera. '[sudo] scons uninstall' - Uninstall Cantera. 'scons test' - Run all tests which did not previously pass or for which the results may have changed. 'scons test-reset' - Reset the passing status of all tests. 'scons test-clean' - Delete files created while running the tests. 'scons test-help' - List available tests. 'scons test-NAME' - Run the test named "NAME". 'scons dump' - Dump the state of the SCons environment to the screen instead of doing , e.g. 'scons build dump'. For debugging purposes. 'scons samples' - Compile the C++ and Fortran samples. 'scons msi' - Build a Windows installer (.msi) for Cantera. 'scons sphinx' - Build the Sphinx documentation 'scons doxygen' - Build the Doxygen documentation You'll notice that cantera provides a ``scons help`` subcommand. Running ``scons help`` prints a list of valid command-line variables. Passing arguments to SCons ^^^^^^^^^^^^^^^^^^^^^^^^^^ Now that you know what arguments the project accepts, you can add them to the package build phase. This is done by overriding ``build_args`` like so: .. code-block:: python def build_args(self, spec, prefix): args = [ f"PREFIX={prefix}", f"ZLIB={spec['zlib'].prefix}", ] if spec.satisfies("+debug"): args.append("DEBUG=yes") else: args.append("DEBUG=no") return args ``SConsPackage`` also provides an ``install_args`` function that you can override to pass additional arguments to ``scons install``. Compiler wrappers ^^^^^^^^^^^^^^^^^ By default, SCons builds all packages in a separate execution environment, and doesn't pass any environment variables from the user environment. Even changes to ``PATH`` are not propagated unless the package developer does so. This is particularly troublesome for Spack's compiler wrappers, which depend on environment variables to manage dependencies and linking flags. In many cases, SCons packages are not compatible with Spack's compiler wrappers, and linking must be done manually. First of all, check the list of valid options for anything relating to environment variables. For example, cantera has the following option: .. code-block:: none * env_vars: [ string ] Environment variables to propagate through to SCons. Either the string "all" or a comma separated list of variable names, e.g. "LD_LIBRARY_PATH,HOME". - default: "LD_LIBRARY_PATH,PYTHONPATH" In the case of cantera, using ``env_vars=all`` allows us to use Spack's compiler wrappers. If you don't see an option related to environment variables, try using Spack's compiler wrappers by passing ``spack_cc``, ``spack_cxx``, and ``spack_fc`` via the ``CC``, ``CXX``, and ``FC`` arguments, respectively. If you pass them to the build and you see an error message like: .. code-block:: none Spack compiler must be run from Spack! Input 'SPACK_PREFIX' is missing. you'll know that the package isn't compatible with Spack's compiler wrappers. In this case, you'll have to use the path to the actual compilers, which are stored in ``self.compiler.cc`` and friends. Note that this may involve passing additional flags to the build to locate dependencies, a task normally done by the compiler wrappers. serf is an example of a package with this limitation. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the SCons build system, see: http://scons.org/documentation.html ================================================ FILE: lib/spack/docs/build_systems/sippackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to using the SIP build system in Spack for creating Python bindings for C and C++ libraries. .. _sippackage: SIP ------ SIP is a tool that makes it very easy to create Python bindings for C and C++ libraries. It was originally developed to create PyQt, the Python bindings for the Qt toolkit, but can be used to create bindings for any C or C++ library. SIP comprises a code generator and a Python module. The code generator processes a set of specification files and generates C or C++ code which is then compiled to create the bindings extension module. The SIP Python module provides support functions to the automatically generated code. Phases ^^^^^^ The ``SIPBuilder`` and ``SIPPackage`` base classes come with the following phases: #. ``configure`` - configure the package #. ``build`` - build the package #. ``install`` - install the package By default, these phases run: .. code-block:: console $ sip-build --verbose --target-dir ... $ make $ make install Important files ^^^^^^^^^^^^^^^ Each SIP package comes with a custom configuration file written in Python. For newer packages, this is called ``project.py``, while in older packages, it may be called ``configure.py``. This script contains instructions to build the project. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ ``SIPPackage`` requires several dependencies. Python and SIP are needed at build-time to run the aforementioned configure script. Python is also needed at run-time to actually use the installed Python library. And as we are building Python bindings for C/C++ libraries, Python is also needed as a link dependency. All of these dependencies are automatically added via the base class. .. code-block:: python extends("python", type=("build", "link", "run")) depends_on("py-sip", type="build") Passing arguments to ``sip-build`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Each phase comes with a ```` function that can be used to pass arguments to that particular phase. For example, if you need to pass arguments to the configure phase, you can use: .. code-block:: python def configure_args(self): return ["--no-python-dbus"] A list of valid options can be found by running ``sip-build --help``. Testing ^^^^^^^ Just because a package successfully built does not mean that it built correctly. The most reliable test of whether or not the package was correctly installed is to attempt to import all of the modules that get installed. To get a list of modules, run the following command in the site-packages directory: .. code-block:: pycon >>> import setuptools >>> setuptools.find_packages() [ 'PyQt5', 'PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtHelp', 'PyQt5.QtMultimedia', 'PyQt5.QtMultimediaWidgets', 'PyQt5.QtNetwork', 'PyQt5.QtOpenGL', 'PyQt5.QtPrintSupport', 'PyQt5.QtQml', 'PyQt5.QtQuick', 'PyQt5.QtSvg', 'PyQt5.QtTest', 'PyQt5.QtWebChannel', 'PyQt5.QtWebSockets', 'PyQt5.QtWidgets', 'PyQt5.QtXml', 'PyQt5.QtXmlPatterns' ] Large, complex packages like ``py-pyqt5`` will return a long list of packages, while other packages may return an empty list. These packages only install a single ``foo.py`` file. In Python packaging lingo, a "package" is a directory containing files like: .. code-block:: none foo/__init__.py foo/bar.py foo/baz.py whereas a "module" is a single Python file. The ``SIPPackage`` base class automatically detects these module names for you. If, for whatever reason, the module names detected are wrong, you can provide the names yourself by overriding ``import_modules`` like so: .. code-block:: python import_modules = ["PyQt5"] These tests often catch missing dependencies and non-RPATHed libraries. Make sure not to add modules/packages containing the word "test", as these likely won't end up in the installation directory, or may require test dependencies like pytest to be installed. These tests can be triggered by running ``spack install --test=root`` or by running ``spack test run`` after the installation has finished. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the SIP build system, see: * https://www.riverbankcomputing.com/software/sip/intro * https://www.riverbankcomputing.com/static/Docs/sip/ * https://wiki.python.org/moin/SIP ================================================ FILE: lib/spack/docs/build_systems/sourceforgepackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Discover how to use the SourceforgePackage mixin in Spack to automatically generate download URLs for packages hosted on SourceForge. .. _sourceforgepackage: SourceForge ----------- ``SourceforgePackage`` is a `mixin-class `_. It automatically sets the URL based on a list of SourceForge mirrors listed in ``sourceforge_mirror_path``, which defaults to a half dozen known mirrors. Refer to the `package source `__ for the current list of mirrors used by Spack. Methods ^^^^^^^ This package provides a method for populating mirror URLs. **urls** This method returns a list of possible URLs for package source. It is decorated with `property` so its results are treated as a package attribute. Refer to :ref:`mirrors-of-the-main-url` for information on how Spack uses the ``urls`` attribute during fetching. Usage ^^^^^^ This helper package can be added to your package by adding it as a base class of your package and defining the relative location of an archive file for one version of your software. .. code-block:: python :emphasize-lines: 1,3 class MyPackage(AutotoolsPackage, SourceforgePackage): ... sourceforge_mirror_path = "my-package/mypackage.1.0.0.tar.gz" ... Over 40 packages are using ``SourceforgePackage`` this mix-in as of July 2022 so there are multiple packages to choose from if you want to see a real example. ================================================ FILE: lib/spack/docs/build_systems/wafpackage.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Explore the Waf build system in Spack, a Python-based tool for configuring and building software projects without Makefiles. .. _wafpackage: Waf ------ Like SCons, Waf is a general-purpose build system that does not rely on Makefiles to build software. Phases ^^^^^^ The ``WafBuilder`` and ``WafPackage`` base classes come with the following phases: #. ``configure`` - configure the project #. ``build`` - build the project #. ``install`` - install the project By default, these phases run: .. code-block:: console $ python waf configure --prefix=/path/to/installation/prefix $ python waf build $ python waf install Each of these are standard Waf commands and can be found by running: .. code-block:: console $ python waf --help Each phase provides a ```` function that runs: .. code-block:: console $ python waf -j where ```` is the number of parallel jobs to build with. Each phase also has a ```` function that can pass arguments to this call. All of these functions are empty. The ``configure`` phase automatically adds ``--prefix=/path/to/installation/prefix``, so you don't need to add that in the ``configure_args``. Testing ^^^^^^^ ``WafPackage`` also provides ``test`` and ``installtest`` methods, which are run after the ``build`` and ``install`` phases, respectively. By default, these phases do nothing, but you can override them to run package-specific unit tests. .. code-block:: python def installtest(self): with working_dir("test"): pytest = which("py.test") pytest() Important files ^^^^^^^^^^^^^^^ Each Waf package comes with a custom ``waf`` build script, written in Python. This script contains instructions to build the project. The package also comes with a ``wscript`` file. This file is used to override the default ``configure``, ``build``, and ``install`` phases to customize the Waf project. It also allows developers to override the default ``./waf --help`` message. Check this file to find useful information about dependencies and the minimum versions that are supported. Build system dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^ ``WafPackage`` does not require ``waf`` to build. ``waf`` is only needed to create the ``./waf`` script. Since ``./waf`` is a Python script, Python is needed to build the project. ``WafPackage`` adds the following dependency automatically: .. code-block:: python depends_on("python@2.5:", type="build") Waf only supports Python 2.5 and up. Passing arguments to Waf ^^^^^^^^^^^^^^^^^^^^^^^^ As previously mentioned, each phase comes with a ```` function that can be used to pass arguments to that particular phase. For example, if you need to pass arguments to the build phase, you can use: .. code-block:: python def build_args(self, spec, prefix): args = [] if self.run_tests: args.append("--test") return args A list of valid options can be found by running ``./waf --help``. External documentation ^^^^^^^^^^^^^^^^^^^^^^ For more information on the Waf build system, see: https://waf.io/book/ ================================================ FILE: lib/spack/docs/build_systems.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: An overview of the build systems supported by Spack, with links to detailed documentation for each system. .. _build-systems: Build Systems ============= Spack defines a number of classes that understand how to use common `build systems `_ (Makefiles, CMake, etc.). Spack package definitions can inherit these classes in order to streamline their builds. This guide provides information specific to each particular build system. It assumes that you've read the Packaging Guide :doc:`part 1 ` and :doc:`part 2 ` and expands on these ideas for each distinct build system that Spack supports: .. toctree:: :maxdepth: 1 :caption: Make-based build_systems/makefilepackage .. toctree:: :maxdepth: 1 :caption: Make-incompatible build_systems/mavenpackage build_systems/sconspackage build_systems/wafpackage .. toctree:: :maxdepth: 1 :caption: Build-script generation build_systems/autotoolspackage build_systems/cmakepackage build_systems/cachedcmakepackage build_systems/mesonpackage build_systems/qmakepackage build_systems/sippackage .. toctree:: :maxdepth: 1 :caption: Language-specific build_systems/luapackage build_systems/octavepackage build_systems/perlpackage build_systems/pythonpackage build_systems/rpackage build_systems/racketpackage build_systems/rubypackage .. toctree:: :maxdepth: 1 :caption: Other build_systems/bundlepackage build_systems/cudapackage build_systems/custompackage build_systems/inteloneapipackage build_systems/rocmpackage build_systems/sourceforgepackage For reference, the :py:mod:`Build System API docs ` provide a list of build systems and methods/attributes that can be overridden. If you are curious about the implementation of a particular build system, you can view the source code by running: .. code-block:: console $ spack edit --build-system autotools This will open up the ``AutotoolsPackage`` definition in your favorite editor. In addition, if you are working with a less common build system like QMake, SCons, or Waf, it may be useful to see examples of other packages. You can quickly find examples by running: .. code-block:: console $ spack cd --packages $ grep -l QMakePackage */package.py You can then view these packages with ``spack edit``. This guide is intended to supplement the :py:mod:`Build System API docs ` with examples of how to override commonly used methods. It also provides rules of thumb and suggestions for package developers who are unfamiliar with a particular build system. ================================================ FILE: lib/spack/docs/chain.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to chain Spack installations by pointing one Spack instance to another to use its installed packages. Chaining Spack Installations (upstreams.yaml) ============================================= You can point your Spack installation to another Spack installation to use any packages that are installed there. To register the other Spack instance, you can add it as an entry to ``upstreams.yaml`` at any of the :ref:`configuration-scopes`: .. code-block:: yaml upstreams: spack-instance-1: install_tree: /path/to/other/spack/opt/spack spack-instance-2: install_tree: /path/to/another/spack/opt/spack The ``install_tree`` must point to the ``opt/spack`` directory inside of the Spack base directory, or the location of the ``install_tree`` defined in :ref:`config.yaml `. Once the upstream Spack instance has been added, ``spack find`` will automatically check the upstream instance when querying installed packages, and new package installations for the local Spack installation will use any dependencies that are installed in the upstream instance. The upstream Spack instance has no knowledge of the local Spack instance and may not have the same permissions or ownership as the local Spack instance. This has the following consequences: #. Upstream Spack instances are not locked. Therefore, it is up to users to make sure that the local instance is not using an upstream instance when it is being modified. #. Users should not uninstall packages from the upstream instance. Since the upstream instance does not know about the local instance, it cannot prevent the uninstallation of packages that the local instance depends on. Other details about upstream Spack installations: #. If a package is installed both locally and upstream, the local installation will always be used as a dependency. This can occur if the local Spack installs a package which is not present in the upstream, but later on the upstream Spack instance also installs that package. #. If an upstream Spack instance registers and installs an external package, the local Spack instance will treat this the same as a Spack-installed package. This feature will only work if the upstream Spack instance includes the upstream functionality (i.e., if its commit is after March 27, 2019). Using Multiple Upstream Spack Instances --------------------------------------- A single Spack instance can use multiple upstream Spack installations. Spack will search upstream instances in the order that you list them in your configuration. If your Spack installation refers to instances X and Y, in that order, then instance X must list Y as an upstream in its own ``upstreams.yaml``. Using Modules for Upstream Packages ----------------------------------- The local Spack instance does not generate modules for packages that are installed upstream. The local Spack instance can be configured to use the modules generated by the upstream Spack instance. There are two requirements to use the modules created by an upstream Spack instance: firstly, the upstream instance must do a ``spack module tcl refresh``, which generates an index file that maps installed packages to their modules; secondly, the local Spack instance must add a ``modules`` entry to the configuration: .. code-block:: yaml upstreams: spack-instance-1: install_tree: /path/to/other/spack/opt/spack modules: tcl: /path/to/other/spack/share/spack/modules Each time new packages are installed in the upstream Spack instance, the upstream Spack maintainer should run ``spack module tcl refresh`` (or the corresponding command for the type of module that they intend to use). .. note:: Spack can generate modules that :ref:`automatically load ` the modules of dependency packages. Spack cannot currently do this for modules in upstream packages. ================================================ FILE: lib/spack/docs/command_index.in ================================================ ================= Command Reference ================= This is a reference for all commands in the Spack command line interface. The same information is available through :ref:`spack-help`. Commands that also have sections in the main documentation have a link to "More documentation". ================================================ FILE: lib/spack/docs/conf.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # flake8: noqa # -*- coding: utf-8 -*- # # Spack documentation build configuration file, created by # sphinx-quickstart on Mon Dec 9 15:32:41 2013. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import os import subprocess import sys from glob import glob from typing import List from docutils.statemachine import StringList from pygments.formatters.html import HtmlFormatter from pygments.lexer import RegexLexer, default from pygments.token import * from sphinx.domains.python import PythonDomain from sphinx.ext.apidoc import main as sphinx_apidoc from sphinx.highlighting import PygmentsBridge from sphinx.parsers import RSTParser # -- Spack customizations ----------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. link_name = os.path.abspath("_spack_root") if not os.path.exists(link_name): os.symlink(os.path.abspath("../../.."), link_name, target_is_directory=True) # Add the Spack bin directory to the path so that we can use its output in docs. os.environ["SPACK_ROOT"] = os.path.abspath("_spack_root") os.environ["SPACK_USER_CONFIG_PATH"] = os.path.abspath(".spack") os.environ["PATH"] += os.pathsep + os.path.abspath("_spack_root/bin") # Set an environment variable so that colify will print output like it would to # a terminal. os.environ["COLIFY_SIZE"] = "25x120" os.environ["COLUMNS"] = "120" sys.path[0:0] = [ os.path.abspath("_spack_root/lib/spack/"), os.path.abspath(".spack/spack-packages/repos"), ] # Init the package repo with all git history, so "Last updated on" is accurate. subprocess.call(["spack", "repo", "update"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if os.path.exists(".spack/spack-packages/.git/shallow"): subprocess.call( ["git", "fetch", "--unshallow"], cwd=".spack/spack-packages", stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) # Generate a command index if an update is needed subprocess.call( [ "spack", "commands", "--format=rst", "--header=command_index.in", "--update=command_index.rst", *glob("*rst"), ] ) # # Run sphinx-apidoc # # Remove any previous API docs # Read the Docs doesn't clean up after previous builds # Without this, the API Docs will never actually update # apidoc_args = [ "--force", # Overwrite existing files "--no-toc", # Don't create a table of contents file "--output-dir=.", # Directory to place all output "--module-first", # emit module docs before submodule docs ] sphinx_apidoc( apidoc_args + [ "_spack_root/lib/spack/spack", "_spack_root/lib/spack/spack/vendor", "_spack_root/lib/spack/spack/test", "_spack_root/lib/spack/spack/package.py", ] ) sphinx_apidoc( apidoc_args + [ "--implicit-namespaces", ".spack/spack-packages/repos/spack_repo", ".spack/spack-packages/repos/spack_repo/builtin/packages", ".spack/spack-packages/repos/spack_repo/builtin/build_systems/generic.py", ] ) class NoWhitespaceHtmlFormatter(HtmlFormatter): """HTML formatter that suppresses redundant span elements for Text.Whitespace tokens.""" def _get_css_classes(self, ttype): # For Text.Whitespace return an empty string, which avoids # elements from being generated. return "" if ttype is Text.Whitespace else super()._get_css_classes(ttype) class CustomPygmentsBridge(PygmentsBridge): def get_formatter(self, **options): return NoWhitespaceHtmlFormatter(**options) # Use custom HTML formatter to avoid redundant elements. # See https://github.com/pygments/pygments/issues/1905#issuecomment-3170486995. PygmentsBridge.html_formatter = NoWhitespaceHtmlFormatter from spack.llnl.util.lang import classproperty from spack.spec_parser import SpecTokens # replace classproperty.__get__ to return `self` so Sphinx can document it correctly. Otherwise # it evaluates the callback, and it documents the result, which is not what we want. classproperty.__get__ = lambda self, instance, owner: self class SpecLexer(RegexLexer): """A custom lexer for Spack spec strings and spack commands.""" name = "Spack spec" aliases = ["spec"] filenames = [] tokens = { "root": [ # Looks for `$ command`, which may need spec highlighting. (r"^\$\s+", Generic.Prompt, "command"), (r"#.*?\n", Comment.Single), # Alternatively, we just get a literal spec string, so we move to spec mode. We just # look ahead here, without consuming the spec string. (r"(?=\S+)", Generic.Prompt, "spec"), ], "command": [ # A spack install command is followed by a spec string, which we highlight. ( r"spack(?:\s+(?:-[eC]\s+\S+|--?\S+))*\s+(?:install|uninstall|spec|load|unload|find|info|list|versions|providers|mark|diff|add)(?: +(?:--?\S+)?)*", Text, "spec", ), # Comment (r"\s+#.*?\n", Comment.Single, "command_output"), # Escaped newline should leave us in this mode (r".*?\\\n", Text), # Otherwise, it's the end of the command (r".*?\n", Text, "command_output"), ], "command_output": [ (r"^\$\s+", Generic.Prompt, "#pop"), # new command (r"#.*?\n", Comment.Single), # comments (r".*?\n", Generic.Output), # command output ], "spec": [ # New line terminates the spec string (r"\s*?$", Text, "#pop"), # Dependency, with optional virtual assignment specifier (SpecTokens.START_EDGE_PROPERTIES.regex, Name.Variable, "edge_properties"), (SpecTokens.DEPENDENCY.regex, Name.Variable), # versions (SpecTokens.VERSION_HASH_PAIR.regex, Keyword.Pseudo), (SpecTokens.GIT_VERSION.regex, Keyword.Pseudo), (SpecTokens.VERSION.regex, Keyword.Pseudo), # variants (SpecTokens.PROPAGATED_BOOL_VARIANT.regex, Name.Function), (SpecTokens.BOOL_VARIANT.regex, Name.Function), (SpecTokens.PROPAGATED_KEY_VALUE_PAIR.regex, Name.Function), (SpecTokens.KEY_VALUE_PAIR.regex, Name.Function), # filename (SpecTokens.FILENAME.regex, Text), # Package name (SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME.regex, Name.Class), (SpecTokens.UNQUALIFIED_PACKAGE_NAME.regex, Name.Class), # DAG hash (SpecTokens.DAG_HASH.regex, Text), (SpecTokens.WS.regex, Text), # Also stop at unrecognized tokens (without consuming them) default("#pop"), ], "edge_properties": [ (SpecTokens.KEY_VALUE_PAIR.regex, Name.Function), (SpecTokens.END_EDGE_PROPERTIES.regex, Name.Variable, "#pop"), ], } # Enable todo items todo_include_todos = True # # Disable duplicate cross-reference warnings. # class PatchedPythonDomain(PythonDomain): def resolve_xref(self, env, fromdocname, builder, type, target, node, contnode): if "refspecific" in node: del node["refspecific"] return super().resolve_xref(env, fromdocname, builder, type, target, node, contnode) # # Disable tabs to space expansion in code blocks # since Makefiles require tabs. # class NoTabExpansionRSTParser(RSTParser): def parse(self, inputstring, document): if isinstance(inputstring, str): lines = inputstring.splitlines() inputstring = StringList(lines, document.current_source) super().parse(inputstring, document) def add_package_api_version_line(app, what, name: str, obj, options, lines: List[str]): """Add versionadded directive to package API docstrings""" # We're adding versionadded directive here instead of in spack/package.py because most symbols # are re-exported, and we don't want to modify __doc__ of symbols we don't own. if name.startswith("spack.package."): symbol = name[len("spack.package.") :] for version, symbols in spack.package.api.items(): if symbol in symbols: lines.extend(["", f".. versionadded:: {version}"]) break def skip_member(app, what, name, obj, skip, options): # Do not skip (Make)Executable.__call__ if name == "__call__" and "Executable" in obj.__qualname__: return False return skip def setup(sphinx): # autodoc-process-docstring sphinx.connect("autodoc-process-docstring", add_package_api_version_line) sphinx.connect("autodoc-skip-member", skip_member) sphinx.add_domain(PatchedPythonDomain, override=True) sphinx.add_source_parser(NoTabExpansionRSTParser, override=True) sphinx.add_lexer("spec", SpecLexer) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. needs_sphinx = "3.4" # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ "sphinx.ext.autodoc", "sphinx.ext.graphviz", "sphinx.ext.intersphinx", "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.viewcode", "sphinx_copybutton", "sphinx_last_updated_by_git", "sphinx_sitemap", "sphinxcontrib.inkscapeconverter", "sphinxcontrib.programoutput", ] copybutton_exclude = ".linenos, .gp, .go" # Set default graphviz options graphviz_dot_args = [ "-Grankdir=LR", "-Gbgcolor=transparent", "-Nshape=box", "-Nfontname=monaco", "-Nfontsize=10", ] # Get nice vector graphics graphviz_output_format = "svg" # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix of source filenames. source_suffix = ".rst" # The encoding of source files. source_encoding = "utf-8-sig" # The master toctree document. master_doc = "index" # General information about the project. project = "Spack" copyright = "Spack Project Developers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. import spack import spack.package version = ".".join(str(s) for s in spack.spack_version_info[:2]) # The full version, including alpha/beta/rc tags. release = spack.spack_version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # Places to look for .po/.mo files for doc translations # locale_dirs = [] # Sphinx gettext settings gettext_compact = True gettext_uuid = False # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ["_build", "_spack_root", ".spack-env", ".spack", ".venv"] autodoc_mock_imports = ["llnl"] autodoc_default_options = {"no-value": True} autodoc_preserve_defaults = True nitpicky = True nitpick_ignore = [ # Python classes that intersphinx is unable to resolve ("py:class", "argparse.HelpFormatter"), ("py:class", "concurrent.futures._base.Executor"), ("py:class", "hashlib._Hash"), ("py:class", "multiprocessing.context.BaseContext"), ("py:class", "posix.DirEntry"), # Spack classes that are private and we don't want to expose ("py:class", "spack_repo.builtin.build_systems._checks.BuilderWithDefaults"), ("py:class", "spack.repo._PrependFileLoader"), # Spack classes that intersphinx is unable to resolve ("py:class", "GitOrStandardVersion"), ("py:class", "spack.bootstrap._common.QueryInfo"), ("py:class", "spack.filesystem_view.SimpleFilesystemView"), ("py:class", "spack.spec.ArchSpec"), ("py:class", "spack.spec.DependencySpec"), ("py:class", "spack.spec.InstallStatus"), ("py:class", "spack.spec.SpecfileReaderBase"), ("py:class", "spack.traverse.EdgeAndDepth"), ("py:class", "spack.vendor.archspec.cpu.microarchitecture.Microarchitecture"), ("py:class", "spack.vendor.jinja2.Environment"), ("py:class", "SpecFiltersFactory"), # TypeVar that is not handled correctly ("py:class", "spack.llnl.util.lang.ClassPropertyType"), ("py:class", "spack.llnl.util.lang.K"), ("py:class", "spack.llnl.util.lang.KT"), ("py:class", "spack.llnl.util.lang.T"), ("py:class", "spack.llnl.util.lang.V"), ("py:class", "spack.llnl.util.lang.VT"), ("py:obj", "spack.llnl.util.lang.ClassPropertyType"), ("py:obj", "spack.llnl.util.lang.K"), ("py:obj", "spack.llnl.util.lang.KT"), ("py:obj", "spack.llnl.util.lang.V"), ("py:obj", "spack.llnl.util.lang.VT"), ("py:class", "_P"), ("py:class", "spack.util.web._R"), ] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "furo" # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = ["_themes"] # Google Search Console verification file html_extra_path = ["google5fda5f94b4ffb8de.html"] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. html_theme_options = { "sidebar_hide_name": True, "light_logo": "spack-logo-text.svg", "dark_logo": "spack-logo-white-text.svg", } # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. html_favicon = "_spack_root/share/spack/logo/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. html_last_updated_fmt = "%b %d, %Y" pygments_style = "default" pygments_dark_style = "monokai" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # html_additional_pages = {} # If false, no module index is generated. # html_domain_indices = True # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, links to the reST sources are added to the pages. # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = False # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Base URL for the documentation, used to generate for better indexing html_baseurl = "https://spack.readthedocs.io/en/latest/" # Output file base name for HTML help builder. htmlhelp_basename = "Spackdoc" # Sitemap settings sitemap_show_lastmod = True sitemap_url_scheme = "{link}" sitemap_excludes = ["search.html", "_modules/*"] # -- Options for LaTeX output -------------------------------------------------- latex_engine = "lualatex" latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [("index", "Spack.tex", "Spack Documentation", "", "manual")] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # If true, show page references after internal links. # latex_show_pagerefs = False # If true, show URL addresses after external links. # latex_show_urls = False # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [("index", "spack", "Spack Documentation", ["Todd Gamblin"], 1)] # If true, show URL addresses after external links. # man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( "index", "Spack", "Spack Documentation", "Todd Gamblin", "Spack", "One line description of project.", "Miscellaneous", ) ] # Documents to append as an appendix to all manuals. # texinfo_appendices = [] # If false, no module index is generated. # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. # texinfo_show_urls = 'footnote' # -- Extension configuration ------------------------------------------------- # sphinx.ext.intersphinx intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} rst_epilog = f""" .. |package_api_version| replace:: v{spack.package_api_version[0]}.{spack.package_api_version[1]} .. |min_package_api_version| replace:: v{spack.min_package_api_version[0]}.{spack.min_package_api_version[1]} .. |spack_version| replace:: {spack.spack_version} """ html_static_path = ["_static"] html_css_files = ["css/custom.css"] html_context = {} if os.environ.get("READTHEDOCS", "") == "True": html_context["READTHEDOCS"] = True ================================================ FILE: lib/spack/docs/config_yaml.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. _config-yaml: .. meta:: :description lang=en: A detailed guide to the config.yaml file in Spack, which allows you to set core configuration options like installation paths, build parallelism, and trusted sources. Spack Settings (config.yaml) ============================ Spack's basic configuration options are set in ``config.yaml``. You can see the default settings by looking at ``etc/spack/defaults/config.yaml``: .. literalinclude:: _spack_root/etc/spack/defaults/base/config.yaml :language: yaml These settings can be overridden in ``etc/spack/config.yaml`` or ``~/.spack/config.yaml``. See :ref:`configuration-scopes` for details. ``install_tree:root`` --------------------- The location where Spack will install packages and their dependencies. The default is ``$spack/opt/spack``. ``projections`` --------------- .. warning:: Modifying projections of the install tree is strongly discouraged. By default, Spack installs all packages into a unique directory relative to the install tree root with the following layout: .. code-block:: text {architecture.platform}-{architecture.target}/{name}-{version}-{hash} In very rare cases, it may be necessary to reduce the length of this path. For example, very old versions of the Intel compiler are known to segfault when input paths are too long: .. code-block:: console : internal error: ** The compiler has encountered an unexpected problem. ** Segmentation violation signal raised. ** Access violation or stack overflow. Please contact Intel Support for assistance. Another case is Python and R packages with many runtime dependencies, which can result in very large ``PYTHONPATH`` and ``R_LIBS`` environment variables. This can cause the ``execve`` system call to fail with ``E2BIG``, preventing processes from starting. For this reason, Spack allows users to modify the installation layout through custom projections. For example: .. code-block:: yaml config: install_tree: root: $spack/opt/spack projections: all: "{name}/{version}/{hash:16}" would install packages into subdirectories using only the package name, version, and a hash length of 16 characters. Notice that reducing the hash length increases the likelihood of hash collisions. ``build_stage`` -------------------- Spack is designed to run from a user home directory, and on many systems, the home directory is a (slow) network file system. On most systems, building in a temporary file system is faster. Usually, there is also more space available in the temporary location than in the home directory. If the username is not already in the path, Spack will append the value of ``$user`` to the selected ``build_stage`` path. .. warning:: We highly recommend specifying ``build_stage`` paths that distinguish between staging and other activities to ensure ``spack clean`` does not inadvertently remove unrelated files. Spack prepends ``spack-stage-`` to temporary staging directory names to reduce this risk. Using a combination of ``spack`` and or ``stage`` in each specified path, as shown in the default settings and documented examples, will add another layer of protection. By default, Spack's ``build_stage`` is configured like this: .. code-block:: yaml build_stage: - $tempdir/$user/spack-stage - ~/.spack/stage This can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage. The list is searched in order, and Spack will use the first directory to which it has write access. Specifying `~/.spack/stage` first will ensure each user builds in their home directory. The historic Spack stage path `$spack/var/spack/stage` will build directly inside the Spack instance. See :ref:`config-file-variables` for more on ``$tempdir`` and ``$spack``. When Spack builds a package, it creates a temporary directory within the ``build_stage``. After the package is successfully installed, Spack deletes the temporary directory it used to build. Unsuccessful builds are not deleted, but you can manually purge them with ``spack clean --stage``. .. note:: The build will fail if there is no writable directory in the ``build_stage`` list, where any user- and site-specific setting will be searched first. ``source_cache`` -------------------- Location to cache downloaded tarballs and repositories. By default, these are stored in ``$spack/var/spack/cache``. These are stored indefinitely by default and can be purged with ``spack clean --downloads``. .. _Misc Cache: ``misc_cache`` -------------------- Temporary directory to store long-lived cache files, such as indices of packages available in repositories. Defaults to ``~/.spack/cache``. Can be purged with ``spack clean --misc-cache``. In some cases, e.g., if you work with many Spack instances or many different versions of Spack, it makes sense to have a cache per instance or per version. You can do that by changing the value to either: * ``~/.spack/$spack_instance_id/cache`` for per-instance caches, or * ``~/.spack/$spack_short_version/cache`` for per-spack-version caches. ``verify_ssl`` -------------------- When set to ``true`` (default), Spack will verify certificates of remote hosts when making ``ssl`` connections. Set to ``false`` to disable, and tools like ``curl`` will use their ``--insecure`` options. Disabling this can expose you to attacks. Use at your own risk. ``ssl_certs`` -------------------- Path to custom certificates for SSL verification. The value can be a filesystem path, or an environment variable that expands to an absolute file path. The default value is set to the environment variable ``SSL_CERT_FILE`` to use the same syntax used by many other applications that automatically detect custom certificates. When ``url_fetch_method:curl``, the ``config:ssl_certs`` should resolve to a single file. Spack will then set the environment variable ``CURL_CA_BUNDLE`` in the subprocess calling ``curl``. If additional ``curl`` arguments are required, they can be set in the config, e.g., ``url_fetch_method:'curl -k -q'``. If ``url_fetch_method:urllib``, then files and directories are supported, i.e., ``config:ssl_certs:$SSL_CERT_FILE`` or ``config:ssl_certs:$SSL_CERT_DIR`` will work. In all cases, the expanded path must be absolute for Spack to use the certificates. Certificates relative to an environment can be created by prepending the path variable with the Spack configuration variable ``$env``. ``checksum`` -------------------- When set to ``true``, Spack verifies downloaded source code using a checksum and will refuse to build packages that it cannot verify. Set to ``false`` to disable these checks. Disabling this can expose you to attacks. Use at your own risk. ``locks`` -------------------- When set to ``true``, concurrent instances of Spack will use locks to avoid modifying the install tree, database file, etc. If ``false``, Spack will disable all locking, but you must **not** run concurrent instances of Spack. For file systems that do not support locking, you should set this to ``false`` and run one Spack instance at a time; otherwise, we recommend enabling locks. ``dirty`` -------------------- By default, Spack unsets variables in your environment that can change the way packages build. This includes ``LD_LIBRARY_PATH``, ``CPATH``, ``LIBRARY_PATH``, ``DYLD_LIBRARY_PATH``, and others. By default, builds are ``clean``, but on some machines, compilers and other tools may need custom ``LD_LIBRARY_PATH`` settings to run. You can set ``dirty`` to ``true`` to skip the cleaning step and make all builds "dirty" by default. Be aware that this will reduce the reproducibility of builds. .. _build-jobs: ``build_jobs`` -------------- Unless overridden in a package or on the command line, Spack builds all packages in parallel. The default parallelism is equal to the number of cores available to the process, up to 16 (the default of ``build_jobs``). For a build system that uses Makefiles, ``spack install`` runs: - ``make -j``, when ``build_jobs`` is less than the number of cores available - ``make -j``, when ``build_jobs`` is greater or equal to the number of cores available If you work on a shared login node or have a strict ulimit, it may be necessary to set the default to a lower value. By setting ``build_jobs`` to 4, for example, commands like ``spack install`` will run ``make -j4`` instead of using every core. To build all software in serial, set ``build_jobs`` to 1. Note that specifying the number of jobs on the command line always takes priority, so that ``spack install -j`` always runs ``make -j``, even when that exceeds the number of cores available. ``ccache`` -------------------- When set to ``true``, Spack will use ccache to cache compiles. This is useful specifically in two cases: (1) when using ``spack dev-build`` and (2) when building the same package with many different variants. The default is ``false``. When enabled, Spack will look inside your ``PATH`` for a ``ccache`` executable and stop if it is not found. Some systems come with ``ccache``, but it can also be installed using ``spack install ccache``. ``ccache`` comes with reasonable defaults for cache size and location. (See the *Configuration settings* section of ``man ccache`` to learn more about the default settings and how to change them.) Please note that we currently disable ccache's ``hash_dir`` feature to avoid an issue with the stage directory (see https://github.com/spack/spack/pull/3761#issuecomment-294352232). ``shared_linking:type`` ----------------------- Controls whether Spack embeds ``RPATH`` or ``RUNPATH`` attributes in ELF binaries so that they can find their dependencies. This has no effect on macOS. Two options are allowed: 1. ``rpath`` uses ``RPATH`` and forces the ``--disable-new-tags`` flag to be passed to the linker. 2. ``runpath`` uses ``RUNPATH`` and forces the ``--enable-new-tags`` flag to be passed to the linker. ``RPATH`` search paths have higher precedence than ``LD_LIBRARY_PATH``, and ``ld.so`` will search for libraries in transitive RPATHs of parent objects. ``RUNPATH`` search paths have lower precedence than ``LD_LIBRARY_PATH``, and ``ld.so`` will ONLY search for dependencies in the ``RUNPATH`` of the loading object. DO NOT MIX the two options within the same install tree. ``shared_linking:bind`` ----------------------- This is an *experimental option* that controls whether Spack embeds absolute paths to needed shared libraries in ELF executables and shared libraries on Linux. Setting this option to ``true`` has two advantages: 1. **Improved startup time**: when running an executable, the dynamic loader does not have to search for needed libraries. They are loaded directly. 2. **Reliability**: libraries loaded at runtime are those that were linked during the build. This minimizes the risk of accidentally picking up system libraries. In the current implementation, Spack sets the soname (shared object name) of libraries to their install path upon installation. This has two implications: 1. Binding does not apply to libraries installed *before* the option was enabled. 2. Disabling the option does *not* prevent binding of libraries installed when the option was still enabled. It is also worth noting that: 1. Applications relying on ``dlopen(3)`` will continue to work, even when they open a library by name. This is because RPATHs are retained in binaries also when ``bind`` is enabled. 2. ``LD_PRELOAD`` continues to work for the typical use case of overriding symbols, such as preloading a library with a more efficient ``malloc``. However, the preloaded library will be loaded *in addition to*, rather than *in place of*, another library with the same name -- which can be problematic in rare cases where libraries rely on a particular ``init`` or ``fini`` order. .. note:: In some cases, packages provide *stub libraries* that only contain an interface for linking but lack an implementation for runtime. An example of this is ``libcuda.so``, provided by the CUDA toolkit; it can be used to link against, but the library needed at runtime is the one installed with the CUDA driver. To avoid binding those libraries, they can be marked as non-bindable using a property in the package: .. code-block:: python class Example(Package): non_bindable_shared_objects = ["libinterface.so"] ``install_status`` ---------------------- When set to ``true``, Spack will show information about its current progress as well as the current and total package numbers. Progress is shown both in the terminal title and inline. Setting it to ``false`` will not show any progress information. To work properly, this requires your terminal to reset its title after Spack has finished its work; otherwise, Spack's status information will remain in the terminal's title indefinitely. Most terminals should already be set up this way and clear Spack's status information. ``aliases`` ----------- Aliases can be used to define new Spack commands. They can be either shortcuts for longer commands or include specific arguments for convenience. For instance, if users want to use ``spack install``'s ``-v`` argument all the time, they can create a new alias called ``inst`` that will always call ``install -v``: .. code-block:: yaml aliases: inst: install -v ``concretization_cache:enable`` ------------------------------- When set to ``true``, Spack will utilize a cache of solver outputs from successful concretization runs. When enabled, Spack will check the concretization cache prior to running the solver. If a previous request to solve a given problem is present in the cache, Spack will load the concrete specs and other solver data from the cache rather than running the solver. Specs not previously concretized will be added to the cache on a successful solve. The cache additionally holds solver statistics, so commands like ``spack solve`` will still return information about the run that produced a given solver result. This cache is a subcache of the :ref:`Misc Cache` and as such will be cleaned when the Misc Cache is cleaned. When ``false`` or omitted, all concretization requests will be performed from scratch ``concretization_cache:url`` ---------------------------- Path to the location where Spack will root the concretization cache. Currently this only supports paths on the local filesystem. Default location is under the :ref:`Misc Cache` at: ``$misc_cache/concretization`` ``concretization_cache:entry_limit`` ------------------------------------ Sets a limit on the number of concretization results that Spack will cache. The limit is evaluated after each concretization run; if Spack has stored more results than the limit allows, the oldest concretization results are pruned until 10% of the limit has been removed. Setting this value to 0 disables automatic pruning. It is expected that users will be responsible for maintaining this cache. ``concretization_cache:size_limit`` ----------------------------------- Sets a limit on the size of the concretization cache in bytes. The limit is evaluated after each concretization run; if Spack has stored more results than the limit allows, the oldest concretization results are pruned until 10% of the limit has been removed. Setting this value to 0 disables automatic pruning. It is expected that users will be responsible for maintaining this cache. ================================================ FILE: lib/spack/docs/configuration.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to configure Spack using its flexible YAML-based system. This guide covers the different configuration scopes and provides links to detailed documentation for each configuration file, helping you customize Spack to your specific needs. .. _configuration: Configuration Files =================== Spack has many configuration files. Here is a quick list of them, in case you want to skip directly to specific docs: * :ref:`concretizer.yaml ` * :ref:`config.yaml ` * :ref:`include.yaml ` * :ref:`mirrors.yaml ` * :ref:`modules.yaml ` * :ref:`packages.yaml ` (including :ref:`compiler configuration `) * :ref:`repos.yaml ` * :ref:`toolchains.yaml ` You can also add any of these as inline configuration in the YAML manifest file (``spack.yaml``) describing an :ref:`environment `. YAML Format ----------- Spack configuration files are written in YAML. We chose YAML because it's human-readable but also versatile in that it supports dictionaries, lists, and nested sections. For more details on the format, see `yaml.org `_. Here is an example ``config.yaml`` file: .. code-block:: yaml config: install_tree: root: $spack/opt/spack build_stage: - $tempdir/$user/spack-stage - ~/.spack/stage Each Spack configuration file is nested under a top-level section corresponding to its name. So, ``config.yaml`` starts with ``config:``, ``mirrors.yaml`` starts with ``mirrors:``, etc. .. tip:: Validation and autocompletion of Spack config files can be enabled in your editor using `JSON Schema Store `_. .. _configuration-scopes: Configuration Scopes -------------------- Spack pulls configuration data from files in several directories. There are multiple configuration scopes. From lowest to highest precedence: #. **defaults**: Stored in ``$(prefix)/etc/spack/defaults/``. These are the "factory" settings. Users should generally not modify the settings here, but should override them in other configuration scopes. The defaults here will change from version to version of Spack. #. **system**: Stored in ``/etc/spack/``. These are settings for this machine or for all machines on which this file system is mounted. The system scope overrides the defaults scope. It can be used for settings idiosyncratic to a particular machine, such as the locations of compilers or external packages. Be careful when modifying this scope, as changes here affect all Spack users on a machine. Before putting configuration here, instead consider using the ``site`` scope, which only affects the spack instance it's part of. #. **site**: Stored in ``$(prefix)/etc/spack/site/``. Settings here affect only *this instance* of Spack, and they override the defaults and system scopes. The site scope is intended for site-wide settings on multi-user machines (e.g., for a common Spack instance). #. **plugin**: Read from a Python package's entry points. Settings here affect all instances of Spack running with the same Python installation. This scope takes higher precedence than site, system, and default scopes. #. **user**: Stored in the home directory: ``~/.spack/``. These settings affect all instances of Spack and take higher precedence than site, system, plugin, or defaults scopes. #. **spack**: Stored in ``$(prefix)/etc/spack/``. Settings here affect only *this instance* of Spack, and they override ``user`` and lower configuration scopes. This is intended for project-specific or single-user spack installations. This is the topmost built-in spack scope, and modifying it gives you full control over configuration scopes. For example, it defines the ``user``, ``site``, and ``system`` scopes, so you can use it to remove them completely if you want. #. **environment**: When using Spack :ref:`environments`, Spack reads additional configuration from the environment file. See :ref:`environment-configuration` for further details on these scopes. Environment scopes can be referenced from the command line as ``env:name`` (e.g., to reference environment ``foo``, use ``env:foo``). #. **custom**: Stored in a custom directory specified by ``--config-scope``. If multiple scopes are listed on the command line, they are ordered from lowest to highest precedence. #. **command line**: Build settings specified on the command line take precedence over all other scopes. Each configuration directory may contain several configuration files, such as ``config.yaml``, ``packages.yaml``, or ``mirrors.yaml``. When configurations conflict, settings from higher-precedence scopes override lower-precedence settings. All of these except ``spack`` and ``defaults`` are initially empty, so you don't have to think about the others unless you need them. The most commonly used scopes are ``environment``, ``user``, and ``spack``. If you forget, you can always see the available configuration scopes in order of precedence with the ``spack config scopes`` command:: > spack config scopes -p Scope Path command_line spack /home/username/spack/etc/spack user /home/username/.spack/ site /home/username/spack/etc/spack/site/ defaults /home/username/spack/etc/spack/defaults/ defaults:darwin /home/username/spack/etc/spack/defaults/darwin/ defaults:base /home/username/spack/etc/spack/defaults/base/ _builtin Commands that modify scopes (e.g., ``spack compilers``, ``spack repo``, ``spack external find``, etc.) take a ``--scope=`` parameter that you can use to control which scope is modified. By default, they modify the highest-precedence available scope that is not read-only (like `defaults`). .. _custom-scopes: Custom scopes ^^^^^^^^^^^^^ You may add configuration scopes directly on the command line with the ``--config-scope`` argument, or ``-C`` for short. Custom command-line scopes override any active environments, as well as the ``defaults``, ``system``, ``site``, ``user``, and ``spack`` scopes, For example, the following adds two configuration scopes, named ``scope-a`` and ``scope-b``, to a ``spack spec`` command: .. code-block:: spec $ spack -C ~/myscopes/scope-a -C ~/myscopes/scope-b spec ncurses Custom scopes come *after* the ``spack`` command and *before* the subcommand, and they specify a single path to a directory containing configuration files. You can add the same configuration files to that directory that you can add to any other scope (e.g., ``config.yaml``, ``packages.yaml``, etc.). If multiple scopes are provided: #. Each must be preceded with the ``--config-scope`` or ``-C`` flag. #. They must be ordered from lowest to highest precedence. Example: scopes for release and development """"""""""""""""""""""""""""""""""""""""""" Suppose that you need to support simultaneous building of release and development versions of ``mypackage``, where ``mypackage`` depends on ``pkg-a``, which in turn depends on ``pkg-b``. You could create the following files: .. code-block:: yaml :caption: ``~/myscopes/release/packages.yaml`` :name: code-example-release-packages-yaml packages: mypackage: prefer: ["@1.7"] pkg-a: prefer: ["@2.3"] pkg-b: prefer: ["@0.8"] .. code-block:: yaml :caption: ``~/myscopes/develop/packages.yaml`` :name: code-example-develop-packages-yaml packages: mypackage: prefer: ["@develop"] pkg-a: prefer: ["@develop"] pkg-b: prefer: ["@develop"] You can switch between ``release`` and ``develop`` configurations using configuration arguments. You would type ``spack -C ~/myscopes/release`` when you want to build the designated release versions of ``mypackage``, ``pkg-a``, and ``pkg-b``, and you would type ``spack -C ~/myscopes/develop`` when you want to build all of these packages at the ``develop`` version. Example: swapping MPI providers """"""""""""""""""""""""""""""" Suppose that you need to build two software packages, ``pkg-a`` and ``pkg-b``. For ``pkg-b`` you want a newer Python version and a different MPI implementation than for ``pkg-a``. You can create different configuration scopes for use with ``pkg-a`` and ``pkg-b``: .. code-block:: yaml :caption: ``~/myscopes/pkg-a/packages.yaml`` :name: code-example-pkg-a-packages-yaml packages: python: require: ["@3.11"] mpi: require: [openmpi] .. code-block:: yaml :caption: ``~/myscopes/pkg-b/packages.yaml`` :name: code-example-pkg-b-packages-yaml packages: python: require: ["@3.13"] mpi: require: [mpich] .. _plugin-scopes: Plugin scopes ^^^^^^^^^^^^^ .. note:: Python version >= 3.8 is required to enable plugin configuration. Spack can be made aware of configuration scopes that are installed as part of a Python package. To do so, register a function that returns the scope's path to the ``"spack.config"`` entry point. Consider the Python package ``my_package`` that includes Spack configurations: .. code-block:: console my-package/ ├── src │ ├── my_package │ │ ├── __init__.py │ │ └── spack/ │ │ │ └── config.yaml └── pyproject.toml Adding the following to ``my_package``'s ``pyproject.toml`` will make ``my_package``'s ``spack/`` configurations visible to Spack when ``my_package`` is installed: .. code-block:: toml [project.entry_points."spack.config"] my_package = "my_package:get_config_path" The function ``my_package.get_config_path`` (matching the entry point definition) in ``my_package/__init__.py`` might look like: .. code-block:: python import importlib.resources def get_config_path(): dirname = importlib.resources.files("my_package").joinpath("spack") if dirname.exists(): return str(dirname) .. _platform-scopes: Platform-specific Configuration ------------------------------- .. warning:: Prior to v1.0, each scope above -- except environment scopes -- had a corresponding platform-specific scope (e.g., ``defaults/linux``, ``system/windows``). This can now be accomplished through a suitably placed :ref:`include.yaml ` file. There is often a need for platform-specific configuration settings. For example, on most platforms, GCC is the preferred compiler. However, on macOS (darwin), Clang often works for more packages, and is set as the default compiler. This configuration is set in ``$(prefix)/etc/spack/defaults/darwin/packages.yaml``, which is included by ``$(prefix)/etc/spack/defaults/include.yaml``. Since it is an included configuration of the ``defaults`` scope, settings in the ``defaults`` scope will take precedence. For example, if ``$(prefix)/etc/spack/defaults/include.yaml`` contains: .. code-block:: yaml include: - path: "${platform}" optional: true - path: base then, on macOS (``darwin``), configuration settings for files under the ``$(prefix)/etc/spack/defaults/darwin`` directory would be picked up if they are present. Because ``${platform}`` is above the ``base`` include in the list, ``${platform}`` settings will override anything in ``base`` if there are conflicts. .. note:: You can get the name to use for ```` by running ``spack arch --platform``. Platform-specific configuration files can similarly be set up for any other scope by creating an ``include.yaml`` similar to the one above for ``defaults`` -- under the appropriate configuration paths (see :ref:`config-overrides`) and creating a subdirectory with the platform name that contains the configurations. .. _config-scope-precedence: Scope Precedence ---------------- When Spack queries for configuration parameters, it searches in higher-precedence scopes first. So, settings in a higher-precedence file can override those with the same key in a lower-precedence one. For list-valued settings, Spack merges lists by *prepending* items from higher-precedence configurations to items from lower-precedence configurations by default. Completely ignoring lower-precedence configuration options is supported with the ``::`` notation for keys (see :ref:`config-overrides` below). .. note:: Settings in a scope take precedence over those provided in any included configuration files (i.e., files listed in :ref:`include.yaml ` or an ``include:`` section in ``spack.yaml``). There are also special notations for string concatenation and precedence override: * ``+:`` will force *prepending* strings or lists. For lists, this is the default behavior. * ``-:`` works similarly, but for *appending* values. See :ref:`config-prepend-append` for more details. Simple keys ^^^^^^^^^^^ Let's look at an example of overriding a single key in a Spack configuration file. If your configurations look like this: .. code-block:: yaml :caption: ``$(prefix)/etc/spack/defaults/config.yaml`` :name: code-example-defaults-config-yaml config: install_tree: root: $spack/opt/spack build_stage: - $tempdir/$user/spack-stage - ~/.spack/stage .. code-block:: yaml :caption: ``~/.spack/config.yaml`` :name: code-example-user-config-yaml config: install_tree: root: /some/other/directory Spack will only override ``install_tree`` in the ``config`` section, and will take the site preferences for other settings. You can see the final, combined configuration with the ``spack config get `` command: .. code-block:: console :emphasize-lines: 3 $ spack config get config config: install_tree: root: /some/other/directory build_stage: - $tempdir/$user/spack-stage - ~/.spack/stage .. _config-prepend-append: String Concatenation ^^^^^^^^^^^^^^^^^^^^ Above, the user ``config.yaml`` *completely* overrides specific settings in the default ``config.yaml``. Sometimes, it is useful to add a suffix/prefix to a path or name. To do this, you can use the ``-:`` notation for *append* string concatenation at the end of a key in a configuration file. For example: .. code-block:: yaml :emphasize-lines: 1 :caption: ``~/.spack/config.yaml`` :name: code-example-append-install-tree config: install_tree: root-: /my/custom/suffix/ Spack will then append to the lower-precedence configuration under the ``root`` key: .. code-block:: console $ spack config get config config: install_tree: root: /some/other/directory/my/custom/suffix build_stage: - $tempdir/$user/spack-stage - ~/.spack/stage Similarly, ``+:`` can be used to *prepend* to a path or name: .. code-block:: yaml :emphasize-lines: 1 :caption: ``~/.spack/config.yaml`` :name: code-example-prepend-install-tree config: install_tree: root+: /my/custom/suffix/ .. _config-overrides: Overriding entire sections ^^^^^^^^^^^^^^^^^^^^^^^^^^ Above, the user ``config.yaml`` only overrides specific settings in the default ``config.yaml``. Sometimes, it is useful to *completely* override lower-precedence settings. To do this, you can use *two* colons at the end of a key in a configuration file. For example: .. code-block:: yaml :emphasize-lines: 1 :caption: ``~/.spack/config.yaml`` :name: code-example-override-config-section config:: install_tree: root: /some/other/directory Spack will ignore all lower-precedence configuration under the ``config::`` section: .. code-block:: console $ spack config get config config: install_tree: root: /some/other/directory List-valued settings ^^^^^^^^^^^^^^^^^^^^ Let's revisit the ``config.yaml`` example one more time. The ``build_stage`` setting's value is an ordered list of directories: .. code-block:: yaml :caption: ``$(prefix)/etc/spack/defaults/config.yaml`` :name: code-example-defaults-build-stage config: build_stage: - $tempdir/$user/spack-stage - ~/.spack/stage Suppose the user configuration adds its *own* list of ``build_stage`` paths: .. code-block:: yaml :caption: ``~/.spack/config.yaml`` :name: code-example-user-build-stage config: build_stage: - /lustre-scratch/$user/spack - ~/mystage Spack will first look at the paths in the defaults ``config.yaml``, then the paths in the user's ``~/.spack/config.yaml``. The list in the higher-precedence scope is *prepended* to the defaults. ``spack config get config`` shows the result: .. code-block:: console :emphasize-lines: 5-8 $ spack config get config config: install_tree: root: /some/other/directory build_stage: - /lustre-scratch/$user/spack - ~/mystage - $tempdir/$user/spack-stage - ~/.spack/stage As in :ref:`config-overrides`, the higher-precedence scope can *completely* override the lower-precedence scope using ``::``. So if the user config looked like this: .. code-block:: yaml :emphasize-lines: 1 :caption: ``~/.spack/config.yaml`` :name: code-example-override-build-stage config: build_stage:: - /lustre-scratch/$user/spack - ~/mystage The merged configuration would look like this: .. code-block:: console :emphasize-lines: 5-6 $ spack config get config config: install_tree: root: /some/other/directory build_stage: - /lustre-scratch/$user/spack - ~/mystage .. _config-file-variables: Config File Variables --------------------- Spack understands several variables which can be used in config file paths wherever they appear. There are three sets of these variables: Spack-specific variables, environment variables, and user path variables. Spack-specific variables and environment variables are both indicated by prefixing the variable name with ``$``. User path variables are indicated at the start of the path with ``~`` or ``~user``. Spack-specific variables ^^^^^^^^^^^^^^^^^^^^^^^^ Spack understands over a dozen special variables. These are: * ``$env``: name of the currently active :ref:`environment ` * ``$spack``: path to the prefix of this Spack installation * ``$tempdir``: default system temporary directory (as specified in Python's `tempfile.tempdir `_ variable. * ``$user``: name of the current user * ``$user_cache_path``: user cache directory (``~/.spack`` unless :ref:`overridden `) * ``$architecture``: the architecture triple of the current host, as detected by Spack. * ``$arch``: alias for ``$architecture``. * ``$platform``: the platform of the current host, as detected by Spack. * ``$operating_system``: the operating system of the current host, as detected by the ``distro`` Python module. * ``$os``: alias for ``$operating_system``. * ``$target``: the ISA target for the current host, as detected by ArchSpec. E.g. ``skylake`` or ``neoverse-n1``. * ``$target_family``. The target family for the current host, as detected by ArchSpec. E.g. ``x86_64`` or ``aarch64``. * ``$date``: the current date in the format YYYY-MM-DD * ``$spack_short_version``: the Spack version truncated to the first components. Note that, as with shell variables, you can write these as ``$varname`` or with braces to distinguish the variable from surrounding characters: ``${varname}``. Their names are also case insensitive, meaning that ``$SPACK`` works just as well as ``$spack``. These special variables are substituted first, so any environment variables with the same name will not be used. Environment variables ^^^^^^^^^^^^^^^^^^^^^ After Spack-specific variables are evaluated, environment variables are expanded. These are formatted like Spack-specific variables, e.g., ``${varname}``. You can use this to insert environment variables in your Spack configuration. User home directories ^^^^^^^^^^^^^^^^^^^^^ Spack performs Unix-style tilde expansion on paths in configuration files. This means that tilde (``~``) will expand to the current user's home directory, and ``~user`` will expand to a specified user's home directory. The ``~`` must appear at the beginning of the path, or Spack will not expand it. .. _configuration_environment_variables: Environment Modifications ------------------------- Spack allows users to prescribe custom environment modifications in a few places within its configuration files. Every time these modifications are allowed, they are specified as a dictionary, like in the following example: .. code-block:: yaml environment: set: LICENSE_FILE: "/path/to/license" unset: - CPATH - LIBRARY_PATH append_path: PATH: "/new/bin/dir" The possible actions that are permitted are ``set``, ``unset``, ``append_path``, ``prepend_path``, and finally ``remove_path``. They all require a dictionary of variable names mapped to the values used for the modification, with the exception of ``unset``, which requires just a list of variable names. No particular order is ensured for the execution of each of these modifications. Seeing Spack's Configuration ---------------------------- With so many scopes overriding each other, it can sometimes be difficult to understand what Spack's final configuration looks like. Spack provides two useful ways to view the final "merged" version of any configuration file: ``spack config get`` and ``spack config blame``. .. _cmd-spack-config-get: ``spack config get`` ^^^^^^^^^^^^^^^^^^^^ ``spack config get`` shows a fully merged configuration file, taking into account all scopes. For example, to see the fully merged ``config.yaml``, you can type: .. code-block:: console $ spack config get config config: debug: false checksum: true verify_ssl: true dirty: false build_jobs: 8 install_tree: root: $spack/opt/spack template_dirs: - $spack/templates directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} build_stage: - $tempdir/$user/spack-stage - ~/.spack/stage - $spack/var/spack/stage source_cache: $spack/var/spack/cache misc_cache: ~/.spack/cache locks: true Likewise, this will show the fully merged ``packages.yaml``: .. code-block:: console $ spack config get packages You can use this in conjunction with the ``-C`` / ``--config-scope`` argument to see how your scope will affect Spack's configuration: .. code-block:: console $ spack -C /path/to/my/scope config get packages .. _cmd-spack-config-blame: ``spack config blame`` ^^^^^^^^^^^^^^^^^^^^^^ ``spack config blame`` functions much like ``spack config get``, but it shows exactly which configuration file each setting came from. If you do not know why Spack is behaving a certain way, this command can help you track down the source of the configuration: .. code-block:: console $ spack --insecure -C ./my-scope -C ./my-scope-2 config blame config ==> Warning: You asked for --insecure. Will NOT check SSL certificates. --- config: _builtin debug: False /home/myuser/spack/etc/spack/defaults/config.yaml:72 checksum: True command_line verify_ssl: False ./my-scope-2/config.yaml:2 dirty: False _builtin build_jobs: 8 ./my-scope/config.yaml:2 install_tree: /path/to/some/tree /home/myuser/spack/etc/spack/defaults/config.yaml:23 template_dirs: /home/myuser/spack/etc/spack/defaults/config.yaml:24 - $spack/templates /home/myuser/spack/etc/spack/defaults/config.yaml:28 directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash} /home/myuser/spack/etc/spack/defaults/config.yaml:49 build_stage: /home/myuser/spack/etc/spack/defaults/config.yaml:50 - $tempdir/$user/spack-stage /home/myuser/spack/etc/spack/defaults/config.yaml:51 - ~/.spack/stage /home/myuser/spack/etc/spack/defaults/config.yaml:52 - $spack/var/spack/stage /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $spack/var/spack/cache /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: ~/.spack/cache /home/myuser/spack/etc/spack/defaults/config.yaml:86 locks: True You can see above that the ``build_jobs`` and ``debug`` settings are built-in and are not overridden by a configuration file. The ``verify_ssl`` setting comes from the ``--insecure`` option on the command line. The ``dirty`` and ``install_tree`` settings come from the custom scopes ``./my-scope`` and ``./my-scope-2``, and all other configuration options come from the default configuration files that ship with Spack. .. _local-config-overrides: Overriding Local Configuration ------------------------------ Spack's ``system`` and ``user`` scopes provide ways for administrators and users to set global defaults for all Spack instances, but for use cases where one wants a clean Spack installation, these scopes can be undesirable. For example, users may want to opt out of global system configuration, or they may want to ignore their own home directory settings when running in a continuous integration environment. Spack also, by default, keeps various caches and user data in ``~/.spack``, but users may want to override these locations. Spack provides three environment variables that allow you to override or opt out of configuration locations: * ``SPACK_USER_CONFIG_PATH``: Override the path to use for the ``user`` scope (``~/.spack`` by default). * ``SPACK_SYSTEM_CONFIG_PATH``: Override the path to use for the ``system`` scope (``/etc/spack`` by default). * ``SPACK_DISABLE_LOCAL_CONFIG``: Set this environment variable to completely disable **both** the system and user configuration directories. Spack will then only consider its own defaults and ``site`` configuration locations. And one that allows you to move the default cache location: * ``SPACK_USER_CACHE_PATH``: Override the default path to use for user data (misc_cache, tests, reports, etc.) With these settings, if you want to isolate Spack in a CI environment, you can do this: .. code-block:: console $ export SPACK_DISABLE_LOCAL_CONFIG=true $ export SPACK_USER_CACHE_PATH=/tmp/spack ================================================ FILE: lib/spack/docs/configuring_compilers.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Discover how to configure compilers in Spack, whether by specifying them as externals, or by installing them with Spack. .. _compiler-config: Configuring Compilers ===================== Spack has the ability to build packages with multiple compilers and compiler versions. Compilers can be made available to Spack by: 1. Specifying them as externals in ``packages.yaml``, or 2. Having them installed in the current Spack store, or 3. Having them available as binaries in a build cache For convenience, Spack will automatically detect compilers as externals the first time it needs them, if no compiler is available. .. _cmd-spack-compilers: ``spack compiler list`` ----------------------- You can see which compilers are available to Spack by running ``spack compiler list``: .. code-block:: spec $ spack compiler list ==> Available compilers -- gcc ubuntu20.04-x86_64 --------------------------------------- [e] gcc@10.5.0 [+] gcc@15.1.0 [+] gcc@14.3.0 Compilers marked with an ``[e]`` are system compilers (externals), and those marked with a ``[+]`` have been installed by Spack. Compilers from remote build caches are marked as ``-``, but are not shown by default. To see them you need a specific option: .. code-block:: console $ spack compiler list --remote ==> Available compilers -- gcc ubuntu20.04-x86_64 --------------------------------------- [e] gcc@10.5.0 [+] gcc@15.1.0 [+] gcc@14.3.0 -- gcc ubuntu20.04-x86_64 --------------------------------------- - gcc@12.4.0 Any of these compilers can be used to build Spack packages. More details on how this is done can be found in :ref:`sec-specs`. .. _cmd-spack-compiler-find: ``spack compiler find`` ----------------------- If you do not see a compiler in the list shown by: .. code-block:: console $ spack compiler list but you want to use it with Spack, you can simply run ``spack compiler find`` with the path to where the compiler is installed. For example: .. code-block:: console $ spack compiler find /opt/intel/oneapi/compiler/2025.1/bin/ ==> Added 1 new compiler to /home/user/.spack/packages.yaml intel-oneapi-compilers@2025.1.0 ==> Compilers are defined in the following files: /home/user/.spack/packages.yaml Or you can run ``spack compiler find`` with no arguments to force auto-detection. This is useful if you do not know where compilers are installed, but you know that new compilers have been added to your ``PATH``. For example, you might load a module, like this: .. code-block:: console $ module load gcc/4.9.0 $ spack compiler find ==> Added 1 new compiler to /home/user/.spack/packages.yaml gcc@4.9.0 This loads the environment module for gcc-4.9.0 to add it to ``PATH``, and then it adds the compiler to Spack. .. note:: By default, Spack does not fill in the ``modules:`` field in the ``packages.yaml`` file. If you are using a compiler from a module, then you should add this field manually. See the section on :ref:`compilers-requiring-modules`. .. _cmd-spack-compiler-info: ``spack compiler info`` ----------------------- If you want to see additional information about specific compilers, you can run ``spack compiler info``: .. code-block:: console $ spack compiler info gcc gcc@=8.4.0 languages='c,c++,fortran' arch=linux-ubuntu20.04-x86_64: prefix: /usr compilers: c: /usr/bin/gcc-8 cxx: /usr/bin/g++-8 fortran: /usr/bin/gfortran-8 gcc@=9.4.0 languages='c,c++,fortran' arch=linux-ubuntu20.04-x86_64: prefix: /usr compilers: c: /usr/bin/gcc cxx: /usr/bin/g++ fortran: /usr/bin/gfortran gcc@=10.5.0 languages='c,c++,fortran' arch=linux-ubuntu20.04-x86_64: prefix: /usr compilers: c: /usr/bin/gcc-10 cxx: /usr/bin/g++-10 fortran: /usr/bin/gfortran-10 This shows the details of the compilers that were detected by Spack. Notice also that we didn't have to be too specific about the version. We just said ``gcc``, and we got information about all the matching compilers. Manual configuration of external compilers ------------------------------------------ If auto-detection fails, you can manually configure a compiler by editing your ``packages`` configuration. You can do this by running: .. code-block:: console $ spack config edit packages which will open the file in :ref:`your favorite editor `. Each compiler has an "external" entry in the file with ``extra_attributes``: .. code-block:: yaml packages: gcc: externals: - spec: gcc@10.5.0 languages='c,c++,fortran' prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc-10 cxx: /usr/bin/g++-10 fortran: /usr/bin/gfortran-10 The compiler executables are listed under ``extra_attributes:compilers``, and are keyed by language. Once you save the file, the configured compilers will show up in the list displayed by ``spack compilers``. You can also add compiler flags to manually configured compilers. These flags should be specified in the ``flags`` section of the compiler specification. The valid flags are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, and ``ldlibs``. For example: .. code-block:: yaml packages: gcc: externals: - spec: gcc@10.5.0 languages='c,c++,fortran' prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc-10 cxx: /usr/bin/g++-10 fortran: /usr/bin/gfortran-10 flags: cflags: -O3 -fPIC cxxflags: -O3 -fPIC cppflags: -O3 -fPIC These flags will be treated by Spack as if they were entered from the command line each time this compiler is used. The compiler wrappers then inject those flags into the compiler command. Compiler flags entered from the command line will be discussed in more detail in the following section. Some compilers also require additional environment configuration. Examples include Intel's oneAPI and AMD's AOCC compiler suites, which have custom scripts for loading environment variables and setting paths. These variables should be specified in the ``environment`` section of the compiler specification. The operations available to modify the environment are ``set``, ``unset``, ``prepend_path``, ``append_path``, and ``remove_path``. For example: .. code-block:: yaml packages: intel-oneapi-compilers: externals: - spec: intel-oneapi-compilers@2025.1.0 prefix: /opt/intel/oneapi extra_attributes: compilers: c: /opt/intel/oneapi/compiler/2025.1/bin/icx cxx: /opt/intel/oneapi/compiler/2025.1/bin/icpx fortran: /opt/intel/oneapi/compiler/2025.1/bin/ifx environment: set: MKL_ROOT: "/path/to/mkl/root" unset: # A list of environment variables to unset - CC prepend_path: # Similar for append|remove_path LD_LIBRARY_PATH: /ld/paths/added/by/setvars/sh It is also possible to specify additional ``RPATHs`` that the compiler will add to all executables generated by that compiler. This is useful for forcing certain compilers to RPATH their own runtime libraries so that executables will run without the need to set ``LD_LIBRARY_PATH``: .. code-block:: yaml packages: gcc: externals: - spec: gcc@4.9.3 prefix: /opt/gcc extra_attributes: compilers: c: /opt/gcc/bin/gcc cxx: /opt/gcc/bin/g++ fortran: /opt/gcc/bin/gfortran extra_rpaths: - /path/to/some/compiler/runtime/directory - /path/to/some/other/compiler/runtime/directory .. _compilers-requiring-modules: Compilers Requiring Modules --------------------------- Many installed compilers will work regardless of the environment from which they are called. However, some installed compilers require environment variables to be set in order to run. On typical HPC clusters, these environment modifications are usually delegated to some "module" system. In such a case, you should tell Spack which module(s) to load in order to run the chosen compiler: .. code-block:: yaml packages: gcc: externals: - spec: gcc@10.5.0 languages='c,c++,fortran' prefix: /opt/compilers extra_attributes: compilers: c: /opt/compilers/bin/gcc-10 cxx: /opt/compilers/bin/g++-10 fortran: /opt/compilers/bin/gfortran-10 modules: [gcc/10.5.0] Some compilers require special environment settings to be loaded not just to run, but also to execute the code they build, breaking packages that need to execute code they just compiled. If it's not possible or practical to use a better compiler, you'll need to ensure that environment settings are preserved for compilers like this (i.e., you'll need to load the module or source the compiler's shell script). By default, Spack tries to ensure that builds are reproducible by cleaning the environment before building. If this interferes with your compiler settings, you CAN use ``spack install --dirty`` as a workaround. Note that this MAY interfere with package builds. Build Your Own Compiler ----------------------- If you require a specific compiler and version, you can have Spack build it for you. For example: .. code-block:: spec $ spack install gcc@14 Once the compiler is installed, you can start using it without additional configuration: .. code-block:: spec $ spack install hdf5~mpi %gcc@14 Mixing Compilers ---------------- For more options on configuring Spack to mix different compilers for different languages, see :ref:`the toolchains configuration docs `. To disable mixing (e.g. if you have multiple compilers defined, but want each concretized DAG to use one of them consistently), you can set: .. code-block:: yaml concretizer: compiler_mixing: false This affects root specs and any (transitive) link or run dependencies. Build-only dependencies are allowed to use different compilers (even when this is set). Some packages are difficult to build with high performance compilers, and it may be necessary to enable compiler mixing just for those packages. To enable mixing for specific packages, specify an allow-list in the ``compiler_mixing`` config: .. code-block:: yaml concretizer: compiler_mixing: ["openssl"] Adding ``openssl`` to the compiler mixing allow-list does not allow mixing for dependencies of ``openssl``. ================================================ FILE: lib/spack/docs/containers.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to turn Spack packages and Spack environments into OCI-compatible container images, either by exporting existing installations or by generating recipes for Docker and Singularity. .. _containers: Container Images ================ Whether you want to share applications with others who do not use Spack, deploy on cloud services that run container images, or move workloads to HPC clusters, containers are an effective way to package and distribute software. Spack offers two fundamentally different paradigms for creating container images, each with distinct advantages. You can either export software packages already built on your host system as a container image, or you can generate a traditional recipe file (``Dockerfile`` or Singularity Definition File) to build the software from scratch inside the container. .. list-table:: Comparison of Spack container image creation methods :widths: 15 42 43 :header-rows: 1 * - - :ref:`Container Image Export ` - :ref:`Recipe Generation ` * - **Purpose** - Exports existing installations from the host system as a container image - Runs ``spack install`` to build software from source *inside* the container build process * - **Spack Command** - ``spack buildcache push`` - ``spack containerize`` * - **Reproducibility** - Limited: depends on the host system - High: controlled build environment * - **Input** - Installed Spack packages or environments - A ``spack.yaml`` file * - **Speed** - Faster: copies existing binaries - Slower: typically builds from source * - **Troubleshooting** - Build issues are resolved on the host, where debugging is simpler - Build issues must be resolved inside the container build process * - **Build Tools** - None - Docker, Podman, Singularity, or similar * - **Privileges** - None (rootless) - May require elevated privileges, depending on the container build tool (root) * - **Output destination** - OCI-compatible registry - Local Docker or Singularity image .. _exporting-images: Exporting Spack installations as Container Images ------------------------------------------------- The command .. code-block:: text spack buildcache push [--base-image BASE_IMAGE] [--tag TAG] mirror [specs...] creates and pushes a container image to an OCI-compatible container registry, with the ``mirror`` argument specifying a registry (see below). Think of this command less as "building a container" and more as archiving a working software stack into a portable image. Container images created this way are **minimal**: they contain only runtime dependencies of the specified specs, the base image, and nothing else. Spack itself is *not* included in the resulting image. The arguments are as follows: ``--base-image BASE_IMAGE`` Specifies the base image to use for the container. This should be a minimal Linux distribution with a libc that is compatible with the host system. For example, if your host system is Ubuntu 22.04, you can use ``ubuntu:22.04``, ``ubuntu:24.04``, or newer: the libc in the container image must be at least the version of the host system, assuming ABI compatibility. It is also perfectly fine to use a completely different Linux distribution as long as the libc is compatible. ``--tag TAG`` Specifies a container image tag to use. This tag is used for the image consisting of all specs specified in the command line together. ``mirror`` argument Either the name of a configured OCI registry image (in ``mirrors.yaml``), or a URL specifying the registry and image name. * When pushing to remote registries, you will typically :ref:`specify the name of a registry ` from your Spack configuration. * When pushing to a local registry, you can simply specify a URL like ``oci+http://localhost:5000/[image]``, where ``[image]`` is the name of the image to create, and ``oci+http://`` indicates that the registry does not support HTTPS. ``specs...`` arguments is a list of Spack specs to include in the image. These are packages that have already been installed by Spack. When a Spack environment is activated, only the packages in the environment are included in the image. If no specs are given, and a Spack environment is active, all packages in the environment are included. Spack publishes every individual dependency as a separate image layer, which allows for efficient storage and transfer of images with overlapping dependencies. .. note:: The Docker ``overlayfs2`` storage driver is limited to 128 layers, above which a ``max depth exceeded`` error may be produced when pulling the image. You can hit this limit when exporting container images from larger environments or packages with many dependencies. There are `alternative drivers `_ to work around this limitation. The ``spack buildcache push --base-image ...`` command serves a **dual purpose**: 1. It makes container images available for container runtimes like Docker and Podman. 2. It makes the *same* binaries available :ref:`as a build cache ` for ``spack install``. .. _configuring-container-registries: Container registries ^^^^^^^^^^^^^^^^^^^^ The ``spack buildcache push`` command exports container images directly to an OCI-compatible container registry, such as Docker Hub, GitHub Container Registry (GHCR), Amazon ECR, Google GCR, Azure ACR, or a private registry. These services require authentication, which is configured with the ``spack mirror add`` command: .. code-block:: spec $ spack mirror add \ --oci-username-variable REGISTRY_USER \ --oci-password-variable REGISTRY_TOKEN \ example-registry \ oci://example.com/name/image This registers a mirror named ``example-registry`` in your ``mirrors.yaml`` configuration file that is associated with a container registry and image ``example.com/name/image``. The registry can then be referred to by its name, e.g. ``spack buildcache push example-registry ...``. The ``oci://`` scheme in the URL indicates that this is an OCI-compatible registry with HTTPS support. If you only specify ``oci://name/image``, Spack will assume the registry is hosted on Docker Hub. The ``--oci-username-variable`` and ``--oci-password-variable`` options specify the names of *environment variables* that will be used to authenticate with the registry. Spack does not store your credentials in configuration files; it expects you to set the corresponding environment variables in your shell before running the ``spack buildcache push`` command: .. code-block:: console $ REGISTRY_USER=user REGISTRY_TOKEN=token spack buildcache push ... .. seealso:: The registry password is typically a *personal access token* (PAT) generated on the registry website or a command line tool. In the section :ref:`oci-authentication` we list specific examples for popular registries. If you don't have access to a remote registry, or wish to experiment with container images locally, you can run a *local registry* on your machine and let Spack push to it. This is as simple as running the `official registry image `_ in the background: .. code-block:: console $ docker run -d -p 5000:5000 --name registry registry In this case, it is not necessary to configure a named mirror, you can simply refer to it by URL using ``oci+http://localhost:5000/[image]``, where ``[image]`` is the name of the image to create, and ``oci+http://`` indicates that the registry does not support HTTPS. .. _local-registry-example: Example 1: pushing selected specs as container images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Assume we have ``python@3.13`` and ``cmake@3`` already installed by Spack, and we want to push them as a combined container image ``software_stack:latest`` to a local registry. First we verify that the specs are indeed installed: .. code-block:: spec $ spack find --long python@3.13 cmake@3 -- linux-ubuntu24.04-zen2 / %c,cxx=gcc@13.3.0 ------------------- scpgv2h cmake@3.31.8 n54tvjw python@3.13.5 Since these are the only installations on our system, we can simply refer to them by their spec strings. In case there are multiple installations, we could use ``python/n54tvjw`` and ``cmake/scpgv2h`` to uniquely refer to them by hashes. We now use ``spack buildcache push`` to publish these packages as a container image with ``ubuntu:24.04`` as a base image: .. code-block:: console $ spack buildcache push \ --base-image ubuntu:24.04 \ --tag latest \ oci+http://localhost:5000/software_stack \ python@3.13 cmake@3 They can now be pulled and run with Docker or any other OCI-compatible container runtime: .. code-block:: console $ docker run -it localhost:5000/software_stack:latest root@container-id:/# python3 --version Python 3.13.5 root@container-id:/# cmake --version cmake version 3.31.8 .. _installed-environments-as-containers: Example 2: pushing entire Spack environments as container images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In this example we show how to export an installed :ref:`Spack environment ` as a container image and push it to a remote registry. .. code-block:: spec # Create and install an environment $ spack env create . $ spack -e . add python@3.13 cmake@3 $ spack -e . install # Configure a remote registry $ spack -e . mirror add \ --oci-username-variable REGISTRY_USER \ --oci-password-variable REGISTRY_TOKEN \ container-registry \ oci://example.com/name/image # Push the image $ REGISTRY_USER=user REGISTRY_TOKEN=token \ spack -e . buildcache push \ --update-index \ --base-image ubuntu:24.04 \ --tag my_env \ container-registry The resulting container image can then be run as follows: .. code-block:: console $ docker run -it example.com/name/image:my_env root@container-id:/# python3 --version Python 3.13.5 root@container-id:/# cmake --version cmake version 3.31.8 The advantage of using a Spack environment is that we do not have to specify the individual specs on the command line when pushing the image. With environments, all root specs and their runtime dependencies are included in the container image. If you do specify specs in ``spack buildcache push`` with an environment active, only those matching specs from the environment are included in the image. .. _generating-recipes: Generating recipes for Docker and Singularity --------------------------------------------- Apart from exporting existing installations into container images, Spack can also generate recipes for container images. This is useful if you want to run Spack itself in a sandboxed environment instead of on the host system. This approach requires you to have a container runtime like Docker or Singularity installed on your system, and can only be used using Spack environments. Since recipes need a little more boilerplate than: .. code-block:: docker COPY spack.yaml /environment RUN spack -e /environment install Spack provides a command to generate customizable recipes for container images. Customizations include minimizing the size of the image, installing packages in the base image using the system package manager, and setting up a proper entry point to run the image. .. _cmd-spack-containerize: A Quick Introduction ^^^^^^^^^^^^^^^^^^^^ Consider having a Spack environment like the following: .. code-block:: yaml spack: specs: - gromacs+mpi - mpich Producing a ``Dockerfile`` from it is as simple as changing directories to where the ``spack.yaml`` file is stored and running the following command: .. code-block:: console $ spack containerize > Dockerfile The ``Dockerfile`` that gets created uses multi-stage builds and other techniques to minimize the size of the final image: .. code-block:: docker # Build stage with Spack pre-installed and ready to be used FROM spack/ubuntu-jammy:develop AS builder # What we want to install and how we want to install it # is specified in a manifest file (spack.yaml) RUN mkdir -p /opt/spack-environment && \ set -o noclobber \ && (echo spack: \ && echo ' specs:' \ && echo ' - gromacs+mpi' \ && echo ' - mpich' \ && echo ' concretizer:' \ && echo ' unify: true' \ && echo ' config:' \ && echo ' install_tree:' \ && echo ' root: /opt/software' \ && echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml # Install the software, remove unnecessary deps RUN cd /opt/spack-environment && spack env activate . && spack install --fail-fast && spack gc -y # Strip all the binaries RUN find -L /opt/views/view/* -type f -exec readlink -f '{}' \; | \ xargs file -i | \ grep 'charset=binary' | \ grep 'x-executable\|x-archive\|x-sharedlib' | \ awk -F: '{print $1}' | xargs strip # Modifications to the environment that are necessary to run RUN cd /opt/spack-environment && \ spack env activate --sh -d . > activate.sh # Bare OS image to run the installed executables FROM ubuntu:22.04 COPY --from=builder /opt/spack-environment /opt/spack-environment COPY --from=builder /opt/software /opt/software COPY --from=builder /opt/views /opt/views RUN { \ echo '#!/bin/sh' \ && echo '.' /opt/spack-environment/activate.sh \ && echo 'exec "$@"'; \ } > /entrypoint.sh \ && chmod a+x /entrypoint.sh \ && ln -s /opt/views/view /opt/view ENTRYPOINT [ "/entrypoint.sh" ] CMD [ "/bin/bash" ] The image itself can then be built and run in the usual way with any of the tools suitable for the task. For instance, if we decided to use Docker: .. code-block:: bash $ spack containerize > Dockerfile $ docker build -t myimage . [ ... ] $ docker run -it myimage The various components involved in the generation of the recipe and their configuration are discussed in detail in the sections below. .. _container_spack_images: Official Container Images for Spack ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Container images with Spack preinstalled are available on `Docker Hub `_ and `GitHub Container Registry `_. These images are based on popular distributions and are named accordingly (e.g. ``spack/ubuntu-noble`` for Spack on top of ``ubuntu:24.04``). The table below summarizes the available base images and their corresponding Spack images: .. _containers-supported-os: .. list-table:: Supported base container images :header-rows: 1 * - Base Distribution - Base Image - Spack Image * - Ubuntu 20.04 - ``ubuntu:20.04`` - ``spack/ubuntu-focal`` * - Ubuntu 22.04 - ``ubuntu:22.04`` - ``spack/ubuntu-jammy`` * - Ubuntu 24.04 - ``ubuntu:24.04`` - ``spack/ubuntu-noble`` * - CentOS Stream 9 - ``quay.io/centos/centos:stream9`` - ``spack/centos-stream9`` * - openSUSE Leap - ``opensuse/leap`` - ``spack/leap15`` * - Amazon Linux 2 - ``amazonlinux:2`` - ``spack/amazon-linux`` * - AlmaLinux 8 - ``almalinux:8`` - ``spack/almalinux8`` * - AlmaLinux 9 - ``almalinux:9`` - ``spack/almalinux9`` * - Rocky Linux 8 - ``rockylinux:8`` - ``spack/rockylinux8`` * - Rocky Linux 9 - ``rockylinux:9`` - ``spack/rockylinux9`` * - Fedora Linux 39 - ``fedora:39`` - ``spack/fedora39`` * - Fedora Linux 40 - ``fedora:40`` - ``spack/fedora40`` All container images are tagged with the version of Spack they contain. .. list-table:: Spack container image tags :header-rows: 1 * - Tag - Meaning * - ``:latest`` - Latest *stable* release of Spack * - ``:1`` - Latest ``1.x.y`` release of Spack * - ``:1.0`` - Latest ``1.0.y`` release of Spack * - ``:1.0.2`` - Specific ``1.0.2`` release of Spack * - ``:develop`` - Latest *development* version of Spack These images are available for anyone to use and take care of all the repetitive tasks that are necessary to set up Spack within a container. The container recipes generated by Spack use them as default base images for their ``build`` stage, even though options to use custom base images provided by users are available to accommodate complex use cases. Configuring the Container Recipe ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Any Spack environment can be used for the automatic generation of container recipes. Sensible defaults are provided for things like the base image or the version of Spack used in the image. If finer tuning is needed, it can be obtained by adding the relevant metadata under the ``container`` attribute of environments: .. code-block:: yaml spack: specs: - gromacs+mpi - mpich container: # Select the format of the recipe e.g. docker, # singularity or anything else that is currently supported format: docker # Sets the base images for the stages where Spack builds the # software or where the software gets installed after being built. images: os: "almalinux:9" spack: develop # Whether or not to strip binaries strip: true # Additional system packages that are needed at runtime os_packages: final: - libgomp # Labels for the image labels: app: "gromacs" mpi: "mpich" A detailed description of the options available can be found in the :ref:`container_config_options` section. Setting Base Images ^^^^^^^^^^^^^^^^^^^ The ``images`` subsection is used to select both the image where Spack builds the software and the image where the built software is installed. This attribute can be set in different ways and which one to use depends on the use case at hand. Use Official Spack Images From Dockerhub """""""""""""""""""""""""""""""""""""""" To generate a recipe that uses an official Docker image from the Spack organization to build the software and the corresponding official OS image to install the built software, all the user has to do is specify: 1. An operating system under ``images:os`` 2. A Spack version under ``images:spack`` Any combination of these two values that can be mapped to one of the images discussed in :ref:`container_spack_images` is allowed. For instance, the following ``spack.yaml``: .. code-block:: yaml spack: specs: - gromacs+mpi - mpich container: images: os: almalinux:9 spack: "1.0" uses ``spack/almalinux9:1.0`` and ``almalinux:9`` for the stages where the software is respectively built and installed: .. code-block:: docker # Build stage with Spack pre-installed and ready to be used FROM spack/almalinux9:1.0 AS builder # What we want to install and how we want to install it # is specified in a manifest file (spack.yaml) RUN mkdir -p /opt/spack-environment && \ set -o noclobber \ && (echo spack: \ && echo ' specs:' \ && echo ' - gromacs+mpi' \ && echo ' - mpich' \ && echo ' concretizer:' \ && echo ' unify: true' \ && echo ' config:' \ && echo ' install_tree:' \ && echo ' root: /opt/software' \ && echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml # ... # Bare OS image to run the installed executables FROM quay.io/almalinuxorg/almalinux:9 COPY --from=builder /opt/spack-environment /opt/spack-environment COPY --from=builder /opt/software /opt/software COPY --from=builder /opt/views /opt/views RUN { \ echo '#!/bin/sh' \ && echo '.' /opt/spack-environment/activate.sh \ && echo 'exec "$@"'; \ } > /entrypoint.sh \ && chmod a+x /entrypoint.sh \ && ln -s /opt/views/view /opt/view ENTRYPOINT [ "/entrypoint.sh" ] CMD [ "/bin/bash" ] This is the simplest available method of selecting base images, and we advise its use whenever possible. There are cases, though, where using Spack official images is not enough to fit production needs. In these situations, users can extend the recipe to start with the bootstrapping of Spack at a certain pinned version or manually select which base image to start from in the recipe, as we'll see next. Use a Bootstrap Stage for Spack """"""""""""""""""""""""""""""" In some cases, users may want to pin the commit SHA that is used for Spack to ensure later reproducibility or start from a fork of the official Spack repository to try a bugfix or a feature in an early stage of development. This is possible by being just a little more verbose when specifying information about Spack in the ``spack.yaml`` file: .. code-block:: yaml images: os: amazonlinux:2 spack: # URL of the Spack repository to be used in the container image url: # Either a commit SHA, a branch name, or a tag ref: # If true, turn a branch name or a tag into the corresponding commit # SHA at the time of recipe generation resolve_sha: ``url`` specifies the URL from which to clone Spack and defaults to https://github.com/spack/spack. The ``ref`` attribute can be either a commit SHA, a branch name, or a tag. The default value in this case is to use the ``develop`` branch, but it may change in the future to point to the latest stable release. Finally, ``resolve_sha`` transforms branch names or tags into the corresponding commit SHAs at the time of recipe generation to allow for greater reproducibility of the results at a later time. The list of operating systems that can be used to bootstrap Spack can be obtained with: .. command-output:: spack containerize --list-os .. note:: The ``resolve_sha`` option uses ``git rev-parse`` under the hood and thus requires checking out the corresponding Spack repository in a temporary folder before generating the recipe. Recipe generation may take longer when this option is set to true because of this additional step. Use Custom Images Provided by Users """"""""""""""""""""""""""""""""""" Consider, as an example, building a production-grade image for a CUDA application. The best strategy would probably be to build on top of images provided by the vendor and regard CUDA as an external package. Spack does not currently provide an official image with CUDA configured this way, but users can build it on their own and then configure the environment to explicitly pull it. This requires users to: 1. Specify the image used to build the software under ``images:build`` 2. Specify the image used to install the built software under ``images:final`` A ``spack.yaml`` like the following: .. code-block:: yaml spack: specs: - gromacs@2019.4+cuda build_type=Release - mpich - fftw precision=float packages: cuda: buildable: false externals: - spec: cuda%gcc prefix: /usr/local/cuda container: images: build: custom/cuda-13.0.1-ubuntu22.04:latest final: nvidia/cuda:13.0.1-base-ubuntu22.04 produces, for instance, the following ``Dockerfile``: .. code-block:: docker # Build stage with Spack pre-installed and ready to be used FROM custom/cuda-13.0.1-ubuntu22.04:latest AS builder # What we want to install and how we want to install it # is specified in a manifest file (spack.yaml) RUN mkdir -p /opt/spack-environment && \ set -o noclobber \ && (echo spack: \ && echo ' specs:' \ && echo ' - gromacs@2019.4+cuda build_type=Release' \ && echo ' - mpich' \ && echo ' - fftw precision=float' \ && echo ' packages:' \ && echo ' cuda:' \ && echo ' buildable: false' \ && echo ' externals:' \ && echo ' - spec: cuda%gcc' \ && echo ' prefix: /usr/local/cuda' \ && echo '' \ && echo ' concretizer:' \ && echo ' unify: true' \ && echo ' config:' \ && echo ' install_tree:' \ && echo ' root: /opt/software' \ && echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml # Install the software, remove unnecessary deps RUN cd /opt/spack-environment && spack env activate . && spack install --fail-fast && spack gc -y # Strip all the binaries RUN find -L /opt/views/view/* -type f -exec readlink -f '{}' \; | \ xargs file -i | \ grep 'charset=binary' | \ grep 'x-executable\|x-archive\|x-sharedlib' | \ awk -F: '{print $1}' | xargs strip # Modifications to the environment that are necessary to run RUN cd /opt/spack-environment && \ spack env activate --sh -d . > activate.sh # Bare OS image to run the installed executables FROM nvidia/cuda:13.0.1-base-ubuntu22.04 COPY --from=builder /opt/spack-environment /opt/spack-environment COPY --from=builder /opt/software /opt/software COPY --from=builder /opt/views /opt/views RUN { \ echo '#!/bin/sh' \ && echo '.' /opt/spack-environment/activate.sh \ && echo 'exec "$@"'; \ } > /entrypoint.sh \ && chmod a+x /entrypoint.sh \ && ln -s /opt/views/view /opt/view ENTRYPOINT [ "/entrypoint.sh" ] CMD [ "/bin/bash" ] where the base images for both stages are completely custom. This second mode of selection for base images is more flexible than just choosing an operating system and a Spack version but is also more demanding. Users may need to generate their base images themselves, and it's also their responsibility to ensure that: 1. Spack is available in the ``build`` stage and set up correctly to install the required software 2. The artifacts produced in the ``build`` stage can be executed in the ``final`` stage Therefore, we do not recommend its use in cases that can be otherwise covered by the simplified mode shown first. Singularity Definition Files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In addition to producing recipes in ``Dockerfile`` format, Spack can produce Singularity Definition Files by just changing the value of the ``format`` attribute: .. code-block:: console $ cat spack.yaml spack: specs: - hdf5~mpi container: format: singularity $ spack containerize > hdf5.def $ sudo singularity build hdf5.sif hdf5.def The minimum version of Singularity required to build a SIF (Singularity Image Format) image from the recipes generated by Spack is ``3.5.3``. Extending the Jinja2 Templates ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``Dockerfile`` and the Singularity definition file that Spack can generate are based on a few Jinja2 templates that are rendered according to the Spack environment being containerized. Even though Spack allows a great deal of customization by just setting appropriate values for the configuration options, sometimes that is not enough. In those cases, a user can directly extend the template that Spack uses to render the image to, e.g., set additional environment variables or perform specific operations either before or after a given stage of the build. Let's consider as an example the following structure: .. code-block:: console $ tree /opt/environment /opt/environment ├── data │ └── data.csv ├── spack.yaml ├── data └── templates └── container └── CustomDockerfile containing both the custom template extension and the Spack environment manifest file. To use a custom template, the Spack environment must register the directory containing it and declare its use under the ``container`` configuration: .. code-block:: yaml :emphasize-lines: 7-8,12 spack: specs: - hdf5~mpi concretizer: unify: true config: template_dirs: - /opt/environment/templates container: format: docker depfile: true template: container/CustomDockerfile The template extension can override two blocks, named ``build_stage`` and ``final_stage``, similarly to the example below: .. code-block:: text :caption: /opt/environment/templates/container/CustomDockerfile :emphasize-lines: 3,8 {% extends "container/Dockerfile" %} {% block build_stage %} RUN echo "Start building" {{ super() }} {% endblock %} {% block final_stage %} {{ super() }} COPY data /share/myapp/data {% endblock %} The Dockerfile is generated by running: .. code-block:: console $ spack -e /opt/environment containerize Note that the Spack environment must be active for Spack to read the template. The recipe that gets generated contains the two extra instructions that we added in our template extension: .. code-block:: Dockerfile :emphasize-lines: 4,55 # Build stage with Spack pre-installed and ready to be used FROM spack/ubuntu-jammy:develop AS builder RUN echo "Start building" # What we want to install and how we want to install it # is specified in a manifest file (spack.yaml) RUN mkdir -p /opt/spack-environment && \ set -o noclobber \ && (echo spack: \ && echo ' specs:' \ && echo ' - hdf5~mpi' \ && echo ' concretizer:' \ && echo ' unify: true' \ && echo ' config:' \ && echo ' template_dirs:' \ && echo ' - /tmp/tmp.xvyLqAZpZg' \ && echo ' install_tree:' \ && echo ' root: /opt/software' \ && echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml # Install the software, remove unnecessary deps RUN cd /opt/spack-environment && spack env activate . && spack concretize && spack env depfile -o Makefile && make -j $(nproc) && spack gc -y # Strip all the binaries RUN find -L /opt/views/view/* -type f -exec readlink -f '{}' \; | \ xargs file -i | \ grep 'charset=binary' | \ grep 'x-executable\|x-archive\|x-sharedlib' | \ awk -F: '{print $1}' | xargs strip # Modifications to the environment that are necessary to run RUN cd /opt/spack-environment && \ spack env activate --sh -d . > activate.sh # Bare OS image to run the installed executables FROM ubuntu:22.04 COPY --from=builder /opt/spack-environment /opt/spack-environment COPY --from=builder /opt/software /opt/software COPY --from=builder /opt/views /opt/views RUN { \ echo '#!/bin/sh' \ && echo '.' /opt/spack-environment/activate.sh \ && echo 'exec "$@"'; \ } > /entrypoint.sh \ && chmod a+x /entrypoint.sh \ && ln -s /opt/views/view /opt/view COPY data /share/myapp/data ENTRYPOINT [ "/entrypoint.sh" ] CMD [ "/bin/bash" ] .. _container_config_options: Configuration Reference ^^^^^^^^^^^^^^^^^^^^^^^ The tables below describe all the configuration options that are currently supported to customize the generation of container recipes: .. list-table:: General configuration options for the ``container`` section of ``spack.yaml`` :header-rows: 1 * - Option Name - Description - Allowed Values - Required * - ``format`` - The format of the recipe - ``docker`` or ``singularity`` - Yes * - ``depfile`` - Whether to use a depfile for installation, or not - True or False (default) - No * - ``images:os`` - Operating system used as a base for the image - See :ref:`containers-supported-os` - Yes, if using constrained selection of base images * - ``images:spack`` - Version of Spack used in the ``build`` stage - Valid tags for ``base:image`` - Yes, if using constrained selection of base images * - ``images:spack:url`` - Repository from which Spack is cloned - Any fork of Spack - No * - ``images:spack:ref`` - Reference for the checkout of Spack - Either a commit SHA, a branch name, or a tag - No * - ``images:spack:resolve_sha`` - Resolve branches and tags in ``spack.yaml`` to commits in the generated recipe - True or False (default: False) - No * - ``images:build`` - Image to be used in the ``build`` stage - Any valid container image - Yes, if using custom selection of base images * - ``images:final`` - Image to be used in the ``final`` stage (runtime) - Any valid container image - Yes, if using custom selection of base images * - ``strip`` - Whether to strip binaries - ``true`` (default) or ``false`` - No * - ``os_packages:command`` - Tool used to manage system packages - ``apt``, ``yum``, ``dnf``, ``dnf_epel``, ``zypper``, ``apk``, ``yum_amazon`` - Only with custom base images * - ``os_packages:update`` - Whether or not to update the list of available packages - True or False (default: True) - No * - ``os_packages:build`` - System packages needed at build-time - Valid packages for the current OS - No * - ``os_packages:final`` - System packages needed at run-time - Valid packages for the current OS - No * - ``labels`` - Labels to tag the image - Pairs of key-value strings - No .. list-table:: Configuration options specific to Singularity :header-rows: 1 * - Option Name - Description - Allowed Values - Required * - ``singularity:runscript`` - Content of ``%runscript`` - Any valid script - No * - ``singularity:startscript`` - Content of ``%startscript`` - Any valid script - No * - ``singularity:test`` - Content of ``%test`` - Any valid script - No * - ``singularity:help`` - Description of the image - Description string - No Best Practices ^^^^^^^^^^^^^^ MPI """""" Due to the dependency on Fortran for OpenMPI, which is the Spack default implementation, consider adding ``gfortran`` to the ``apt-get install`` list. Recent versions of OpenMPI will require you to pass ``--allow-run-as-root`` to your ``mpirun`` calls if started as root user inside Docker. For execution on HPC clusters, it can be helpful to import the Docker image into Singularity in order to start a program with an *external* MPI. Otherwise, also add ``openssh-server`` to the ``apt-get install`` list. CUDA """""" Starting from CUDA 9.0, NVIDIA provides minimal CUDA images based on Ubuntu. Please see `their instructions `_. Avoid double-installing CUDA by adding, e.g.: .. code-block:: yaml packages: cuda: externals: - spec: "cuda@9.0.176 arch=linux-ubuntu16-x86_64 %gcc@5.4.0" prefix: /usr/local/cuda buildable: false to your ``spack.yaml``. Users will either need ``nvidia-docker`` or, e.g., Singularity to *execute* device kernels. Docker on Windows and macOS """"""""""""""""""""""""""" On macOS and Windows, Docker runs on a hypervisor that is not allocated much memory by default, and some Spack packages may fail to build due to lack of memory. To work around this issue, consider configuring your Docker installation to use more of your host memory. In some cases, you can also ease the memory pressure on parallel builds by limiting the parallelism in your ``config.yaml``. .. code-block:: yaml config: build_jobs: 2 ================================================ FILE: lib/spack/docs/contribution_guide.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide for developers and administrators on contributing new packages, features, or bug fixes to Spack, covering Git workflows, pull requests, and continuous integration testing. .. _contribution-guide: Contribution Guide ================== This guide is intended for developers or administrators who want to contribute a new package, feature, or bug fix to Spack. It assumes that you have at least some familiarity with Git and GitHub. The guide will show a few examples of contributing workflows and discuss the granularity of pull requests (PRs). It will also discuss the tests your PR must pass in order to be accepted into Spack. First, what is a PR? Quoting `Bitbucket's tutorials `_: Pull requests are a mechanism for a developer to notify team members that they have **completed a feature**. The pull request is more than just a notification -- it's a dedicated forum for discussing the proposed feature. Important is **completed feature**. The changes one proposes in a PR should correspond to one feature, bug fix, extension, etc. One can create PRs with changes relevant to different ideas; however, reviewing such PRs becomes tedious and error-prone. If possible, try to follow the **one-PR-one-package/feature** rule. Branches -------- Spack's ``develop`` branch has the latest contributions. Nearly all pull requests should start from ``develop`` and target ``develop``. There is a branch for each major release series. Release branches originate from ``develop`` and have tags for each point release in the series. For example, ``releases/v0.14`` has tags for ``v0.14.0``, ``v0.14.1``, ``v0.14.2``, etc., versions of Spack. We backport important bug fixes to these branches, but we do not advance the package versions or make other changes that would change the way Spack concretizes dependencies. Currently, the maintainers manage these branches by cherry-picking from ``develop``. See :ref:`releases` for more information. Continuous Integration ---------------------- Spack uses `GitHub Actions `_ for Continuous Integration (CI) testing. This means that every time you submit a pull request, a series of tests will be run to make sure you did not accidentally introduce any bugs into Spack. **Your PR will not be accepted until it passes all of these tests.** While you can certainly wait for the results of these tests after submitting a PR, we recommend that you run them locally to speed up the review process. .. note:: Oftentimes, CI will fail for reasons other than a problem with your PR. For example, ``apt-get``, ``pip``, or ``brew`` (Homebrew) might fail to download one of the dependencies for the test suite, or a transient bug might cause the unit tests to timeout. If any job fails, click the "Details" link and click on the test(s) that is failing. If it does not look like it is failing for reasons related to your PR, you have two options. If you have write permissions for the Spack repository, you should see a "Restart workflow" button on the right-hand side. If not, you can close and reopen your PR to rerun all of the tests. If the same test keeps failing, there may be a problem with your PR. If you notice that every recent PR is failing with the same error message, it may be that an issue occurred with the CI infrastructure, or one of Spack's dependencies put out a new release that is causing problems. If this is the case, please file an issue. We currently test against Python 3.6 and up on both macOS and Linux and perform three types of tests: .. _cmd-spack-unit-test: Unit Tests ^^^^^^^^^^ Unit tests ensure that core Spack features like fetching or spec resolution are working as expected. If your PR only adds new packages or modifies existing ones, there's very little chance that your changes could cause the unit tests to fail. However, if you make changes to Spack's core libraries, you should run the unit tests to make sure you didn't break anything. Since they test things like fetching from VCS repos, the unit tests require `git `_, `mercurial `_, and `subversion `_ to run. Make sure these are installed on your system and can be found in your ``PATH``. All of these can be installed with Spack or with your system package manager. To run *all* of the unit tests, use: .. code-block:: console $ spack unit-test These tests may take several minutes to complete. If you know you are only modifying a single Spack feature, you can run subsets of tests at a time. For example, this would run all the tests in ``lib/spack/spack/test/architecture.py``: .. code-block:: console $ spack unit-test lib/spack/spack/test/architecture.py And this would run the ``test_platform`` test from that file: .. code-block:: console $ spack unit-test lib/spack/spack/test/architecture.py::test_platform This allows you to develop iteratively: make a change, test that change, make another change, test that change, etc. We use `pytest `_ as our tests framework, and these types of arguments are just passed to the ``pytest`` command underneath. See `the pytest docs `_ for more details on test selection syntax. ``spack unit-test`` has a few special options that can help you understand what tests are available. To get a list of all available unit test files, run: .. command-output:: spack unit-test --list :ellipsis: 5 To see a more detailed list of available unit tests, use ``spack unit-test --list-long``: .. command-output:: spack unit-test --list-long :ellipsis: 10 And to see the fully qualified names of all tests, use ``--list-names``: .. command-output:: spack unit-test --list-names :ellipsis: 5 You can combine these with ``pytest`` arguments to restrict which tests you want to know about. For example, to see just the tests in ``architecture.py``: .. command-output:: spack unit-test --list-long lib/spack/spack/test/architecture.py You can also combine any of these options with a ``pytest`` keyword search. See the `pytest usage documentation `_ for more details on test selection syntax. For example, to see the names of all tests that have "spec" or "concretize" somewhere in their names: .. command-output:: spack unit-test --list-names -k "spec and concretize" By default, ``pytest`` captures the output of all unit tests, and it will print any captured output for failed tests. Sometimes it is helpful to see your output interactively while the tests run (e.g., if you add print statements to unit tests). To see the output *live*, use the ``-s`` argument to ``pytest``: .. code-block:: console $ spack unit-test -s --list-long lib/spack/spack/test/architecture.py::test_platform Unit tests are crucial to making sure bugs are not introduced into Spack. If you are modifying core Spack libraries or adding new functionality, please add new unit tests for your feature and consider strengthening existing tests. You will likely be asked to do this if you submit a pull request to the Spack project on GitHub. Check out the `pytest documentation `_ and feel free to ask for guidance on how to write tests! .. note:: You may notice the ``share/spack/qa/run-unit-tests`` script in the repository. This script is designed for CI. It runs the unit tests and reports coverage statistics back to Codecov. If you want to run the unit tests yourself, we suggest you use ``spack unit-test``. Style Tests ^^^^^^^^^^^^ Spack uses `Flake8 `_ to test for `PEP 8 `_ conformance and `mypy `_ for type checking. PEP 8 is a series of style guides for Python that provide suggestions for everything from variable naming to indentation. In order to limit the number of PRs that were mostly style changes, we decided to enforce PEP 8 conformance. Your PR needs to comply with PEP 8 in order to be accepted, and if it modifies the Spack library, it needs to successfully type-check with mypy as well. Testing for compliance with Spack's style is easy. Simply run the ``spack style`` command: .. code-block:: console $ spack style ``spack style`` has a couple advantages over running the tools by hand: #. It only tests files that you have modified since branching off of ``develop``. #. It works regardless of what directory you are in. #. It automatically adds approved exemptions from the ``flake8`` checks. For example, URLs are often longer than 80 characters, so we exempt them from line length checks. We also exempt lines that start with ``homepage =``, ``url =``, ``version()``, ``variant()``, ``depends_on()``, and ``extends()`` in ``package.py`` files. This is now also possible when directly running Flake8 if you can use the ``spack`` formatter plugin included with Spack. More approved Flake8 exemptions can be found `here `_. If all is well, you'll see something like this: .. code-block:: console $ run-flake8-tests Dependencies found. ======================================================= flake8: running flake8 code checks on spack. Modified files: var/spack/repos/spack_repo/builtin/packages/hdf5/package.py var/spack/repos/spack_repo/builtin/packages/hdf/package.py var/spack/repos/spack_repo/builtin/packages/netcdf/package.py ======================================================= Flake8 checks were clean. However, if you are not compliant with PEP 8, Flake8 will complain: .. code-block:: console var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:26: [F401] 'os' imported but unused var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:61: [E303] too many blank lines (2) var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:106: [E501] line too long (92 > 79 characters) Flake8 found errors. Most of the error messages are straightforward, but if you do not understand what they mean, just ask questions about them when you submit your PR. The line numbers will change if you add or delete lines, so simply run ``spack style`` again to update them. .. tip:: Try fixing Flake8 errors in reverse order. This eliminates the need for multiple runs of ``spack style`` just to re-compute line numbers and makes it much easier to fix errors directly off of the CI output. Documentation Tests ^^^^^^^^^^^^^^^^^^^ Spack uses `Sphinx `_ to build its documentation. In order to prevent things like broken links and missing imports, we added documentation tests that build the documentation and fail if there are any warning or error messages. Building the documentation requires several dependencies: * sphinx * sphinxcontrib-programoutput * sphinx-rtd-theme * graphviz * git * mercurial * subversion All of these can be installed with Spack, e.g.: .. code-block:: console $ spack install py-sphinx py-sphinxcontrib-programoutput py-sphinx-rtd-theme graphviz git mercurial subversion .. warning:: Sphinx has `several required dependencies `_. If you are using a Python from Spack and you installed ``py-sphinx`` and friends, you need to make them available to your Python interpreter. The easiest way to do this is to run: .. code-block:: console $ spack load py-sphinx py-sphinx-rtd-theme py-sphinxcontrib-programoutput so that all of the dependencies are added to ``PYTHONPATH``. If you see an error message like: .. code-block:: console Extension error: Could not import extension sphinxcontrib.programoutput (exception: No module named sphinxcontrib.programoutput) make: *** [html] Error 1 that means Sphinx could not find ``py-sphinxcontrib-programoutput`` in your ``PYTHONPATH``. Once all of the dependencies are installed, you can try building the documentation: .. code-block:: console $ cd path/to/spack/lib/spack/docs/ $ make clean $ make If you see any warning or error messages, you will have to correct those before your PR is accepted. If you are editing the documentation, you should be running the documentation tests to make sure there are no errors. Documentation changes can result in some obfuscated warning messages. If you do not understand what they mean, feel free to ask when you submit your PR. .. _spack-builders-and-pipelines: GitLab CI ^^^^^^^^^ Build Cache Stacks """""""""""""""""" Spack welcomes the contribution of software stacks of interest to the community. These stacks are used to test package recipes and generate publicly available build caches. Spack uses GitLab CI for managing the orchestration of build jobs. GitLab Entry Point ~~~~~~~~~~~~~~~~~~ Add a stack entry point to ``share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml``. There are two stages required for each new stack: the generation stage and the build stage. The generate stage is defined using the job template ``.generate`` configured with environment variables defining the name of the stack in ``SPACK_CI_STACK_NAME``, the platform (``SPACK_TARGET_PLATFORM``) and architecture (``SPACK_TARGET_ARCH``) configuration, and the tags associated with the class of runners to build on. .. note:: The ``SPACK_CI_STACK_NAME`` must match the name of the directory containing the stack's ``spack.yaml`` file. .. note:: The platform and architecture variables are specified in order to select the correct configurations from the generic configurations used in Spack CI. The configurations currently available are: * ``.cray_rhel_zen4`` * ``.cray_sles_zen4`` * ``.darwin_aarch64`` * ``.darwin_x86_64`` * ``.linux_aarch64`` * ``.linux_icelake`` * ``.linux_neoverse_n1`` * ``.linux_neoverse_v1`` * ``.linux_neoverse_v2`` * ``.linux_skylake`` * ``.linux_x86_64`` * ``.linux_x86_64_v4`` New configurations can be added to accommodate new platforms and architectures. The build stage is defined as a trigger job that consumes the GitLab CI pipeline generated in the generate stage for this stack. Build stage jobs use the ``.build`` job template, which handles the basic configuration. An example entry point for a new stack called ``my-super-cool-stack`` .. code-block:: yaml .my-super-cool-stack: extends: [".linux_x86_64_v3"] variables: SPACK_CI_STACK_NAME: my-super-cool-stack tags: ["all", "tags", "your", "job", "needs"] my-super-cool-stack-generate: extends: [".generate", ".my-super-cool-stack"] image: my-super-cool-stack-image:0.0.1 my-super-cool-stack-build: extends: [".build", ".my-super-cool-stack"] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml job: my-super-cool-stack-generate strategy: depend needs: - artifacts: true job: my-super-cool-stack-generate Stack Configuration ~~~~~~~~~~~~~~~~~~~ The stack configuration is a Spack environment file with two additional sections added. Stack configurations should be located in ``share/spack/gitlab/cloud_pipelines/stacks//spack.yaml``. The ``ci`` section is generally used to define stack-specific mappings such as image or tags. For more information on what can go into the ``ci`` section, refer to the docs on pipelines. The ``cdash`` section is used for defining where to upload the results of builds. Spack configures most of the details for posting pipeline results to `cdash.spack.io `_. The only requirement in the stack configuration is to define a ``build-group`` that is unique; this is usually the long name of the stack. An example stack that builds ``zlib``. .. code-block:: yaml spack: view: false packages: all: require: ["%gcc", "target=x86_64_v3"] specs: - zlib ci: pipeline-gen: - build-job: image: my-super-cool-stack-image:0.0.1 cdash: build-group: My Super Cool Stack .. note:: The ``image`` used in the ``*-generate`` job must match exactly the ``image`` used in the ``build-job``. When the images do not match, the build job may fail. Registering Runners """"""""""""""""""" Contributing computational resources to Spack's CI build farm is one way to help expand the capabilities and offerings of the public Spack build caches. Currently, Spack utilizes Linux runners from AWS, Google, and the University of Oregon (UO). Runners require four key pieces: * Runner Registration Token * Accurate tags * OIDC Authentication script * GPG keys Minimum GitLab Runner Version: ``16.1.0`` `Installation instructions `_ Registration Token ~~~~~~~~~~~~~~~~~~ The first step to contribute new runners is to open an issue in the `Spack infrastructure `_ project. This will be reported to the Spack infrastructure team, who will guide users through the process of registering new runners for Spack CI. The information needed to register a runner is the motivation for the new resources, a semi-detailed description of the runner, and finally the point of contact for maintaining the software on the runner. The point of contact will then work with the infrastructure team to obtain runner registration token(s) for interacting with Spack's GitLab instance. Once the runner is active, this point of contact will also be responsible for updating the GitLab runner software to keep pace with Spack's GitLab. Tagging ~~~~~~~ In the initial stages of runner registration, it is important to **exclude** the special tag ``spack``. This will prevent the new runner(s) from being picked up for production CI jobs while it is configured and evaluated. Once it is determined that the runner is ready for production use, the ``spack`` tag will be added. Because GitLab has no concept of tag exclusion, runners that provide specialized resources also require specialized tags. For example, a basic CPU-only x86_64 runner may have a tag ``x86_64`` associated with it. However, a runner containing a CUDA-capable GPU may have the tag ``x86_64-cuda`` to denote that it should only be used for packages that will benefit from a CUDA-capable resource. OIDC ~~~~ Spack runners use OIDC authentication for connecting to the appropriate AWS bucket, which is used for coordinating the communication of binaries between build jobs. In order to configure OIDC authentication, Spack CI runners use a Python script with minimal dependencies. This script can be configured for runners as seen here using the ``pre_build_script``. .. code-block:: toml [[runners]] pre_build_script = """ echo 'Executing Spack pre-build setup script' for cmd in "${PY3:-}" python3 python; do if command -v > /dev/null "$cmd"; then export PY3="$(command -v "$cmd")" break fi done if [ -z "${PY3:-}" ]; then echo "Unable to find python3 executable" exit 1 fi $PY3 -c "import urllib.request; urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" $PY3 pre_build.py > envvars . ./envvars rm -f envvars unset GITLAB_OIDC_TOKEN """ GPG Keys ~~~~~~~~ Runners that may be utilized for ``protected`` CI require the registration of an intermediate signing key that can be used to sign packages. For more information on package signing, read :ref:`key_architecture`. Coverage -------- Spack uses `Codecov `_ to generate and report unit test coverage. This helps us tell what percentage of lines of code in Spack are covered by unit tests. Although code covered by unit tests can still contain bugs, it is much less error-prone than code that is not covered by unit tests. Codecov provides `browser extensions `_ for Google Chrome and Firefox. These extensions integrate with GitHub and allow you to see coverage line-by-line when viewing the Spack repository. If you are new to Spack, a great way to get started is to write unit tests to increase coverage! Unlike with CI on GitHub Actions, Codecov tests are not required to pass in order for your PR to be merged. If you modify core Spack libraries, we would greatly appreciate unit tests that cover these changed lines. Otherwise, we have no way of knowing whether or not your changes introduce a bug. If you make substantial changes to the core, we may request unit tests to increase coverage. .. note:: If the only files you modified are package files, we do not care about coverage on your PR. You may notice that the Codecov tests fail even though you did not modify any core files. This means that Spack's overall coverage has increased since you branched off of ``develop``. This is a good thing! If you really want to get the Codecov tests to pass, you can rebase off of the latest ``develop``, but again, this is not required. Git Workflows ------------- Spack is still in the beta stages of development. Most of our users run off of the ``develop`` branch, and fixes and new features are constantly being merged. So, how do you keep up-to-date with upstream while maintaining your own local differences and contributing PRs to Spack? Branching ^^^^^^^^^ The easiest way to contribute a pull request is to make all of your changes on new branches. Make sure your ``develop`` branch is up-to-date and create a new branch off of it: .. code-block:: console $ git checkout develop $ git pull upstream develop $ git branch $ git checkout Here we assume that the local ``develop`` branch tracks the upstream ``develop`` branch of Spack. This is not a requirement, and you could also do the same with remote branches. But for some, it is more convenient to have a local branch that tracks upstream. Normally, we prefer that commits pertaining to a package ```` have a message in the format ``: descriptive message``. It is important to add a descriptive message so that others who might be looking at your changes later (in a year or maybe two) can understand the rationale behind them. Now, you can make your changes while keeping the ``develop`` branch clean. Edit a few files and commit them by running: .. code-block:: console $ git add $ git commit --message Next, push it to your remote fork and create a PR: .. code-block:: console $ git push origin --set-upstream GitHub provides a `tutorial `_ on how to file a pull request. When you send the request, make ``develop`` the destination branch. If you need this change immediately and do not have time to wait for your PR to be merged, you can always work on this branch. But if you have multiple PRs, another option is to maintain a "Frankenstein" branch that combines all of your other branches: .. code-block:: console $ git co develop $ git branch $ git checkout $ git merge This can be done with each new PR you submit. Just make sure to keep this local branch up-to-date with the upstream ``develop`` branch too. Cherry-Picking ^^^^^^^^^^^^^^ What if you made some changes to your local modified ``develop`` branch and already committed them, but later decided to contribute them to Spack? You can use cherry-picking to create a new branch with only these commits. First, check out your local modified ``develop`` branch: .. code-block:: console $ git checkout Now, get the hashes of the commits you want from the output of ``git log``: .. code-block:: console $ git log Next, create a new branch off of the upstream ``develop`` branch and copy the commits that you want in your PR: .. code-block:: console $ git checkout develop $ git pull upstream develop $ git branch $ git checkout $ git cherry-pick $ git push origin --set-upstream Now you can create a PR from the web interface of GitHub. The net result is as follows: #. You patched your local version of Spack and can use it further. #. You "cherry-picked" these changes into a standalone branch and submitted it as a PR upstream. Should you have several commits to contribute, you could follow the same procedure by getting hashes of all of them and cherry-picking them to the PR branch. .. note:: It is important that whenever you change something that might be of importance upstream, create a pull request as soon as possible. Do not wait for weeks or months to do this, because: #. you might forget why you modified certain files. #. it could get difficult to isolate this change into a standalone, clean PR. Rebasing ^^^^^^^^ Other developers are constantly making contributions to Spack, possibly on the same files that your PR changed. If their PR is merged before yours, it can create a merge conflict. This means that your PR can no longer be automatically merged without a chance of breaking your changes. In this case, you will be asked to rebase on top of the latest upstream ``develop`` branch. First, make sure your ``develop`` branch is up-to-date: .. code-block:: console $ git checkout develop $ git pull upstream develop Now, we need to switch to the branch you submitted for your PR and rebase it on top of ``develop``: .. code-block:: console $ git checkout $ git rebase develop Git will likely ask you to resolve conflicts. Edit the file that it says cannot be merged automatically and resolve the conflict. Then, run: .. code-block:: console $ git add $ git rebase --continue You may have to repeat this process multiple times until all conflicts are resolved. Once this is done, simply force push your rebased branch to your remote fork: .. code-block:: console $ git push --force origin Rebasing with cherry-pick ^^^^^^^^^^^^^^^^^^^^^^^^^ You can also perform a rebase using ``cherry-pick``. First, create a temporary backup branch: .. code-block:: console $ git checkout $ git branch tmp If anything goes wrong, you can always go back to your ``tmp`` branch. Now, look at the logs and save the hashes of any commits you would like to keep: .. code-block:: console $ git log Next, go back to the original branch and reset it to ``develop``. Before doing so, make sure that your local ``develop`` branch is up-to-date with upstream: .. code-block:: console $ git checkout develop $ git pull upstream develop $ git checkout $ git reset --hard develop Now you can cherry-pick relevant commits: .. code-block:: console $ git cherry-pick $ git cherry-pick Push the modified branch to your fork: .. code-block:: console $ git push --force origin If everything looks good, delete the backup branch: .. code-block:: console $ git branch --delete --force tmp Re-writing History ^^^^^^^^^^^^^^^^^^ Sometimes you may end up on a branch that has diverged so much from ``develop`` that it cannot easily be rebased. If the current commit history is more of an experimental nature and only the net result is important, you may rewrite the history. First, merge upstream ``develop`` and reset your branch to it. On the branch in question, run: .. code-block:: console $ git merge develop $ git reset develop At this point, your branch will point to the same commit as ``develop``, and thereby the two are indistinguishable. However, all the files that were previously modified will stay as such. In other words, you do not lose the changes you made. Changes can be reviewed by looking at diffs: .. code-block:: console $ git status $ git diff The next step is to rewrite the history by adding files and creating commits: .. code-block:: console $ git add $ git commit --message After all changed files are committed, you can push the branch to your fork and create a PR: .. code-block:: console $ git push origin --set-upstream ================================================ FILE: lib/spack/docs/developer_guide.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A comprehensive guide for developers working on Spack itself, covering the directory structure, code organization, and key concepts like specs and packages. .. _developer_guide: Developer Guide =============== This guide is intended for people who want to work on Spack itself. If you just want to develop packages, see the :doc:`Packaging Guide `. It is assumed that you have read the :ref:`basic-usage` and :doc:`packaging guide ` sections and that you are familiar with the concepts discussed there. Overview -------- Spack is designed with three separate roles in mind: #. **Users**, who need to install software *without* knowing all the details about how it is built. #. **Packagers**, who know how a particular software package is built and encode this information in package files. #. **Developers**, who work on Spack, add new features, and try to make the jobs of packagers and users easier. Users could be end-users installing software in their home directory or administrators installing software to a shared directory on a shared machine. Packagers could be administrators who want to automate software builds or application developers who want to make their software more accessible to users. As you might expect, there are many types of users with different levels of sophistication, and Spack is designed to accommodate both simple and complex use cases for packages. A user who only knows that they need a certain package should be able to type something simple, like ``spack install ``, and get the package that they want. If a user wants to ask for a specific version, use particular compilers, or build several versions with different configurations, then that should be possible with a minimal amount of additional specification. This gets us to the two key concepts in Spack's software design: #. **Specs**: expressions for describing builds of software, and #. **Packages**: Python modules that build software according to a spec. A package is a template for building particular software, and a spec is a descriptor for one or more instances of that template. Users express the configuration they want using a spec, and a package turns the spec into a complete build. The obvious difficulty with this design is that users underspecify what they want. To build a software package, the package object needs a *complete* specification. In Spack, if a spec describes only one instance of a package, then we say it is **concrete**. If a spec could describe many instances (i.e., it is underspecified in one way or another), then we say it is **abstract**. Spack's job is to take an *abstract* spec from the user, find a *concrete* spec that satisfies the constraints, and hand the task of building the software off to the package object. Packages are managed through Spack's **package repositories**, which allow packages to be stored in multiple repositories with different namespaces. The built-in packages are hosted in a separate Git repository and automatically managed by Spack, while custom repositories can be added for organization-specific or experimental packages. The rest of this document describes all the pieces that come together to make that happen. Directory Structure ------------------- So that you can familiarize yourself with the project, we will start with a high-level view of Spack's directory structure: .. code-block:: none spack/ <- installation root bin/ spack <- main spack executable etc/ spack/ <- Spack config files. Can be overridden by files in ~/.spack. var/ spack/ test_repos/ <- contains package repositories for tests cache/ <- saves resources downloaded during installs opt/ spack/ <- packages are installed here lib/ spack/ docs/ <- source for this documentation external/ <- external libs included in Spack distribution spack/ <- spack module; contains Python code build_systems/ <- modules for different build systems cmd/ <- each file in here is a Spack subcommand compilers/ <- compiler description files container/ <- module for spack containerize hooks/ <- hook modules to run at different points modules/ <- modules for Lmod, Tcl, etc. operating_systems/ <- operating system modules platforms/ <- different Spack platforms reporters/ <- reporters like CDash, JUnit schema/ <- schemas to validate data structures solver/ <- the Spack solver test/ <- unit test modules util/ <- common code Spack is designed so that it could live within a `standard UNIX directory hierarchy `_, so ``lib``, ``var``, and ``opt`` all contain a ``spack`` subdirectory in case Spack is installed alongside other software. Most of the interesting parts of Spack live in ``lib/spack``. .. note:: **Package Repositories**: Built-in packages are hosted in a separate Git repository at `spack/spack-packages `_ and are automatically cloned to ``~/.spack/package_repos/`` when needed. The ``var/spack/test_repos/`` directory is used for unit tests only. See :ref:`repositories` for details on package repositories. Spack has *one* directory layout, and there is no installation process. Most Python programs do not look like this (they use ``distutils``, ``setup.py``, etc.), but we wanted to make Spack *very* easy to use. The simple layout spares users from the need to install Spack into a Python environment. Many users do not have write access to a Python installation, and installing an entire new instance of Python to bootstrap Spack would be very complicated. Users should not have to install a big, complicated package to use the thing that is supposed to spare them from the details of big, complicated packages. The end result is that Spack works out of the box: clone it and add ``bin`` to your ``PATH``, and you are ready to go. Code Structure -------------- This section gives an overview of the various Python modules in Spack, grouped by functionality. Package-related modules ^^^^^^^^^^^^^^^^^^^^^^^ :mod:`spack.package_base` Contains the :class:`~spack.package_base.PackageBase` class, which is the superclass for all packages in Spack. :mod:`spack.util.naming` Contains functions for mapping between Spack package names, Python module names, and Python class names. :mod:`spack.directives` *Directives* are functions that can be called inside a package definition to modify the package, like :func:`~spack.directives.depends_on` and :func:`~spack.directives.provides`. See :ref:`dependencies` and :ref:`virtual-dependencies`. :mod:`spack.multimethod` Implementation of the :func:`@when ` decorator, which allows :ref:`multimethods ` in packages. Spec-related modules ^^^^^^^^^^^^^^^^^^^^ :mod:`spack.spec` Contains :class:`~spack.spec.Spec`. Also implements most of the logic for concretization of specs. :mod:`spack.spec_parser` Contains :class:`~spack.spec_parser.SpecParser` and functions related to parsing specs. :mod:`spack.version` Implements a simple :class:`~spack.version.Version` class with simple comparison semantics. It also implements :class:`~spack.version.VersionRange` and :class:`~spack.version.VersionList`. All three are comparable with each other and offer union and intersection operations. Spack uses these classes to compare versions and to manage version constraints on specs. Comparison semantics are similar to the ``LooseVersion`` class in ``distutils`` and to the way RPM compares version strings. :mod:`spack.compilers` Submodules contains descriptors for all valid compilers in Spack. This is used by the build system to set up the build environment. .. warning:: Not yet implemented. Currently has two compiler descriptions, but compilers aren't fully integrated with the build process yet. Build environment ^^^^^^^^^^^^^^^^^ :mod:`spack.stage` Handles creating temporary directories for builds. :mod:`spack.build_environment` This contains utility functions used by the compiler wrapper script, ``cc``. :mod:`spack.directory_layout` Classes that control the way an installation directory is laid out. Create more implementations of this to change the hierarchy and naming scheme in ``$spack_prefix/opt`` Spack Subcommands ^^^^^^^^^^^^^^^^^ :mod:`spack.cmd` Each module in this package implements a Spack subcommand. See :ref:`writing commands ` for details. Unit tests ^^^^^^^^^^ ``spack.test`` Implements Spack's test suite. Add a module and put its name in the test suite in ``__init__.py`` to add more unit tests. Other Modules ^^^^^^^^^^^^^ :mod:`spack.url` URL parsing, for deducing names and versions of packages from tarball URLs. :mod:`spack.error` :class:`~spack.error.SpackError`, the base class for Spack's exception hierarchy. :mod:`spack.llnl.util.tty` Basic output functions for all of the messages Spack writes to the terminal. :mod:`spack.llnl.util.tty.color` Implements a color formatting syntax used by ``spack.tty``. :mod:`spack.llnl.util` In this package are a number of utility modules for the rest of Spack. .. _package-repositories: Package Repositories ^^^^^^^^^^^^^^^^^^^^ Spack's package repositories allow developers to manage packages from multiple sources. Understanding this system is important for developing Spack itself. :mod:`spack.repo` The core module for managing package repositories. Contains the ``Repo`` and ``RepoPath`` classes that handle loading and searching packages from multiple repositories. Built-in packages are stored in a separate Git repository (`spack/spack-packages `_) rather than being included directly in the Spack source tree. This repository is automatically cloned to ``~/.spack/package_repos/`` when needed. Key concepts: * **Repository namespaces**: Each repository has a unique namespace (e.g., ``builtin``) * **Repository search order**: Packages are found by searching repositories in order * **Git-based repositories**: Remote repositories can be automatically cloned and managed * **Repository configuration**: Managed through ``repos.yaml`` configuration files See :ref:`repositories` for complete details on configuring and managing package repositories. .. _package_class_structure: Package class architecture -------------------------- .. note:: This section aims to provide a high-level knowledge of how the package class architecture evolved in Spack, and provides some insights on the current design. Packages in Spack were originally designed to support only a single build system. The overall class structure for a package looked like: .. image:: images/original_package_architecture.png :scale: 60 % :align: center In this architecture the base class ``AutotoolsPackage`` was responsible for both the metadata related to the ``autotools`` build system (e.g. dependencies or variants common to all packages using it), and for encoding the default installation procedure. In reality, a non-negligible number of packages are either changing their build system during the evolution of the project, or using different build systems for different platforms. An architecture based on a single class requires hacks or other workarounds to deal with these cases. To support a model more adherent to reality, Spack v0.19 changed its internal design by extracting the attributes and methods related to building a software into a separate hierarchy: .. image:: images/builder_package_architecture.png :scale: 60 % :align: center In this new format each ``package.py`` contains one ``*Package`` class that gathers all the metadata, and one or more ``*Builder`` classes that encode the installation procedure. A specific builder object is created just before the software is built, so at a time where Spack knows which build system needs to be used for the current installation, and receives a ``package`` object during initialization. Compatibility with single-class format ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Internally, Spack always uses builders to perform operations related to the installation of a specific software. The builders are created in the ``spack.builder.create`` function. .. literalinclude:: _spack_root/lib/spack/spack/builder.py :pyobject: create To achieve backward compatibility with the single-class format Spack creates in this function a special "adapter builder", if no custom builder is detected in the recipe: .. image:: images/adapter.png :scale: 60 % :align: center Overall the role of the adapter is to route access to attributes of methods first through the ``*Package`` hierarchy, and then back to the base class builder. This is schematically shown in the diagram above, where the adapter role is to "emulate" a method resolution order like the one represented by the red arrows. .. _writing-commands: Writing commands ---------------- Adding a new command to Spack is easy. Simply add a ``.py`` file to ``lib/spack/spack/cmd/``, where ```` is the name of the subcommand. At a bare minimum, two functions are required in this file: ``setup_parser()`` ^^^^^^^^^^^^^^^^^^ Unless your command does not accept any arguments, a ``setup_parser()`` function is required to define what arguments and flags your command takes. See the `Argparse documentation `_ for more details on how to add arguments. Some commands have a set of subcommands, like ``spack compiler find`` or ``spack module lmod refresh``. You can add subparsers to your parser to handle this. Check out ``spack edit --command compiler`` for an example of this. Many commands take the same arguments and flags. These arguments should be defined in ``lib/spack/spack/cmd/common/arguments.py`` so that they do not need to be redefined in multiple commands. ``()`` ^^^^^^^^^^^^ In order to run your command, Spack searches for a function with the same name as your command in ``.py``. This is the main method for your command and can call other helper methods to handle common tasks. Remember, before adding a new command, think to yourself whether or not this new command is actually necessary. Sometimes, the functionality you desire can be added to an existing command. Also, remember to add unit tests for your command. If it is not used very frequently, changes to the rest of Spack can cause your command to break without sufficient unit tests to prevent this from happening. Whenever you add/remove/rename a command or flags for an existing command, make sure to update Spack's `Bash tab completion script `_. Writing Hooks ------------- A hook is a callback that makes it easy to design functions that run for different events. We do this by defining hook types and then inserting them at different places in the Spack codebase. Whenever a hook type triggers by way of a function call, we find all the hooks of that type and run them. Spack defines hooks by way of a module in the ``lib/spack/spack/hooks`` directory. This module has to be registered in ``lib/spack/spack/hooks/__init__.py`` so that Spack is aware of it. This section will cover the basic kind of hooks and how to write them. Types of Hooks ^^^^^^^^^^^^^^ The following hooks are currently implemented to make it easy for you, the developer, to add hooks at different stages of a Spack install or similar. If there is a hook that you would like and it is missing, you can propose to add a new one. ``pre_install(spec)`` """"""""""""""""""""" A ``pre_install`` hook is run within the install subprocess, directly before the installation starts. It expects a single argument of a spec. ``post_install(spec, explicit=None)`` """"""""""""""""""""""""""""""""""""" A ``post_install`` hook is run within the install subprocess, directly after the installation finishes, but before the build stage is removed and the spec is registered in the database. It expects two arguments: the spec and an optional boolean indicating whether this spec is being installed explicitly. ``pre_uninstall(spec)`` and ``post_uninstall(spec)`` """""""""""""""""""""""""""""""""""""""""""""""""""" These hooks are currently used for cleaning up module files after uninstall. Adding a New Hook Type ^^^^^^^^^^^^^^^^^^^^^^ Adding a new hook type is very simple! In ``lib/spack/spack/hooks/__init__.py``, you can simply create a new ``HookRunner`` that is named to match your new hook. For example, let's say you want to add a new hook called ``post_log_write`` to trigger after anything is written to a logger. You would add it as follows: .. code-block:: python # pre/post install and run by the install subprocess pre_install = HookRunner("pre_install") post_install = HookRunner("post_install") # hooks related to logging post_log_write = HookRunner("post_log_write") # <- here is my new hook! You then need to decide what arguments your hook would expect. Since this is related to logging, let's say that you want a message and level. That means that when you add a Python file to the ``lib/spack/spack/hooks`` folder with one or more callbacks intended to be triggered by this hook, you might use your new hook as follows: .. code-block:: python def post_log_write(message, level): """Do something custom with the message and level every time we write to the log """ print("running post_log_write!") To use the hook, we would call it as follows somewhere in the logic to do logging. In this example, we use it outside of a logger that is already defined: .. code-block:: python import spack.hooks # We do something here to generate a logger and message spack.hooks.post_log_write(message, logger.level) This is not to say that this would be the best way to implement an integration with the logger (you would probably want to write a custom logger, or you could have the hook defined within the logger), but it serves as an example of writing a hook. Unit tests ---------- Unit testing ------------ Debugging Unit Tests in CI -------------------------- Spack runs its CI for unit tests via Github Actions from the Spack repo. The unit tests are run for each platform Spack supports, Windows, Linux, and MacOS. It may be the case that a unit test fails or passes on just one of these platforms. When the platform is one the PR author does not have access to, it can be difficult to reproduce, diagnose, and fix a CI failure. Thankfully, PR authors can take advantage of a Github Actions Action to gain temporary access to the failing platform from the context of their PRs. Simply copy the following Github actions yaml stanza into `the GHA workflow file `__ in the `steps` section of whatever unit test needs debugging. .. code-block:: yaml - name: Setup tmate session uses: mxschmitt/action-tmate@c0afd6f790e3a5564914980036ebf83216678101 Ideally this would be inserted somewhere after GHA checks out Spack and does any setup, but before the unit tests themselves are run. You can of course put this stanza after the unit-tests, but then you'll be stuck waiting for the unit tests to complete (potentially up to ~30m) and will need to add additional logic to the yaml in the case where the unit tests fail. For example, if you were to add this step to the Linux unit test CI, it would look something like: .. code-block:: yaml - name: Bootstrap clingo if: ${{ matrix.concretizer == 'clingo' }} env: SPACK_PYTHON: python run: | . share/spack/setup-env.sh spack bootstrap disable spack-install spack bootstrap now spack -v solve zlib - name: Setup tmate session uses: mxschmitt/action-tmate@c0afd6f790e3a5564914980036ebf83216678101 - name: Run unit tests env: SPACK_PYTHON: python SPACK_TEST_PARALLEL: 4 COVERAGE: true COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }} UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.14' }} run: |- share/spack/qa/run-unit-tests Note that the ssh session comes after Spack does its setup but before it runs the unit tests. Once this step is present in the job definition, it will be triggered for each CI run. This action provides access to an SSH server running on the GHA runner that is hosting a given CI run. As the action runs, you should observe output similar to: .. code-block:: console ssh 5RjFs7LPdtwGG8cwSPkGrdMNg@sfo2.tmate.io https://tmate.io/t/5RjFs7LPdtwGG8cwSPkGrdMNg The first line is the ssh command necessary to connect to the server, the second line is a tmate web UI that also provides access to the ssh server on the runner. .. note:: The web UI has occasionally been unresponsive, if it does not respond within ~10s, you'll need to use your local ssh utility. Once connected via SSH, you have the same level of access to the machine that the CI job's user does. Spack's source should be available already (depending on where the step was inserted). So you can just setup the shell to run Spack via the setup scripts and then debug as needed. .. note:: If you have configured your Github profile with SSH keys, the action will be aware of this and require those keys to access the SSH session. .. note:: If you are on Windows you'll be dropped into an MSYS shell, Spack is not supported inside MSYS, so it is strongly recommended to drop into a CMD or powershell prompt. You will have access to this ssh session for as long as Github allows a job to be alive. Once you have finished debugging, remove this action from the Github actions workflow. If you want to continue a workflow and you are inside a session, just create a empty file with the name continue either in the root directory or in the project directory. This action has a few option to configure behavior like ssh key handling, tmate server, detached mode, etc. For more on how to use those options, see the actions docs at https://github.com/mxschmitt/action-tmate Developer environment --------------------- .. warning:: This is an experimental feature. It is expected to change and you should not use it in a production environment. When installing a package, we currently have support to export environment variables to specify adding debug flags to the build. By default, a package installation will build without any debug flags. However, if you want to add them, you can export: .. code-block:: console export SPACK_ADD_DEBUG_FLAGS=true spack install zlib If you want to add custom flags, you should export an additional variable: .. code-block:: console export SPACK_ADD_DEBUG_FLAGS=true export SPACK_DEBUG_FLAGS="-g" spack install zlib These environment variables will eventually be integrated into Spack so they are set from the command line. Developer commands ------------------ .. _cmd-spack-doc: ``spack doc`` ^^^^^^^^^^^^^ .. _cmd-spack-style: ``spack style`` ^^^^^^^^^^^^^^^ ``spack style`` exists to help the developer check imports and style with mypy, Flake8, isort, and (soon) Black. To run all style checks, simply do: .. code-block:: console $ spack style To run automatic fixes for isort, you can do: .. code-block:: console $ spack style --fix You do not need any of these Python packages installed on your system for the checks to work! Spack will bootstrap install them from packages for your use. ``spack unit-test`` ^^^^^^^^^^^^^^^^^^^ See the :ref:`contributor guide section ` on ``spack unit-test``. .. _cmd-spack-python: ``spack python`` ^^^^^^^^^^^^^^^^ ``spack python`` is a command that lets you import and debug things as if you were in a Spack interactive shell. Without any arguments, it is similar to a normal interactive Python shell, except you can import ``spack`` and any other Spack modules: .. code-block:: console $ spack python >>> from spack.version import Version >>> a = Version("1.2.3") >>> b = Version("1_2_3") >>> a == b True >>> c = Version("1.2.3b") >>> c > a True >>> If you prefer using an IPython interpreter, given that IPython is installed, you can specify the interpreter with ``-i``: .. code-block:: console $ spack python -i ipython In [1]: With either interpreter you can run a single command: .. code-block:: console $ spack python -c 'from spack.concretize import concretize_one; concretize_one("python")' ... $ spack python -i ipython -c 'from spack.concretize import concretize_one; concretize_one("python")' Out[1]: ... or a file: .. code-block:: console $ spack python ~/test_fetching.py $ spack python -i ipython ~/test_fetching.py just like you would with the normal Python command. .. _cmd-spack-blame: ``spack blame`` ^^^^^^^^^^^^^^^ ``spack blame`` is a way to quickly see contributors to packages or files in Spack's source tree. For built-in packages, this shows contributors to the package files in the separate ``spack/spack-packages`` repository. You should provide a target package name or file name to the command. Here is an example asking to see contributions for the package "python": .. code-block:: console $ spack blame python LAST_COMMIT LINES % AUTHOR EMAIL 2 weeks ago 3 0.3 Mickey Mouse a month ago 927 99.7 Minnie Mouse 2 weeks ago 930 100.0 By default, you will get a table view (shown above) sorted by date of contribution, with the most recent contribution at the top. If you want to sort instead by percentage of code contribution, then add ``-p``: .. code-block:: console $ spack blame -p python And to see the Git blame view, add ``-g`` instead: .. code-block:: console $ spack blame -g python Finally, to get a JSON export of the data, add ``--json``: .. code-block:: console $ spack blame --json python .. _cmd-spack-url: ``spack url`` ^^^^^^^^^^^^^ A package containing a single URL can be used to download several different versions of the package. If you have ever wondered how this works, all of the magic is in :mod:`spack.url`. This module contains methods for extracting the name and version of a package from its URL. The name is used by ``spack create`` to guess the name of the package. By determining the version from the URL, Spack can replace it with other versions to determine where to download them from. The regular expressions in ``parse_name_offset`` and ``parse_version_offset`` are used to extract the name and version, but they are not perfect. In order to debug Spack's URL parsing support, the ``spack url`` command can be used. .. _cmd-spack-url-parse: ``spack url parse`` """"""""""""""""""" If you need to debug a single URL, you can use the following command: .. command-output:: spack url parse http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.0.tar.gz You will notice that the name and version of this URL are correctly detected, and you can even see which regular expressions it was matched to. However, you will notice that when it substitutes the version number in, it does not replace the ``2.2`` with ``9.9`` where we would expect ``9.9.9b`` to live. This particular package may require a ``list_url`` or ``url_for_version`` function. This command also accepts a ``--spider`` flag. If provided, Spack searches for other versions of the package and prints the matching URLs. .. _cmd-spack-url-list: ``spack url list`` """""""""""""""""" This command lists every URL in every package in Spack. If given the ``--color`` and ``--extrapolation`` flags, it also colors the part of the string that it detected to be the name and version. The ``--incorrect-name`` and ``--incorrect-version`` flags can be used to print URLs that were not being parsed correctly. .. _cmd-spack-url-summary: ``spack url summary`` """"""""""""""""""""" This command attempts to parse every URL for every package in Spack and prints a summary of how many of them are being correctly parsed. It also prints a histogram showing which regular expressions are being matched and how frequently: .. command-output:: spack url summary This command is essential for anyone adding or changing the regular expressions that parse names and versions. By running this command before and after the change, you can make sure that your regular expression fixes more packages than it breaks. Profiling --------- To profile Spack, use Python's built-in `cProfile `_ module directly: .. code-block:: console $ python3 -m cProfile -s cumtime bin/spack find $ python3 -m cProfile -o profile.out bin/spack find .. _releases: Releases -------- This section documents Spack's release process. It is intended for project maintainers, as the tasks described here require maintainer privileges on the Spack repository. For others, we hope this section at least provides some insight into how the Spack project works. .. _release-branches: Release branches ^^^^^^^^^^^^^^^^ There are currently two types of Spack releases: :ref:`minor releases ` (``1.1.0``, ``1.2.0``, etc.) and :ref:`patch releases ` (``1.1.1``, ``1.1.2``, ``1.1.3``, etc.). Here is a diagram of how Spack release branches work: .. code-block:: text o branch: develop (latest version, v1.2.0.dev0) | o | o branch: releases/v1.1, tag: v1.1.1 o | | o tag: v1.1.0 o | | o |/ o | o | o branch: releases/v1.0, tag: v1.0.2 o | | o tag: v1.0.1 o | | o tag: v1.0.0 o | | o |/ o The ``develop`` branch has the latest contributions, and nearly all pull requests target ``develop``. The ``develop`` branch will report that its version is that of the next **minor** release with a ``.dev0`` suffix. Each Spack release series also has a corresponding branch, e.g., ``releases/v1.1`` has ``v1.1.x`` versions of Spack, and ``releases/v1.0`` has ``v1.0.x`` versions. A minor release is the first tagged version on a release branch. Patch releases are back-ported from develop onto release branches. This is typically done by cherry-picking bugfix commits off of ``develop``. To avoid version churn for users of a release series, patch releases **should not** make changes that would change the concretization of packages. They should generally only contain fixes to the Spack core. However, sometimes priorities are such that new functionality needs to be added to a patch release. Both minor and patch releases are tagged. As a convenience, we also tag the latest release as ``releases/latest``, so that users can easily check it out to get the latest stable version. See :ref:`updating-latest-release` for more details. .. admonition:: PEP 440 compliance :class: note Spack releases up to ``v0.17`` were merged back into the ``develop`` branch to ensure that release tags would appear among its ancestors. Since ``v0.18`` we opted to have a linear history of the ``develop`` branch, for reasons explained `here `_. At the same time, we converted to using `PEP 440 `_ compliant versions. Scheduling work for releases ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We schedule work for **minor releases** through `milestones `_ and `GitHub Projects `_, while **patch releases** use `labels `_. While there can be multiple milestones open at a given time, only one is usually active. Its name corresponds to the next major/minor version, for example ``v1.1.0``. Important issues and pull requests should be assigned to this milestone by core developers, so that they are not forgotten at the time of release. The milestone is closed when the release is made, and a new milestone is created for the next major/minor release, if not already there. Bug reports in GitHub issues are automatically labelled ``bug`` and ``triage``. Spack developers assign one of the labels ``impact-low``, ``impact-medium`` or ``impact-high``. This will make the issue appear in the `Triaged bugs `_ project board. Important issues should be assigned to the next milestone as well, so they appear at the top of the project board. Spack's milestones are not firm commitments so we move work between releases frequently. If we need to make a release and some tasks are not yet done, we will simply move them to the next minor release milestone, rather than delaying the release to complete them. Backporting bug fixes ^^^^^^^^^^^^^^^^^^^^^ When a bug is fixed in the ``develop`` branch, it is often necessary to backport the fix to one (or more) of the ``releases/vX.Y`` branches. Only the release manager is responsible for doing backports, but Spack maintainers are responsible for labelling pull requests (and issues if no bug fix is available yet) with ``vX.Y.Z`` labels. The labels should correspond to the future patch versions that the bug fix should be backported to. Backports are done publicly by the release manager using a pull request named ``Backports vX.Y.Z``. This pull request is opened from the ``backports/vX.Y.Z`` branch, targets the ``releases/vX.Y`` branch and contains a (growing) list of cherry-picked commits from the ``develop`` branch. Typically there are one or two backport pull requests open at any given time. .. _minor-releases: Making minor releases ^^^^^^^^^^^^^^^^^^^^^ Assuming all required work from the milestone is completed, the steps to make the minor release are: #. `Create a new milestone `_ for the next major/minor release. #. `Create a new label `_ for the next patch release. #. Move any optional tasks that are not done to the next milestone. #. Create a branch for the release, based on ``develop``: .. code-block:: console $ git checkout -b releases/v1.1 develop For a version ``vX.Y.Z``, the branch's name should be ``releases/vX.Y``. That is, you should create a ``releases/vX.Y`` branch if you are preparing the ``X.Y.0`` release. #. Remove the ``dev0`` development release segment from the version tuple in ``lib/spack/spack/__init__.py``. The version number itself should already be correct and should not be modified. #. Update ``CHANGELOG.md`` with major highlights in bullet form. Use proper Markdown formatting, like `this example from v1.0.0 `_. #. Push the release branch to GitHub. #. Make sure CI passes on the release branch, including: * Regular unit tests * Build tests * The E4S pipeline at `gitlab.spack.io `_ If CI is not passing, submit pull requests to ``develop`` as normal and keep rebasing the release branch on ``develop`` until CI passes. #. Make sure the entire documentation is up to date. If documentation is outdated, submit pull requests to ``develop`` as normal and keep rebasing the release branch on ``develop``. #. Bump the minor version in the ``develop`` branch. Create a pull request targeting the ``develop`` branch, bumping the minor version in ``lib/spack/spack/__init__.py`` with a ``dev0`` release segment. For instance, when you have just released ``v1.1.0``, set the version to ``(1, 2, 0, 'dev0')`` on ``develop``. #. Follow the steps in :ref:`publishing-releases`. #. Follow the steps in :ref:`updating-latest-release`. #. Follow the steps in :ref:`announcing-releases`. .. _patch-releases: Making patch releases ^^^^^^^^^^^^^^^^^^^^^ To make the patch release process both efficient and transparent, we use a *backports pull request* which contains cherry-picked commits from the ``develop`` branch. The majority of the work is to cherry-pick the bug fixes, which ideally should be done as soon as they land on ``develop``; this ensures cherry-picking happens in order and makes conflicts easier to resolve since the changes are fresh in the mind of the developer. The backports pull request is always titled ``Backports vX.Y.Z`` and is labelled ``backports``. It is opened from a branch named ``backports/vX.Y.Z`` and targets the ``releases/vX.Y`` branch. The first commit on the ``backports/vX.Y.Z`` branch should update the Spack version to ``X.Y.Z.dev0``, and should have the commit message ``set version to X.Y.Z.dev0``. This ensures that if users check out an intermediate commit between two patch releases, Spack reports the version correctly. Whenever a pull request labelled ``vX.Y.Z`` is merged, cherry-pick the associated squashed commit on ``develop`` to the ``backports/vX.Y.Z`` branch. For pull requests that were rebased (or not squashed), cherry-pick each associated commit individually. Never force-push to the ``backports/vX.Y.Z`` branch. .. warning:: Sometimes you may **still** get merge conflicts even if you have cherry-picked all the commits in order. This generally means there is some other intervening pull request that the one you are trying to pick depends on. In these cases, you will need to make a judgment call regarding those pull requests. Consider the number of affected files and/or the resulting differences. 1. If the changes are small, you might just cherry-pick it. 2. If the changes are large, then you may decide that this fix is not worth including in a patch release, in which case you should remove the label from the pull request. Remember that large, manual backports are seldom the right choice for a patch release. When all commits are cherry-picked in the ``backports/vX.Y.Z`` branch, make the patch release as follows: #. `Create a new label `_ ``vX.Y.{Z+1}`` for the next patch release. #. Replace the label ``vX.Y.Z`` with ``vX.Y.{Z+1}`` for all PRs and issues that are not yet done. #. Manually push a single commit with commit message ``Set version to vX.Y.Z`` to the ``backports/vX.Y.Z`` branch, that both bumps the Spack version number and updates the changelog: 1. Bump the version in ``lib/spack/spack/__init__.py``. 2. Update ``CHANGELOG.md`` with a list of the changes. This is typically a summary of the commits you cherry-picked onto the release branch. See `the changelog from v1.0.2 `_. #. Make sure CI passes on the **backports pull request**, including: * Regular unit tests * Build tests * The E4S pipeline at `gitlab.spack.io `_ #. Merge the ``Backports vX.Y.Z`` PR with the **Rebase and merge** strategy. This is needed to keep track in the release branch of all the commits that were cherry-picked. #. Make sure CI passes on the last commit of the **release branch**. #. In the rare case you need to include additional commits in the patch release after the backports PR is merged, it is best to delete the last commit ``Set version to vX.Y.Z`` from the release branch with a single force-push, open a new backports PR named ``Backports vX.Y.Z (2)``, and repeat the process. Avoid repeated force-pushes to the release branch. #. Follow the steps in :ref:`publishing-releases`. #. Follow the steps in :ref:`updating-latest-release`. #. Follow the steps in :ref:`announcing-releases`. #. Submit a PR to update the ``CHANGELOG.md`` in the ``develop`` branch with the addition of this patch release. .. _publishing-releases: Publishing a release on GitHub ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #. Create the release in GitHub. * Go to `github.com/spack/spack/releases `_ and click ``Draft a new release``. * Set ``Tag version`` to the name of the tag that will be created. The name should start with ``v`` and contain *all three* parts of the version (e.g., ``v1.1.0`` or ``v1.1.1``). * Set ``Target`` to the ``releases/vX.Y`` branch (e.g., ``releases/v1.0``). * Set ``Release title`` to ``vX.Y.Z`` to match the tag (e.g., ``v1.0.1``). * Paste the latest release Markdown from your ``CHANGELOG.md`` file as the text. * Save the draft so you can keep coming back to it as you prepare the release. #. When you are ready to finalize the release, click ``Publish release``. #. Immediately after publishing, go back to `github.com/spack/spack/releases `_ and download the auto-generated ``.tar.gz`` file for the release. It is the ``Source code (tar.gz)`` link. #. Click ``Edit`` on the release you just made and attach the downloaded release tarball as a binary. This does two things: #. Makes sure that the hash of our releases does not change over time. GitHub sometimes annoyingly changes the way they generate tarballs that can result in the hashes changing if you rely on the auto-generated tarball links. #. Gets download counts on releases visible through the GitHub API. GitHub tracks downloads of artifacts, but *not* the source links. See the `releases page `_ and search for ``download_count`` to see this. #. Go to `readthedocs.org `_ and activate the release tag. This builds the documentation and makes the released version selectable in the versions menu. .. _updating-latest-release: Updating `releases/latest` ^^^^^^^^^^^^^^^^^^^^^^^^^^ If the new release is the **highest** Spack release yet, you should also tag it as ``releases/latest``. For example, suppose the highest release is currently ``v1.1.3``: * If you are releasing ``v1.1.4`` or ``v1.2.0``, then you should tag it with ``releases/latest``, as these are higher than ``v1.1.3``. * If you are making a new release of an **older** minor version of Spack, e.g., ``v1.0.5``, then you should not tag it as ``releases/latest`` (as there are newer major/minor versions). To do so, first fetch the latest tag created on GitHub, since you may not have it locally: .. code-block:: console $ git fetch --force git@github.com:spack/spack tag vX.Y.Z Then tag ``vX.Y.Z`` as ``releases/latest`` and push the individual tag to GitHub. .. code-block:: console $ git tag --force releases/latest vX.Y.Z $ git push --force git@github.com:spack/spack releases/latest The ``--force`` argument to ``git tag`` makes Git overwrite the existing ``releases/latest`` tag with the new one. Do **not** use the ``--tags`` flag when pushing, as this will push *all* local tags. .. _announcing-releases: Announcing a release ^^^^^^^^^^^^^^^^^^^^ We announce releases in all of the major Spack communication channels. Publishing the release takes care of GitHub. The remaining channels are X, Slack, and the mailing list. Here are the steps: #. Announce the release on X. * Compose the tweet on the ``@spackpm`` account per the ``spack-twitter`` slack channel. * Be sure to include a link to the release's page on GitHub. You can base the tweet on `this example `_. #. Announce the release on Slack. * Compose a message in the ``#announcements`` Slack channel (`spackpm.slack.com `_). * Preface the message with ``@channel`` to notify even those people not currently logged in. * Be sure to include a link to the tweet above. The tweet will be shown inline so that you do not have to retype your release announcement. #. Announce the release on the Spack mailing list. * Compose an email to the Spack mailing list. * Be sure to include a link to the release's page on GitHub. * It is also helpful to include some information directly in the email. You can base your announcement on this `example email `_. Once you have completed the above steps, congratulations, you are done! You have finished making the release! ================================================ FILE: lib/spack/docs/env_vars_yaml.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to modify shell environment variables within a Spack environment using the env_vars.yaml file. .. _env-vars-yaml: Environment Variable Settings (env_vars.yaml) ============================================= Spack allows you to include shell environment variable modifications for a Spack environment by including an ``env_vars.yaml`` file. Environment variables can be modified by setting, unsetting, appending, and prepending variables in the shell environment. The changes to the shell environment will take effect when the Spack environment is activated. For example: .. code-block:: yaml env_vars: set: ENVAR_TO_SET_IN_ENV_LOAD: "FOO" unset: - ENVAR_TO_UNSET_IN_ENV_LOAD prepend_path: PATH_LIST: "path/to/prepend" append_path: PATH_LIST: "path/to/append" remove_path: PATH_LIST: "path/to/remove" ================================================ FILE: lib/spack/docs/environments.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to use Spack environments to manage reproducible software stacks, making it easy to share and recreate specific sets of packages and their dependencies. .. _environments: Environments (spack.yaml, spack.lock) ===================================== An environment is used to group a set of specs intended for some purpose to be built, rebuilt, and deployed in a coherent fashion. Environments define aspects of the installation of the software, such as: #. *which* specs to install; #. *how* those specs are configured; and #. *where* the concretized software will be installed. Aggregating this information into an environment for processing has advantages over the *à la carte* approach of building and loading individual Spack modules. With environments, you concretize, install, or load (activate) all of the specs with a single command. Concretization fully configures the specs and dependencies of the environment in preparation for installing the software. This is a more robust solution than ad-hoc installation scripts. And you can share an environment or even re-use it on a different computer. Environment definitions, especially *how* specs are configured, allow the software to remain stable and repeatable even when Spack packages are upgraded. Changes are only picked up when the environment is explicitly re-concretized. Defining *where* specs are installed supports a filesystem view of the environment. Yet Spack maintains a single installation of the software that can be re-used across multiple environments. Activating an environment determines *when* all of the associated (and installed) specs are loaded so limits the software loaded to those specs actually needed by the environment. Spack can even generate a script to load all modules related to an environment. Other packaging systems also provide environments that are similar in some ways to Spack environments; for example, `Conda environments `_ or `Python Virtual Environments `_. Spack environments provide some distinctive features though: #. A spec installed "in" an environment is no different from the same spec installed anywhere else in Spack. #. Spack environments may contain more than one spec of the same package. Spack uses a "manifest and lock" model similar to `Bundler gemfiles `_ and other package managers. The environment's user input file (or manifest), is named ``spack.yaml``. The lock file, which contains the fully configured and concretized specs, is named ``spack.lock``. .. _environments-using: Using Environments ------------------ Here we follow a typical use case of creating, concretizing, installing and loading an environment. .. _cmd-spack-env-create: Creating a managed Environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An environment is created by: .. code-block:: console $ spack env create myenv The directory ``$SPACK_ROOT/var/spack/environments/myenv`` is created to manage the environment. .. note:: By default, all managed environments are stored in the ``$SPACK_ROOT/var/spack/environments`` folder. This location can be changed by setting the ``environments_root`` variable in ``config.yaml``. Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and ``spack.lock`` file under ``$SPACK_ROOT/var/spack/environments/myenv``. User interaction occurs through the ``spack.yaml`` file and the Spack commands that affect it. Metadata and, by default, the view are stored in the ``.spack-env`` directory. When the environment is concretized, Spack creates the ``spack.lock`` file with the fully configured specs and dependencies for the environment. The ``.spack-env`` subdirectory also contains: * ``repo/``: A subdirectory acting as the repo consisting of the Spack packages used in the environment. It allows the environment to build the same, in theory, even on different versions of Spack with different packages! * ``logs/``: A subdirectory containing the build logs for the packages in this environment. Spack Environments can also be created from another environment. Environments can be created from the manifest file (the user input), the lockfile, or the entire environment at once. Create an environment from a manifest using: .. code-block:: console $ spack env create myenv spack.yaml The resulting environment is guaranteed to have the same root specs as the original but may concretize differently in the presence of different explicit or default configuration settings (e.g., a different version of Spack or for a different user account). Environments created from a manifest will copy any included configs from relative paths inside the environment. Relative paths from outside the environment will cause errors, and absolute paths will be kept absolute. For example, if ``spack.yaml`` includes: .. code-block:: yaml spack: include: [./config.yaml] then the created environment will have its own copy of the file ``config.yaml`` copied from the location in the original environment. Create an environment from a ``spack.lock`` file using: .. code-block:: console $ spack env create myenv spack.lock The resulting environment, when on the same or a compatible machine, is guaranteed to initially have the same concrete specs as the original. Create an environment from an entire environment using either the environment name or path: .. code-block:: console $ spack env create myenv /path/to/env $ spack env create myenv2 myenv The resulting environment will include the concrete specs from the original if the original is concretized (as when created from a lockfile) and all of the config options and abstract specs specified in the original (as when created from a manifest file). It will also include any other files included in the environment directory, such as repos or source code, as they could be referenced in the environment by relative path. .. note:: Environment creation also accepts a full path to the file. If the path is not under the ``$SPACK_ROOT/var/spack/environments`` directory then the source is referred to as an :ref:`independent environment `. The name of an environment can be a nested path to help organize environments via subdirectories. .. code-block:: console $ spack env create projectA/configA/myenv This will create a managed environment under ``$environments_root/projectA/configA/myenv``. Changing ``environment_root`` can therefore also be used to make a whole group of nested environments available. .. _cmd-spack-env-activate: .. _cmd-spack-env-deactivate: Activating an Environment ^^^^^^^^^^^^^^^^^^^^^^^^^ To activate an environment, use the following command: .. code-block:: console $ spack env activate myenv By default, the ``spack env activate`` will load the view associated with the environment into the user environment. The ``-v, --with-view`` argument ensures this behavior, and the ``-V, --without-view`` argument activates the environment without changing the user environment variables. The ``-p`` option to the ``spack env activate`` command modifies the user's prompt to begin with the environment name in brackets. .. code-block:: console $ spack env activate -p myenv [myenv] $ ... The ``activate`` command can also be used to create a new environment, if it is not already defined, by adding the ``--create`` flag. Managed and independent environments can both be created using the same flags that `spack env create` accepts. If an environment already exists then Spack will simply activate it and ignore the create-specific flags. .. code-block:: console $ spack env activate --create -p myenv # ... # [creates if myenv does not exist yet] # ... [myenv] $ ... To deactivate an environment, use the command: .. code-block:: console $ spack env deactivate or the shortcut alias .. code-block:: console $ despacktivate If the environment was activated with its view, deactivating the environment will remove the view from the user environment. .. _independent_environments: Independent Environments ^^^^^^^^^^^^^^^^^^^^^^^^ Independent environments can be located in any directory outside of Spack. .. note:: When uninstalling packages, Spack asks the user to confirm the removal of packages that are still used in a managed environment. This is not the case for independent environments. To create an independent environment, use one of the following commands: .. code-block:: console $ spack env create --dir my_env $ spack env create ./my_env As a shorthand, you can also create an independent environment upon activation if it does not already exist: .. code-block:: console $ spack env activate --create ./my_env For convenience, Spack can also place an independent environment in a temporary directory for you: .. code-block:: console $ spack env activate --temp Environment-Aware Commands ^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack commands are environment-aware. For example, the ``find`` command shows only the specs in the active environment if an environment has been activated. Otherwise it shows all specs in the Spack instance. The same rule applies to the ``install`` and ``uninstall`` commands. .. code-block:: spec $ spack find ==> 0 installed packages $ spack install zlib@1.2.11 [+] q6cqrdt zlib@1.2.11 ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv (12s) $ spack env activate myenv $ spack find ==> In environment myenv ==> No root specs ==> 0 installed packages $ spack install zlib@1.2.8 [+] yfc7epf zlib@1.2.8 ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x (12s) ==> Updating view at ~/spack/var/spack/environments/myenv/.spack-env/view $ spack find ==> In environment myenv ==> Root specs zlib@1.2.8 ==> 1 installed package -- linux-rhel7-broadwell / gcc@8.1.0 ---------------------------- zlib@1.2.8 $ despacktivate $ spack find ==> 2 installed packages -- linux-rhel7-broadwell / gcc@8.1.0 ---------------------------- zlib@1.2.8 zlib@1.2.11 Note that when we installed the abstract spec ``zlib@1.2.8``, it was presented as a root of the environment. All explicitly installed packages will be listed as roots of the environment. All of the Spack commands that act on the list of installed specs are environment-aware in this way, including ``install``, ``uninstall``, ``find``, ``extensions``, etc. In the :ref:`environment-configuration` section we will discuss environment-aware commands further. .. _cmd-spack-add: Adding Abstract Specs ^^^^^^^^^^^^^^^^^^^^^ An abstract spec is the user-specified spec before Spack applies defaults or dependency information. You can add abstract specs to an environment using the ``spack add`` command. This adds the abstract spec as a root of the environment in the ``spack.yaml`` file. The most important component of an environment is a list of abstract specs. Adding abstract specs does not immediately install anything, nor does it affect the ``spack.lock`` file. To update the lockfile, the environment must be :ref:`re-concretized `, and to update any installations, the environment must be :ref:`(re)installed `. The ``spack add`` command is environment-aware. It adds the spec to the currently active environment. An error is generated if there isn't an active environment. .. code-block:: spec $ spack env activate myenv $ spack add mpileaks or .. code-block:: spec $ spack -e myenv add python .. note:: All environment-aware commands can also be called using the ``spack -e`` flag to specify the environment. .. _cmd-spack-concretize: Concretizing ^^^^^^^^^^^^ Once user specs have been added to an environment, they can be concretized. There are three different modes of operation to concretize an environment, explained in detail in :ref:`environments_concretization_config`. Regardless of which mode of operation is chosen, the following command will ensure all of the root specs are concretized according to the constraints that are prescribed in the configuration: .. code-block:: console [myenv]$ spack concretize In the case of specs that are not concretized together, the command above will concretize only the specs that were added and not yet concretized. Forcing a re-concretization of all of the specs can be done by adding the ``-f`` option: .. code-block:: console [myenv]$ spack concretize -f Without the option, Spack guarantees that already concretized specs are unchanged in the environment. The ``concretize`` command does not install any packages. For packages that have already been installed outside of the environment, the process of adding the spec and concretizing is identical to installing the spec assuming it concretizes to the exact spec that was installed outside of the environment. The ``spack find`` command can show concretized specs separately from installed specs using the ``-c`` (``--concretized``) flag. .. code-block:: console [myenv]$ spack add zlib [myenv]$ spack concretize [myenv]$ spack find -c ==> In environment myenv ==> Root specs zlib ==> Concretized roots -- linux-rhel7-x86_64 / gcc@4.9.3 ------------------------------- zlib@1.2.11 ==> 0 installed packages .. _installing-environment: Installing an Environment ^^^^^^^^^^^^^^^^^^^^^^^^^ In addition to adding individual specs to an environment, one can install the entire environment at once using the command .. code-block:: console [myenv]$ spack install If the environment has been concretized, Spack will install the concretized specs. Otherwise, ``spack install`` will concretize the environment before installing the concretized specs. .. note:: Every ``spack install`` process builds one package at a time with multiple build jobs, controlled by the ``-j`` flag and the ``config:build_jobs`` option (see :ref:`build-jobs`). To speed up environment builds further, independent packages can be installed in parallel by launching more Spack instances. For example, the following will build at most four packages in parallel using three background jobs: .. code-block:: console [myenv]$ spack install & spack install & spack install & spack install Another option is to generate a ``Makefile`` and run ``make -j`` to control the number of parallel install processes. See :ref:`cmd-spack-env-depfile` for details. As it installs, ``spack install`` creates symbolic links in the ``logs/`` directory in the environment, allowing for easy inspection of build logs related to that environment. The ``spack install`` command also stores a Spack repo containing the ``package.py`` file used at install time for each package in the ``repos/`` directory in the environment. The ``--no-add`` option can be used in a concrete environment to tell Spack to install specs already present in the environment but not to add any new root specs to the environment. For root specs provided to ``spack install`` on the command line, ``--no-add`` is the default, while for dependency specs, it is optional. In other words, if there is an unambiguous match in the active concrete environment for a root spec provided to ``spack install`` on the command line, Spack does not require you to specify the ``--no-add`` option to prevent the spec from being added again. At the same time, a spec that already exists in the environment, but only as a dependency, will be added to the environment as a root spec without the ``--no-add`` option. .. _cmd-spack-develop: Developing Packages in a Spack Environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``spack develop`` command allows one to develop Spack packages in an environment. It will configure Spack to install the package from local source. By default, ``spack develop`` will also clone the package to a subdirectory in the environment for the local source. These choices can be overridden with the ``--path`` argument, and the ``--no-clone`` argument. Relative paths provided to the ``--path`` argument will be resolved relative to the environment directory. All of these options are recorded in the environment manifest, although default values may be left implied. .. code-block:: console $ spack develop --path src/foo foo@develop $ cat `spack location -e`/spack.yaml spack: ... develop foo: spec: foo@develop path: src/foo When ``spack develop`` is run in a concretized environment, Spack will modify the concrete specs in the environment to reflect the modified provenance. Any package built from local source will have a ``dev_path`` variant, and the hash of any dependent of those packages will be modified to reflect the change. The value of the ``dev_path`` variant will be the absolute path to the package source directory. If the develop spec conflicts with the concrete specs in the environment, Spack will raise an exception and require the ``spack develop --no-modify-concrete-specs`` option, followed by a ``spack concretize --force`` to apply the ``dev_path`` variant and constraints from the develop spec. When concretizing an environment with develop specs, the version, variants, and other attributes of the spec provided to the ``spack develop`` command will be treated as constraints by the concretizer (in addition to any constraints from the packages ``specs`` list). If the ``develop`` configuration for the package does not include a spec version, Spack will choose the **highest** version of the package. This means that any "infinity" versions (``develop``, ``main``, etc.) will be preferred for specs marked with the ``spack develop`` command, which is different from the standard Spack behavior to prefer the highest **numeric** version. These packages will have an automatic ``dev_path`` variant added by the concretizer, with a value of the absolute path to the local source Spack is building from. Spack will ensure the package and its dependents are rebuilt any time the environment is installed if the package's local source code has been modified. Spack's native implementation is to check if ``mtime`` is newer than the installation. A custom check can be created by overriding the ``detect_dev_src_change`` method in your package class. This is particularly useful for projects using custom Spack repos to drive development and want to optimize performance. When ``spack develop`` is run without any arguments, Spack will clone any develop specs in the environment for which the specified path does not exist. When working deep in the graph it is often desirable to have multiple specs marked as ``develop`` so you don't have to restage and/or do full rebuilds each time you call ``spack install``. The ``--recursive`` flag can be used in these scenarios to ensure that all the dependents of the initial spec you provide are also marked as develop specs. The ``--recursive`` flag requires a pre-concretized environment so the graph can be traversed from the supplied spec all the way to the root specs. For packages with ``git`` attributes, git branches, tags, and commits can also be used as valid concrete versions (see :ref:`version-specifier`). This means that for a package ``foo``, ``spack develop foo@git.main`` will clone the ``main`` branch of the package, and ``spack install`` will install from that git clone if ``foo`` is in the environment. Further development on ``foo`` can be tested by re-installing the environment, and eventually committed and pushed to the upstream git repo. If the package being developed supports out-of-source builds then users can use the ``--build_directory`` flag to control the location and name of the build directory. This is a shortcut to set the ``package_attributes:build_directory`` in the ``packages`` configuration (see :ref:`assigning-package-attributes`). The supplied location will become the build-directory for that package in all future builds. .. admonition:: Potential pitfalls of setting the build directory :class: warning Spack does not check for out-of-source build compatibility with the packages and so the onus of making sure the package supports out-of-source builds is on the user. For example, most ``autotool`` and ``makefile`` packages do not support out-of-source builds while all ``CMake`` packages do. Understanding these nuances is up to the software developers and we strongly encourage developers to only redirect the build directory if they understand their package's build-system. Modifying Specs in an Environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``spack change`` command allows the user to change individual specs in a Spack environment. By default, ``spack change`` operates on the abstract specs of an environment. The command a list of spec arguments. For each argument, the root spec with the same name as the provided spec is modified to satisfy the provided spec. For example, in an environment with the root spec ``hdf5+mpi+fortran``, then .. code-block:: console spack change hdf5~mpi+cxx will change the root spec to ``hdf5~mpi+cxx+fortran``. When more complex matching semantics are necessary, the ``--match-spec`` argument replaces the spec name as the selection criterion. When using the ``--match-spec`` argument, the spec name is not required. In the same environment, .. code-block:: console spack change --match-spec "+fortran" +hl will constrain the ``hdf5`` spec to ``+hl``. By default, the ``spack change`` command will result in an error and no change to the environment if it will modify more than one abstract spec. Use the ``--all`` option to allow ``spack change`` to modify multiple abstract specs. The ``--concrete`` option allows ``spack change`` to modify the concrete specs of an environment as well as the abstract specs. Multiple concrete specs may be modified, even for a change that modifies only a single abstract spec. The ``--all`` option does not affect how many concrete specs may be modified. .. warning:: Concrete specs are modified without any constraints from the packages. The ``spack change --concrete`` command may create invalid specs that will not build properly if applied without caution. The ``--concrete-only`` option allows for modifying concrete specs without modifying abstract specs. It allows changes to be applied to non-root nodes in the environment, and other changes that do not modify any root specs. Loading ^^^^^^^ Once an environment has been installed, the following creates a load script for it: .. code-block:: console $ spack env loads -r This creates a file called ``loads`` in the environment directory. Sourcing that file in Bash will make the environment available to the user, and can be included in ``.bashrc`` files, etc. The ``loads`` file may also be copied out of the environment, renamed, etc. .. _environment_include_concrete: Including Concrete Environments ------------------------------- Spack can create an environment that includes information from already concretized environments. You can think of the new environment as a combination of existing environments. It uses information from the existing environments' ``spack.lock`` files in the creation of the new environment. When such an environment is concretized it will generate its own ``spack.lock`` file that contains relevant information from the included environments. Creating combined concrete environments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To create a combined concrete environment, you must have at least one existing concrete environment. You will use the command ``spack env create`` with the argument ``--include-concrete`` followed by the name or path of the environment you'd like to include. Here is an example of how to create a combined environment from the command line:: $ spack env create myenv $ spack -e myenv add python $ spack -e myenv concretize $ spack env create --include-concrete myenv combined_env You can also include concrete environments directly in the ``spack.yaml`` file. It involves adding the absolute paths to the concrete environments ``spack.lock`` under the new environment's ``include`` heading. Spack-specific configuration variables, such as ``$spack``, and environment variables can be used in the include paths as long as the expression expands to an absolute path. (See :ref:`config-file-variables` for more information.) For example, .. code-block:: yaml spack: include: - /absolute/path/to/environment1/spack.lock - $spack/../path/to/environment2/spack.lock specs: [] concretizer: unify: true will include the specs from ``environment1`` and ``environment2`` where the second environment's path is the absolute path of the directory that is relative to the spack root. .. note:: Once the ``spack.yaml`` file is updated you must concretize the new environment to get the concrete specs from the included environments. This will produce the combined ``spack.lock`` file. Updating a combined environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you want changes made to one of the included environments reflected in the combined environment, then you will need to re-concretize the included environment **then** the combined environment for the change to be incorporated. For example:: $ spack env create myenv $ spack -e myenv add python $ spack -e myenv concretize $ spack env create --include-concrete myenv combined_env $ spack -e myenv find ==> In environment myenv ==> Root specs python ==> 0 installed packages $ spack -e combined_env find ==> In environment combined_env ==> No root specs ==> Included specs python ==> 0 installed packages Here we see that ``combined_env`` contains the python package from ``myenv`` environment. But if we were to add another spec to ``myenv``, ``combined_env`` will not know about the other spec. .. code-block:: spec $ spack -e myenv add perl $ spack -e myenv concretize $ spack -e myenv find ==> In environment myenv ==> Root specs perl python ==> 0 installed packages $ spack -e combined_env find ==> In environment combined_env ==> No root specs ==> Included specs python ==> 0 installed packages It isn't until you run the ``spack concretize`` command that the combined environment will get the updated information from the re-concretized ``myenv``. .. code-block:: console $ spack -e combined_env concretize $ spack -e combined_env find ==> In environment combined_env ==> No root specs ==> Included specs perl python ==> 0 installed packages .. _environment-configuration: Configuring Environments ------------------------ A variety of Spack behaviors are changed through Spack configuration files, covered in more detail in the :ref:`configuration` section. Spack Environments provide an additional level of configuration scope between the custom scope and the user scope discussed in the configuration documentation. There are two ways to include configuration information in a Spack Environment: #. Inline in the ``spack.yaml`` file #. Included in the ``spack.yaml`` file from another file. Many Spack commands also affect configuration information in files automatically. Those commands take a ``--scope`` argument, and the environment can be specified by ``env:NAME`` (to affect environment ``foo``, set ``--scope env:foo``). These commands will automatically manipulate configuration inline in the ``spack.yaml`` file. Inline configurations ^^^^^^^^^^^^^^^^^^^^^ Inline environment-scope configuration is done using the same yaml format as standard Spack configuration scopes, covered in the :ref:`configuration` section. Each section is contained under a top-level yaml object with its name. For example, a ``spack.yaml`` manifest file containing some package preference configuration (as in a ``packages.yaml`` file) could contain: .. code-block:: yaml spack: # ... packages: all: providers: mpi: [openmpi] # ... This configuration sets the default ``mpi`` provider to be ``openmpi``. Included configurations ^^^^^^^^^^^^^^^^^^^^^^^ Spack environments allow an ``include`` heading in their yaml schema. This heading pulls in external configuration files and applies them to the environment. .. code-block:: yaml spack: include: - environment/relative/path/to/config.yaml - path: https://github.com/path/to/raw/config/compilers.yaml sha256: 26e871804a92cd07bb3d611b31b4156ae93d35b6a6d6e0ef3a67871fcb1d258b - /absolute/path/to/packages.yaml - path: /path/to/$os/$target/environment optional: true - path: /path/to/os-specific/config-dir when: os == "ventura" Included configuration files are required *unless* they are explicitly optional or the entry's condition evaluates to ``false``. Optional includes are specified with the ``optional`` clause and conditional with the ``when`` clause. (See :ref:`include-yaml` for more information on optional and conditional entries.) Files are listed using paths to individual files or directories containing them. Path entries may be absolute or relative to the environment or specified as URLs. URLs to individual files must link to the **raw** form of the file's contents (e.g., `GitHub `_ or `GitLab `_) **and** include a valid sha256 for the file. Only the ``file``, ``ftp``, ``http`` and ``https`` protocols (or schemes) are supported. Spack-specific, environment and user path variables can be used. (See :ref:`config-file-variables` for more information.) .. warning:: Recursive includes are not currently processed in a breadth-first manner so the value of a configuration option that is altered by multiple included files may not be what you expect. This will be addressed in a future update. Configuration precedence ^^^^^^^^^^^^^^^^^^^^^^^^ Inline configurations take precedence over included configurations, so you don't have to change shared configuration files to make small changes to an individual environment. Included configurations listed earlier will have higher precedence, as the included configs are applied in reverse order. Manually Editing the Specs List ------------------------------- The list of abstract/root specs in the environment is maintained in the ``spack.yaml`` manifest under the heading ``specs``. .. code-block:: yaml spack: specs: - ncview - netcdf - nco - py-sphinx Appending to this list in the yaml is identical to using the ``spack add`` command from the command line. However, there is more power available from the yaml file. .. _environments_concretization_config: Spec concretization ^^^^^^^^^^^^^^^^^^^ An environment can be concretized in three different modes and the behavior active under any environment is determined by the ``concretizer:unify`` configuration option. The *default* mode is to unify all specs: .. code-block:: yaml spack: specs: - hdf5+mpi - zlib@1.2.8 concretizer: unify: true This means that any package in the environment corresponds to a single concrete spec. In the above example, when ``hdf5`` depends down the line of ``zlib``, it is required to take ``zlib@1.2.8`` instead of a newer version. This mode of concretization is particularly useful when environment views are used: if every package occurs in only one flavor, it is usually possible to merge all install directories into a view. A downside of unified concretization is that it can be overly strict. For example, a concretization error would happen when both ``hdf5+mpi`` and ``hdf5~mpi`` are specified in an environment. The second mode is to *unify when possible*: this makes concretization of root specs more independent. Instead of requiring reuse of dependencies across different root specs, it is only maximized: .. code-block:: yaml spack: specs: - hdf5~mpi - hdf5+mpi - zlib@1.2.8 concretizer: unify: when_possible This means that both ``hdf5`` installations will use ``zlib@1.2.8`` as a dependency even if newer versions of that library are available. The third mode of operation is to concretize root specs entirely independently by disabling unified concretization: .. code-block:: yaml spack: specs: - hdf5~mpi - hdf5+mpi - zlib@1.2.8 concretizer: unify: false In this example ``hdf5`` is concretized separately, and does not consider ``zlib@1.2.8`` as a constraint or preference. Instead, it will take the latest possible version. The last two concretization options are typically useful for system administrators and user support groups providing a large software stack for their HPC center. .. note:: The ``concretizer:unify`` config option was introduced in Spack 0.18 to replace the ``concretization`` property. For reference, ``concretization: together`` is replaced by ``concretizer:unify:true``, and ``concretization: separately`` is replaced by ``concretizer:unify:false``. .. admonition:: Re-concretization of user specs The ``spack concretize`` command without additional arguments will *not* change any previously concretized specs. This may prevent it from finding a solution when using ``unify: true``, and it may prevent it from finding a minimal solution when using ``unify: when_possible``. You can force Spack to ignore the existing concrete environment with ``spack concretize -f``. .. _environment-spec-matrices: Spec Matrices ^^^^^^^^^^^^^ Entries in the ``specs`` list can be individual abstract specs or a spec matrix. A spec matrix is a yaml object containing multiple lists of specs, and evaluates to the cross-product of those specs. Spec matrices also contain an ``excludes`` directive, which eliminates certain combinations from the evaluated result. The following two environment manifests are identical: .. code-block:: yaml spack: specs: - zlib %gcc@7.1.0 - zlib %gcc@4.9.3 - libelf %gcc@7.1.0 - libelf %gcc@4.9.3 - libdwarf %gcc@7.1.0 - cmake .. code-block:: yaml spack: specs: - matrix: - [zlib, libelf, libdwarf] - ["%gcc@7.1.0", "%gcc@4.9.3"] exclude: - libdwarf%gcc@4.9.3 - cmake Spec matrices can be used to install swaths of software across various toolchains. .. _spec-list-references: Spec List References ^^^^^^^^^^^^^^^^^^^^ The last type of possible entry in the specs list is a reference. The Spack Environment manifest yaml schema contains an additional heading ``definitions``. Under definitions is an array of yaml objects. Each object has one or two fields. The one required field is a name, and the optional field is a ``when`` clause. The named field is a spec list. The spec list uses the same syntax as the ``specs`` entry. Each entry in the spec list can be a spec, a spec matrix, or a reference to an earlier named list. References are specified using the ``$`` sigil, and are "splatted" into place (i.e. the elements of the referent are at the same level as the elements listed separately). As an example, the following two manifest files are identical. .. code-block:: yaml spack: definitions: - first: [libelf, libdwarf] - compilers: ["%gcc", "%intel"] - second: - $first - matrix: - [zlib] - [$compilers] specs: - $second - cmake .. code-block:: yaml spack: specs: - libelf - libdwarf - zlib%gcc - zlib%intel - cmake .. note:: Named spec lists in the definitions section may only refer to a named list defined above itself. Order matters. In short files like the example, it may be easier to simply list the included specs. However for more complicated examples involving many packages across many toolchains, separately factored lists make environments substantially more manageable. Additionally, the ``-l`` option to the ``spack add`` command allows one to add to named lists in the definitions section of the manifest file directly from the command line. The ``when`` directive can be used to conditionally add specs to a named list. The ``when`` directive takes a string of Python code referring to a restricted set of variables, and evaluates to a boolean. The specs listed are appended to the named list if the ``when`` string evaluates to ``True``. In the following snippet, the named list ``compilers`` is ``["%gcc", "%clang", "%intel"]`` on ``x86_64`` systems and ``["%gcc", "%clang"]`` on all other systems. .. code-block:: yaml spack: definitions: - compilers: ["%gcc", "%clang"] - when: arch.satisfies("target=x86_64:") compilers: ["%intel"] .. note:: Any definitions with the same named list with true ``when`` clauses (or absent ``when`` clauses) will be appended together The valid variables for a ``when`` clause are: #. ``platform``. The platform string of the default Spack architecture on the system. #. ``os``. The OS string of the default Spack architecture on the system. #. ``target``. The target string of the default Spack architecture on the system. #. ``architecture`` or ``arch``. A Spack spec satisfying the default Spack architecture on the system. This supports querying via the ``satisfies`` method, as shown above. #. ``arch_str``. The architecture string of the default Spack architecture on the system. #. ``re``. The standard regex module in Python. #. ``env``. The user environment (usually ``os.environ`` in Python). #. ``hostname``. The hostname of the system (if ``hostname`` is an executable in the user's PATH). SpecLists as Constraints ^^^^^^^^^^^^^^^^^^^^^^^^ Dependencies and compilers in Spack can be both packages in an environment and constraints on other packages. References to SpecLists allow a shorthand to treat packages in a list as either a compiler or a dependency using the ``$%`` or ``$^`` syntax respectively. For example, the following environment has three root packages: ``gcc@8.1.0``, ``mvapich2@2.3.1 %gcc@8.1.0``, and ``hdf5+mpi %gcc@8.1.0 ^mvapich2@2.3.1``. .. code-block:: yaml spack: definitions: - compilers: [gcc@8.1.0] - mpis: [mvapich2@2.3.1] - packages: [hdf5+mpi] specs: - $compilers - matrix: - [$mpis] - [$%compilers] - matrix: - [$packages] - [$^mpis] - [$%compilers] This allows for a much-needed reduction in redundancy between packages and constraints. .. _environment-spec-groups: Spec Groups ^^^^^^^^^^^ .. versionadded:: 1.2 Environments can be organized with named spec groups, enabling you to apply localized configuration overrides and establish concretization dependencies. This is extremely useful in a couple of common scenarios, as detailed below. .. _environment-spec-groups-bootstrapping-compiler: Building and using a compiler in a single environment """"""""""""""""""""""""""""""""""""""""""""""""""""" A common use case is to build a recent compiler on top of an existing system and then compile a stack of software with it. For instance, assume we are interested in building ``hdf5`` and ``libtree`` with ``gcc@15.2``. The following manifest file would do exactly that: .. code-block:: yaml spack: specs: - group: compiler specs: - gcc@15.2 - group: apps needs: [compiler] specs: - hdf5 %gcc@15.2 - libtree %gcc@15.2 The ``group:`` attribute allows to name a group of specs, which are then listed under the ``specs:`` attribute in the same object. The simplest example is the ``compiler`` group composed of just the ``gcc@15.2`` spec. To express dependencies among groups of specs the ``needs:`` attribute is used, which is a list of names corresponding to the groups we depend on. The way this works is that group dependencies are always concretized *before* the current group, and their specs are *always* available for reuse when the current group is concretized. .. _environment-spec-groups-configuring-groups: Configuring a group of specs """""""""""""""""""""""""""" Another common scenario is the deployment of different configurations (e.g. CUDA enabled vs. ROCm enabled) of the same set of software. As an example, assume we want to install ``gromacs`` and ``quantum-espresso`` for both ``target=x86_64_v3`` and ``target=x86_64_v4``. That can be done with the following manifest file: .. code-block:: yaml spack: - group: apps-x86_64_v3 specs: - gromacs - quantum-espresso override: packages: all: prefer: - target=x86_64_v3 - group: apps-x86_64_v4 specs: - gromacs - quantum-espresso override: packages: all: prefer: - target=x86_64_v4 The ``override:`` attribute allows us to override the configuration for a single group of specs. The overridden part is always added as the *topmost* scope when the current group is concretized. This ensures the override always takes precedence over other sources of configuration. .. _environment-spec-groups-explicit: Controlling garbage collection with ``explicit: false`` """"""""""""""""""""""""""""""""""""""""""""""""""""""" By default every spec group is treated as a set of *explicit* roots. This means its specs are preserved by ``spack gc`` even when nothing else depends on them. Setting ``explicit: false`` on a group marks its specs as *implicit*, making them eligible for garbage collection once no other installed spec depends on them: .. code-block:: yaml spack: specs: - group: compiler explicit: false specs: - gcc@15.2 - group: apps needs: [compiler] specs: - hdf5 %gcc@15.2 - libtree %gcc@15.2 After the apps are installed, ``spack gc`` will remove the compiler once no installed spec has a link or run dependency on it. .. note:: Flipping ``explicit: false`` on a group that has already been installed does **not** retroactively update the database record for the already-installed specs. The flag takes effect only for specs installed, or re-installed, after the change. To immediately mark an existing spec as implicit, use ``spack mark -i ``. Modifying Environment Variables ------------------------------- Spack Environments can modify the active shell's environment variables when activated. The environment can be configured to set, unset, prepend, or append using ``env_vars`` configuration in ``spack.yaml``: .. code-block:: yaml spack: env_vars: set: ENVAR_TO_SET_IN_ENV_LOAD: "FOO" unset: - ENVAR_TO_UNSET_IN_ENV_LOAD prepend_path: PATH_LIST: "path/to/prepend" append_path: PATH_LIST: "path/to/append" remove_path: PATH_LIST: "path/to/remove" Environment Views ----------------- Spack Environments can have an associated filesystem view, which is a directory with a more traditional structure ``/bin``, ``/lib``, ``/include`` in which all files of the installed packages are linked. By default a view is created for each environment, thanks to the ``view: true`` option in the ``spack.yaml`` manifest file: .. code-block:: yaml spack: specs: [perl, python] view: true The view is created in a hidden directory ``.spack-env/view`` relative to the environment. If you've used ``spack env activate``, you may have already interacted with this view. Spack prepends its ``/bin`` dir to ``PATH`` when the environment is activated, so that you can directly run executables from all installed packages in the environment. Views are highly customizable: you can control where they are put, modify their structure, include and exclude specs, change how files are linked, and you can even generate multiple views for a single environment. .. _configuring_environment_views: Minimal view configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^ The minimal configuration .. code-block:: yaml spack: # ... view: true lets Spack generate a single view with default settings under the ``.spack-env/view`` directory of the environment. Another short way to configure a view is to specify just where to put it: .. code-block:: yaml spack: # ... view: /path/to/view Views can also be disabled by setting ``view: false``. .. _cmd-spack-env-view: Advanced view configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^ One or more **view descriptors** can be defined under ``view``, keyed by a name. The example from the previous section with ``view: /path/to/view`` is equivalent to defining a view descriptor named ``default`` with a ``root`` attribute: .. code-block:: yaml spack: # ... view: default: # name of the view root: /path/to/view # view descriptor attribute The ``default`` view descriptor name is special: when you ``spack env activate`` your environment, this view will be used to update (among other things) your ``PATH`` variable. View descriptors must contain the root of the view, and optionally projections, ``select`` and ``exclude`` lists and link information via ``link`` and ``link_type``. As a more advanced example, in the following manifest file snippet we define a view named ``mpis``, rooted at ``/path/to/view`` in which all projections use the package name, version, and compiler name to determine the path for a given package. This view selects all packages that depend on MPI, and excludes those built with the GCC compiler at version 8.5. The root specs with their (transitive) link and run type dependencies will be put in the view due to the ``link: all`` option, and the files in the view will be symlinks to the Spack install directories. .. code-block:: yaml spack: # ... view: mpis: root: /path/to/view select: [^mpi] exclude: ["%gcc@8.5"] projections: all: "{name}/{version}-{compiler.name}" link: all link_type: symlink The default for the ``select`` and ``exclude`` values is to select everything and exclude nothing. The default projection is the default view projection (``{}``). The ``link`` attribute allows the following values: #. ``link: all`` include root specs with their transitive run and link type dependencies (default); #. ``link: run`` include root specs with their transitive run type dependencies; #. ``link: roots`` include root specs without their dependencies. The ``link_type`` defaults to ``symlink`` but can also take the value of ``hardlink`` or ``copy``. .. tip:: The option ``link: run`` can be used to create small environment views for Python packages. Python will be able to import packages *inside* of the view even when the environment is not activated, and linked libraries will be located *outside* of the view thanks to rpaths. From the command line, the ``spack env create`` command takes an argument ``--with-view [PATH]`` that sets the path for a single, default view. If no path is specified, the default path is used (``view: true``). The argument ``--without-view`` can be used to create an environment without any view configured. The ``spack env view`` command can be used to manage views of an environment. The subcommand ``spack env view enable`` will add a view named ``default`` to an environment. It takes an optional argument to specify the path for the new default view. The subcommand ``spack env view disable`` will remove the view named ``default`` from an environment if one exists. The subcommand ``spack env view regenerate`` will regenerate the views for the environment. This will apply any updates in the environment configuration that have not yet been applied. .. _view_projections: View Projections """""""""""""""" The default projection into a view is to link every package into the root of the view. The projections attribute is a mapping of partial specs to spec format strings, defined by the :meth:`~spack.spec.Spec.format` function, as shown in the example below: .. code-block:: yaml projections: zlib: "{name}-{version}" ^mpi: "{name}-{version}/{^mpi.name}-{^mpi.version}-{compiler.name}-{compiler.version}" all: "{name}-{version}/{compiler.name}-{compiler.version}" Projections also permit environment and Spack configuration variable expansions as shown below: .. code-block:: yaml projections: all: "{name}-{version}/{compiler.name}-{compiler.version}/$date/$SYSTEM_ENV_VARIABLE" where ``$date`` is the Spack configuration variable that will expand with the ``YYYY-MM-DD`` format and ``$SYSTEM_ENV_VARIABLE`` is an environment variable defined in the shell. The entries in the projections configuration file must all be either specs or the keyword ``all``. For each spec, the projection used will be the first non-``all`` entry that the spec satisfies, or ``all`` if there is an entry for ``all`` and no other entry is satisfied by the spec. Where the keyword ``all`` appears in the file does not matter. Given the example above, the spec ``zlib@1.2.8`` will be linked into ``/my/view/zlib-1.2.8/``, the spec ``hdf5@1.8.10+mpi %gcc@4.9.3 ^mvapich2@2.2`` will be linked into ``/my/view/hdf5-1.8.10/mvapich2-2.2-gcc-4.9.3``, and the spec ``hdf5@1.8.10~mpi %gcc@4.9.3`` will be linked into ``/my/view/hdf5-1.8.10/gcc-4.9.3``. If the keyword ``all`` does not appear in the projections configuration file, any spec that does not satisfy any entry in the file will be linked into the root of the view as in a single-prefix view. Any entries that appear below the keyword ``all`` in the projections configuration file will not be used, as all specs will use the projection under ``all`` before reaching those entries. Group of Specs """""""""""""" Views can also be applied to a selected list of :ref:`spec groups `. This can be done by specifying the ``group:`` attribute in the view configuration. For instance, with the following manifest: .. code-block:: yaml spack: concretizer: unify: true packages: all: require: - target=x86_64_v4 specs: - group: compiler specs: - gcc@15.2 - group: apps needs: [compiler] specs: - hdf5~mpi %gcc@15.2 - libtree %gcc@15.2 view: apps: root: ./views/apps group: apps The view will only contain entries from the ``apps`` group, and will not include specs from the ``compiler`` group. Activating environment views ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``spack env activate `` command has two effects: 1. It activates the environment so that further Spack commands such as ``spack install`` will run in the context of the environment. 2. It activates the view so that environment variables such as ``PATH`` are updated to include the view. Without further arguments, the ``default`` view of the environment is activated. If a view with a different name has to be activated, ``spack env activate --with-view `` can be used instead. You can also activate the environment without modifying further environment variables using ``--without-view``. The environment variables affected by the ``spack env activate`` command and the paths that are used to update them are determined by the :ref:`prefix inspections ` defined in your modules configuration; the defaults are summarized in the following table. =================== ========= Variable Paths =================== ========= PATH bin MANPATH man, share/man ACLOCAL_PATH share/aclocal PKG_CONFIG_PATH lib/pkgconfig, lib64/pkgconfig, share/pkgconfig CMAKE_PREFIX_PATH . =================== ========= Each of these paths are appended to the view root, and added to the relevant variable if the path exists. For this reason, it is not recommended to use non-default projections with the default view of an environment. The ``spack env deactivate`` command will remove the active view of the Spack environment from the user's environment variables. .. _cmd-spack-env-depfile: Generating Depfiles from Environments ------------------------------------------ Spack can generate ``Makefile``\s to make it easier to build multiple packages in an environment in parallel. .. note:: Since Spack v1.1, there is a new experimental installer that supports package-level parallelism out of the box with POSIX jobserver support. You can enable it with ``spack config add config:installer:new``. This new installer may provide a simpler alternative to the ``spack env depfile`` workflow described in this section for users primarily interested in speeding up environment installations. Generated ``Makefile``\s expose targets that can be included in existing ``Makefile``\s, to allow other targets to depend on the environment installation. A typical workflow is as follows: .. code-block:: spec $ spack env create -d . $ spack -e . add perl $ spack -e . concretize $ spack -e . env depfile -o Makefile $ make -j64 This generates a ``Makefile`` from a concretized environment in the current working directory, and ``make -j64`` installs the environment, exploiting parallelism across packages as much as possible. Spack respects the Make jobserver and forwards it to the build environment of packages, meaning that a single ``-j`` flag is enough to control the load, even when packages are built in parallel. By default the following phony convenience targets are available: - ``make all``: installs the environment (default target); - ``make clean``: cleans files used by make, but does not uninstall packages. .. tip:: GNU Make version 4.3 and above have great support for output synchronization through the ``-O`` and ``--output-sync`` flags, which ensure that output is printed orderly per package install. To get synchronized output with colors, use ``make -j SPACK_COLOR=always --output-sync=recurse``. Specifying dependencies on generated ``make`` targets ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An interesting question is how to include generated ``Makefile``\s in your own ``Makefile``\s. This comes up when you want to install an environment that provides executables required in a command for a make target of your own. The example below shows how to accomplish this: the ``env`` target specifies the generated ``spack/env`` target as a prerequisite, meaning that the environment gets installed and is available for use in the ``env`` target. .. code-block:: Makefile SPACK ?= spack .PHONY: all clean env all: env spack.lock: spack.yaml $(SPACK) -e . concretize -f env.mk: spack.lock $(SPACK) -e . env depfile -o $@ --make-prefix spack env: spack/env $(info environment installed!) clean: rm -rf spack.lock env.mk spack/ ifeq (,$(filter clean,$(MAKECMDGOALS))) include env.mk endif This works as follows: when ``make`` is invoked, it first "remakes" the missing include ``env.mk`` as there is a target for it. This triggers concretization of the environment and makes Spack output ``env.mk``. At that point the generated target ``spack/env`` becomes available through ``include env.mk``. As it is typically undesirable to remake ``env.mk`` as part of ``make clean``, the include is conditional. .. note:: When including generated ``Makefile``\s, it is important to use the ``--make-prefix`` flag and use the non-phony target ``/env`` as prerequisite, instead of the phony target ``/all``. Building a subset of the environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The generated ``Makefile``\s contain install targets for each spec, identified by ``--``. This allows you to install only a subset of the packages in the environment. When packages are unique in the environment, it's enough to know the name and let tab-completion fill out the version and hash. The following phony targets are available: ``install/`` to install the spec with its dependencies, and ``install-deps/`` to *only* install its dependencies. This can be useful when certain flags should only apply to dependencies. Below we show a use case where a spec is installed with verbose output (``spack install --verbose``) while its dependencies are installed silently: .. code-block:: console $ spack env depfile -o Makefile # Install dependencies in parallel, only show a log on error. $ make -j16 install-deps/python-3.11.0- SPACK_INSTALL_FLAGS=--show-log-on-error # Install the root spec with verbose output. $ make -j16 install/python-3.11.0- SPACK_INSTALL_FLAGS=--verbose Adding post-install hooks ^^^^^^^^^^^^^^^^^^^^^^^^^ Another advanced use-case of generated ``Makefile``\s is running a post-install command for each package. These "hooks" could be anything from printing a post-install message, running tests, or pushing just-built binaries to a build cache. This can be accomplished through the generated ``[/]SPACK_PACKAGE_IDS`` variable. Assuming we have an active and concrete environment, we generate the associated ``Makefile`` with a prefix ``example``: .. code-block:: console $ spack env depfile -o env.mk --make-prefix example And we now include it in a different ``Makefile``, in which we create a target ``example/push/%`` with ``%`` referring to a package identifier. This target depends on the particular package installation. In this target we automatically have the target-specific ``HASH`` and ``SPEC`` variables at our disposal. They are respectively the spec hash (excluding leading ``/``), and a human-readable spec. Finally, we have an entry point target ``push`` that will update the build cache index once every package is pushed. Note how this target uses the generated ``example/SPACK_PACKAGE_IDS`` variable to define its prerequisites. .. code-block:: Makefile SPACK ?= spack BUILDCACHE_DIR = $(CURDIR)/tarballs .PHONY: all all: push include env.mk example/push/%: example/install/% @mkdir -p $(dir $@) $(info About to push $(SPEC) to a buildcache) $(SPACK) -e . buildcache push --only=package $(BUILDCACHE_DIR) /$(HASH) @touch $@ push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS)) $(info Updating the buildcache index) $(SPACK) -e . buildcache update-index $(BUILDCACHE_DIR) $(info Done!) @touch $@ ================================================ FILE: lib/spack/docs/environments_basics.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to use Spack environments to manage reproducible software stacks on a local machine. Spack Environments ================== Spack is a powerful package manager designed for the complex software needs of supercomputers. These same robust features for managing versions and dependencies also make it an excellent tool for local development on a laptop or workstation. If you are used to tools like Conda, Homebrew or pip for managing local command-line tools and development projects, you will find Spack environments to be a powerful and flexible alternative. Spack environments allow you to create self-contained, reproducible software collections, a concept similar to Conda environments and Python's virtual environments. Unlike other package managers, Spack environments do not contain copies of the software themselves. Instead, they reference installations in the Spack store, which is a central location where Spack keeps all installed packages. This means that multiple environments can share the same package installations, saving disk space and reducing duplication. In this section, we will walk through creating a simple environment to manage a personal software stack. Creating and Activating an Environment -------------------------------------- First, let's create and activate a new environment. This places you "inside" the environment, so all subsequent Spack commands apply to it by default. .. code-block:: console $ spack env create myenv ==> Created environment myenv in /path/to/spack/var/spack/environments/myenv $ spack env activate myenv Here, *myenv* is the name of our new environment. You can verify you are in the environment using: .. code-block:: console $ spack env status ==> In environment myenv Adding Specs to the Environment ------------------------------- Now that our environment is active, we can add the packages we want to install. Let's say we want a newer version of curl and a few Python libraries. .. code-block:: spec $ spack add curl@8 python py-numpy py-scipy py-matplotlib You can add packages one at a time or all at once. Notice that we didn't need to specify the environment name, as Spack knows we are working inside ``myenv``. These packages are now added to the environment's manifest file, ``spack.yaml``. You can view the manifest at any time by running: .. code-block:: console $ spack config edit This will open your ``spack.yaml``, which should look like this: .. code-block:: yaml :caption: Example ``spack.yaml`` for our environment # This is a Spack Environment file. # # It describes a set of packages to be installed, along with # configuration settings. spack: # add package specs to the `specs` list specs: - curl@8 - python - py-numpy - py-scipy - py-matplotlib view: true concretizer: unify: true The ``view: true`` setting tells Spack to create a single directory where all executables, libraries, etc., are symlinked together, similar to a traditional Unix prefix. By default, this view is located inside the environment directory. Installing the Software ----------------------- With our specs defined, the next step is to have Spack solve the dependency graph. This is called "concretization." .. code-block:: console $ spack concretize ==> Concretized ... ... Spack will find a consistent set of versions and dependencies for the packages you requested. Once this is done, you can install everything with a single command: .. code-block:: console $ spack install Spack will now download, build, and install all the necessary packages. After the installation is complete, the environment's view is automatically updated. Because the environment is active, your ``PATH`` and other variables are already configured. You can verify the installation: .. code-block:: console $ which python3 /path/to/spack/var/spack/environments/myenv/.spack-env/view/bin/python3 When you are finished working in the environment, you can deactivate it: .. code-block:: console $ spack env deactivate Keeping Up With Updates ----------------------- Over time, you may want to update the packages in your environment to their latest versions. Spack makes this easy. First, update Spack's package repository to make the latest package versions available: .. code-block:: console $ spack repo update Then, activate the environment, re-concretize and reinstall. .. code-block:: console $ spack env activate myenv $ spack concretize --fresh-roots --force $ spack install The ``--fresh-roots`` flag tells the concretizer to prefer the latest available package versions you've added explicitly to the environment, while allowing existing dependencies to remain unchanged if possible. Alternatively, you can use the ``--fresh`` flag to prefer the latest versions of all packages including dependencies, but that might lead to longer install times and more changes. The ``--force`` flag allows it to overwrite the previously solved dependencies. The ``install`` command is smart and will only build packages that are not already installed for the new configuration. Cleaning Up Old Packages ------------------------ After an update, you may have old, unused packages taking up space. You can safely remove any package that is no longer part of an environment's dependency tree. .. code-block:: console $ spack gc --except-any-environment This runs Spack's garbage collector, which will find and uninstall any package versions that are no longer referenced by *any* of your environments. Removing the Environment ------------------------ If you no longer need an environment, you can completely remove it. First, ensure the environment is not active: .. code-block:: console $ spack env deactivate Then, remove the environment. .. code-block:: console $ spack env rm myenv This removes the environment's directory and its view, but the packages that were installed for it remain in the Spack store. To actually remove the installations from the Spack store and free up disk space, you can run the garbage collector again. .. code-block:: console $ spack gc --except-any-environment This command will safely uninstall any packages that are no longer referenced by any of your remaining environments. Next steps ---------- Spack has many other features for managing software environments. See :doc:`environments` for more advanced usage. ================================================ FILE: lib/spack/docs/extensions.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Discover how to extend Spack's core functionality by creating custom commands and plugins. Custom Extensions ================= *Spack extensions* allow you to add custom subcommands to the ``spack`` command. This is extremely useful when developing and maintaining a command whose purpose is too specific to be included in the Spack codebase. It's also useful for evolving a command through its early stages before starting a discussion to merge it upstream. From Spack's point of view, an extension is any path in your filesystem that respects the following naming and layout for files: .. code-block:: console spack-scripting/ # The top level directory must match the format 'spack-{extension_name}' ├── pytest.ini # Optional file if the extension ships its own tests ├── scripting # Folder that may contain modules that are needed for the extension commands │ ├── cmd # Folder containing extension commands │ │ └── filter.py # A new command that will be available │ └── functions.py # Module with internal details ├── tests # Tests for this extension │ ├── conftest.py │ └── test_filter.py └── templates # Templates that may be needed by the extension In the example above, the extension is named *scripting*. It adds an additional command (``spack filter``) and unit tests to verify its behavior. The extension can import any core Spack module in its implementation. When loaded by the ``spack`` command, the extension itself is imported as a Python package in the ``spack.extensions`` namespace. In the example above, since the extension is named "scripting", the corresponding Python module is ``spack.extensions.scripting``. The code for this example extension can be obtained by cloning the corresponding git repository: .. code-block:: console $ git -C /tmp clone https://github.com/spack/spack-scripting.git Configure Spack to Use Extensions --------------------------------- To make your current Spack instance aware of extensions you should add their root paths to ``config.yaml``. In the case of our example, this means ensuring that: .. code-block:: yaml config: extensions: - /tmp/spack-scripting is part of your configuration file. Once this is set up, any command that the extension provides will be available from the command line: .. code-block:: console $ spack filter --help usage: spack filter [-h] [--installed | --not-installed] [--explicit | --implicit] [--output OUTPUT] ... filter specs based on their properties positional arguments: specs specs to be filtered optional arguments: -h, --help show this help message and exit --installed select installed specs --not-installed select specs that are not yet installed --explicit select specs that were installed explicitly --implicit select specs that are not installed or were installed implicitly --output OUTPUT where to dump the result The corresponding unit tests can be run giving the appropriate options to ``spack unit-test``: .. code-block:: console $ spack unit-test --extension=scripting ========================================== test session starts =========================================== platform linux -- Python 3.11.5, pytest-7.4.3, pluggy-1.3.0 rootdir: /home/culpo/github/spack-scripting configfile: pytest.ini testpaths: tests plugins: xdist-3.5.0 collected 5 items tests/test_filter.py ..... [100%] ========================================== slowest 30 durations ========================================== 2.31s setup tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0] 0.57s call tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2] 0.56s call tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4] 0.54s call tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3] 0.54s call tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1] 0.48s call tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0] 0.01s setup tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4] 0.01s setup tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2] 0.01s setup tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1] 0.01s setup tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3] (5 durations < 0.005s hidden. Use -vv to show these durations.) =========================================== 5 passed in 5.06s ============================================ Registering Extensions via Entry Points --------------------------------------- .. note:: Python version >= 3.8 is required to register extensions via entry points. Spack can be made aware of extensions that are installed as part of a Python package. To do so, register a function that returns the extension path, or paths, to the ``"spack.extensions"`` entry point. Consider the Python package ``my_package`` that includes a Spack extension: .. code-block:: console my-package/ ├── src │ ├── my_package │ │ └── __init__.py │ └── spack-scripting/ # the spack extensions └── pyproject.toml adding the following to ``my_package``'s ``pyproject.toml`` will make the ``spack-scripting`` extension visible to Spack when ``my_package`` is installed: .. code-block:: toml [project.entry_points."spack.extensions"] my_package = "my_package:get_extension_path" The function ``my_package.get_extension_path`` in ``my_package/__init__.py`` might look like .. code-block:: python import importlib.resources def get_extension_path(): dirname = importlib.resources.files("my_package").joinpath("spack-scripting") if dirname.exists(): return str(dirname) ================================================ FILE: lib/spack/docs/features.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: An overview of the key features that distinguish Spack from other package managers, including simple installation, custom configurations, and non-destructive installs. Feature Overview ================ This is a high-level overview of features that make Spack different from other `package managers `_ and `port systems `_. Simple package installation --------------------------- Installing the default version of a package is simple. This will install the latest version of the ``mpileaks`` package and all of its dependencies: .. code-block:: spec $ spack install mpileaks Custom versions & configurations -------------------------------- Spack allows installation to be customized. Users can specify the version, compile-time options, and target architecture, all on the command line. .. code-block:: spec # Install a particular version by appending @ $ spack install hdf5@1.14 # Add special compile-time options by name $ spack install hdf5@1.14 api=v110 # Add special boolean compile-time options with + $ spack install hdf5@1.14 +hl # Add compiler flags using the conventional names $ spack install hdf5@1.14 cflags="-O3 -floop-block" # Target a specific micro-architecture $ spack install hdf5@1.14 target=icelake Users can specify as many or as few options as they care about. Spack will fill in the unspecified values with sensible defaults. Customize dependencies ---------------------- Spack allows *dependencies* of a particular installation to be customized extensively. Users can specify both *direct* dependencies of a package, using the ``%`` sigil, or *transitive* dependencies, using the ``^`` sigil: .. code-block:: spec # Install hdf5 using gcc@15 as a compiler (direct dependency of hdf5) $ spack install hdf5@1.14 %gcc@15 # Install hdf5 using hwloc with CUDA enabled (transitive dependency) $ spack install hdf5@1.14 ^hwloc+cuda The expression on the command line can be as simple or as complicated as the user needs: .. code-block:: spec # Install hdf5 compiled with gcc@15, linked to mpich compiled with gcc@14 $ spack install hdf5@1.14 %gcc@15 ^mpich %gcc@14 Non-destructive installs ------------------------ Spack installs every unique package/dependency configuration into its own prefix, so new installs will not break existing ones. Packages can peacefully coexist ------------------------------- Spack avoids library misconfiguration by using ``RPATH`` to link dependencies. When a user links a library or runs a program, it is tied to the dependencies it was built with, so there is no need to manipulate ``LD_LIBRARY_PATH`` at runtime. Unprivileged user installs -------------------------- Spack does not require administrator privileges to install packages. You can install software in any directory you choose, making it easy to manage packages in your home directory or shared project locations without needing sudo access. From source and binary ---------------------- Spack's core strength is creating highly customized, optimized software builds from source code. While it's primarily a from-source package manager, it also supports fast binary installations through build caches. Contributing is easy -------------------- To contribute a new package, all Spack needs is a URL for the source archive. The ``spack create`` command will create a boilerplate package file, and the package authors can fill in specific build steps in pure Python. For example, this command: .. code-block:: console $ spack create https://ftp.osuosl.org/pub/blfs/conglomeration/libelf/libelf-0.8.13.tar.gz creates a simple Python file: .. code-block:: python from spack.package import * class Libelf(AutotoolsPackage): """FIXME: Put a proper description of your package here.""" # FIXME: Add a proper url for your package's homepage here. homepage = "https://www.example.com" url = "https://ftp.osuosl.org/pub/blfs/conglomeration/libelf/libelf-0.8.13.tar.gz" # FIXME: Add a list of GitHub accounts to # notify when the package is updated. # maintainers("github_user1", "github_user2") version("0.8.13", sha256="591a9b4ec81c1f2042a97aa60564e0cb79d041c52faa7416acb38bc95bd2c76d") # FIXME: Add dependencies if required. # depends_on("foo") def configure_args(self): # FIXME: Add arguments other than --prefix # FIXME: If not needed delete this function args = [] return args It doesn't take much Python coding to get from there to a working package: .. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/libelf/package.py :lines: 5- Understanding Spack's scope --------------------------- Spack is a package manager designed for performance and customization of software. To clarify its role and prevent common misconceptions, it's helpful to understand what falls outside of its current scope: 1. Spack is a user-space tool, not an operating system. It runs on top of your existing OS (like Linux, macOS, or Windows) and complements the system's native package manager (like ``yum`` or ``apt``), but does not replace it. Spack relies on the host system for essentials like the C runtime libraries. Building a software stack with a custom `libc` is a planned future capability but is not yet implemented. 2. Spack performs native builds, not cross-compilation. It builds software for the same processor architecture it is running on. Support for cross-compilation (e.g., building for an ARM processor on an x86 machine) is a planned future capability but is not yet implemented. ================================================ FILE: lib/spack/docs/frequently_asked_questions.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Answers to common Spack questions, including version and variant selection, package preferences, compiler configuration, and concretizer behavior, with practical YAML and command-line examples. Frequently Asked Questions ========================== This page contains answers to frequently asked questions about Spack. If you have questions that are not answered here, feel free to ask on `Slack `_ or `GitHub Discussions `_. If you've learned the answer to a question that you think should be here, please consider contributing to this page. .. _faq-concretizer-precedence: Why does Spack pick particular versions and variants? ----------------------------------------------------- This question comes up in a variety of forms: 1. Why does Spack seem to ignore my package preferences from ``packages.yaml`` configuration? 2. Why does Spack toggle a variant instead of using the default from the ``package.py`` file? The short answer is that Spack always picks an optimal configuration based on a complex set of criteria\ [#f1]_. These criteria are more nuanced than always choosing the latest versions or default variants. .. note:: As a rule of thumb: requirements + constraints > strong preferences > reuse > preferences > defaults. The following set of criteria (from lowest to highest precedence) explains common cases where concretization output may seem surprising at first. 1. :ref:`Package preferences ` configured in ``packages.yaml`` override variant defaults from ``package.py`` files, and influence the optimal ordering of versions. Preferences are specified as follows: .. code-block:: yaml packages: foo: version: [1.0, 1.1] variants: ~mpi 2. :ref:`Reuse concretization ` configured in ``concretizer.yaml`` overrides preferences, since it's typically faster to reuse an existing spec than to build a preferred one from sources. When build caches are enabled, specs may be reused from a remote location too. Reuse concretization is configured as follows: .. code-block:: yaml concretizer: reuse: dependencies # other options are 'true' and 'false' 3. :ref:`Strong preferences ` configured in ``packages.yaml`` are higher priority than reuse, and can be used to strongly prefer a specific version or variant, without erroring out if it's not possible. Strong preferences are specified as follows: .. code-block:: yaml packages: foo: prefer: - "@1.1: ~mpi" 4. :ref:`Package requirements ` configured in ``packages.yaml``, and constraints from the command line as well as ``package.py`` files override all of the above. Requirements are specified as follows: .. code-block:: yaml packages: foo: require: - "@1.2: +mpi" conflict: - "@1.4" Requirements and constraints restrict the set of possible solutions, while reuse behavior and preferences influence what an optimal solution looks like. How do I use a specific compiler? --------------------------------- When you have multiple compilers available in :ref:`spack-compiler-list`, and want to build your packages with a specific one, you have the following options: 1. Specify your compiler preferences globally for all packages in configuration files. 2. Specify them on the level of individual specs, like ``pkg %gcc@15`` or ``pkg %c,cxx=gcc@15``. We'll explore both options in more detail. Specific compiler for all packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you want to use a specific compiler for all packages, it's best to use :ref:`strong preferences in packages.yaml config `. The following example prefers GCC 15 for all languages ``c``, ``cxx``, and ``fortran``: .. code-block:: yaml :caption: Recommended: *prefer* a specific compiler :name: code-example-prefer-compiler packages: c: prefer: - gcc@15 cxx: prefer: - gcc@15 fortran: prefer: - gcc@15 You can also replace ``prefer:`` with ``require:`` if you want Spack to produce an error if the preferred compiler cannot be used. See also :ref:`the previous FAQ entry `. In Spack, the languages ``c``, ``cxx`` and ``fortran`` are :ref:`virtual packages `, on which packages depend if they need a compiler for that language. Compiler packages provide these language virtuals. When you specify these strong preferences, Spack determines whether the package depends on any of the language virtuals, and if so, it applies the associated compiler spec when possible. What is **not recommended** is to define ``%gcc`` as a required dependency of all packages: .. code-block:: yaml :caption: Incorrect: requiring a dependency on a compiler for all packages :name: code-example-typical-mistake-require-compiler packages: all: require: - "%gcc@15" This is *incorrect*, because some packages do not need a compiler at all (e.g. pure Python packages). Specific compiler for individual specs ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If different parts of your software stack need to be built with different compilers, it's best to specify compilers as dependencies of the relevant specs (whether on the command line or in Spack environments). .. code-block:: spec :caption: Example of specifying different compilers for different specs :name: console-example-different-compilers $ spack install foo %gcc@15 ^bar %intel-oneapi-compilers What this means is that ``foo`` will depend on GCC 15, while ``bar`` will depend on ``intel-oneapi-compilers``. You can also be more specific about what compiler to use for a particular language: .. code-block:: spec :caption: Example of specifying different compilers for different languages :name: console-example-different-languages $ spack install foo %c,cxx=gcc@15 %fortran=intel-oneapi-compilers These input specs can be simplified using :doc:`toolchains_yaml`. See also :ref:`pitfalls-without-toolchains` for common mistakes to avoid. .. rubric:: Footnotes .. [#f1] The exact list of criteria can be retrieved with the :ref:`spack-solve` command. ================================================ FILE: lib/spack/docs/getting_help.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Find out how to get help with Spack, including using the spack help command. Getting Help ============ .. _cmd-spack-help: ``spack help`` -------------- If you don't find what you need here, the ``help`` subcommand will print out a list of *all* of Spack's options and subcommands: .. command-output:: spack help Adding an argument, e.g., ``spack help ``, will print out usage information for a particular subcommand: .. command-output:: spack help install Alternatively, you can use ``spack --help`` in place of ``spack help``, or ``spack --help`` to get help on a particular subcommand. ================================================ FILE: lib/spack/docs/getting_started.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A beginner's guide to Spack, walking you through the initial setup, basic commands, and core concepts to get you started with managing software. .. _getting_started: Getting Started =============== Getting Spack is easy. You can clone it from the `GitHub repository `_ using this command: .. code-block:: console $ git clone --depth=2 https://github.com/spack/spack.git This will create a directory called ``spack``. Once you have cloned Spack, we recommend sourcing the appropriate script for your shell. For *bash*, *zsh* and *sh* users: .. code-block:: console $ . spack/share/spack/setup-env.sh For *csh* and *tcsh* users: .. code-block:: console $ source spack/share/spack/setup-env.csh For *fish* users: .. code-block:: console $ . spack/share/spack/setup-env.fish Now you're ready to use Spack! List packages you can install ----------------------------- Once Spack is ready, you can list all the packages it knows about with the following command: .. code-block:: spec $ spack list If you want to get more information on a specific package, for instance ``hdf5``, you can use: .. code-block:: spec $ spack info hdf5 This command shows information about ``hdf5``, including a brief description, the versions of the package Spack knows about, and all the options you can activate when installing. As you can see, it's quite simple to gather basic information on packages before you install them! .. admonition:: Slowdown on the very first command :class: warning The first command you run with Spack may take a while, as Spack builds caches to speed up future commands. Installing your first package ----------------------------- To install most packages, Spack needs a compiler suite to be available. To search your machine for available compilers, you can run: .. code-block:: console $ spack compiler find The command shows users whether any compilers were found and where their configuration is stored. If the search was successful, you can now list known compilers, and get an output similar to the following: .. code-block:: console $ spack compiler list ==> Available compilers -- gcc ubuntu20.04-x86_64 --------------------------------------- [e] gcc@9.4.0 [e] gcc@8.4.0 [e] gcc@10.5.0 If no compilers were found, you need to either: * Install further prerequisites, see :ref:`verify-spack-prerequisites`, and repeat the search above. * Register a build cache that provides a compiler already available as a binary Once a compiler is available, you can proceed installing your first package: .. code-block:: spec $ spack install tcl The output of this command should look similar to the following: .. code-block:: text [e] zmjbkxx gcc@10.5.0 /usr (0s) [e] rawvy4p glibc@2.31 /usr (0s) [+] 5qfbgng compiler-wrapper@1.0 /home/spack/.local/spack/opt/linux-icelake/compiler-wrapper-1.0-5qfbgngzoqcjfbwrjn2vh75fr3g25c35 (0s) [+] vchaib2 gcc-runtime@10.5.0 /home/spack/.local/spack/opt/linux-icelake/gcc-runtime-10.5.0-vchaib2njqlk2cud4a2n33tabq526qjj (0s) [+] vzazvty gmake@4.4.1 /home/spack/.local/spack/opt/linux-icelake/gmake-4.4.1-vzazvtyn5cjdmg3vkkuau35x7hzu7pyl (12s) [+] soedrhb zlib-ng@2.3.3 /home/spack/.local/spack/opt/linux-icelake/zlib-ng-2.3.3-soedrhbnpeordiixaib6utcple6tpgya (3s) [+] u6nztpk tcl@8.6.17 /home/spack/.local/spack/opt/linux-icelake/tcl-8.6.17-u6nztpkhzbga4ul665qqhxucxqk3cins (49s) Congratulations! You just installed your first package with Spack! Use the software you just installed ----------------------------------- Once you have installed ``tcl``, you can immediately use it by starting the ``tclsh`` with its absolute path: .. code-block:: console $ /home/spack/.local/spack/opt/linux-icelake/tcl-8.6.17-u6nztpkhzbga4ul665qqhxucxqk3cins/bin/tclsh >% echo "Hello world!" Hello world! This works, but using such a long absolute path is not the most convenient way to run an executable. The simplest way to have ``tclsh`` available on the command line is: .. code-block:: spec $ spack load tcl The environment of the current shell has now been modified, and you can run .. code-block:: console $ tclsh directly. To undo these modifications, you can: .. code-block:: spec $ spack unload tcl .. admonition:: Environments and views :class: tip :doc:`Spack Environments ` are a better way to install and load a set of packages that are frequently used together. The discussion of this topic goes beyond this ``Getting Started`` guide, and we refer to :ref:`environments` for more information. Next steps ---------- This section helped you get Spack installed and running quickly. There are further resources in the documentation that cover both basic and advanced topics in more detail: Basic Usage 1. :ref:`basic-usage` 2. :ref:`compiler-config` 3. :doc:`environments_basics` Advanced Topics 1. :ref:`toolchains` 2. :ref:`cmd-spack-audit` 3. :ref:`cmd-spack-verify` ================================================ FILE: lib/spack/docs/google5fda5f94b4ffb8de.html ================================================ google-site-verification: google5fda5f94b4ffb8de.html ================================================ FILE: lib/spack/docs/gpu_configuration.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to configuring Spack to use external GPU support, including ROCm and CUDA installations, as well as the OpenGL API. Using External GPU Support ========================== Many packages come with a ``+cuda`` or ``+rocm`` variant. With no added configuration, Spack will download and install the needed components. It may be preferable to use existing system support: the following sections help with using a system installation of GPU libraries. Using an External ROCm Installation ----------------------------------- Spack breaks down ROCm into many separate component packages. The following is an example ``packages.yaml`` that organizes a consistent set of ROCm components for use by dependent packages: .. code-block:: yaml packages: all: variants: amdgpu_target=gfx90a hip: buildable: false externals: - spec: hip@5.3.0 prefix: /opt/rocm-5.3.0/hip hsa-rocr-dev: buildable: false externals: - spec: hsa-rocr-dev@5.3.0 prefix: /opt/rocm-5.3.0/ comgr: buildable: false externals: - spec: comgr@5.3.0 prefix: /opt/rocm-5.3.0/ hipsparse: buildable: false externals: - spec: hipsparse@5.3.0 prefix: /opt/rocm-5.3.0/ hipblas: buildable: false externals: - spec: hipblas@5.3.0 prefix: /opt/rocm-5.3.0/ rocblas: buildable: false externals: - spec: rocblas@5.3.0 prefix: /opt/rocm-5.3.0/ rocprim: buildable: false externals: - spec: rocprim@5.3.0 prefix: /opt/rocm-5.3.0/rocprim/ This is in combination with the following compiler definition: .. code-block:: yaml packages: llvm-amdgpu: externals: - spec: llvm-amdgpu@=5.3.0 prefix: /opt/rocm-5.3.0 extra_attributes: compilers: c: /opt/rocm-5.3.0/bin/amdclang cxx: /opt/rocm-5.3.0/bin/amdclang++ This includes the following considerations: - Each of the listed externals specifies ``buildable: false`` to force Spack to use only the externals we defined. - ``spack external find`` can automatically locate some of the ``hip``/``rocm`` packages, but not all of them, and furthermore not in a manner that guarantees a complementary set if multiple ROCm installations are available. - The ``prefix`` is the same for several components, but note that others require listing one of the subdirectories as a prefix. Using an External CUDA Installation ----------------------------------- CUDA is split into fewer components and is simpler to specify: .. code-block:: yaml packages: all: variants: - cuda_arch=70 cuda: buildable: false externals: - spec: cuda@11.0.2 prefix: /opt/cuda/cuda-11.0.2/ where ``/opt/cuda/cuda-11.0.2/lib/`` contains ``libcudart.so``. Using an External OpenGL API ---------------------------- Depending on whether we have a graphics card or not, we may choose to use OSMesa or GLX to implement the OpenGL API. If a graphics card is unavailable, OSMesa is recommended and can typically be built with Spack. However, if we prefer to utilize the system GLX tailored to our graphics card, we need to declare it as an external. Here's how to do it: .. code-block:: yaml packages: libglx: require: [opengl] opengl: buildable: false externals: - prefix: /usr/ spec: opengl@4.6 Note that the prefix has to be the root of both the libraries and the headers (e.g., ``/usr``), not the path to the ``lib`` directory. To know which spec for OpenGL is available, use ``cd /usr/include/GL && grep -Ri gl_version``. ================================================ FILE: lib/spack/docs/images/packaging.excalidrawlib ================================================ { "type": "excalidrawlib", "version": 2, "source": "https://excalidraw.com", "libraryItems": [ { "status": "unpublished", "elements": [ { "type": "rectangle", "version": 601, "versionNonce": 158569138, "isDeleted": false, "id": "8MYJkzMoNEhDhGH1FB83g", "fillStyle": "hachure", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 445.75, "y": 129, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 736, "height": 651, "seed": 448140078, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664627195460, "link": null, "locked": false }, { "type": "rectangle", "version": 195, "versionNonce": 1239338030, "isDeleted": false, "id": "2CKbNSYnk0z80hSe6axnR", "fillStyle": "hachure", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 470.25, "y": 164, "strokeColor": "#000000", "backgroundColor": "#228be6", "width": 495, "height": 455, "seed": 566918834, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "id": "IU_VoaKHNHswI8HaxNWt5", "type": "arrow" } ], "updated": 1664627105795, "link": null, "locked": false }, { "type": "rectangle", "version": 403, "versionNonce": 56919410, "isDeleted": false, "id": "XUzv2kfpdxMahaSVVS42X", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 509.25, "y": 407.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 354909550, "groupIds": [ "LYqioPcAzrIgJBDV3IaDA", "SsaCg2uTI9sJjhD323wkh" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "71z_J7hoepiXas8Fk5x0B", "type": "arrow" }, { "id": "IU_VoaKHNHswI8HaxNWt5", "type": "arrow" } ], "updated": 1664627099901, "link": null, "locked": false }, { "type": "text", "version": 300, "versionNonce": 925254318, "isDeleted": false, "id": "lkCxvsSEn-AuBHtfj1N0d", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 547.25, "y": 441, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 321, "height": 45, "seed": 1361827954, "groupIds": [ "LYqioPcAzrIgJBDV3IaDA", "SsaCg2uTI9sJjhD323wkh" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664627099902, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "AutotoolsPackage", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "AutotoolsPackage" }, { "type": "rectangle", "version": 377, "versionNonce": 1733756722, "isDeleted": false, "id": "aCDb2PgRdoFKA8e-GqQzR", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 509.25, "y": 200, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 175218606, "groupIds": [ "WEeFev8dTdo9KgzR3hPki", "SsaCg2uTI9sJjhD323wkh" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "71z_J7hoepiXas8Fk5x0B", "type": "arrow" } ], "updated": 1664627099902, "link": null, "locked": false }, { "type": "text", "version": 161, "versionNonce": 585481454, "isDeleted": false, "id": "fXYOlmw0CV0WFTNUDity0", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 627.75, "y": 233.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 160, "height": 45, "seed": 1186724402, "groupIds": [ "WEeFev8dTdo9KgzR3hPki", "SsaCg2uTI9sJjhD323wkh" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664627099902, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "ArpackNg", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "ArpackNg" }, { "type": "arrow", "version": 290, "versionNonce": 890458354, "isDeleted": false, "id": "71z_J7hoepiXas8Fk5x0B", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 707.8516807799414, "y": 403, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 0, "height": 85, "seed": 247298542, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664627099902, "link": null, "locked": false, "startBinding": { "focus": 0.02318227093169459, "gap": 3, "elementId": "XUzv2kfpdxMahaSVVS42X" }, "endBinding": { "focus": -0.02318227093169459, "gap": 6, "elementId": "aCDb2PgRdoFKA8e-GqQzR" }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 0, -85 ] ] }, { "type": "text", "version": 673, "versionNonce": 1429991214, "isDeleted": false, "id": "bsoYa0EVTdXYsTx5nsFJk", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 783.25, "y": 518.3821170339361, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 164, "height": 90, "seed": 1633805298, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "id": "IU_VoaKHNHswI8HaxNWt5", "type": "arrow" } ], "updated": 1664627099902, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Package \nHierarchy", "baseline": 77, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Package \nHierarchy" }, { "type": "rectangle", "version": 903, "versionNonce": 1712814318, "isDeleted": false, "id": "qRi5xNnAOqg-SFwtYBpoN", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 510.25, "y": 657.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 1226050606, "groupIds": [ "-wCL8N0qNvseDw29hpA8g" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "IU_VoaKHNHswI8HaxNWt5", "type": "arrow" } ], "updated": 1664627118807, "link": null, "locked": false }, { "type": "text", "version": 623, "versionNonce": 492299954, "isDeleted": false, "id": "9h25d9NB-Q9Wc79boMEnC", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 552.25, "y": 691, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 313, "height": 45, "seed": 186946994, "groupIds": [ "-wCL8N0qNvseDw29hpA8g" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664627118807, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Builder Forwarder", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Builder Forwarder" }, { "type": "text", "version": 1188, "versionNonce": 351671150, "isDeleted": false, "id": "IlomIIocRvEmmYro4MZ68", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1002.75, "y": 168.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 157, "height": 90, "seed": 1428885362, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664627188273, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Package\n Wrapper", "baseline": 77, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Package\n Wrapper" }, { "type": "arrow", "version": 832, "versionNonce": 1121332014, "isDeleted": false, "id": "IU_VoaKHNHswI8HaxNWt5", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "dotted", "roughness": 2, "opacity": 100, "angle": 0, "x": 707.7778281289579, "y": 653.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 7.847537838213611, "height": 130.23576593212783, "seed": 1301783086, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664627118807, "link": null, "locked": false, "startBinding": { "elementId": "qRi5xNnAOqg-SFwtYBpoN", "focus": 0.013062197564634722, "gap": 4 }, "endBinding": { "elementId": "XUzv2kfpdxMahaSVVS42X", "focus": 0.056574233332975385, "gap": 3.7642340678721666 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ -7.847537838213611, -130.23576593212783 ] ] } ], "id": "mulubEO9Lw-HgC00sx7G-", "created": 1664627205632 }, { "status": "unpublished", "elements": [ { "type": "rectangle", "version": 360, "versionNonce": 699609906, "isDeleted": false, "id": "ai3MIBTq8Rkokk4d2NJ_k", "fillStyle": "hachure", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 441.5, "y": 56, "strokeColor": "#000000", "backgroundColor": "#228be6", "width": 479, "height": 642, "seed": 725687342, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926148, "link": null, "locked": false }, { "type": "rectangle", "version": 327, "versionNonce": 1239118706, "isDeleted": false, "id": "7tuXfM91g28UGae9gJkis", "fillStyle": "hachure", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 993.25, "y": 53, "strokeColor": "#000000", "backgroundColor": "#228be6", "width": 479, "height": 642, "seed": 860539570, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "id": "F6E1EQxM-PyPeNjQXH6NZ", "type": "arrow" } ], "updated": 1664623054904, "link": null, "locked": false }, { "type": "rectangle", "version": 482, "versionNonce": 616506034, "isDeleted": false, "id": "TmgDkNmbU86sH2Ssf1mL2", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1030.75, "y": 503.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 329380206, "groupIds": [ "rqi4zfKDNJjqgRyIIknBO" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "RQl1RtMzUcPE_zXHt8Ldm", "type": "arrow" }, { "id": "F6E1EQxM-PyPeNjQXH6NZ", "type": "arrow" }, { "id": "Iey2r9ev3NqXShFhDRa3t", "type": "arrow" } ], "updated": 1664623131360, "link": null, "locked": false }, { "type": "text", "version": 377, "versionNonce": 1649618094, "isDeleted": false, "id": "M6LF3AKrGIzDW8p00PLeg", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1068.75, "y": 537, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 321, "height": 45, "seed": 1690477682, "groupIds": [ "rqi4zfKDNJjqgRyIIknBO" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926151, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "AutotoolsPackage", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "AutotoolsPackage" }, { "type": "rectangle", "version": 466, "versionNonce": 378147058, "isDeleted": false, "id": "-34MaUc1fQDbeqLTRUx91", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1030.625, "y": 296, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 964531118, "groupIds": [ "TtAdfrQjw8FIlPZMGmWhX" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "RQl1RtMzUcPE_zXHt8Ldm", "type": "arrow" }, { "id": "7czUS_PAuM5hdRJoQRDRT", "type": "arrow" }, { "id": "Iey2r9ev3NqXShFhDRa3t", "type": "arrow" } ], "updated": 1664623131360, "link": null, "locked": false }, { "type": "text", "version": 250, "versionNonce": 1826973422, "isDeleted": false, "id": "85YHNomCStJoIV17Sp0A6", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1093.625, "y": 329.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 271, "height": 45, "seed": 1436108338, "groupIds": [ "TtAdfrQjw8FIlPZMGmWhX" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926151, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "builtin.ArpackNg", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "builtin.ArpackNg" }, { "type": "arrow", "version": 476, "versionNonce": 1270564594, "isDeleted": false, "id": "RQl1RtMzUcPE_zXHt8Ldm", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1233.8516807799415, "y": 499, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 0, "height": 85, "seed": 1613426158, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926151, "link": null, "locked": false, "startBinding": { "elementId": "TmgDkNmbU86sH2Ssf1mL2", "focus": 0.023182270931695163, "gap": 4.5 }, "endBinding": { "elementId": "-34MaUc1fQDbeqLTRUx91", "focus": -0.02381199385360952, "gap": 6 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 0, -85 ] ] }, { "type": "text", "version": 693, "versionNonce": 1438013742, "isDeleted": false, "id": "wSIdF9zegc69r2D38BVMs", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1061.75, "y": 632.3821170339361, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 335, "height": 45, "seed": 1052094450, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926151, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Old-style packages", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Old-style packages" }, { "type": "rectangle", "version": 556, "versionNonce": 1760787058, "isDeleted": false, "id": "lYxakYKLpAmo_DvzDJ27b", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1030.625, "y": 95, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 1302932978, "groupIds": [ "-WCCzMWoqGFfWxksMC6LG" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "RQl1RtMzUcPE_zXHt8Ldm", "type": "arrow" }, { "id": "8Z8HX6DlXqC-qL-63w1ol", "type": "arrow" }, { "id": "ia8wHuSmOVJLvGe5blR5g", "type": "arrow" }, { "id": "7czUS_PAuM5hdRJoQRDRT", "type": "arrow" } ], "updated": 1664623123836, "link": null, "locked": false }, { "type": "text", "version": 341, "versionNonce": 1412367214, "isDeleted": false, "id": "hF1874wuKYmbBjYAQwrVJ", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1088.125, "y": 128.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 282, "height": 45, "seed": 524182062, "groupIds": [ "-WCCzMWoqGFfWxksMC6LG" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926152, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "myrepo.ArpackNg", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "myrepo.ArpackNg" }, { "type": "arrow", "version": 593, "versionNonce": 214413938, "isDeleted": false, "id": "8Z8HX6DlXqC-qL-63w1ol", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "dotted", "roughness": 2, "opacity": 100, "angle": 0, "x": 1226.4453379157953, "y": 297.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 2.434529927712447, "height": 84, "seed": 1326581486, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926152, "link": null, "locked": false, "startBinding": null, "endBinding": { "elementId": "lYxakYKLpAmo_DvzDJ27b", "focus": -0.00782655608584947, "gap": 6.5 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 2.434529927712447, -84 ] ] }, { "type": "rectangle", "version": 733, "versionNonce": 390297266, "isDeleted": false, "id": "G4--cV2YGQSrSijvYiNDB", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 482.5, "y": 507, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 85080878, "groupIds": [ "qZhg7KFANDHKWmTH71Lm0", "FSKOW2oS76ubMa6DTOrDh" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "RQl1RtMzUcPE_zXHt8Ldm", "type": "arrow" }, { "id": "BkpnKUCjV1uqDGHPNuNZK", "type": "arrow" }, { "id": "aQdIO4VQx_J6SzCz-xt64", "type": "arrow" }, { "id": "F6E1EQxM-PyPeNjQXH6NZ", "type": "arrow" } ], "updated": 1664623061069, "link": null, "locked": false }, { "type": "text", "version": 577, "versionNonce": 2001681906, "isDeleted": false, "id": "MbNSUrN26Lx1aERuxunnt", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 541.5, "y": 540.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 279, "height": 45, "seed": 950326962, "groupIds": [ "qZhg7KFANDHKWmTH71Lm0", "FSKOW2oS76ubMa6DTOrDh" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926152, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Default Builder", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Default Builder" }, { "type": "rectangle", "version": 722, "versionNonce": 1372930162, "isDeleted": false, "id": "WIS84sS48dCmi8q81Hh9F", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 482.5, "y": 99, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 1977410350, "groupIds": [ "_CQwHz-xftDZzy8u9u4YO", "FSKOW2oS76ubMa6DTOrDh" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "RQl1RtMzUcPE_zXHt8Ldm", "type": "arrow" }, { "id": "BkpnKUCjV1uqDGHPNuNZK", "type": "arrow" }, { "id": "aQdIO4VQx_J6SzCz-xt64", "type": "arrow" }, { "id": "ia8wHuSmOVJLvGe5blR5g", "type": "arrow" } ], "updated": 1664623105535, "link": null, "locked": false }, { "type": "text", "version": 531, "versionNonce": 1851174834, "isDeleted": false, "id": "qIbTXN1LbDYGZzSceYynz", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 540.5, "y": 132.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 281, "height": 45, "seed": 221818546, "groupIds": [ "_CQwHz-xftDZzy8u9u4YO", "FSKOW2oS76ubMa6DTOrDh" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926152, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Adapter Builder", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Adapter Builder" }, { "type": "arrow", "version": 85, "versionNonce": 50141422, "isDeleted": false, "id": "aQdIO4VQx_J6SzCz-xt64", "fillStyle": "solid", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "x": 670, "y": 505, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 2, "height": 291, "seed": 417372974, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926152, "link": null, "locked": false, "startBinding": { "elementId": "G4--cV2YGQSrSijvYiNDB", "focus": -0.05731267980406219, "gap": 2 }, "endBinding": { "elementId": "WIS84sS48dCmi8q81Hh9F", "focus": 0.04321344955983103, "gap": 3 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 2, -291 ] ] }, { "type": "arrow", "version": 720, "versionNonce": 1494556718, "isDeleted": false, "id": "ia8wHuSmOVJLvGe5blR5g", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 4.706831282597808, "x": 932.4285606227004, "y": 52.69401049592016, "strokeColor": "#c92a2a", "backgroundColor": "transparent", "width": 47.10077935537049, "height": 145.9883132350331, "seed": 314146734, "groupIds": [], "strokeSharpness": "round", "boundElements": [], "updated": 1664623039605, "link": null, "locked": false, "startBinding": { "elementId": "WIS84sS48dCmi8q81Hh9F", "focus": 0.6597923311816741, "gap": 2.0985583595166872 }, "endBinding": { "elementId": "lYxakYKLpAmo_DvzDJ27b", "focus": -0.6857137990945498, "gap": 3.0336827015810286 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 45.89517648378751, 72.7218231059162 ], [ -1.2056028715829825, 145.9883132350331 ] ] }, { "type": "text", "version": 727, "versionNonce": 549636846, "isDeleted": false, "id": "JRrvIVZ9KAv56BYbRbCLA", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 527.5, "y": 633.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 295, "height": 45, "seed": 2130028978, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664622926153, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Builder Hierarchy", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Builder Hierarchy" }, { "type": "text", "version": 281, "versionNonce": 777063918, "isDeleted": false, "id": "BBj29IYUUwcEAk0aGGgEe", "fillStyle": "solid", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "x": 746, "y": 2, "strokeColor": "#c92a2a", "backgroundColor": "#ced4da", "width": 438, "height": 35, "seed": 344107566, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664623034966, "link": null, "locked": false, "fontSize": 28, "fontFamily": 1, "text": "Defer to the old-style package", "baseline": 25, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Defer to the old-style package" }, { "type": "arrow", "version": 864, "versionNonce": 353999662, "isDeleted": false, "id": "F6E1EQxM-PyPeNjQXH6NZ", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 1.5656833824867196, "x": 932.5276780900645, "y": 511.2079252998286, "strokeColor": "#c92a2a", "backgroundColor": "transparent", "width": 47.10077935537049, "height": 145.9883132350331, "seed": 2119154546, "groupIds": [], "strokeSharpness": "round", "boundElements": [], "updated": 1664623061069, "link": null, "locked": false, "startBinding": { "elementId": "TmgDkNmbU86sH2Ssf1mL2", "focus": 0.700636908798286, "gap": 3.7338363313426726 }, "endBinding": { "elementId": "G4--cV2YGQSrSijvYiNDB", "focus": -0.7137516210459195, "gap": 1.5235945037890133 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 45.89517648378751, 72.7218231059162 ], [ -1.2056028715829825, 145.9883132350331 ] ] }, { "type": "text", "version": 318, "versionNonce": 1988243186, "isDeleted": false, "id": "VIOq-st9nvReenpiJkr7q", "fillStyle": "solid", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 0, "opacity": 100, "angle": 0, "x": 828, "y": 724.5, "strokeColor": "#c92a2a", "backgroundColor": "#ced4da", "width": 274, "height": 70, "seed": 2086072882, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664623095297, "link": null, "locked": false, "fontSize": 28, "fontFamily": 1, "text": "Fall-back to the \nAdapter base class", "baseline": 60, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Fall-back to the \nAdapter base class" }, { "type": "arrow", "version": 971, "versionNonce": 1844256174, "isDeleted": false, "id": "7czUS_PAuM5hdRJoQRDRT", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 6.272294617229998, "x": 1433.5276780900645, "y": 163.20792529982862, "strokeColor": "#c92a2a", "backgroundColor": "transparent", "width": 47.10077935537049, "height": 145.9883132350331, "seed": 142056302, "groupIds": [], "strokeSharpness": "round", "boundElements": [], "updated": 1664623123836, "link": null, "locked": false, "startBinding": { "elementId": "lYxakYKLpAmo_DvzDJ27b", "focus": -0.8331982906950285, "gap": 5.098981289624589 }, "endBinding": { "elementId": "-34MaUc1fQDbeqLTRUx91", "focus": 0.7587321286266477, "gap": 5.483331940596372 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 45.89517648378751, 72.7218231059162 ], [ -1.2056028715829825, 145.9883132350331 ] ] }, { "type": "arrow", "version": 1075, "versionNonce": 2073112366, "isDeleted": false, "id": "Iey2r9ev3NqXShFhDRa3t", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 6.272294617229998, "x": 1434.451400933309, "y": 387.7559332541056, "strokeColor": "#c92a2a", "backgroundColor": "transparent", "width": 47.10077935537049, "height": 145.9883132350331, "seed": 840513518, "groupIds": [], "strokeSharpness": "round", "boundElements": [], "updated": 1664623131360, "link": null, "locked": false, "startBinding": { "elementId": "-34MaUc1fQDbeqLTRUx91", "focus": -0.7723329153292293, "gap": 6.037577244264867 }, "endBinding": { "elementId": "TmgDkNmbU86sH2Ssf1mL2", "focus": 0.808011962769455, "gap": 6.296927895236422 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 45.89517648378751, 72.7218231059162 ], [ -1.2056028715829825, 145.9883132350331 ] ] } ], "id": "sJP5ES4-kuhrqaBed7Feh", "created": 1664623142493 }, { "status": "unpublished", "elements": [ { "type": "rectangle", "version": 351, "versionNonce": 94847218, "isDeleted": false, "id": "QfhQQY4Kvx8RLvCd6qXsx", "fillStyle": "hachure", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1011.5, "y": 249, "strokeColor": "#000000", "backgroundColor": "#228be6", "width": 479, "height": 438, "seed": 1024685106, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347442, "link": null, "locked": false }, { "type": "rectangle", "version": 156, "versionNonce": 2082406190, "isDeleted": false, "id": "rMQqqzkSZsBVWvOk137wO", "fillStyle": "hachure", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 511, "y": 247, "strokeColor": "#000000", "backgroundColor": "#228be6", "width": 479, "height": 438, "seed": 250617778, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347443, "link": null, "locked": false }, { "type": "rectangle", "version": 392, "versionNonce": 414601906, "isDeleted": false, "id": "h2lcAgJBn6WsPKAj3vWS8", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 545.5, "y": 490.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 721668433, "groupIds": [ "ETPwHpdW1CXh0DtqZ_2na" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "r2Lq0kGXd6aTn5T-ki1aL", "type": "arrow" } ], "updated": 1664612347443, "link": null, "locked": false }, { "type": "text", "version": 293, "versionNonce": 848488814, "isDeleted": false, "id": "eaxk_MzyrjAjXKf0vmFuU", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 583.5, "y": 524, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 321, "height": 45, "seed": 1324675135, "groupIds": [ "ETPwHpdW1CXh0DtqZ_2na" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347443, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "AutotoolsPackage", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "AutotoolsPackage" }, { "type": "rectangle", "version": 370, "versionNonce": 595405938, "isDeleted": false, "id": "6TAhmS7GKN_ppUHjSVGLb", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 545.5, "y": 283, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 2083634783, "groupIds": [ "biKtN87UToAb_UBhyub5I" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "r2Lq0kGXd6aTn5T-ki1aL", "type": "arrow" } ], "updated": 1664612347443, "link": null, "locked": false }, { "type": "text", "version": 155, "versionNonce": 1066372014, "isDeleted": false, "id": "xyXchzGRLKRPuMVGo17mr", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 664, "y": 316.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 160, "height": 45, "seed": 2066951921, "groupIds": [ "biKtN87UToAb_UBhyub5I" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347443, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "ArpackNg", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "ArpackNg" }, { "type": "arrow", "version": 285, "versionNonce": 1807928882, "isDeleted": false, "id": "r2Lq0kGXd6aTn5T-ki1aL", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 748.6016807799414, "y": 486, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 0, "height": 85, "seed": 1479060383, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347443, "link": null, "locked": false, "startBinding": { "elementId": "h2lcAgJBn6WsPKAj3vWS8", "focus": 0.02318227093169459, "gap": 3 }, "endBinding": { "elementId": "6TAhmS7GKN_ppUHjSVGLb", "focus": -0.02318227093169459, "gap": 6 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 0, -85 ] ] }, { "type": "text", "version": 572, "versionNonce": 1094575598, "isDeleted": false, "id": "pUx1_v_UyKhu5zXISU4-f", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 653, "y": 619.3821170339361, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 182, "height": 45, "seed": 1608256017, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347443, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Metadata", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Metadata" }, { "type": "rectangle", "version": 734, "versionNonce": 1401317810, "isDeleted": false, "id": "4YBPHTc5sQiOKGM9NOZwg", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1045.5, "y": 490.5, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 396.99999999999994, "height": 112, "seed": 1687989426, "groupIds": [ "lxE4hLtUAF2X7993lwk8q" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "M8cWqpsa0-iwN_cVJeXEQ", "type": "arrow" } ], "updated": 1664612347443, "link": null, "locked": false }, { "type": "text", "version": 436, "versionNonce": 1572061806, "isDeleted": false, "id": "P2U0ucf_QPvJcOWlMLp2K", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1183, "y": 524, "strokeColor": "#000000", "backgroundColor": "#ced4da", "width": 122, "height": 45, "seed": 276038958, "groupIds": [ "lxE4hLtUAF2X7993lwk8q" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347443, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Builder", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Builder" }, { "type": "arrow", "version": 489, "versionNonce": 1663911086, "isDeleted": false, "id": "M8cWqpsa0-iwN_cVJeXEQ", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "dashed", "roughness": 2, "opacity": 100, "angle": 0, "x": 942, "y": 337, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 303, "height": 143, "seed": 1698960686, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347443, "link": null, "locked": false, "startBinding": null, "endBinding": { "elementId": "4YBPHTc5sQiOKGM9NOZwg", "focus": 0.04820781382766574, "gap": 10.5 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "dot", "points": [ [ 0, 0 ], [ 295, 0 ], [ 303, 143 ] ] }, { "type": "text", "version": 841, "versionNonce": 2059173614, "isDeleted": false, "id": "QGyg9pXnTgByg9Lw9oZKC", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1043.5, "y": 621.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 401, "height": 45, "seed": 1012078510, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664612347443, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Installation Procedure", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Installation Procedure" } ], "id": "tezI4Q4gBH7mr-Q_us1KO", "created": 1664612353293 }, { "status": "unpublished", "elements": [ { "type": "rectangle", "version": 273, "versionNonce": 1078330865, "isDeleted": false, "id": "h2lcAgJBn6WsPKAj3vWS8", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 545.5, "y": 489, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 396.99999999999994, "height": 112, "seed": 721668433, "groupIds": [ "ETPwHpdW1CXh0DtqZ_2na" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "r2Lq0kGXd6aTn5T-ki1aL", "type": "arrow" } ], "updated": 1664534889868, "link": null, "locked": false }, { "type": "text", "version": 174, "versionNonce": 1400524191, "isDeleted": false, "id": "eaxk_MzyrjAjXKf0vmFuU", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 583.5, "y": 522.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 321, "height": 45, "seed": 1324675135, "groupIds": [ "ETPwHpdW1CXh0DtqZ_2na" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664534889868, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "AutotoolsPackage", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "AutotoolsPackage" }, { "type": "text", "version": 108, "versionNonce": 438728849, "isDeleted": false, "id": "xyXchzGRLKRPuMVGo17mr", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 664, "y": 316.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 160, "height": 45, "seed": 2066951921, "groupIds": [ "1wm7ikIN28k9zdVSKTLKQ" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664540120970, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "ArpackNg", "baseline": 32, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "ArpackNg" }, { "type": "rectangle", "version": 322, "versionNonce": 1389146591, "isDeleted": false, "id": "6TAhmS7GKN_ppUHjSVGLb", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 545.5, "y": 283, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 396.99999999999994, "height": 112, "seed": 2083634783, "groupIds": [ "1wm7ikIN28k9zdVSKTLKQ" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "r2Lq0kGXd6aTn5T-ki1aL", "type": "arrow" } ], "updated": 1664534889868, "link": null, "locked": false }, { "type": "arrow", "version": 94, "versionNonce": 787416433, "isDeleted": false, "id": "r2Lq0kGXd6aTn5T-ki1aL", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 748.6016807799414, "y": 486, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 0, "height": 85, "seed": 1479060383, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664534889868, "link": null, "locked": false, "startBinding": { "elementId": "h2lcAgJBn6WsPKAj3vWS8", "focus": 0.02318227093169459, "gap": 3 }, "endBinding": { "elementId": "6TAhmS7GKN_ppUHjSVGLb", "focus": -0.02318227093169459, "gap": 6 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 0, -85 ] ] }, { "type": "text", "version": 227, "versionNonce": 117980031, "isDeleted": false, "id": "pUx1_v_UyKhu5zXISU4-f", "fillStyle": "hachure", "strokeWidth": 4, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 969, "y": 386.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 442, "height": 90, "seed": 1608256017, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1664534908931, "link": null, "locked": false, "fontSize": 36, "fontFamily": 1, "text": "Metadata \n+ Installation Procedure", "baseline": 77, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Metadata \n+ Installation Procedure" } ], "id": "_c7AOn60omrTlppZHlLQh", "created": 1664540190548 }, { "status": "unpublished", "elements": [ { "type": "rectangle", "version": 367, "versionNonce": 963584621, "isDeleted": false, "id": "oAei2n-Ha1gpjnYdK7AwC", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 240.5, "y": 642.75, "strokeColor": "#000000", "backgroundColor": "#228be6", "width": 392, "height": 80, "seed": 701868237, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "id": "slfbd0bbRqA8648kZ5fns", "type": "text" }, { "id": "slfbd0bbRqA8648kZ5fns", "type": "text" }, { "type": "text", "id": "slfbd0bbRqA8648kZ5fns" } ], "updated": 1663329462351, "link": null, "locked": false }, { "type": "text", "version": 373, "versionNonce": 1698441027, "isDeleted": false, "id": "slfbd0bbRqA8648kZ5fns", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 245.5, "y": 670.25, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 382, "height": 25, "seed": 1179637379, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663329462351, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "Execute the installation process", "baseline": 18, "textAlign": "center", "verticalAlign": "middle", "containerId": "oAei2n-Ha1gpjnYdK7AwC", "originalText": "Execute the installation process" }, { "type": "rectangle", "version": 208, "versionNonce": 844908259, "isDeleted": false, "id": "cLwg2WXUit_OTQmXLIdIW", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 815.5, "y": 517.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 392, "height": 80, "seed": 557411811, "groupIds": [ "D1SCf714tngJFHk8TFX8T" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "SpG_8HxzMHjM2HYK6Fgwx", "type": "arrow" } ], "updated": 1663329462351, "link": null, "locked": false }, { "type": "text", "version": 274, "versionNonce": 1704611021, "isDeleted": false, "id": "1r8FMl26VYSKpPKlHA_Oc", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 916.5, "y": 545, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 207, "height": 25, "seed": 961881101, "groupIds": [ "D1SCf714tngJFHk8TFX8T" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663329462351, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "CMakeBuilder.cmake()", "baseline": 18, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "CMakeBuilder.cmake()" }, { "type": "rectangle", "version": 264, "versionNonce": 295137923, "isDeleted": false, "id": "CSwjuAw6Nl67sqQ6p21ty", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 815.5, "y": 642.75, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 392, "height": 80, "seed": 1011629069, "groupIds": [ "D1SCf714tngJFHk8TFX8T" ], "strokeSharpness": "sharp", "boundElements": [ { "id": "SpG_8HxzMHjM2HYK6Fgwx", "type": "arrow" }, { "id": "zvmLoAH5oICRD5og-pBvu", "type": "arrow" } ], "updated": 1663329462351, "link": null, "locked": false }, { "type": "text", "version": 466, "versionNonce": 196160301, "isDeleted": false, "id": "WX4axTU0IR7PJb0GkR-jq", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 922.5, "y": 670.25, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 193, "height": 25, "seed": 716117827, "groupIds": [ "D1SCf714tngJFHk8TFX8T" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663329462351, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "CMakeBuilder.build()", "baseline": 18, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "CMakeBuilder.build()" }, { "type": "rectangle", "version": 301, "versionNonce": 1545420173, "isDeleted": false, "id": "coUXke3Fv_DpjqG9zgEjQ", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 815.5, "y": 768, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 392, "height": 80, "seed": 1934529891, "groupIds": [ "D1SCf714tngJFHk8TFX8T" ], "strokeSharpness": "sharp", "boundElements": [ { "type": "text", "id": "yVIbU03yFYvpXnh9xIgET" }, { "id": "zvmLoAH5oICRD5og-pBvu", "type": "arrow" } ], "updated": 1663329462351, "link": null, "locked": false }, { "type": "text", "version": 273, "versionNonce": 1837690307, "isDeleted": false, "id": "yVIbU03yFYvpXnh9xIgET", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 820.5, "y": 795.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 382, "height": 25, "seed": 1611291683, "groupIds": [ "D1SCf714tngJFHk8TFX8T" ], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663329462351, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "CMakeBuilder.install()", "baseline": 18, "textAlign": "center", "verticalAlign": "middle", "containerId": "coUXke3Fv_DpjqG9zgEjQ", "originalText": "CMakeBuilder.install()" }, { "type": "arrow", "version": 564, "versionNonce": 1041761261, "isDeleted": false, "id": "SpG_8HxzMHjM2HYK6Fgwx", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1209, "y": 558.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 96, "height": 109, "seed": 732445197, "groupIds": [ "D1SCf714tngJFHk8TFX8T" ], "strokeSharpness": "round", "boundElements": [], "updated": 1663329462351, "link": null, "locked": false, "startBinding": { "elementId": "cLwg2WXUit_OTQmXLIdIW", "focus": -0.7327371048252911, "gap": 1.5 }, "endBinding": { "elementId": "CSwjuAw6Nl67sqQ6p21ty", "focus": 0.6494341563786008, "gap": 2.5 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 96, 54 ], [ 1, 109 ] ] }, { "type": "arrow", "version": 642, "versionNonce": 1380728163, "isDeleted": false, "id": "zvmLoAH5oICRD5og-pBvu", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1216, "y": 680, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 98, "height": 124.33745608356844, "seed": 708861581, "groupIds": [ "D1SCf714tngJFHk8TFX8T" ], "strokeSharpness": "round", "boundElements": [], "updated": 1663329462351, "link": null, "locked": false, "startBinding": { "elementId": "CSwjuAw6Nl67sqQ6p21ty", "focus": -0.7839018302828619, "gap": 8.5 }, "endBinding": { "elementId": "coUXke3Fv_DpjqG9zgEjQ", "focus": 0.7841576120638036, "gap": 6.5 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 96, 54 ], [ -2, 124.33745608356844 ] ] }, { "type": "text", "version": 613, "versionNonce": 909390253, "isDeleted": false, "id": "fAHH1YdSlMq8ioLIj36Of", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 644, "y": 353.7484662576685, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 166, "height": 567.2515337423315, "seed": 1455993539, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663329499644, "link": null, "locked": false, "fontSize": 493.16790307261914, "fontFamily": 2, "text": "{", "baseline": 454.2515337423315, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "{" } ], "id": "KBV_I9pxrJD2zPuaP6vBc", "created": 1663329511286 }, { "status": "unpublished", "elements": [ { "type": "rectangle", "version": 93, "versionNonce": 42296109, "isDeleted": false, "id": "cLwg2WXUit_OTQmXLIdIW", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 625.5, "y": 298, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 392, "height": 80, "seed": 557411811, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "id": "SpG_8HxzMHjM2HYK6Fgwx", "type": "arrow" } ], "updated": 1663324636434, "link": null, "locked": false }, { "type": "text", "version": 99, "versionNonce": 1537897869, "isDeleted": false, "id": "1r8FMl26VYSKpPKlHA_Oc", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 726.5, "y": 325.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 179, "height": 25, "seed": 961881101, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663324636434, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "Fetch source files", "baseline": 18, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Fetch source files" }, { "type": "rectangle", "version": 149, "versionNonce": 1653290435, "isDeleted": false, "id": "CSwjuAw6Nl67sqQ6p21ty", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 625.5, "y": 423.25, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 392, "height": 80, "seed": 1011629069, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "id": "SpG_8HxzMHjM2HYK6Fgwx", "type": "arrow" }, { "id": "zvmLoAH5oICRD5og-pBvu", "type": "arrow" } ], "updated": 1663324636434, "link": null, "locked": false }, { "type": "text", "version": 272, "versionNonce": 1195260909, "isDeleted": false, "id": "WX4axTU0IR7PJb0GkR-jq", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 645.5, "y": 450.75, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 352, "height": 25, "seed": 716117827, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663324636434, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "Expand them in the stage directory", "baseline": 18, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Expand them in the stage directory" }, { "type": "rectangle", "version": 185, "versionNonce": 2143651171, "isDeleted": false, "id": "coUXke3Fv_DpjqG9zgEjQ", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 625.5, "y": 548.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 392, "height": 80, "seed": 1934529891, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "type": "text", "id": "yVIbU03yFYvpXnh9xIgET" }, { "id": "zvmLoAH5oICRD5og-pBvu", "type": "arrow" }, { "id": "5yqrFWV-hhJ4RoVewqAC0", "type": "arrow" } ], "updated": 1663324636434, "link": null, "locked": false }, { "type": "text", "version": 135, "versionNonce": 1833580109, "isDeleted": false, "id": "yVIbU03yFYvpXnh9xIgET", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 630.5, "y": 563.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 382, "height": 50, "seed": 1611291683, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663324636434, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "Set the stage directory as the \ncurrent working directory", "baseline": 43, "textAlign": "center", "verticalAlign": "middle", "containerId": "coUXke3Fv_DpjqG9zgEjQ", "originalText": "Set the stage directory as the current working directory" }, { "type": "rectangle", "version": 253, "versionNonce": 1704770627, "isDeleted": false, "id": "tBTBRiEA6AJABK4wnKF_-", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 625.5, "y": 673.75, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 392, "height": 80, "seed": 1257829773, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "id": "GoE9udjDQxUqdsYCUnbVI", "type": "text" }, { "type": "text", "id": "GoE9udjDQxUqdsYCUnbVI" }, { "id": "5yqrFWV-hhJ4RoVewqAC0", "type": "arrow" }, { "id": "v-9Voh5erXQ8iqoQ_9BVO", "type": "arrow" } ], "updated": 1663324636434, "link": null, "locked": false }, { "type": "text", "version": 194, "versionNonce": 1557028205, "isDeleted": false, "id": "GoE9udjDQxUqdsYCUnbVI", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 630.5, "y": 701.25, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 382, "height": 25, "seed": 895792579, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663324636434, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "Fork a new build environment", "baseline": 18, "textAlign": "center", "verticalAlign": "middle", "containerId": "tBTBRiEA6AJABK4wnKF_-", "originalText": "Fork a new build environment" }, { "type": "rectangle", "version": 321, "versionNonce": 1675770851, "isDeleted": false, "id": "oAei2n-Ha1gpjnYdK7AwC", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 625.5, "y": 799, "strokeColor": "#000000", "backgroundColor": "#228be6", "width": 392, "height": 80, "seed": 701868237, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [ { "id": "slfbd0bbRqA8648kZ5fns", "type": "text" }, { "id": "slfbd0bbRqA8648kZ5fns", "type": "text" }, { "type": "text", "id": "slfbd0bbRqA8648kZ5fns" }, { "id": "v-9Voh5erXQ8iqoQ_9BVO", "type": "arrow" } ], "updated": 1663324636434, "link": null, "locked": false }, { "type": "text", "version": 328, "versionNonce": 1868179405, "isDeleted": false, "id": "slfbd0bbRqA8648kZ5fns", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 630.5, "y": 826.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 382, "height": 25, "seed": 1179637379, "groupIds": [], "strokeSharpness": "sharp", "boundElements": [], "updated": 1663324636434, "link": null, "locked": false, "fontSize": 20, "fontFamily": 1, "text": "Execute the installation process", "baseline": 18, "textAlign": "center", "verticalAlign": "middle", "containerId": "oAei2n-Ha1gpjnYdK7AwC", "originalText": "Execute the installation process" }, { "type": "arrow", "version": 221, "versionNonce": 1777917731, "isDeleted": false, "id": "SpG_8HxzMHjM2HYK6Fgwx", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1019, "y": 339, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 96, "height": 109, "seed": 732445197, "groupIds": [], "strokeSharpness": "round", "boundElements": [], "updated": 1663324636434, "link": null, "locked": false, "startBinding": { "elementId": "cLwg2WXUit_OTQmXLIdIW", "focus": -0.7533277870216306, "gap": 7 }, "endBinding": { "elementId": "CSwjuAw6Nl67sqQ6p21ty", "focus": 0.7554869684499315, "gap": 6 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 96, 54 ], [ 1, 109 ] ] }, { "type": "arrow", "version": 299, "versionNonce": 309789379, "isDeleted": false, "id": "zvmLoAH5oICRD5og-pBvu", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1026, "y": 460.5, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 98, "height": 124.33745608356844, "seed": 708861581, "groupIds": [], "strokeSharpness": "round", "boundElements": [], "updated": 1663324636435, "link": null, "locked": false, "startBinding": { "elementId": "CSwjuAw6Nl67sqQ6p21ty", "focus": -0.7021630615640598, "gap": 12 }, "endBinding": { "elementId": "coUXke3Fv_DpjqG9zgEjQ", "focus": 0.8530521262002744, "gap": 12 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 96, 54 ], [ -2, 124.33745608356844 ] ] }, { "type": "arrow", "version": 301, "versionNonce": 914472685, "isDeleted": false, "id": "5yqrFWV-hhJ4RoVewqAC0", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1019, "y": 586.6789496258876, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 99, "height": 123.78306157234579, "seed": 642378381, "groupIds": [], "strokeSharpness": "round", "boundElements": [], "updated": 1663324636435, "link": null, "locked": false, "startBinding": { "elementId": "coUXke3Fv_DpjqG9zgEjQ", "focus": -0.6501663893510815, "gap": 7 }, "endBinding": { "elementId": "tBTBRiEA6AJABK4wnKF_-", "focus": 0.8705418381344308, "gap": 8 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 99, 42.82105037411236 ], [ 10.000000000000227, 123.78306157234579 ] ] }, { "type": "arrow", "version": 351, "versionNonce": 984592995, "isDeleted": false, "id": "v-9Voh5erXQ8iqoQ_9BVO", "fillStyle": "hachure", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 2, "opacity": 100, "angle": 0, "x": 1031, "y": 714.8662394200408, "strokeColor": "#000000", "backgroundColor": "transparent", "width": 90, "height": 137.7637151210173, "seed": 698547757, "groupIds": [], "strokeSharpness": "round", "boundElements": [], "updated": 1663324636435, "link": null, "locked": false, "startBinding": { "elementId": "tBTBRiEA6AJABK4wnKF_-", "focus": -0.6014975041597337, "gap": 10 }, "endBinding": { "elementId": "oAei2n-Ha1gpjnYdK7AwC", "focus": 0.9573045267489712, "gap": 12 }, "lastCommittedPoint": null, "startArrowhead": null, "endArrowhead": "triangle", "points": [ [ 0, 0 ], [ 90, 33.633760579959244 ], [ 4, 137.7637151210173 ] ] } ], "id": "RzNgncGu1938Ma5Teh6qZ", "created": 1663324659550 } ] } ================================================ FILE: lib/spack/docs/include_yaml.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to use include directives to modularize your Spack YAML configuration files for better organization and reusability. .. _include-yaml: Include Settings (include.yaml) =============================== Spack allows you to include configuration files through ``include.yaml``, or in the ``include:`` section in an environment. You can specify includes using local paths, remote paths, and ``git`` URLs. Included paths become configuration scopes in Spack and can even be used to override built-in scopes. Local files ~~~~~~~~~~~ You can include a single configuration file or an entire configuration *scope* like this: .. code-block:: yaml include: - /path/to/a/required/config.yaml - $MY_SPECIAL_CONFIG_FILE - path: $HOME/path/to/my/project/packages.yaml - path: /path/to/$os/$target/config optional: true - path: /path/to/os-specific/config-dir when: os == "ventura" Included paths may be absolute, relative (to the configuration file), specified as URLs, or provided in environment variables (e.g., ``$MY_SPECIAL_CONFIG_FILE``). * ``optional``: Spack will raise an error when an included configuration file does not exist, *unless* it is explicitly made ``optional: true``, like the second path above. * ``when``: Configuration scopes can also be included *conditionally* with ``when``. ``when:`` conditions are evaluated as described for :ref:`Spec List References `. The same conditions and variables in :ref:`Spec List References ` can be used for conditional activation in the ``when`` clauses. Remote file URLs ~~~~~~~~~~~~~~~~ Only the ``ftp``, ``http``, and ``https`` protocols (or schemes) are supported for remote file URLs. Spack-specific, environment, and user path variables can be used. (See :ref:`config-file-variables` for more information.) A ``sha256`` is required. For example, suppose you have a ``/etc/spack/include.yaml`` file that specifies a remote ``config.yaml`` file as follows:: include: - path: https://github.com/path/to/raw/config/config.yaml sha256: 26e871804a92cd07bb3d611b31b4156ae93d35b6a6d6e0ef3a67871fcb1d258b The ``config.yaml`` file is downloaded to a subdirectory of ``/etc/spack``. The contents of the downloaded file are read and included in Spack's configuration when Spack configuration files are processed. .. note:: You can check the destination of the downloaded file by running: ``spack config scopes -p``. .. warning:: Remote file URLs must link to the **raw** form of the file's contents (e.g., `GitHub `_ or `GitLab `_). If the directory containing the ``include.yaml`` file is not writable when the remote file is downloaded, then the destination will be a temporary directory. ``git`` repository files ~~~~~~~~~~~~~~~~~~~~~~~~ You can also include configuration files from a ``git`` repository. The ``branch``, ``commit``, or ``tag`` to be checked out is required. A list of relative paths in which to find the configuration files is also required. Inclusion of the repository (and its paths) can be optional or conditional. If you want to control the :ref:`name of the configuration scope `, you can provide a ``name``. For example, suppose we only want to include the ``config.yaml`` and ``packages.yaml`` files from the `spack/spack-configs `_ repository's ``USC/config`` directory when using the ``centos7`` operating system. And we want the configuration scope name to start ``common``. We could then configure the include in, for example, the user scope include file (i.e., ``$HOME/.spack/include.yaml`` by default), as follows:: include: - name: common git: https://github.com/spack/spack-configs.git branch: main when: os == "centos7" paths: - USC/config/config.yaml - USC/config/packages.yaml .. note:: The git URL could be specified through an environment variable (e.g., ``$MY_USC_CONFIG_URL``). If the condition is satisfied, then the ``main`` branch of the repository will be cloned -- under ``$HOME/.spack/includes`` -- when configuration scopes are initially created. Once cloned, the settings for the two files under the ``USC/config`` directory will be integrated into Spack's configuration. In this example, the new scopes and their paths can be seen by running:: $ spack config scopes -p Scope Path command_line spack /Users/username/spack/etc/spack/ user /Users/username/.spack/ common:USC/config/config.yaml /Users/username/.spack/includes/common/USC/config/config.yaml common:USC/config/packages.yaml /Users/username/.spack/includes/common/USC/config/packages.yaml site /Users/username/spack/etc/spack/site/ system /etc/spack/ defaults /Users/username/spack/etc/spack/defaults/ defaults:darwin /Users/username/spack/etc/spack/defaults/darwin/ defaults:base /Users/username/spack/etc/spack/defaults/base/ _builtin Since there are two unique paths, each results in a separate configuration scope. If only the ``USC/config`` directory was listed under ``paths``, then there would be only one configuration scope, named ``USC``, and the configuration settings from all of the configuration files within that directory would be integrated. .. versionadded:: 1.1 ``git:``, ``branch:``, ``commit:``, and ``tag:`` attributes. .. versionadded:: 1.2 ``name:`` attribute and git environment variable support. Precedence ~~~~~~~~~~ Using ``include:`` adds the included files as a configuration scope *below* the including file. This is so that you can override settings from files you include. If you want one file to take precedence over another, you can put the include with higher precedence earlier in the list: .. code-block:: yaml include: - /path/to/higher/precedence/scope/ - /path/to/middle/precedence/scope/ - git: https://github.com/org/git-repo-scope commit: 95c59784bd02ea248bf905d79d063df38e087b19 ``prefer_modify`` ^^^^^^^^^^^^^^^^^ When you use commands like ``spack compiler find``, ``spack external find``, ``spack config edit`` or ``spack config add``, they modify the topmost writable scope in the current configuration. Scopes can tell Spack to prefer to edit their included scopes instead, using ``prefer_modify``: .. code-block:: yaml include: - name: "preferred" path: /path/to/scope/we/want-to-write prefer_modify: true Now, if the including scope is the highest precedence scope and would otherwise be selected automatically by one of these commands, they will instead prefer to edit ``preferred``. The including scope can still be modified by using the ``--scope`` argument (e.g., ``spack compiler find --scope NAME``). .. warning:: Recursive includes are not currently processed in a breadth-first manner, so the value of a configuration option that is altered by multiple included files may not be what you expect. This will be addressed in a future update. .. versionadded:: 1.1 The ``prefer_modify:`` attribute. Overriding local paths ~~~~~~~~~~~~~~~~~~~~~~ Optionally, you can enable a local path to be overridden by an environment variable using ``path_override_env_var:``: .. code-block:: yaml include: - path_override_env_var: SPECIAL_CONFIG_PATH path: /path/to/special/config.yaml Here, If ``SPECIAL_CONFIG_PATH`` is set, its value will be used as the path. If not, Spack will instead use the ``path:`` specified in configuration. .. note:: ``path_override_env_var:`` is currently only supported for ``path:`` includes, not ``git:`` includes. .. versionadded:: 1.1 The ``path_override_env_var:`` attribute. .. _named-config-scopes: Named configuration scopes ~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, the included scope names are constructed by appending ``:`` and the included scope's basename to the parent scope name. For example, Spack's own ``defaults`` scope includes a ``base`` scope and a platform-specific scope:: $ spack config scopes -p Scope Path command_line spack /home/username/spack/etc/spack/ user /home/username/.spack/ site /home/username/spack/etc/spack/site/ defaults /home/username/spack/etc/spack/defaults/ defaults:darwin /home/username/spack/etc/spack/defaults/darwin/ defaults:base /home/username/spack/etc/spack/defaults/base/ _builtin You can see ``defaults`` and the included ``defaults:base`` and ``defaults:darwin`` scopes here. If you want to define your own name for an included scope, you can supply an optional ``name:`` argument when you include it: .. code-block:: yaml spack: include: - path: foo name: myscope You can see the ``myscope`` name when we activate this environment:: > spack -e ./env config scopes -p Scope Path command_line env:/home/username/env /home/username/env/spack.yaml/ myscope /home/username/env/foo/ spack /home/username/spack/etc/spack/ user /home/username/.spack/ site /home/username/spack/etc/spack/site/ defaults /home/username/spack/etc/spack/defaults/ defaults:darwin /home/username/spack/etc/spack/defaults/darwin/ defaults:base /home/username/spack/etc/spack/defaults/base/ _builtin You can now use the argument ``myscope`` to refer to this, for example with ``spack config --scope myscope add ...``. Built-in configuration scopes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The default ``user``, ``system``, and ``site`` scopes are defined using ``include:`` in ``$spack/etc/spack/include.yaml``: .. literalinclude:: _spack_root/etc/spack/include.yaml :language: yaml You can see that all three of these scopes are given meaningful names, and all three are ``optional``, i.e., they'll be ignored if their directories do not exist. The ``user`` and ``system`` scopes can also be disabled by setting ``SPACK_DISABLE_LOCAL_CONFIG``. Finally, the ``user`` scope can be overridden with a path in ``SPACK_USER_CONFIG_PATH`` if it is set. Overriding scopes by name ^^^^^^^^^^^^^^^^^^^^^^^^^ Configuration scopes have unique names. This means that you can use the ``name:`` attribute to *replace* a builtin scope. If you supply an environment like this: .. code-block:: yaml spack: include: - path: foo name: user The newly included ``user`` scope will *completely* override the builtin ``user`` scope:: > spack -e ~/env config scopes -p Scope Path command_line env:/home/username/env /home/username/env/spack.yaml/ user /home/username/env/foo/ spack /home/username/spack/etc/spack/ site /home/username/spack/etc/spack/site/ defaults /home/username/spack/etc/spack/defaults/ defaults:darwin /home/username/spack/etc/spack/defaults/darwin/ defaults:base /home/username/spack/etc/spack/defaults/base/ _builtin .. warning:: Overriding the ``defaults`` scope can have **very** unexpected consequences and is not advised. .. versionadded:: 1.1 The ``name:`` attribute. Overriding built-in scopes with ``include::`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In some cases, you may want to override *all* of the built-in configuration scopes. The ``user`` and ``system`` scopes depend on the user and the machine on which Spack is running, and they can end up bringing in unexpected configuration settings in surprising ways. If you want to eliminate them completely from an environment, you can write: .. code-block:: yaml spack: include:: [] This overrides all scopes except the ``defaults`` that Spack needs in order to function. You can see that ``spack``, ``user``, and ``site`` are overridden:: > spack -e ~/env config scopes -vp Scope Type Status Path command_line internal active env:/home/username/env env,path active /home/username/env/spack.yaml/ spack path override /home/username/spack/etc/spack/ user include,path override /home/username/.spack/ site include,path override /home/username/spack/etc/spack/site/ defaults path active /home/username/spack/etc/spack/defaults/ defaults:darwin include,path active /home/username/spack/etc/spack/defaults/darwin/ defaults:base include,path active /home/username/spack/etc/spack/defaults/base/ _builtin internal active And if you run ``spack config blame``, the settings from these scopes will no longer show up. ``defaults`` are not overridden as they are needed by Spack to function. This allows you to create completely isolated environments that do not bring in external settings. .. versionadded:: 1.1 ``include::`` with two colons for overriding. ================================================ FILE: lib/spack/docs/index.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) Spack documentation master file, created by sphinx-quickstart on Mon Dec 9 15:32:41 2013. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. .. meta:: :description lang=en: Documentation of Spack, a flexible package manager for high-performance computing, designed to support multiple versions and configurations of software on a wide variety of platforms. Spack =================== Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms and environments. It was designed for large supercomputing centers, where many users and application teams share common installations of software on clusters with exotic architectures, using libraries that do not have a standard ABI. Spack is non-destructive: installing a new version does not break existing installations, so many configurations can coexist on the same system. Most importantly, Spack is *simple*. It offers a simple *spec* syntax so that users can specify versions and configuration options concisely. Spack is also simple for package authors: package files are written in pure Python, and specs allow package authors to maintain a single file for many different builds of the same package. See the :doc:`features` for examples and highlights. Get Spack from the `GitHub repository `_ and install your first package: .. code-block:: console $ git clone --depth=2 https://github.com/spack/spack.git $ cd spack/bin $ ./spack install libelf .. note:: ``--depth=2`` prunes the git history to reduce the size of the Spack installation. If you're new to Spack and want to start using it, see :doc:`getting_started`, or refer to the full manual below. .. toctree:: :maxdepth: 2 :caption: Introduction features getting_started spec_syntax installing_prerequisites windows .. toctree:: :maxdepth: 2 :caption: Basic Usage package_fundamentals installing configuring_compilers environments_basics frequently_asked_questions getting_help .. toctree:: :maxdepth: 2 :caption: Links Tutorial (spack-tutorial.rtfd.io) Packages (packages.spack.io) Binaries (binaries.spack.io) .. toctree:: :maxdepth: 2 :caption: Configuration configuration config_yaml packages_yaml toolchains_yaml build_settings repositories mirrors chain module_file_support include_yaml env_vars_yaml .. toctree:: :maxdepth: 2 :caption: Reference environments containers binary_caches bootstrapping command_index extensions pipelines signing gpu_configuration .. toctree:: :maxdepth: 2 :caption: Contributing packaging_guide_creation packaging_guide_build packaging_guide_testing packaging_guide_advanced build_systems roles_and_responsibilities contribution_guide developer_guide package_review_guide .. toctree:: :maxdepth: 2 :caption: Advanced Topics advanced_topics .. toctree:: :maxdepth: 2 :caption: API Docs Spack Package API Spack Builtin Repo Spack API Docs Indices and tables ------------------ * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: lib/spack/docs/installing.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how Spack installs packages: the interactive terminal UI, parallelism via a POSIX jobserver, multi-process installs, background execution, and handling build failures. .. _installing: Installing Packages =================== This page covers the ``spack install`` experience in detail, including the interactive terminal UI (TUI), parallelism, background execution, and handling build failures. Before diving in, ensure you are familiar with :doc:`package_fundamentals` for basic usage and spec syntax. .. versionadded:: 1.2 The TUI and POSIX jobserver are new in Spack 1.2 and require a Unix-like platform. Interactive terminal UI ----------------------- By default, ``spack install`` shows live progress inline in the terminal. Completed packages scroll into terminal history, while active builds update dynamically below the progress header. Every package in the install plan is shown with its current status: .. code-block:: text $ spack install -j16 python [+] abc1234 zlib@1.3.1 /home/user/spack/opt/spack/... (4s) [+] def5678 pkgconf@2.2.0 /home/user/spack/opt/spack/... (6s) [+] 9ab0123 ncurses@6.5 /home/user/spack/opt/spack/... (23s) Progress: 3/7 +/-: 4 jobs /: filter v: logs n/p: next/prev [/] cde4567 readline@8.2 configure (11s) [/] fgh8901 openssl@3.4.1 build (18s) Status indicators: * ``[+]`` finished successfully * ``[x]`` failed * ``[/]``, ``[-]``, ``[\]``, ``[|]`` building (rotating spinner) * ``[e]`` external **Log-following mode**: press ``v`` to switch from the overview to a live view of build output. Press ``v``, ``q``, or ``Esc`` to return to the overview. While in log-following mode, press ``n`` / ``p`` to cycle to the next or previous build. Press ``/``, type a pattern, and press ``Enter`` to jump to a matching build (``Esc`` cancels the filter). When a build fails, press ``v`` to see a parsed error summary and the path to the full log. Parallelism ----------- Spack controls parallelism at two levels: the number of build jobs shared across all packages (``-j``), and the number of packages building concurrently (``-p``). Build-level parallelism (``-j``) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``-j`` flag controls the **total** number of concurrent build jobs via a POSIX jobserver. All build processes (``make``, ``cmake``, ``ninja``, etc.) share the same jobserver, so ``-j16`` means at most 16 build jobs across *all* packages combined. This is the primary concurrency knob. .. code-block:: console $ spack install -j16 python Spack creates a POSIX jobserver compatible with GNU Make's jobserver protocol. Child build systems automatically respect it through ``MAKEFLAGS``, so total CPU usage stays bounded regardless of how many packages are building concurrently. .. note:: If an external jobserver is already present in ``MAKEFLAGS``, for example when Spack itself is invoked from inside a larger ``make`` build, Spack attaches to the existing jobserver instead of creating its own. Package-level parallelism (``-p``) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``-p`` / ``--concurrent-packages`` flag limits how many packages can be in the build queue simultaneously. By default there is no limit, and packages are started as jobserver tokens become available. .. code-block:: console $ spack install -j16 -p4 python This builds with 16 total make-jobs but never more than 4 packages at once. Dynamic adjustment ^^^^^^^^^^^^^^^^^^ You can adjust parallelism while a build is running: * Press ``+`` to add a job (increases ``-j`` by 1) * Press ``-`` to remove a job (decreases ``-j`` by 1) When reducing parallelism, Spack waits for currently running jobs to finish before the new limit takes effect; it does not kill active processes. The progress header shows the adjustment in progress, e.g. ``+/-: 4=>2 jobs``, until the actual count reaches the target. Multi-process and multi-node installs -------------------------------------- Multiple ``spack install`` processes can safely run concurrently, whether on the same machine or across multiple nodes in a cluster with a shared filesystem. Spack coordinates through :ref:`per-prefix filesystem locks `: before building a package, the process acquires an exclusive lock on its install prefix. If another process already holds the lock, Spack waits rather than building a second copy. When a process encounters a prefix that was already installed, it simply skips it and moves on to the next install. For best results on a cluster, it's recommended to limit per-process package-level parallelism (e.g., ``spack install -p2``) for better load balancing. Non-interactive mode -------------------- When the controlling process is not a tty, such as in CI pipelines, when redirecting output to a file, or when running in the background, Spack skips the TUI and prints simple line-based status updates instead. Use ``spack install -v`` to also print build output. You can also background builds: * **Suspend and resume**: press ``Ctrl-Z`` to suspend the install, then ``bg`` to let it continue in the background or ``fg`` to bring it back. Child builds are paused while suspended, and resumed when continued in the background or foreground. The TUI is suppressed while backgrounded and restored on ``fg``. * **Start in the background**: run ``spack install ... &`` to skip the TUI entirely and build in the background from the start. .. tip:: You don't need a new terminal or SSH session to keep a build running — just suspend it with ``Ctrl-Z`` and ``bg``, then continue working. Handling failures ----------------- By default, Spack continues building other packages when one fails (best-effort). Use ``--fail-fast`` to stop immediately on the first failure. .. code-block:: console $ spack install --fail-fast python Failed builds show ``[x]`` in the overview. Navigate to a failed build and press ``v`` to see a parsed error summary and the path to the full log. See :ref:`spack install ` for the full set of flags related to debugging and controlling build behavior. ================================================ FILE: lib/spack/docs/installing_prerequisites.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Find instructions on how to install the necessary prerequisites for Spack on various operating systems, including Linux and macOS. .. _verify-spack-prerequisites: Spack Prerequisites =================== Spack relies on a few basic utilities to be present on the system where it runs, depending on the operating system. To install them, follow the instructions below. Linux ----- For **Debian** and **Ubuntu** users: .. code-block:: console $ apt update $ apt install file bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 tar unzip xz-utils zstd For **RHEL**, **AlmaLinux**, and **Rocky Linux** users: .. code-block:: console $ dnf install epel-release $ dnf install file bzip2 ca-certificates git gzip patch python3 tar unzip xz zstd gcc gcc-c++ gcc-gfortran macOS ----- On macOS, the Command Line Tools package is required, and the full Xcode suite may be necessary for some packages, such as Qt and apple-gl. To install Xcode, you can use the following command: .. code-block:: console $ xcode-select --install For most packages, the Xcode command-line tools are sufficient. However, some packages like ``qt`` require the full Xcode suite. You can check to see which you have installed by running: .. code-block:: console $ xcode-select -p If the output is: .. code-block:: none /Applications/Xcode.app/Contents/Developer you already have the full Xcode suite installed. If the output is: .. code-block:: none /Library/Developer/CommandLineTools you only have the command-line tools installed. The full Xcode suite can be installed through the App Store. Make sure to launch the Xcode application and accept the license agreement before using Spack. It may ask you to install additional components. Alternatively, the Xcode license can be accepted through the command line: .. code-block:: console $ sudo xcodebuild -license accept ================================================ FILE: lib/spack/docs/mirrors.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Discover how to set up and manage mirrors in Spack to provide a local repository of tarballs for offline package fetching. .. _mirrors: Mirrors (mirrors.yaml) ====================== Some sites may not have access to the internet for fetching packages. These sites will need a local repository of tarballs from which they can get their files. Spack has support for this with *mirrors*. A mirror is a URL that points to a directory, either on the local filesystem or on some server, containing tarballs for all of Spack's packages. Here's an example of a mirror's directory structure: .. code-block:: none mirror/ cmake/ cmake-2.8.10.2.tar.gz dyninst/ dyninst-8.1.1.tgz dyninst-8.1.2.tgz libdwarf/ libdwarf-20130126.tar.gz libdwarf-20130207.tar.gz libdwarf-20130729.tar.gz libelf/ libelf-0.8.12.tar.gz libelf-0.8.13.tar.gz libunwind/ libunwind-1.1.tar.gz mpich/ mpich-3.0.4.tar.gz mvapich2/ mvapich2-1.9.tgz The structure is very simple. There is a top-level directory. The second level directories are named after packages, and the third level contains tarballs for each package, named after each package. .. note:: Archives are **not** named exactly the way they were in the package's fetch URL. They have the form ``-.``, where ```` is Spack's name for the package, ```` is the version of the tarball, and ```` is whatever format the package's fetch URL contains. In order to make mirror creation reasonably fast, we copy the tarball in its original format to the mirror directory, but we do not standardize on a particular compression algorithm, because this would potentially require expanding and recompressing each archive. .. _cmd-spack-mirror: ``spack mirror`` ---------------- Mirrors are managed with the ``spack mirror`` command. The help for ``spack mirror`` looks like this: .. command-output:: spack help mirror The ``create`` command actually builds a mirror by fetching all of its packages from the internet and checksumming them. The other three commands are for managing mirror configuration. They control the URL(s) from which Spack downloads its packages. .. _cmd-spack-mirror-create: ``spack mirror create`` ----------------------- You can create a mirror using the ``spack mirror create`` command, assuming you're on a machine where you can access the internet. The command will iterate through all of Spack's packages and download the safe ones into a directory structure like the one above. Here is what it looks like: .. code-block:: console $ spack mirror create libelf libdwarf ==> Created new mirror in spack-mirror-2014-06-24 ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz ########################################################## 81.6% ==> Checksum passed for libelf@0.8.13 ==> Added libelf@0.8.13 ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.12.tar.gz ###################################################################### 98.6% ==> Checksum passed for libelf@0.8.12 ==> Added libelf@0.8.12 ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130207.tar.gz ###################################################################### 97.3% ==> Checksum passed for libdwarf@20130207 ==> Added libdwarf@20130207 ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130126.tar.gz ######################################################## 78.9% ==> Checksum passed for libdwarf@20130126 ==> Added libdwarf@20130126 ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130729.tar.gz ############################################################# 84.7% ==> Added libdwarf@20130729 ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130729.tar.gz to mirror ==> Added python@2.7.8. ==> Successfully updated mirror in spack-mirror-2015-02-24. Archive stats: 0 already present 5 added 0 failed to fetch. Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and copy it over to the machine you want it hosted on. Custom package sets ^^^^^^^^^^^^^^^^^^^ Normally, ``spack mirror create`` downloads all the archives it has checksums for. If you want to only create a mirror for a subset of packages, you can do that by supplying a list of package specs on the command line after ``spack mirror create``. For example, this command: .. code-block:: console $ spack mirror create libelf@0.8.12: boost@1.44: Will create a mirror for libelf versions greater than or equal to 0.8.12 and boost versions greater than or equal to 1.44. Mirror files ^^^^^^^^^^^^ If you have a *very* large number of packages you want to mirror, you can supply a file with specs in it, one per line: .. code-block:: console $ cat specs.txt libdwarf libelf@0.8.12: boost@1.44: boost@1.39.0 ... $ spack mirror create --file specs.txt ... This is useful if there is a specific suite of software managed by your site. Mirror environment ^^^^^^^^^^^^^^^^^^ To create a mirror of all packages required by a concrete environment, activate the environment and run ``spack mirror create -a``. This is especially useful to create a mirror of an environment that was concretized on another machine. Optionally specify ``-j `` to control the number of workers used to create a full mirror. If not specified, the optimal number of workers is determined dynamically. For a full mirror, the number of workers used is the minimum of 16 workers, available CPU cores, and number of packages to mirror. For individual packages, 1 worker is used. .. code-block:: console [remote] $ spack env create myenv [remote] $ spack env activate myenv [remote] $ spack add ... [remote] $ spack concretize $ sftp remote:/spack/var/environment/myenv/spack.lock $ spack env create myenv spack.lock $ spack env activate myenv $ spack mirror create -a .. _cmd-spack-mirror-add: ``spack mirror add`` -------------------- Once you have a mirror, you need to let Spack know about it. This is relatively simple. First, figure out the URL for the mirror. If it's a directory, you can use a file URL like this one: .. code-block:: none file://$HOME/spack-mirror-2014-06-24 That points to the directory on the local filesystem. If it were on a web server, you could use a URL like this one: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 Spack will use the URL as the root for all of the packages it fetches. You can tell your Spack installation to use that mirror like this: .. code-block:: console $ spack mirror add local_filesystem file://$HOME/spack-mirror-2014-06-24 Each mirror has a name so that you can refer to it again later. .. _cmd-spack-mirror-list: ``spack mirror list`` --------------------- To see all the mirrors Spack knows about, run ``spack mirror list``: .. code-block:: console $ spack mirror list local_filesystem file:///home/username/spack-mirror-2014-06-24 .. _cmd-spack-mirror-remove: ``spack mirror remove`` ----------------------- To remove a mirror by name, run: .. code-block:: console $ spack mirror remove local_filesystem $ spack mirror list ==> No mirrors configured. Mirror precedence ----------------- Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``: .. code-block:: yaml mirrors: local_filesystem: file:///home/username/spack-mirror-2014-06-24 remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 If you want to change the order in which mirrors are searched for packages, you can edit this file and reorder the sections. Spack will search the topmost mirror first and the bottom-most mirror last. .. _caching: Local Default Cache ------------------- Spack caches resources that are downloaded as part of installations. The cache is a valid Spack mirror: it uses the same directory structure and naming scheme as other Spack mirrors (so it can be copied anywhere and referenced with a URL like other mirrors). The mirror is maintained locally (within the Spack installation directory) at :file:`var/spack/cache/`. It is always enabled (and is always searched first when attempting to retrieve files for an installation) but can be cleared with ``spack clean --misc-cache``; the cache directory can also be deleted manually without issue. Caching includes retrieved tarball archives and source control repositories, but only resources with an associated digest or commit ID (e.g. a revision number for SVN) will be cached. ================================================ FILE: lib/spack/docs/module_file_support.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to configure and customize module file generation in Spack for Environment Modules and Lmod. .. _modules: Modules (modules.yaml) ====================== The use of module systems to manage user environments in a controlled way is a common practice at HPC centers that is sometimes embraced also by individual programmers on their development machines. To support this common practice Spack integrates with `Environment Modules `_ and `Lmod `_ by providing post-install hooks that generate module files and commands to manipulate them. Modules are one of several ways you can use Spack packages. For other options that may fit your use case better, you should also look at :ref:`spack load ` and :ref:`environments `. Quick start ----------- In the current version of Spack, module files are not generated by default. To get started, you can generate module files for all currently installed packages by running either .. code-block:: console $ spack module tcl refresh or .. code-block:: console $ spack module lmod refresh Spack can also generate module files for all future installations automatically through the following configuration: .. code-block:: console $ spack config add modules:default:enable:[tcl] or .. code-block:: console $ spack config add modules:default:enable:[lmod] Assuming you have a module system installed, you should now be able to use the ``module`` command to interact with them: .. code-block:: console $ module avail --------------------------------------------------------------- ~/spack/share/spack/modules/linux-ubuntu14-x86_64 --------------------------------------------------------------- autoconf/2.69-gcc-4.8-qextxkq hwloc/1.11.6-gcc-6.3.0-akcisez m4/1.4.18-gcc-4.8-ev2znoc openblas/0.2.19-gcc-6.3.0-dhkmed6 py-setuptools/34.2.0-gcc-6.3.0-fadur4s automake/1.15-gcc-4.8-maqvukj isl/0.18-gcc-4.8-afi6taq m4/1.4.18-gcc-6.3.0-uppywnz openmpi/2.1.0-gcc-6.3.0-go2s4z5 py-six/1.10.0-gcc-6.3.0-p4dhkaw binutils/2.28-gcc-4.8-5s7c6rs libiconv/1.15-gcc-4.8-at46wg3 mawk/1.3.4-gcc-4.8-acjez57 openssl/1.0.2k-gcc-4.8-dkls5tk python/2.7.13-gcc-6.3.0-tyehea7 bison/3.0.4-gcc-4.8-ek4luo5 libpciaccess/0.13.4-gcc-6.3.0-gmufnvh mawk/1.3.4-gcc-6.3.0-ostdoms openssl/1.0.2k-gcc-6.3.0-gxgr5or readline/7.0-gcc-4.8-xhufqhn bzip2/1.0.6-gcc-4.8-iffrxzn libsigsegv/2.11-gcc-4.8-pp2cvte mpc/1.0.3-gcc-4.8-g5mztc5 pcre/8.40-gcc-4.8-r5pbrxb readline/7.0-gcc-6.3.0-zzcyicg bzip2/1.0.6-gcc-6.3.0-bequudr libsigsegv/2.11-gcc-6.3.0-7enifnh mpfr/3.1.5-gcc-4.8-o7xm7az perl/5.24.1-gcc-4.8-dg5j65u sqlite/3.8.5-gcc-6.3.0-6zoruzj cmake/3.7.2-gcc-6.3.0-fowuuby libtool/2.4.6-gcc-4.8-7a523za mpich/3.2-gcc-6.3.0-dmvd3aw perl/5.24.1-gcc-6.3.0-6uzkpt6 tar/1.29-gcc-4.8-wse2ass curl/7.53.1-gcc-4.8-3fz46n6 libtool/2.4.6-gcc-6.3.0-n7zmbzt ncurses/6.0-gcc-4.8-dcpe7ia pkg-config/0.29.2-gcc-4.8-ib33t75 tcl/8.6.6-gcc-4.8-tfxzqbr expat/2.2.0-gcc-4.8-mrv6bd4 libxml2/2.9.4-gcc-4.8-ryzxnsu ncurses/6.0-gcc-6.3.0-ucbhcdy pkg-config/0.29.2-gcc-6.3.0-jpgubk3 util-macros/1.19.1-gcc-6.3.0-xorz2x2 flex/2.6.3-gcc-4.8-yf345oo libxml2/2.9.4-gcc-6.3.0-rltzsdh netlib-lapack/3.6.1-gcc-6.3.0-js33dog py-appdirs/1.4.0-gcc-6.3.0-jxawmw7 xz/5.2.3-gcc-4.8-mew4log gcc/6.3.0-gcc-4.8-24puqve lmod/7.4.1-gcc-4.8-je4srhr netlib-scalapack/2.0.2-gcc-6.3.0-5aidk4l py-numpy/1.12.0-gcc-6.3.0-oemmoeu xz/5.2.3-gcc-6.3.0-3vqeuvb gettext/0.19.8.1-gcc-4.8-yymghlh lua/5.3.4-gcc-4.8-im75yaz netlib-scalapack/2.0.2-gcc-6.3.0-hjsemcn py-packaging/16.8-gcc-6.3.0-i2n3dtl zip/3.0-gcc-4.8-rwar22d gmp/6.1.2-gcc-4.8-5ub2wu5 lua-luafilesystem/1_6_3-gcc-4.8-wkey3nl netlib-scalapack/2.0.2-gcc-6.3.0-jva724b py-pyparsing/2.1.10-gcc-6.3.0-tbo6gmw zlib/1.2.11-gcc-4.8-pgxsxv7 help2man/1.47.4-gcc-4.8-kcnqmau lua-luaposix/33.4.0-gcc-4.8-mdod2ry netlib-scalapack/2.0.2-gcc-6.3.0-rgqfr6d py-scipy/0.19.0-gcc-6.3.0-kr7nat4 zlib/1.2.11-gcc-6.3.0-7cqp6cj The names should look familiar, as they resemble the output from ``spack find``. For example, you could type the following command to load the ``cmake`` module: .. code-block:: console $ module load cmake/3.7.2-gcc-6.3.0-fowuuby Neither of these is particularly pretty, easy to remember, or easy to type. Luckily, Spack offers many facilities for customizing the module scheme used at your site. Module file customization ------------------------- The table below summarizes the essential information associated with the different file formats that can be generated by Spack: +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ | | Hierarchical | **Default root directory** | **Default template file** | **Compatible tools** | +===========+==============+==============================+==============================================+======================+ | ``tcl`` | No | share/spack/modules | share/spack/templates/modules/modulefile.tcl | Env. Modules/Lmod | +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ | ``lmod`` | Yes | share/spack/lmod | share/spack/templates/modules/modulefile.lua | Lmod | +-----------+--------------+------------------------------+----------------------------------------------+----------------------+ Spack ships with sensible defaults for the generation of module files, but you can customize many aspects of it to accommodate package or site specific needs. In general you can override or extend the default behavior by: 1. overriding certain callback APIs in the Python packages 2. writing specific rules in the ``modules.yaml`` configuration file 3. writing your own templates to override or extend the defaults The former method lets you express changes in the run-time environment that are needed to use the installed software properly, e.g. injecting variables from language interpreters into their extensions. The latter two instead permit to fine tune the filesystem layout, content and creation of module files to meet site specific conventions. .. _overide-api-calls-in-package-py: Setting environment variables dynamically in ``package.py`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are two methods that you can implement in any ``package.py`` to dynamically affect the content of the module files generated by Spack. The most important one is ``setup_run_environment``, which can be used to set environment variables in the module file that depend on the spec: .. code-block:: python def setup_run_environment(self, env: EnvironmentModifications) -> None: if self.spec.satisfies("+foo"): env.set("FOO", "bar") The second, less commonly used, is ``setup_dependent_run_environment(self, env, dependent_spec)``, which allows a dependency to set variables in the module file of its dependents. This is typically used in packages like ``python``, ``r``, or ``perl`` to prepend the dependent's prefix to the search path of the interpreter (``PYTHONPATH``, ``R_LIBS``, ``PERL5LIB`` resp.), so it can locate the packages at runtime. For example, a simplified version of the ``python`` package could look like this: .. code-block:: python def setup_dependent_run_environment( self, env: EnvironmentModifications, dependent_spec: Spec ) -> None: if dependent_spec.package.extends(self.spec): env.prepend_path("PYTHONPATH", dependent_spec.prefix.lib.python) and would make any package that ``extends("python")`` have its library directory added to the ``PYTHONPATH`` environment variable in the module file. It's much more convenient to set this variable here, than to repeat it in every Python extension's ``setup_run_environment`` method. .. _modules-yaml: The ``modules.yaml`` config file and module sets ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The configuration files that control module generation behavior are named ``modules.yaml``. The default configuration looks like this: .. literalinclude:: _spack_root/etc/spack/defaults/base/modules.yaml :language: yaml You can define one or more **module sets**, each of which can be configured separately with regard to install location, naming scheme, inclusion and exclusion, autoloading, et cetera. The default module set is aptly named ``default``. All :ref:`Spack commands that operate on modules ` apply to the ``default`` module set, unless another module set is specified explicitly (with the ``--name`` flag). Changing the modules root ^^^^^^^^^^^^^^^^^^^^^^^^^ As shown in the table above, the default module root for ``lmod`` is ``$spack/share/spack/lmod`` and the default root for ``tcl`` is ``$spack/share/spack/modules``. This can be overridden for any module set by changing the ``roots`` key of the configuration. .. code-block:: yaml modules: default: roots: tcl: /path/to/install/tcl/modules my_custom_lmod_modules: roots: lmod: /path/to/install/custom/lmod/modules # ... This configuration will create two module sets. The default module set will install its ``tcl`` modules to ``/path/to/install/tcl/modules`` (and still install its lmod modules, if any, to the default location). The set ``my_custom_lmod_modules`` will install its lmod modules to ``/path/to/install/custom/lmod/modules`` (and still install its tcl modules, if any, to the default location). By default, an architecture-specific directory is added to the root directory. A module set may override that behavior by setting the ``arch_folder`` config value to ``False``. .. code-block:: yaml modules: default: roots: tcl: /path/to/install/tcl/modules arch_folder: false Obviously, having multiple module sets install modules to the default location could be confusing to users of your modules. In the next section, we will discuss enabling and disabling module types (module file generators) for each module set. Automatically generating module files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack can be configured to automatically generate module files as part of package installation. This is done by adding the desired module systems to the ``enable`` list. .. code-block:: yaml modules: default: enable: - tcl - lmod Configuring ``tcl`` and ``lmod`` modules ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can configure the behavior of either module system separately, under a key corresponding to the generator being customized: .. code-block:: yaml modules: default: tcl: # contains environment modules specific customizations lmod: # contains lmod specific customizations In general, the configuration options that you can use in ``modules.yaml`` will either change the layout of the module files on the filesystem, or they will affect their content. For the latter point it is possible to use anonymous specs to fine tune the set of packages on which the modifications should be applied. .. _autoloading-dependencies: Autoloading and hiding dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A module file should set the variables that are needed for an application to work. But since an application often has many dependencies, where should all the environment variables for those be set? In Spack the rule is that each package sets the runtime variables that are needed by the package itself, and no more. This way, dependencies can be loaded standalone too, and duplication of environment variables is avoided. That means however that if you want to use an application, you need to load the modules for all its dependencies. Of course this is not something you would want users to do manually. Since Spack knows the dependency graph of every package, it can easily generate module files that automatically load the modules for its dependencies recursively. It is enabled by default for both Lmod and Environment Modules under the ``autoload: direct`` config option. The former system has builtin support through the ``depends_on`` function, the latter simply uses a ``module load`` statement. Both module systems (at least in newer versions) do reference counting, so that if a module is loaded by two different modules, it will only be unloaded after the others are. The ``autoload`` key accepts the values: * ``none``: no autoloading * ``run``: autoload direct *run* type dependencies * ``direct``: autoload direct *link and run* type dependencies * ``all``: autoload all dependencies In case of ``run`` and ``direct``, a ``module load`` triggers a recursive load. The ``direct`` option is most correct: there are cases where pure link dependencies need to set variables for themselves, or need to have variables of their own dependencies set. In practice however, ``run`` is often sufficient, and may make ``module load`` snappier. The ``all`` option is discouraged and seldomly used. A common complaint about autoloading is the large number of modules that are visible to the user. Spack has a solution for this as well: ``hide_implicits: true``. This ensures that only those packages you've explicitly installed are exposed by ``module avail``, but still allows for autoloading of hidden dependencies. Lmod should support hiding implicits in general, while Environment Modules requires version 4.7 or higher. .. note:: If supported by your module system, we highly encourage the following configuration that enables autoloading and hiding of implicits. It ensures all runtime variables are set correctly, including those for dependencies, without overwhelming the user with a large number of available modules. Further, it makes it easier to get readable module names without collisions, see the section below on :ref:`modules-projections`. .. code-block:: yaml modules: default: tcl: hide_implicits: true all: autoload: direct # or `run` lmod: hide_implicits: true all: autoload: direct # or `run` .. _anonymous_specs: Setting environment variables for selected packages in config ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In the configuration file you can filter particular specs, and make further changes to the environment variables that go into their module files. This is very powerful when you want to avoid :ref:`modifying the package itself `, or when you want to set certain variables on multiple selected packages at once. For instance, in the snippet below: .. code-block:: yaml modules: default: tcl: # The keyword `all` selects every package all: environment: set: BAR: "bar" # This anonymous spec selects any package that # depends on mpi. The double colon at the # end clears the set of rules that matched so far. ^mpi:: environment: prepend_path: PATH: "{^mpi.prefix}/bin" set: BAR: "baz" # Selects any zlib package zlib: environment: prepend_path: LD_LIBRARY_PATH: "foo" # Selects zlib compiled with gcc@4.8 zlib%gcc@4.8: environment: unset: - FOOBAR you are instructing Spack to set the environment variable ``BAR=bar`` for every module, unless the associated spec satisfies the abstract dependency ``^mpi`` in which case ``BAR=baz``, and the directory containing the respective MPI executables is prepended to the ``PATH`` variable. In addition in any spec that satisfies ``zlib`` the value ``foo`` will be prepended to ``LD_LIBRARY_PATH`` and in any spec that satisfies ``zlib%gcc@4.8`` the variable ``FOOBAR`` will be unset. .. admonition:: Note: order does matter :class: note The modifications associated with the ``all`` keyword are always evaluated first, no matter where they appear in the configuration file. All the other changes to environment variables for matching specs are evaluated from top to bottom. .. warning:: As general advice, it's often better to set as few unnecessary variables as possible. For example, the following seemingly innocent and potentially useful configuration .. code-block:: yaml all: environment: set: "{name}_ROOT": "{prefix}" sets ``BINUTILS_ROOT`` to its prefix in modules for ``binutils``, which happens to break the ``gcc`` compiler: it uses this variable as its default search path for certain object files and libraries, and by merely setting it, everything fails to link. Exclude or include specific module files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can use anonymous specs also to prevent module files from being written or to force them to be written. Consider the case where you want to hide from users all the boilerplate software that you had to build in order to bootstrap a new compiler. Suppose for instance that ``gcc@4.4.7`` is the compiler provided by your system. If you write a configuration file like: .. code-block:: yaml modules: default: tcl: include: ["gcc", "llvm"] # include will have precedence over exclude exclude: ["%gcc@4.4.7"] # Assuming gcc@4.4.7 is the system compiler you will prevent the generation of module files for any package that is compiled with ``gcc@4.4.7``, with the only exception of any ``gcc`` or any ``llvm`` installation. It is safe to combine ``exclude`` and ``autoload`` :ref:`mentioned above `. When ``exclude`` prevents a module file to be generated for a dependency, the ``autoload`` feature will simply not generate a statement to load it. .. _modules-projections: Customize the naming of modules ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The names of environment modules generated by Spack are not always easy to fully comprehend due to the long hash in the name. There are three module configuration options to help with that. The first is a global setting to adjust the hash length. It can be set anywhere from 0 to 32 and has a default length of 7. This is the representation of the hash in the module file name and does not affect the size of the package hash. Be aware that the smaller the hash length the more likely naming conflicts will occur. The following snippet shows how to set hash length in the module file names: .. code-block:: yaml modules: default: tcl: hash_length: 7 .. tip:: Using ``hide_implicits: true`` (see :ref:`autoloading-dependencies`) vastly reduces the number modules exposed to the user. The hidden modules always contain the hash in their name, and are not influenced by the ``hash_length`` setting. Hidden implicits thus make it easier to use a short hash length or no hash at all, without risking name conflicts. To help make module names more readable, and to help alleviate name conflicts with a short hash, one can use the ``suffixes`` option in the modules configuration file. This option will add strings to modules that match a spec. For instance, the following config options, .. code-block:: yaml modules: default: tcl: all: suffixes: ^python@3: "python{^python.version.up_to_2}" ^openblas: "openblas" will add a ``python3.12`` to module names of packages compiled with Python 3.12, and similarly for all specs depending on ``python@3``. This is useful to know which version of Python a set of Python extensions is associated with. Likewise, the ``openblas`` string is attached to any program that has ``openblas`` in the spec, most likely via the ``+blas`` variant specification. The most heavyweight solution to module naming is to change the entire naming convention for module files. This uses the projections format covered in :ref:`view_projections`. .. code-block:: yaml modules: default: tcl: projections: all: "{name}/{version}-{compiler.name}-{compiler.version}-module" ^mpi: "{name}/{version}-{^mpi.name}-{^mpi.version}-{compiler.name}-{compiler.version}-module" will create module files that are nested in directories by package name, contain the version and compiler name and version, and have the word ``module`` before the hash for all specs that do not depend on ``mpi``, and will have the same information plus the MPI implementation name and version for all packages that depend on ``mpi``. When specifying module names by projection for Lmod modules, we recommend NOT including names of dependencies (e.g., MPI, compilers) that are already in the Lmod hierarchy. .. note:: Tcl and Lua modules also allow for explicit conflicts between module files. .. code-block:: yaml modules: default: enable: - tcl tcl: projections: all: "{name}/{version}-{compiler.name}-{compiler.version}" all: conflict: - "{name}" - "intel/14.0.1" will create module files that will conflict with ``intel/14.0.1`` and with the base directory of the same module, effectively preventing the possibility to load two or more versions of the same software at the same time. The tokens that are available for use in this directive are those understood by the :meth:`~spack.spec.Spec.format` method. For Lmod and Environment Modules versions prior to 4.2, it is important to express the conflict on both module files conflicting with each other. .. admonition:: Note: Lmod hierarchical module files :class: note When ``lmod`` is activated Spack will generate a set of hierarchical lua module files that are understood by Lmod. The hierarchy always contains the ``Core`` and ``Compiler`` layers, but can be extended to include any package or virtual package in Spack. A case that could be useful in practice is for instance: .. code-block:: yaml modules: default: enable: - lmod lmod: core_compilers: - "gcc@4.8" core_specs: - "r" hierarchy: - "mpi" - "lapack" - "python" that will generate a hierarchy in which the ``python``, ``lapack`` and ``mpi`` layer can be switched independently. This allows a site to build the same libraries or applications against different implementations of ``mpi`` and ``lapack``, and with different versions of those implementations and of ``python``, and let Lmod switch safely from among the resulting installs. All packages built with a compiler in ``core_compilers`` and all packages that satisfy a spec in ``core_specs`` will be put in the ``Core`` hierarchy of the lua modules. .. admonition:: Warning: consistency of core packages :class: warning The user is responsible for maintaining consistency among core packages, as ``core_specs`` bypasses the hierarchy that allows Lmod to safely switch between coherent software stacks. .. admonition:: Warning: deep hierarchies :class: warning For hierarchies that are deeper than three layers ``lmod spider`` may have some issues. See `this discussion on the Lmod project `_. .. _customize-env-modifications: Customize environment modifications ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can control which prefixes in a Spack package are added to environment variables with the ``prefix_inspections`` section; this section maps relative prefixes to the list of environment variables which should be updated with those prefixes. The ``prefix_inspections`` configuration is different from other settings in that a ``prefix_inspections`` configuration at the ``modules`` level of the configuration file applies to all module sets. This allows users to make general overrides to the default inspections and customize them per-module-set. .. code-block:: yaml modules: prefix_inspections: ./bin: - PATH ./man: - MANPATH ./: - CMAKE_PREFIX_PATH Prefix inspections are only applied if the relative path inside the installation prefix exists. In this case, for a Spack package ``foo`` installed to ``/spack/prefix/foo``, if ``foo`` installs executables to ``bin`` but no manpages in ``man``, the generated module file for ``foo`` would update ``PATH`` to contain ``/spack/prefix/foo/bin`` and ``CMAKE_PREFIX_PATH`` to contain ``/spack/prefix/foo``, but would not update ``MANPATH``. The default list of environment variables in this config section includes ``PATH``, ``MANPATH``, ``ACLOCAL_PATH``, ``PKG_CONFIG_PATH`` and ``CMAKE_PREFIX_PATH``, as well as ``DYLD_FALLBACK_LIBRARY_PATH`` on macOS. On Linux however, the corresponding ``LD_LIBRARY_PATH`` variable is *not* set, because it affects the behavior of system executables too. .. note:: In general, the ``LD_LIBRARY_PATH`` variable is not required when using packages built with Spack, thanks to the use of RPATH. Some packages may still need the variable, which is best handled on a per-package basis instead of globally, as explained in :ref:`overide-api-calls-in-package-py`. There is a special case for prefix inspections relative to environment views. If all of the following conditions hold for a module set configuration: #. The configuration is for an :ref:`environment ` and will never be applied outside the environment, #. The environment in question is configured to use a view, #. The :ref:`environment view is configured ` with a projection that ensures every package is linked to a unique directory, then the module set may be configured to create modules relative to the environment view. This is specified by the ``use_view`` configuration option in the module set. If ``True``, the module set is constructed relative to the default view of the environment. Otherwise, the value must be the name of the environment view relative to which to construct modules, or ``False-ish`` to disable the feature explicitly (the default is ``False``). If the ``use_view`` value is set in the config, then the prefix inspections for the package are done relative to the package's path in the view. .. code-block:: yaml spack: modules: view_relative_modules: use_view: my_view prefix_inspections: ./bin: - PATH view: my_view: root: /path/to/my/view projections: all: "{name}-{hash}" The ``spack`` key is relevant to :ref:`environment ` configuration, and the view key is discussed in detail in the section on :ref:`Configuring environment views `. With this configuration the generated module for package ``foo`` would set ``PATH`` to include ``/path/to/my/view/foo-/bin`` instead of ``/spack/prefix/foo/bin``. The ``use_view`` option is useful when deploying a large software stack to users who are likely to inspect the modules to find full paths to software, when it is desirable to present the users with a simpler set of paths than those generated by the Spack install tree. Filter out environment modifications ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Modifications to certain environment variables in module files are there by default, for instance because they are generated by prefix inspections. If you want to prevent modifications to some environment variables, you can do so by using the ``exclude_env_vars``: .. code-block:: yaml modules: default: tcl: all: filter: # Exclude changes to any of these variables exclude_env_vars: ["CPATH", "LIBRARY_PATH"] The configuration above will generate module files that will not contain modifications to either ``CPATH`` or ``LIBRARY_PATH``. Select default modules ^^^^^^^^^^^^^^^^^^^^^^ By default, when multiple modules of the same name share a directory, the highest version number will be the default module. This behavior of the ``module`` command can be overridden with a symlink named ``default`` to the desired default module. If you wish to configure default modules with Spack, add a ``defaults`` key to your modules configuration: .. code-block:: yaml modules: my-module-set: tcl: defaults: - gcc@10.2.1 - hdf5@1.2.10+mpi+hl%gcc These defaults may be arbitrarily specific. For any package that satisfies a default, Spack will generate the module file in the appropriate path, and will generate a default symlink to the module file as well. .. warning:: If Spack is configured to generate multiple default packages in the same directory, the last modulefile to be generated will be the default module. .. _maintaining-module-files: Maintaining Module Files ------------------------ Each type of module file has a command with the same name associated with it. The actions these commands permit are usually associated with the maintenance of a production environment. Here's, for instance, a sample of the features of the ``spack module tcl`` command: .. command-output:: spack module tcl --help .. _cmd-spack-module-refresh: Refresh the set of modules ^^^^^^^^^^^^^^^^^^^^^^^^^^ The subcommand that regenerates module files to update their content or their layout is ``refresh``: .. command-output:: spack module tcl refresh --help A set of packages can be selected using anonymous specs for the optional ``constraint`` positional argument. Optionally the entire tree can be deleted before regeneration if the change in layout is radical. .. _cmd-spack-module-rm: Delete module files ^^^^^^^^^^^^^^^^^^^ If instead what you need is just to delete a few module files, then the right subcommand is ``rm``: .. command-output:: spack module tcl rm --help .. note:: We care about your module files! Every modification done on modules that are already existing will ask for a confirmation by default. If the command is used in a script it is possible though to pass the ``-y`` argument, that will skip this safety measure. .. _modules-in-shell-scripts: Using Spack modules in shell scripts ------------------------------------ To enable additional Spack commands for loading and unloading module files, and to add the correct path to ``MODULEPATH``, you need to source the appropriate setup file. Assuming Spack is installed in ``$SPACK_ROOT``, run the appropriate command for your shell: .. code-block:: console # For bash/zsh/sh $ . $SPACK_ROOT/share/spack/setup-env.sh # For tcsh/csh $ source $SPACK_ROOT/share/spack/setup-env.csh # For fish $ . $SPACK_ROOT/share/spack/setup-env.fish If you want to have Spack's shell support available on the command line at any login you can put this source line in one of the files that are sourced at startup (like ``.profile``, ``.bashrc`` or ``.cshrc``). Be aware that the shell startup time may increase slightly as a result. .. _cmd-spack-module-loads: ``spack module tcl loads`` ^^^^^^^^^^^^^^^^^^^^^^^^^^ In some cases, it is desirable to use a Spack-generated module, rather than relying on Spack's built-in user-environment modification capabilities. To translate a spec into a module name, use ``spack module tcl loads`` or ``spack module lmod loads`` depending on the module system desired. To load not just a module, but also all the modules it depends on, use the ``--dependencies`` option. This is not required for most modules because Spack builds binaries with RPATH support. However, not all packages use RPATH to find their dependencies: this can be true in particular for Python extensions, which are currently *not* built with RPATH. Scripts to load modules recursively may be made with the command: .. code-block:: console $ spack module tcl loads --dependencies An equivalent alternative using `process substitution `_ is: .. code-block:: console $ source <( spack module tcl loads --dependencies ) Module Commands for Shell Scripts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Although Spack is flexible, the ``module`` command is much faster. This can become an issue when emitting a series of ``spack load`` commands inside a shell script. By adding the ``--dependencies`` flag, ``spack module tcl loads`` may also be used to generate code that can be cut-and-pasted into a shell script. For example: .. code-block:: console $ spack module tcl loads --dependencies py-numpy git # bzip2@1.0.6%gcc@4.9.3=linux-x86_64 module load bzip2/1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx # ncurses@6.0%gcc@4.9.3=linux-x86_64 module load ncurses/6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv # zlib@1.2.8%gcc@4.9.3=linux-x86_64 module load zlib/1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z # sqlite@3.8.5%gcc@4.9.3=linux-x86_64 module load sqlite/3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr # readline@6.3%gcc@4.9.3=linux-x86_64 module load readline/6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3 # python@3.5.1%gcc@4.9.3=linux-x86_64 module load python/3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi # py-setuptools@20.5%gcc@4.9.3=linux-x86_64 module load py-setuptools/20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2 # py-nose@1.3.7%gcc@4.9.3=linux-x86_64 module load py-nose/1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64 module load openblas/0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64 module load py-numpy/1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r # curl@7.47.1%gcc@4.9.3=linux-x86_64 module load curl/7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi # autoconf@2.69%gcc@4.9.3=linux-x86_64 module load autoconf/2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4 # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64 module load cmake/3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t # expat@2.1.0%gcc@4.9.3=linux-x86_64 module load expat/2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64 module load git/2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd The script may be further edited by removing unnecessary modules. Module Prefixes ^^^^^^^^^^^^^^^ On some systems, modules are automatically prefixed with a certain string; ``spack module tcl loads`` needs to know about that prefix when it issues ``module load`` commands. Add the ``--prefix`` option to your ``spack module tcl loads`` commands if this is necessary. For example, consider the following on one system: .. code-block:: console $ module avail linux-SuSE11-x86_64/antlr/2.7.7-gcc-5.3.0-bdpl46y $ spack module tcl loads antlr # WRONG! # antlr@2.7.7%gcc@5.3.0~csharp+cxx~java~python arch=linux-SuSE11-x86_64 module load antlr/2.7.7-gcc-5.3.0-bdpl46y $ spack module tcl loads --prefix linux-SuSE11-x86_64/ antlr # antlr@2.7.7%gcc@5.3.0~csharp+cxx~java~python arch=linux-SuSE11-x86_64 module load linux-SuSE11-x86_64/antlr/2.7.7-gcc-5.3.0-bdpl46y ================================================ FILE: lib/spack/docs/package_api.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: An overview of the Spack Package API, a stable interface for package authors to interact with the Spack framework. Spack Package API ================= This document describes the Spack Package API (:mod:`spack.package`), the stable interface for Spack package authors. It is assumed you have already read the :doc:`Spack Packaging Guide `. The Spack Package API is the *only* module in the Spack codebase considered public API. It re-exports essential functions and classes from various Spack modules, allowing package authors to import them directly from :mod:`spack.package` without needing to know Spack's internal structure. Spack Package API Versioning ---------------------------- The current Package API version is |package_api_version|, defined in :attr:`spack.package_api_version`. Notice that the Package API is versioned independently from Spack itself: * The **minor version** is incremented when new functions or classes are exported from :mod:`spack.package`. * The **major version** is incremented when functions or classes are removed or have breaking changes to their signatures (a rare occurrence). This independent versioning allows package authors to utilize new Spack features without waiting for a new Spack release. Compatibility between Spack and :doc:`package repositories ` is managed as follows: * Package repositories declare their minimum required Package API version in their ``repo.yaml`` file using the ``api: vX.Y`` format. * Spack checks if the declared API version falls within its supported range, specifically between :attr:`spack.min_package_api_version` and :attr:`spack.package_api_version`. Spack version |spack_version| supports package repositories with a Package API version between |min_package_api_version| and |package_api_version|, inclusive. Changelog --------- **v2.4** *(Spack v1.0.3)* * The ``%%`` operator can be used on input specs to set propagated preferences, which is particularly useful for ``unify: false`` environments. **v2.3** *(Spack v1.0.3)* * The :func:`~spack.package.version` directive now supports the ``git_sparse_paths`` parameter, allowing sparse checkouts when fetching from git repositories. **v2.2** *(Spack v1.0.0)* * Renamed implicit builder attributes with backward compatibility: * ``legacy_buildsystem`` to ``default_buildsystem``, * ``legacy_methods`` to ``package_methods``, * ``legacy_attributes`` to ``package_attributes``, * ``legacy_long_methods`` to ``package_long_methods``. * Exported :class:`~spack.package.GenericBuilder`, :class:`~spack.package.Package`, and :class:`~spack.package.BuilderWithDefaults` from :mod:`spack.package`. * Exported numerous utility functions and classes for file operations, library/header search, macOS/Windows support, compiler detection, and build system helpers. **v2.1** *(Spack v1.0.0)* * Exported :class:`~spack.package.CompilerError` and :class:`~spack.package.SpackError` from :mod:`spack.package`. Spack Package API Reference --------------------------- .. automodule:: spack.package :members: :show-inheritance: :undoc-members: :no-value: ================================================ FILE: lib/spack/docs/package_fundamentals.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn the fundamental Spack commands for managing software packages, including how to find, inspect, install, and remove them. .. _basic-usage: Package Fundamentals ==================== Spack provides a comprehensive ecosystem of software packages that you can install. In this section you'll learn: 1. How to discover which packages are available, 2. How to get detailed information about specific packages, 3. How to install/uninstall packages, and 4. How to discover, and use, software that has been installed .. _basic-list-and-info-packages: Listing Available Packages -------------------------- To install software with Spack, you need to know what software is available. You can search for available packages on the `packages.spack.io `_ website or by using the ``spack list`` command. .. _cmd-spack-list: ``spack list`` ^^^^^^^^^^^^^^ The ``spack list`` command prints out a list of all of the packages Spack can install: .. code-block:: spec $ spack list Packages are listed by name in alphabetical order. A pattern can be used to narrow the list, and the following rules apply: * A pattern with no wildcards (``*`` or ``?``) is treated as if it starts and ends with ``*`` * All patterns are case-insensitive To search for all packages whose names contain the word ``sql`` you can run the following command: .. code-block:: spec $ spack list sql A few options are also provided for more specific searches. For instance, it is possible to search the description of packages for a match. A way to list all the packages whose names or descriptions contain the word ``quantum`` is the following: .. code-block:: spec $ spack list -d quantum .. _cmd-spack-info: ``spack info`` ^^^^^^^^^^^^^^ To get more information about a particular package from `spack list`, use `spack info`. Just supply the name of a package: .. command-output:: spack info mpich :language: spec Most of the information is self-explanatory. The *safe versions* are versions for which Spack knows the checksum. Spack uses this checksum to verify that the versions are downloaded without errors or malicious changes. :ref:`Dependencies ` and :ref:`virtual dependencies ` are described in more detail later. .. _cmd-spack-versions: ``spack versions`` ^^^^^^^^^^^^^^^^^^ To see *more* available versions of a package, run ``spack versions``. For example: .. command-output:: spack versions libelf :language: spec There are two sections in the output. *Safe versions* are versions for which Spack has a checksum on file. It can verify that these versions are downloaded correctly. In many cases, Spack can also show you what versions are available out on the web -- these are *remote versions*. Spack gets this information by scraping it directly from package web pages. Depending on the package and how its releases are organized, Spack may or may not be able to find remote versions. .. _cmd-spack-providers: ``spack providers`` ^^^^^^^^^^^^^^^^^^^ You can see what packages provide a particular virtual package using ``spack providers``. If you wanted to see what packages provide ``mpi``, you would just run: .. command-output:: spack providers mpi :language: spec And if you *only* wanted to see packages that provide MPI-2, you would add a version specifier to the spec: .. command-output:: spack providers mpi@2 :language: spec Notice that the package versions that provide insufficient MPI versions are now filtered out. Installing and Uninstalling --------------------------- .. _cmd-spack-install: ``spack install`` ^^^^^^^^^^^^^^^^^ ``spack install`` will install any package shown by ``spack list``. For example, to install the latest version of the ``mpileaks`` package, you might type this: .. code-block:: spec $ spack install mpileaks If ``mpileaks`` depends on other packages, Spack will install the dependencies first. It then fetches the ``mpileaks`` tarball, expands it, verifies that it was downloaded without errors, builds it, and installs it in its own directory under ``$SPACK_ROOT/opt``. .. code-block:: spec $ spack install mpileaks ... dependency build output ... [+] ph7pbnh mpileaks@1.0 ~/spack/opt/linux-rhel7-broadwell/gcc-8.1.0/mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 (5s) The last line, with the ``[+]``, indicates where the package is installed. Building a specific version ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack can also build *specific versions* of a package. To do this, just add ``@`` after the package name, followed by a version: .. code-block:: spec $ spack install mpich@3.0.4 Any number of versions of the same package can be installed at once without interfering with each other. This is useful for multi-user sites, as installing a version that one user needs will not disrupt existing installations for other users. In addition to different versions, Spack can customize the compiler, compile-time options (variants), compiler flags, and target architecture of an installation. Spack is unique in that it can also configure the *dependencies* a package is built with. For example, two configurations of the same version of a package, one built with boost 1.39.0, and the other version built with version 1.43.0, can coexist. This can all be done on the command line using the *spec* syntax. Spack calls the descriptor used to refer to a particular package configuration a **spec**. In the commands above, ``mpileaks`` and ``mpileaks@3.0.4`` are both valid *specs*. We'll talk more about how you can use them to customize an installation in :ref:`sec-specs`. Reusing installed dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, when you run ``spack install``, Spack tries hard to reuse existing installations as dependencies, either from a local store or from remote build caches, if configured. This minimizes unwanted rebuilds of common dependencies, in particular if you update Spack frequently. In case you want the latest versions and configurations to be installed instead, you can add the ``--fresh`` option: .. code-block:: spec $ spack install --fresh mpich Reusing installations in this mode is "accidental" and happens only if there's a match between existing installations and what Spack would have installed anyway. You can use the ``spack spec -I mpich`` command to see what will be reused and what will be built before you install. You can configure Spack to use the ``--fresh`` behavior by default in ``concretizer.yaml``: .. code-block:: yaml concretizer: reuse: false .. _cmd-spack-uninstall: ``spack uninstall`` ^^^^^^^^^^^^^^^^^^^ To uninstall a package, run ``spack uninstall ``. This will ask the user for confirmation before completely removing the directory in which the package was installed. .. code-block:: spec $ spack uninstall mpich If there are still installed packages that depend on the package to be uninstalled, Spack will refuse to uninstall it. To uninstall a package and every package that depends on it, you may give the ``--dependents`` option. .. code-block:: spec $ spack uninstall --dependents mpich will display a list of all the packages that depend on ``mpich`` and, upon confirmation, will uninstall them in the correct order. A command like .. code-block:: spec $ spack uninstall mpich may be ambiguous if multiple ``mpich`` configurations are installed. For example, if both ``mpich@3.0.2`` and ``mpich@3.1`` are installed, ``mpich`` could refer to either one. Because it cannot determine which one to uninstall, Spack will ask you either to provide a version number to remove the ambiguity or use the ``--all`` option to uninstall all matching packages. You may force uninstall a package with the ``--force`` option .. code-block:: spec $ spack uninstall --force mpich but you risk breaking other installed packages. In general, it is safer to remove dependent packages *before* removing their dependencies or to use the ``--dependents`` option. .. _cmd-spack-gc: Garbage collection ^^^^^^^^^^^^^^^^^^ When Spack builds software from sources, it often installs tools that are needed only to build or test other software. These are not necessary at runtime. To support cases where removing these tools can be a benefit, Spack provides the ``spack gc`` ("garbage collector") command, which will uninstall all unneeded packages: .. code-block:: console $ spack find ==> 24 installed packages -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ---------------------- autoconf@2.69 findutils@4.6.0 libiconv@1.16 libszip@2.1.1 m4@1.4.18 openjpeg@2.3.1 pkgconf@1.6.3 util-macros@1.19.1 automake@1.16.1 gdbm@1.18.1 libpciaccess@0.13.5 libtool@2.4.6 mpich@3.3.2 openssl@1.1.1d readline@8.0 xz@5.2.4 cmake@3.16.1 hdf5@1.10.5 libsigsegv@2.12 libxml2@2.9.9 ncurses@6.1 perl@5.30.0 texinfo@6.5 zlib@1.2.11 $ spack gc ==> The following packages will be uninstalled: -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ---------------------- vn47edz autoconf@2.69 6m3f2qn findutils@4.6.0 ubl6bgk libtool@2.4.6 pksawhz openssl@1.1.1d urdw22a readline@8.0 ki6nfw5 automake@1.16.1 fklde6b gdbm@1.18.1 b6pswuo m4@1.4.18 k3s2csy perl@5.30.0 lp5ya3t texinfo@6.5 ylvgsov cmake@3.16.1 5omotir libsigsegv@2.12 leuzbbh ncurses@6.1 5vmfbrq pkgconf@1.6.3 5bmv4tg util-macros@1.19.1 ==> Do you want to proceed? [y/N] y [ ... ] $ spack find ==> 9 installed packages -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ---------------------- hdf5@1.10.5 libiconv@1.16 libpciaccess@0.13.5 libszip@2.1.1 libxml2@2.9.9 mpich@3.3.2 openjpeg@2.3.1 xz@5.2.4 zlib@1.2.11 In the example above, ``spack gc`` scans the package database. It keeps only the packages that were explicitly installed by a user, along with their required ``link`` and ``run`` dependencies (including transitive dependencies). All other packages, such as build-only dependencies or orphaned packages, are identified as "garbage" and removed. You can check :ref:`cmd-spack-find-metadata` to see how to query for explicitly installed packages or :ref:`dependency-types` for a more thorough treatment of dependency types. .. _cmd-spack-mark: Marking packages explicit or implicit ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, Spack will mark packages a user installs as explicitly installed, while all of its dependencies will be marked as implicitly installed. Packages can be marked manually as explicitly or implicitly installed by using ``spack mark``. This can be used in combination with ``spack gc`` to clean up packages that are no longer required. .. code-block:: spec $ spack install m4 ==> 29005: Installing libsigsegv [...] ==> 29005: Installing m4 [...] $ spack install m4 ^libsigsegv@2.11 ==> 39798: Installing libsigsegv [...] ==> 39798: Installing m4 [...] $ spack find -d ==> 4 installed packages -- linux-fedora32-haswell / gcc@10.1.1 -------------------------- libsigsegv@2.11 libsigsegv@2.12 m4@1.4.18 libsigsegv@2.12 m4@1.4.18 libsigsegv@2.11 $ spack gc ==> There are no unused specs. Spack's store is clean. $ spack mark -i m4 ^libsigsegv@2.11 ==> m4@1.4.18 : marking the package implicit $ spack gc ==> The following packages will be uninstalled: -- linux-fedora32-haswell / gcc@10.1.1 -------------------------- 5fj7p2o libsigsegv@2.11 c6ensc6 m4@1.4.18 ==> Do you want to proceed? [y/N] In the example above, we ended up with two versions of ``m4`` because they depend on different versions of ``libsigsegv``. ``spack gc`` will not remove any of the packages because both versions of ``m4`` have been installed explicitly and both versions of ``libsigsegv`` are required by the ``m4`` packages. ``spack mark`` can also be used to implement upgrade workflows. The following example demonstrates how ``spack mark`` and ``spack gc`` can be used to only keep the current version of a package installed. When updating Spack via ``git pull``, new versions for either ``libsigsegv`` or ``m4`` might be introduced. This will cause Spack to install duplicates. Because we only want to keep one version, we mark everything as implicitly installed before updating Spack. If there is no new version for either of the packages, ``spack install`` will simply mark them as explicitly installed, and ``spack gc`` will not remove them. .. code-block:: spec $ spack install m4 ==> 62843: Installing libsigsegv [...] ==> 62843: Installing m4 [...] $ spack mark -i -a ==> m4@1.4.18 : marking the package implicit $ git pull [...] $ spack install m4 [...] ==> m4@1.4.18 : marking the package explicit [...] $ spack gc ==> There are no unused specs. Spack's store is clean. When using this workflow for installations that contain more packages, care must be taken to either only mark selected packages or issue ``spack install`` for all packages that should be kept. You can check :ref:`cmd-spack-find-metadata` to see how to query for explicitly or implicitly installed packages. .. _nondownloadable: Non-Downloadable Tarballs ^^^^^^^^^^^^^^^^^^^^^^^^^ The tarballs for some packages cannot be automatically downloaded by Spack. This could be for a number of reasons: #. The author requires users to manually accept a license agreement before downloading (e.g., ``jdk`` and ``galahad``). #. The software is proprietary and cannot be downloaded on the open Internet. To install these packages, one must create a mirror and manually add the tarballs in question to it (see :ref:`mirrors`): #. Create a directory for the mirror. You can create this directory anywhere you like, it does not have to be inside ``~/.spack``: .. code-block:: console $ mkdir ~/.spack/manual_mirror #. Register the mirror with Spack by creating ``~/.spack/mirrors.yaml``: .. code-block:: yaml mirrors: manual: file://~/.spack/manual_mirror #. Put your tarballs in it. Tarballs should be named ``/-.tar.gz``. For example: .. code-block:: console $ ls -l manual_mirror/galahad -rw-------. 1 me me 11657206 Jun 21 19:25 galahad-2.60003.tar.gz #. Install as usual: .. code-block:: console $ spack install galahad Seeing Installed Packages ------------------------- We know that ``spack list`` shows you the names of available packages, but how do you figure out which are already installed? .. _cmd-spack-find: ``spack find`` ^^^^^^^^^^^^^^ ``spack find`` shows the *specs* of installed packages. A spec is like a name, but it has a version, compiler, architecture, and build options associated with it. In Spack, you can have many installations of the same package with different specs. Running ``spack find`` with no arguments lists installed packages: .. code-block:: spec $ spack find ==> 74 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- ImageMagick@6.8.9-10 libdwarf@20130729 py-dateutil@2.4.0 adept-utils@1.0 libdwarf@20130729 py-ipython@2.3.1 atk@2.14.0 libelf@0.8.12 py-matplotlib@1.4.2 boost@1.55.0 libelf@0.8.13 py-nose@1.3.4 bzip2@1.0.6 libffi@3.1 py-numpy@1.9.1 cairo@1.14.0 libmng@2.0.2 py-pygments@2.0.1 callpath@1.0.2 libpng@1.6.16 py-pyparsing@2.0.3 cmake@3.0.2 libtiff@4.0.3 py-pyside@1.2.2 dbus@1.8.6 libtool@2.4.2 py-pytz@2014.10 dbus@1.9.0 libxcb@1.11 py-setuptools@11.3.1 dyninst@8.1.2 libxml2@2.9.2 py-six@1.9.0 fontconfig@2.11.1 libxml2@2.9.2 python@2.7.8 freetype@2.5.3 llvm@3.0 qhull@1.0 gdk-pixbuf@2.31.2 memaxes@0.5 qt@4.8.6 glib@2.42.1 mesa@8.0.5 qt@5.4.0 graphlib@2.0.0 mpich@3.0.4 readline@6.3 gtkplus@2.24.25 mpileaks@1.0 sqlite@3.8.5 harfbuzz@0.9.37 mrnet@4.1.0 stat@2.1.0 hdf5@1.8.13 ncurses@5.9 tcl@8.6.3 icu@54.1 netcdf@4.3.3 tk@src jpeg@9a openssl@1.0.1h vtk@6.1.0 launchmon@1.0.1 pango@1.36.8 xcb-proto@1.11 lcms@2.6 pixman@0.32.6 xz@5.2.0 libdrm@2.4.33 py-dateutil@2.4.0 zlib@1.2.8 -- linux-debian7-x86_64 / gcc@4.9.2 -------------------------------- libelf@0.8.10 mpich@3.0.4 Packages are divided into groups according to their architecture and compiler. Within each group, Spack tries to keep the view simple and only shows the version of installed packages. .. _cmd-spack-find-metadata: Viewing more metadata """""""""""""""""""""""""""""""" ``spack find`` can filter the package list based on the package name, spec, or a number of properties of their installation status. For example, missing dependencies of a spec can be shown with ``--missing``, deprecated packages can be included with ``--deprecated``, packages that were explicitly installed with ``spack install `` can be singled out with ``--explicit``, and those that have been pulled in only as dependencies with ``--implicit``. In some cases, there may be different configurations of the *same* version of a package installed. For example, there are two installations of ``libdwarf@20130729`` above. We can look at them in more detail using ``spack find --deps`` and by asking only to show ``libdwarf`` packages: .. code-block:: spec $ spack find --deps libdwarf ==> 2 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 ^libelf@0.8.12 libdwarf@20130729-b52fac98 ^libelf@0.8.13 Now we see that the two instances of ``libdwarf`` depend on *different* versions of ``libelf``: 0.8.12 and 0.8.13. This view can become complicated for packages with many dependencies. If you just want to know whether two packages' dependencies differ, you can use ``spack find --long``: .. code-block:: spec $ spack find --long libdwarf ==> 2 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 libdwarf@20130729-b52fac98 Now the ``libdwarf`` installs have hashes after their names. These are hashes over all of the dependencies of each package. If the hashes are the same, then the packages have the same dependency configuration. If you want to know the path where each package is installed, you can use ``spack find --paths``: .. code-block:: spec $ spack find --paths ==> 74 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- ImageMagick@6.8.9-10 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd adept-utils@1.0 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/adept-utils@1.0-5adef8da atk@2.14.0 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/atk@2.14.0-3d09ac09 boost@1.55.0 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/boost@1.55.0 bzip2@1.0.6 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/bzip2@1.0.6 cairo@1.14.0 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/cairo@1.14.0-fcc2ab44 callpath@1.0.2 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/callpath@1.0.2-5dce4318 ... You can restrict your search to a particular package by supplying its name: .. code-block:: spec $ spack find --paths libelf -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libelf@0.8.11 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.11 libelf@0.8.12 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.12 libelf@0.8.13 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.13 Spec queries """""""""""""""""""""""""""""""" ``spack find`` actually does a lot more than this. You can use *specs* to query for specific configurations and builds of each package. If you want to find only libelf versions greater than version 0.8.12, you could say: .. code-block:: spec $ spack find libelf@0.8.12: -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libelf@0.8.12 libelf@0.8.13 Finding just the versions of ``libdwarf`` built with a particular version of libelf would look like this: .. code-block:: spec $ spack find --long libdwarf ^libelf@0.8.12 ==> 1 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 We can also search for packages that have a certain attribute. For example, ``spack find libdwarf +debug`` will show only installations of ``libdwarf`` with the 'debug' compile-time option enabled. The full spec syntax is discussed in detail in :ref:`sec-specs`. Machine-readable output """""""""""""""""""""""""""""""" If you only want to see very specific things about installed packages, Spack has some options for you. ``spack find --format`` can be used to output only specific fields: .. code-block:: console $ spack find --format "{name}-{version}-{hash}" autoconf-2.69-icynozk7ti6h4ezzgonqe6jgw5f3ulx4 automake-1.16.1-o5v3tc77kesgonxjbmeqlwfmb5qzj7zy bzip2-1.0.6-syohzw57v2jfag5du2x4bowziw3m5p67 bzip2-1.0.8-zjny4jwfyvzbx6vii3uuekoxmtu6eyuj cmake-3.15.1-7cf6onn52gywnddbmgp7qkil4hdoxpcb ... or: .. code-block:: console $ spack find --format "{hash:7}" icynozk o5v3tc7 syohzw5 zjny4jw 7cf6onn ... This uses the same syntax as described in the documentation for :meth:`~spack.spec.Spec.format` -- you can use any of the options there. This is useful for passing metadata about packages to other command-line tools. Alternatively, if you want something even more machine readable, you can output each spec as JSON records using ``spack find --json``. This will output metadata on specs and all dependencies as JSON: .. code-block:: spec $ spack find --json sqlite@3.28.0 [ { "name": "sqlite", "hash": "3ws7bsihwbn44ghf6ep4s6h4y2o6eznv", "version": "3.28.0", "arch": { "platform": "darwin", "platform_os": "mojave", "target": "x86_64" }, "compiler": { "name": "apple-clang", "version": "10.0.0" }, "namespace": "builtin", "parameters": { "fts": true, "functions": false, "cflags": [], "cppflags": [], "cxxflags": [], "fflags": [], "ldflags": [], "ldlibs": [] }, "dependencies": { "readline": { "hash": "722dzmgymxyxd6ovjvh4742kcetkqtfs", "type": [ "build", "link" ] } } }, ... ] You can use this with tools like `jq `_ to quickly create JSON records structured the way you want: .. code-block:: console $ spack find --json sqlite@3.28.0 | jq -C '.[] | { name, version, hash }' { "name": "sqlite", "version": "3.28.0", "hash": "3ws7bsihwbn44ghf6ep4s6h4y2o6eznv" } { "name": "readline", "version": "7.0", "hash": "722dzmgymxyxd6ovjvh4742kcetkqtfs" } { "name": "ncurses", "version": "6.1", "hash": "zvaa4lhlhilypw5quj3akyd3apbq5gap" } .. _cmd-spack-diff: ``spack diff`` ^^^^^^^^^^^^^^ It's often the case that you have two versions of a spec that you need to disambiguate. Let's say that we've installed two variants of ``zlib``, one with and one without the optimize variant: .. code-block:: spec $ spack install zlib $ spack install zlib -optimize When we do ``spack find``, we see the two versions. .. code-block:: spec $ spack find zlib ==> 2 installed packages -- linux-ubuntu20.04-skylake / gcc@9.3.0 ------------------------ zlib@1.2.11 zlib@1.2.11 Let's say we want to uninstall ``zlib``. We run the command and quickly encounter a problem because two versions are installed. .. code-block:: spec $ spack uninstall zlib ==> Error: zlib matches multiple packages: -- linux-ubuntu20.04-skylake / gcc@9.3.0 ------------------------ efzjziy zlib@1.2.11 sl7m27m zlib@1.2.11 ==> Error: You can either: a) use a more specific spec, or b) specify the spec by its hash (e.g. `spack uninstall /hash`), or c) use `spack uninstall --all` to uninstall ALL matching specs. Oh no! We can see from the above that we have two different versions of ``zlib`` installed, and the only difference between the two is the hash. This is a good use case for ``spack diff``, which can easily show us the "diff" or set difference between properties for two packages. Let's try it out. Because the only difference we see in the ``spack find`` view is the hash, let's use ``spack diff`` to look for more detail. We will provide the two hashes: .. code-block:: diff $ spack diff /efzjziy /sl7m27m --- zlib@1.2.11efzjziyc3dmb5h5u5azsthgbgog5mj7g +++ zlib@1.2.11sl7m27mzkbejtkrajigj3a3m37ygv4u2 @@ variant_value @@ - zlib optimize False + zlib optimize True The output is colored and written in the style of a git diff. This means that you can copy and paste it into a GitHub markdown as a code block with language "diff" and it will render nicely! Here is an example: .. code-block:: diff --- zlib@1.2.11/efzjziyc3dmb5h5u5azsthgbgog5mj7g +++ zlib@1.2.11/sl7m27mzkbejtkrajigj3a3m37ygv4u2 @@ variant_value @@ - zlib optimize False + zlib optimize True Awesome! Now let's read the diff. It tells us that our first ``zlib`` was built with ``~optimize`` (``False``) and the second was built with ``+optimize`` (``True``). You can't see it in the docs here, but the output above is also colored based on the content being an addition (+) or subtraction (-). This is a small example, but you will be able to see differences for any attributes on the installation spec. Running ``spack diff A B`` means we'll see which spec attributes are on ``B`` but not on ``A`` (green) and which are on ``A`` but not on ``B`` (red). Here is another example with an additional difference type, ``version``: .. code-block:: diff $ spack diff python@2.7.8 python@3.8.11 --- python@2.7.8/tsxdi6gl4lihp25qrm4d6nys3nypufbf +++ python@3.8.11/yjtseru4nbpllbaxb46q7wfkyxbuvzxx @@ variant_value @@ - python patches a8c52415a8b03c0e5f28b5d52ae498f7a7e602007db2b9554df28cd5685839b8 + python patches 0d98e93189bc278fbc37a50ed7f183bd8aaf249a8e1670a465f0db6bb4f8cf87 @@ version @@ - openssl 1.0.2u + openssl 1.1.1k - python 2.7.8 + python 3.8.11 Let's say that we were only interested in one kind of attribute above, ``version``. We can ask the command to only output this attribute. To do this, you'd add the ``--attribute`` for attribute parameter, which defaults to all. Here is how you would filter to show just versions: .. code-block:: diff $ spack diff --attribute version python@2.7.8 python@3.8.11 --- python@2.7.8/tsxdi6gl4lihp25qrm4d6nys3nypufbf +++ python@3.8.11/yjtseru4nbpllbaxb46q7wfkyxbuvzxx @@ version @@ - openssl 1.0.2u + openssl 1.1.1k - python 2.7.8 + python 3.8.11 And you can add as many attributes as you'd like with multiple ``--attribute`` arguments (for lots of attributes, you can use ``-a`` for short). Finally, if you want to view the data as JSON (and possibly pipe into an output file), just add ``--json``: .. code-block:: spec $ spack diff --json python@2.7.8 python@3.8.11 This data will be much longer because along with the differences for ``A`` vs. ``B`` and ``B`` vs. ``A``, the JSON output also shows the intersection. Using Installed Packages ------------------------ As you've seen, Spack packages are installed into long paths with hashes, and you need a way to get them into your path. Spack has three different ways to solve this problem, which fit different use cases: 1. Spack provides :ref:`environments `, and views, with which you can "activate" a number of related packages all at once. This is likely the best method for most use cases. 2. Spack can generate :ref:`environment modules `, which are commonly used on supercomputing clusters. Module files can be generated for every installation automatically, and you can customize how this is done. 3. For one-off use, Spack provides the :ref:`spack load ` command .. _cmd-spack-load: .. _cmd-spack-unload: ``spack load / unload`` ^^^^^^^^^^^^^^^^^^^^^^^ If you sourced the appropriate shell script, as shown in :ref:`getting_started`, you can use the ``spack load`` command to quickly add a package to your ``PATH``. For example, this will add the ``mpich`` package built with ``gcc`` to your path: .. code-block:: spec $ spack install mpich %gcc@4.4.7 # ... wait for install ... $ spack load mpich %gcc@4.4.7 $ which mpicc ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4/bin/mpicc These commands will add appropriate directories to your ``PATH`` and ``MANPATH`` according to the :ref:`prefix inspections ` defined in your modules configuration. When you no longer want to use a package, you can type ``spack unload``: .. code-block:: spec $ spack unload mpich %gcc@4.4.7 Ambiguous specs """"""""""""""" If a spec used with load/unload is ambiguous (i.e., more than one installed package matches it), then Spack will warn you: .. code-block:: spec $ spack load libelf ==> Error: libelf matches multiple packages. Matching packages: qmm4kso libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64 cd2u6jt libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64 Use a more specific spec You can either type the ``spack load`` command again with a fully qualified argument, or you can add just enough extra constraints to identify one package. For example, above, the key differentiator is that one ``libelf`` is built with the Intel compiler, while the other used ``gcc``. You could therefore just type: .. code-block:: spec $ spack load libelf %intel To identify just the one built with the Intel compiler. If you want to be *very* specific, you can load it by its hash. For example, to load the first ``libelf`` above, you would run: .. code-block:: spec $ spack load /qmm4kso To see which packages that you have loaded into your environment, you would use ``spack find --loaded``. .. code-block:: console $ spack find --loaded ==> 2 installed packages -- linux-debian7 / gcc@4.4.7 ------------------------------------ libelf@0.8.13 -- linux-debian7 / intel@15.0.0 --------------------------------- libelf@0.8.13 You can also use ``spack load --list`` to get the same output, but it does not have the full set of query options that ``spack find`` offers. We'll learn more about Spack's spec syntax in :ref:`a later section `. .. _extensions: Spack environments ^^^^^^^^^^^^^^^^^^ Spack can install a large number of Python packages. Their names are typically prefixed with ``py-``. Installing and using them is no different from any other package: .. code-block:: spec $ spack install py-numpy $ spack load py-numpy $ python3 >>> import numpy The ``spack load`` command sets the ``PATH`` variable so that the correct Python executable is used and makes sure that ``numpy`` and its dependencies can be located in the ``PYTHONPATH``. Spack is different from other Python package managers in that it installs every package into its *own* prefix. This is in contrast to ``pip``, which installs all packages into the same prefix, whether in a virtual environment or not. For many users, **virtual environments** are more convenient than repeated ``spack load`` commands, particularly when working with multiple Python packages. Fortunately, Spack supports environments itself, which together with a view are no different from Python virtual environments. The recommended way of working with Python extensions such as ``py-numpy`` is through :ref:`Environments `. The following example creates a Spack environment with ``numpy`` in the current working directory. It also puts a filesystem view in ``./view``, which is a more traditional combined prefix for all packages in the environment. .. code-block:: spec $ spack env create --with-view view --dir . $ spack -e . add py-numpy $ spack -e . concretize $ spack -e . install Now you can activate the environment and start using the packages: .. code-block:: console $ spack env activate . $ python3 >>> import numpy The environment view is also a virtual environment, which is useful if you are sharing the environment with others who are unfamiliar with Spack. They can either use the Python executable directly: .. code-block:: console $ ./view/bin/python3 >>> import numpy or use the activation script: .. code-block:: console $ source ./view/bin/activate $ python3 >>> import numpy In general, there should not be much difference between ``spack env activate`` and using the virtual environment. The main advantage of ``spack env activate`` is that it knows about more packages than just Python packages, and it may set additional runtime variables that are not covered by the virtual environment activation script. See :ref:`environments` for a more in-depth description of Spack environments and customizations to views. ================================================ FILE: lib/spack/docs/package_review_guide.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: This is a guide for people who review package pull requests and includes criteria for them to be merged into the develop branch. .. _package-review-guide: Package Review Guide ==================== Package reviews are performed with the goals of minimizing build errors and making packages as **uniform and stable** as possible. This section establishes guidelines to help assess Spack community `package repository `_ pull requests (PRs). It describes the considerations and actions to be taken when reviewing new and updated `Spack packages `_. In some cases, there are possible solutions to common issues. How to use this guide --------------------- Whether you are a :ref:`Package Reviewer `, :ref:`Maintainer `, or :ref:`Committer `, this guide highlights relevant aspects to consider when reviewing package pull requests. If you are a :ref:`Package Contributor ` (or simply ``Contributor``), you may also find the information and solutions useful in your work. While we provide information on what to look for, the changes themselves should drive the actual review process. .. note:: :ref:`Confirmation of successful package builds ` of **all** affected versions can reduce the amount of effort needed to review a PR. However, packaging conventions and the combinatorial nature of versions and directives mean each change should still be checked. Reviewing a new package ~~~~~~~~~~~~~~~~~~~~~~~ If the pull request includes a new package, then focus on answering the following questions: * Should the :ref:`package ` be added to the repository? * Does the package :ref:`structure ` conform to conventions? * Are the directives and their options correct? * Do all :ref:`automated checks ` pass? If not, are there easy-to-resolve CI and/or test issues that can be addressed or does the submitter need to investigate the failures? * Is there :ref:`confirmation ` that every version builds successfully on at least one platform? Refer to the relevant sections below for more guidance. Reviewing changes to an existing package ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If the pull request includes changes to an existing package, then focus on answering the following questions: * Are there any changes to the package :ref:`structure ` and, if so, do they conform to conventions? * If there are new or updated directives, then are they and their options correct? * If there are changes to the :ref:`url or its equivalent `, are the older versions still correct? * If there are changes to the :ref:`git or equivalent URL `, do older branches exist in the new location? * Do all :ref:`automated checks ` pass? If not, are there easy-to-resolve CI and/or test issues that can be addressed or does the submitter need to investigate the failures? * Is there :ref:`confirmation ` that every new version builds successfully on at least one platform? Refer to the relevant sections below for more guidance. .. _suitable_package: Package suitability ------------------- It is rare that a package would be considered inappropriate for inclusion in the public `Spack package `_ repository. One exception is making packages for standard Perl modules. **Action.** Should you find the software is not appropriate, ask that the package be removed from the PR if it is one of multiple affected files or suggest the PR be closed. In either case, explain the reason for the request. CORE Perl modules ~~~~~~~~~~~~~~~~~ In general, modules that are part of the standard installation for all listed Perl versions (i.e., ``CORE``) should **not be implemented or contributed**. Details on the exceptions and process for checking Perl modules can be found in the :ref:`Perl build system ` documentation. .. _structure_reviews: Package structure ----------------- The `convention `_ for structuring Spack packages has metadata (key properties) listed first followed by directives then methods: * :ref:`url_equivalent_reviews`; * :ref:`vcs_url_reviews`; * :ref:`maintainers_reviews`; * :ref:`license_reviews`; * :ref:`version_reviews`; * :ref:`variant_reviews`; * :ref:`depends_on_reviews`; * :ref:`packaging_conflicts` and :ref:`packaging_requires` directives; then * methods. `Groupings `_ using ``with`` context managers can affect the order of dependency, conflict, and requires directives to some degree. However, they do cut down on visual clutter and make packages more readable. **Action.** If you see clear deviations from the convention, request that they be addressed. When in doubt, ask others with merge privileges for advice. .. _url_equivalent_reviews: ``url``, ``url_for_version``, or URL equivalent ----------------------------------------------- Changes to URLs may invalidate existing versions, which should be checked when there is a URL-related modification. All packages have a URL, though for some :ref:`build-systems` it is derived automatically and not visible in the package. Reasons :ref:`versions ` may become invalid include: * the new URL does not support Spack version extrapolation; * the addition of or changes to ``url_for_version`` involve checks of the ``spec``'s version instead of the ``version`` argument or the (usually older) versions are not covered; * extrapolation of the derived URL no longer matches that of older versions; and * the older versions are no longer available. **Action.** Checking existing version directives with checksums can usually be done manually with the modified package using `spack checksum `_. **Solutions.** Options for resolving the problem that can be suggested for investigation depend on the source. In simpler cases involving ``url`` or ``url_for_version``, invalid versions can sometimes be corrected by ensuring all versions are covered by ``url_for_version``. Alternatively, especially for older versions, the version-specific URL can be added as an argument to the ``version`` directive. Sometimes the derived URLs of versions on the hosting system can vary. This commonly happens with Python packages. For example, the case of one or more letters in the package name may change at some point (e.g., `py-sphinx `_). Also, dashes may be replaced with underscores (e.g., `py-scikit-build `_). In some cases, both changes can occur for the same package. As these examples illustrate, it is sometimes possible to add a ``url_for_version`` method to override the default derived URL to ensure the correct one is returned. If older versions are no longer available and there is a chance someone has the package in a build cache, the usual approach is to first suggest :ref:`deprecating ` them in the package. .. _vcs_url_reviews: ``git``, ``hg``, ``svn``, or ``cvs`` ------------------------------------ If the :ref:`repository-specific URL ` for fetching branches or the version control system (VCS) equivalent changes, there is a risk that the listed versions are no longer accessible. **Action.** You may need to check the new source repository to confirm the presence of all of the listed versions. .. _maintainers_reviews: ``maintainers`` directive ------------------------- **Action.** If the new package does not have a :ref:`maintainers ` directive, ask the Contributor to add one. .. note:: This request is optional for existing packages. Be prepared for them to refuse. .. _license_reviews: ``license`` directive --------------------- **Action.** If the new package does not have a :ref:`license ` directive, ask the Contributor to investigate and add it. .. note:: This request is optional for existing packages. Be prepared for them to refuse. .. _version_reviews: ``version`` directives ---------------------- In general, Spack packages are expected to be built from source code. There are a few exceptions (e.g., :ref:`BundlePackage `). Typically every package will have at least one :ref:`version ` directive. The goals of reviewing version directives are to confirm that versions are listed in the proper order **and** that the arguments for new and updated versions are correct. .. note:: Additions and removals of version directives should generally trigger a review of :ref:`dependencies `. ``version`` directive order ~~~~~~~~~~~~~~~~~~~~~~~~~~~ By :ref:`convention ` version directives should be listed in descending order, from newest to oldest. If branch versions are included, then they should be listed first. **Action.** When versions are being added, check the ordering of the directives. Request that the directives be re-ordered if any of the directives do not conform to the convention. .. note:: Edge cases, such as manually downloaded software, may be difficult to confirm. Checksums, commits, tags, and branches ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ **Checksums, commits, and tags** Normally these version arguments are automatically validated by GitHub Actions using `spack ci verify-versions `_. **Action.** Review the PR's ``verify-checksums`` precheck to confirm. If necessary, checksums can usually be manually confirmed using `spack checksum `_. .. warning:: From a security and reproducibility standpoint, it is important that Spack be able to verify downloaded source. This is accomplished using a hash (e.g., checksum or commit). See :ref:`checksum verification ` for more information. Exceptions are allowed in rare cases, such as software supplied from reputable vendors. When in doubt, ask others with merge privileges for advice. **Tags** If a ``tag`` is provided without a ``commit``, the downloaded software will not be trusted. **Action.** Suggest that the ``commit`` argument be included in the ``version`` directive. **Branches** Confirming new branch versions involves checking that the branches exist in the repository *and* that the version and branch names are consistent. Let's take each in turn. **Action.** Confirming branch existence often involves checking the source repository though is not necessary if there is confirmation that the branch was built successfully from the package. In general, the version and branch names should match. When they do not, it is sometimes the result of people not being aware of how Spack handles :ref:`version-comparison`. **Action.** If there is a name mismatch, especially for the most common branch names (e.g., ``develop``, ``main``, and ``master``), ask why and suggest the arguments be changed such that they match the actual branch name. **Manual downloads** **Action.** Since these can be difficult to confirm, it is acceptable to rely on the package's Maintainers, if any. Deprecating versions ~~~~~~~~~~~~~~~~~~~~ If someone is deprecating versions, it is good to find out why. Sometimes there are concerns, such as security or lack of availability. **Action.** Suggest the Contributor review the :ref:`deprecation guidelines ` before finalizing the changes if they haven't already explained why they made the choice in the PR description or comments. .. _variant_reviews: ``variant`` directives ---------------------- :ref:`Variants ` represent build options so any changes involving these directives should be reflected elsewhere in the package. Adding or modifying variants ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ **Action.** Confirm that new or modified variants are actually used in the package. The most common uses are additions and changes to: * :ref:`dependencies `; * configure options; and/or * build arguments. Removing or disabling variants ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If the variant is still relevant to listed version directives, it may be preferable to adjust or add `conditions `_. **Action.** Consider asking why the variant (or build option) is being removed and suggest making it conditional when it is still relevant. .. warning:: If the default value of a variant is changed in the PR, then there is a risk that other packages relying on that value will no longer build as others expect. This may be worth noting in the review. .. _depends_on_reviews: ``depends_on`` directives ------------------------- :ref:`Dependencies ` represent software that must be installed before the package builds or is able to work correctly. Updating dependencies ~~~~~~~~~~~~~~~~~~~~~ It is important that dependencies reflect the requirements of listed versions. They only need to be checked in a review when versions are being added or removed or the dependencies are being changed. **Action.** Dependencies affected by such changes should be confirmed, when possible, and *at least* when the Contributor is not a Maintainer of the package. **Solutions.** In some cases, the needed change may be as simple as ensuring the version ranges (see :ref:`version_compatibility`) and/or variant options in the dependency are accurate. In others, one or more of the dependencies needed by new versions are missing and need to be added. Or there may be dependencies that are no longer relevant when versions requiring them are removed, meaning the dependencies should be removed as well. For example, it is not uncommon for Python package dependencies to be out of date when new versions are added. In this case, check Python package dependencies by following the build system `guidelines `_. .. tip:: In general, refer to the relevant dependencies section, if any, for the package’s :ref:`build-systems` for guidance. Updating language and compiler dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When :ref:`language and compiler dependencies ` were introduced, their ``depends_on`` directives were derived from the source for existing packages. These dependencies are flagged with ``# generated`` comments when they have not been confirmed. Unfortunately, the generated dependencies are not always complete or necessarily required. **Action.** If these dependencies are being updated, ask that the ``# generated`` comments be removed if the Contributor can confirm they are relevant. Definitely make sure Contributors do **not** include ``# generated`` on the dependencies they are adding to the package. .. _automated_checks_reviews: Failed automated checks ----------------------- All PRs are expected to pass **at least the required** automated checks. Style failures ~~~~~~~~~~~~~~ The PR may fail one or more style checks. **Action.** If the failure is due to issues raised by the ``black`` style checker *and* the PR is otherwise ready to be merged, you can add ``@spackbot fix style`` in a comment to see if Spack will fix the errors. Otherwise, inform the Contributor that the style failures need to be addressed. CI stack failures ~~~~~~~~~~~~~~~~~ Existing packages **may** be included in GitLab CI pipelines through inclusion in one or more `stacks `_. **Action.** It is worth checking **at least a sampling** of the failed job logs, if present, to determine the possible cause and take or suggest an action accordingly. **CI Runner Failures** Sometimes CI runners time out or the pods become unavailable. **Action.** If that is the case, the resolution may be as simple as restarting the pipeline by adding a ``@spackbot run pipeline`` comment. Otherwise, the Contributor will need to investigate and resolve the problem. **Stand-alone Test Failures** Sometimes :ref:`stand-alone tests ` could be causing the build job to time out. If the tests take too long, the issue could be that the package is running too many and/or long running tests. Or the tests may be trying to use resources (e.g., a batch scheduler) that are not available on runners. **Action.** If the tests for a package are hanging, at a minimum create a `new issue `_ if there is not one already, to flag the package. **(Temporary) Solution.** Look at the package implementation to see if the tests are using a batch scheduler or there appear to be too many or long running tests. If that is the case, then a pull request should be created in the ``spack/spack-packages`` repository that adds the package to the ``broken-tests-packages`` list in the `ci configuration `_. Once the fix PR is merged, then the affected PR can be rebased to pick up the change. .. _build_success_reviews: Successful builds ----------------- Is there evidence that the package builds successfully on at least one platform? For a new package, we would ideally have confirmation for every version; whereas, we would want confirmation of only the affected versions for changes to an existing package. Acceptable forms of confirmation are **one or more of**: * the Contributor or another reviewer explicitly confirms that a successful build of **each new version on at least one platform**; * the software is built successfully by Spack CI by **at least one of the CI stacks**; and * **at least one Maintainer** explicitly confirms they are able to successfully build the software. Individuals are expected to update the PR description or add a comment to explicitly confirm the builds. You may need to check the CI stacks and/or outputs to confirm that there is a stack that builds the new version. .. note:: When builds are confirmed by individuals, we would prefer the output of ``spack debug report`` be included in either the PR description or a comment. ================================================ FILE: lib/spack/docs/packages_yaml.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to customizing package settings in Spack using the packages.yaml file, including configuring compilers, specifying external packages, package requirements, and permissions. .. _packages-config: Package Settings (packages.yaml) ================================ Spack allows you to customize how your software is built through the ``packages.yaml`` file. Using it, you can make Spack prefer particular implementations of virtual dependencies (e.g., MPI or BLAS/LAPACK), or you can make it prefer to build with particular compilers. You can also tell Spack to use *external* software installations already present on your system. At a high level, the ``packages.yaml`` file is structured like this: .. code-block:: yaml packages: package1: # settings for package1 package2: # settings for package2 # ... all: # settings that apply to all packages. You can either set build preferences specifically for *one* package, or you can specify that certain settings should apply to *all* packages. The types of settings you can customize are described in detail below. Spack's build defaults are in the default ``etc/spack/defaults/packages.yaml`` file. You can override them in ``~/.spack/packages.yaml`` or ``etc/spack/packages.yaml``. For more details on how this works, see :ref:`configuration-scopes`. .. _sec-external-packages: External packages ----------------- Spack can be configured to use externally-installed packages rather than building its own packages. This may be desirable if machines ship with system packages, such as a customized MPI, which should be used instead of Spack building its own MPI. External packages are configured through the ``packages.yaml`` file. Here's an example of an external configuration: .. code-block:: yaml packages: openmpi: externals: - spec: "openmpi@1.4.3~debug" prefix: /opt/openmpi-1.4.3 - spec: "openmpi@1.4.3+debug" prefix: /opt/openmpi-1.4.3-debug This example lists two installations of OpenMPI, one with debug information, and one without. If Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the pre-installed OpenMPI in the given directory. Note that the specified path is the top-level install prefix, not the ``bin`` subdirectory. ``packages.yaml`` can also be used to specify modules to load instead of the installation prefixes. The following example says that module ``CMake/3.7.2`` provides CMake version 3.7.2. .. code-block:: yaml cmake: externals: - spec: cmake@3.7.2 modules: - CMake/3.7.2 Each ``packages.yaml`` begins with a ``packages:`` attribute, followed by a list of package names. To specify externals, add an ``externals:`` attribute under the package name, which lists externals. Each external should specify a ``spec:`` string that should be as well-defined as reasonably possible. If a package lacks a spec component, such as missing a compiler or package version, then Spack will guess the missing component based on its most-favored packages, and it may guess incorrectly. .. _cmd-spack-external-find: Automatically find external packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can run the :ref:`spack external find ` command to search for system-provided packages and add them to ``packages.yaml``. After running this command your ``packages.yaml`` may include new entries: .. code-block:: yaml packages: cmake: externals: - spec: cmake@3.17.2 prefix: /usr Generally this is useful for detecting a small set of commonly-used packages; for now this is generally limited to finding build-only dependencies. Specific limitations include: * Packages are not discoverable by default: For a package to be discoverable with ``spack external find``, it needs to add special logic. See :ref:`here ` for more details. * The logic does not search through module files, it can only detect packages with executables defined in ``PATH``; you can help Spack locate externals which use module files by loading any associated modules for packages that you want Spack to know about before running ``spack external find``. * Spack does not overwrite existing entries in the package configuration: If there is an external defined for a spec at any configuration scope, then Spack will not add a new external entry (``spack config blame packages`` can help locate all external entries). Prevent packages from being built from sources ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Adding an external spec in ``packages.yaml`` allows Spack to use an external location, but it does not prevent Spack from building packages from sources. In the above example, Spack might choose for many valid reasons to start building and linking with the latest version of OpenMPI rather than continue using the pre-installed OpenMPI versions. To prevent this, the ``packages.yaml`` configuration also allows packages to be flagged as non-buildable. The previous example could be modified to be: .. code-block:: yaml packages: openmpi: externals: - spec: "openmpi@1.4.3~debug" prefix: /opt/openmpi-1.4.3 - spec: "openmpi@1.4.3+debug" prefix: /opt/openmpi-1.4.3-debug buildable: false The addition of the ``buildable`` flag tells Spack that it should never build its own version of OpenMPI from sources, and it will instead always rely on a pre-built OpenMPI. .. note:: If ``concretizer:reuse`` is on (see :ref:`concretizer-options` for more information on that flag) pre-built specs are taken from: the local store, an upstream store, a registered buildcache and externals in ``packages.yaml``. If ``concretizer:reuse`` is off, only external specs in ``packages.yaml`` are included in the list of pre-built specs. If an external module is specified as not buildable, then Spack will load the external module into the build environment which can be used for linking. The ``buildable`` attribute does not need to be paired with external packages. It could also be used alone to forbid packages that may be buggy or otherwise undesirable. Non-buildable virtual packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Virtual packages in Spack can also be specified as not buildable, and external implementations can be provided. In the example above, OpenMPI is configured as not buildable, but Spack will often prefer other MPI implementations over the externally available OpenMPI. Spack can be configured with every MPI provider not buildable individually, but more conveniently: .. code-block:: yaml packages: mpi: buildable: false openmpi: externals: - spec: "openmpi@1.4.3~debug" prefix: /opt/openmpi-1.4.3 - spec: "openmpi@1.4.3+debug" prefix: /opt/openmpi-1.4.3-debug Spack can then use any of the listed external implementations of MPI to satisfy a dependency, and will choose among them depending on the compiler and architecture. In cases where the concretizer is configured to reuse specs, and other ``mpi`` providers (available via stores or build caches) are not desirable, Spack can be configured to require specs matching only the available externals: .. code-block:: yaml packages: mpi: buildable: false require: - one_of: - "openmpi@1.4.3~debug" - "openmpi@1.4.3+debug" openmpi: externals: - spec: "openmpi@1.4.3~debug" prefix: /opt/openmpi-1.4.3 - spec: "openmpi@1.4.3+debug" prefix: /opt/openmpi-1.4.3-debug This configuration prevents any spec using MPI and originating from stores or build caches to be reused, unless it matches the requirements under ``packages:mpi:require``. For more information on requirements see :ref:`package-requirements`. Specifying dependencies among external packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ External packages frequently have dependencies on other software components. Explicitly modeling these relationships provides Spack with a more complete representation of the software stack. This ensures that: - Runtime environments include all necessary components. - Build-time dependencies are accurately represented. This comprehensive view, in turn, enables Spack to more reliably build software that depends on these externals. Spack provides two methods for configuring dependency relationships among externals, each offering different trade-offs between conciseness and explicit control: - An "inline" spec syntax. - A structured YAML configuration that is more verbose but also more explicit. The following sections will detail both approaches. Dependencies using inline spec syntax """"""""""""""""""""""""""""""""""""" Spack allows you to define external package dependencies using the standard spec syntax directly within your package configuration. This approach is concise and leverages the familiar spec syntax that you already use elsewhere in Spack. When configuring an external package with dependencies using the spec syntax, you can include dependency specifications directly in the main ``spec:`` field: .. code-block:: yaml # Specification for the following DAG: # # o mpileaks@2.3 # |\ # | o callpath@1.0 # |/ # o mpich@3.0.4 packages: mpileaks: externals: - spec: "mpileaks@2.3~debug+opt %mpich@3 %callpath" prefix: /user/path callpath: externals: - spec: "callpath@1.0 %mpi=mpich" prefix: /user/path mpich: externals: - spec: "mpich@3.0.4" prefix: /user/path In this example ``mpileaks`` depends on both ``mpich`` and ``callpath``. Spack will parse the ``mpileaks`` spec string, and create the appropriate dependency relationships automatically. Users *need* to ensure that each dependency maps exactly to a single other external package. In case multiple externals can satisfy the same dependency, or in case no external can satisfy a dependency, Spack will error and point to the configuration line causing the issue. Whenever no information is given about the dependency type, Spack will infer it from the current package recipe. For instance, the dependencies in the configuration above are inferred to be of ``build,link`` type from the recipe of ``mpileaks`` and ``callpath``: .. code-block:: console $ spack -m spec --types -l --cover edges mpileaks [e] oelprl6 [ ] mpileaks@2.3~debug+opt+shared+static build_system=generic platform=linux os=ubuntu20.04 target=icelake [e] jdhzy2t [bl ] ^callpath@1.0 build_system=generic platform=linux os=ubuntu20.04 target=icelake [e] pgem3yp [bl ] ^mpich@3.0.4~debug build_system=generic platform=linux os=ubuntu20.04 target=icelake [e] pgem3yp [bl ] ^mpich@3.0.4~debug build_system=generic platform=linux os=ubuntu20.04 target=icelake When inferring the dependency types, Spack will also infer virtuals if they are not already specified. This method's conciseness comes with a strict requirement: each dependency must resolve to a single, unambiguous external package. This makes the approach suitable for simple or temporary configurations. In larger, more dynamic environments, however, it can become a maintenance challenge, as adding new external packages over time may require frequent updates to existing specs to preserve their uniqueness. Dependencies using YAML configuration """"""""""""""""""""""""""""""""""""" While the spec syntax offers a concise way to specify dependencies, Spack's YAML-based explicit dependency configuration provides more control and clarity, especially for complex dependency relationships. This approach uses the ``dependencies:`` field to precisely define each dependency relationship. The example in the previous section, written using the YAML configuration, becomes: .. code-block:: yaml # Specification for the following DAG: # # o mpileaks@2.3 # |\ # | o callpath@1.0 # |/ # o mpich@3.0.4 packages: mpileaks: externals: - spec: "mpileaks@2.3~debug+opt" prefix: /user/path dependencies: - id: callpath_id deptypes: link - spec: mpich deptypes: - "build" - "link" virtuals: "mpi" callpath: externals: - spec: "callpath@1.0" prefix: /user/path id: callpath_id dependencies: - spec: mpich deptypes: - "build" - "link" virtuals: "mpi" mpich: externals: - spec: "mpich@3.0.4" prefix: /user/path Each dependency can be specified either by: - A ``spec:`` that matches an available external package, like in the previous case, or by - An ``id`` that explicitly references another external package. Using the ``id`` provides an unambiguous reference to a specific external package, which is essential for differentiating between externals that have similar specs but differ, for example, only by their installation prefix. The dependency types can be specified in the optional ``deptypes`` field, while virtuals can be specified in the optional ``virtuals`` field. As before, when the dependency types are not specified, Spack will infer them from the package recipe. .. _extra-attributes-for-externals: Extra attributes for external packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Sometimes external packages require additional attributes to be used effectively. This information can be defined on a per-package basis and stored in the ``extra_attributes`` section of the external package configuration. In addition to per-package information, this section can be used to define environment modifications to be performed whenever the package is used. For example, if an external package is built without ``rpath`` support, it may require ``LD_LIBRARY_PATH`` settings to find its dependencies. This could be configured as follows: .. code-block:: yaml packages: mpich: externals: - spec: mpich@3.3 +hwloc prefix: /path/to/mpich extra_attributes: environment: prepend_path: LD_LIBRARY_PATH: /path/to/hwloc/lib64 See :ref:`configuration_environment_variables` for more information on how to configure environment modifications in Spack config files. .. _configuring-system-compilers-as-external-packages: Configuring system compilers as external packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In Spack, compilers are treated as packages like any other. This means that you can also configure system compilers as external packages and use them in Spack. Spack automatically detects system compilers and configures them in ``packages.yaml`` for you. You can also run :ref:`spack-compiler-find` to find and configure new system compilers. When configuring compilers as external packages, you need to set a few :ref:`extra attributes ` for them to work properly. The ``compilers`` extra attribute field is required to clarify which paths within the compiler prefix are used for which languages: .. code-block:: yaml packages: gcc: externals: - spec: gcc@10.5.0 languages='c,c++,fortran' prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc-10 cxx: /usr/bin/g++-10 fortran: /usr/bin/gfortran-10 Other fields accepted by compilers under ``extra_attributes`` are ``flags``, ``environment``, ``extra_rpaths``, and ``implicit_rpaths``. .. code-block:: yaml packages: gcc: externals: - spec: gcc@10.5.0 languages='c,c++,fortran' prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc-10 cxx: /usr/bin/g++-10 fortran: /usr/bin/gfortran-10 flags: cflags: -O3 fflags: -g -O2 environment: set: GCC_ROOT: /usr prepend_path: PATH: /usr/unusual_path_for_ld/bin implicit_rpaths: - /usr/lib/gcc extra_rpaths: - /usr/lib/unusual_gcc_path The ``flags`` attribute specifies compiler flags to apply to every spec that depends on this compiler. The accepted flag types are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, and ``ldlibs``. In the example above, every spec compiled with this compiler will pass the flags ``-g -O2`` to ``/usr/bin/gfortran-10`` and will pass the flag ``-O3`` to ``/usr/bin/gcc-10``. The ``environment`` attribute specifies user environment modifications to apply before every time the compiler is invoked. The available operations are ``set``, ``unset``, ``prepend_path``, ``append_path``, and ``remove_path``. In the example above, Spack will set ``GCC_ROOT=/usr`` and set ``PATH=/usr/unusual_path_for_ld/bin:$PATH`` before handing control to the build system that will use this compiler. The ``extra_rpaths`` and ``implicit_rpaths`` fields specify additional paths to pass as rpaths to the linker when using this compiler. The ``implicit_rpaths`` field is filled in automatically by Spack when detecting compilers, and the ``extra_rpaths`` field is available for users to configure necessary rpaths that have not been detected by Spack. In addition, paths from ``extra_rpaths`` are added as library search paths for the linker. In the example above, both ``/usr/lib/gcc`` and ``/usr/lib/unusual_gcc_path`` would be added as rpaths to the linker, and ``-L/usr/lib/unusual_gcc_path`` would be added as well. .. _package-requirements: Package Requirements -------------------- Spack can be configured to always use certain compilers, package versions, and variants during concretization through package requirements. Package requirements are useful when you find yourself repeatedly specifying the same constraints on the command line, and wish that Spack respects these constraints whether you mention them explicitly or not. Another use case is specifying constraints that should apply to all root specs in an environment, without having to repeat the constraint everywhere. Apart from that, requirements config is more flexible than constraints on the command line, because it can specify constraints on packages *when they occur* as a dependency. In contrast, on the command line it is not possible to specify constraints on dependencies while also keeping those dependencies optional. .. seealso:: FAQ: :ref:`Why does Spack pick particular versions and variants? ` Requirements syntax ^^^^^^^^^^^^^^^^^^^ The package requirements configuration is specified in ``packages.yaml``, keyed by package name and expressed using the Spec syntax. In the simplest case you can specify attributes that you always want the package to have by providing a single spec string to ``require``: .. code-block:: yaml packages: libfabric: require: "@1.13.2" In the above example, ``libfabric`` will always build with version 1.13.2. If you need to compose multiple configuration scopes ``require`` accepts a list of strings: .. code-block:: yaml packages: libfabric: require: - "@1.13.2" - "%gcc" In this case ``libfabric`` will always build with version 1.13.2 **and** using GCC as a compiler. For more complex use cases, require accepts also a list of objects. These objects must have either a ``any_of`` or a ``one_of`` field, containing a list of spec strings, and they can optionally have a ``when`` and a ``message`` attribute: .. code-block:: yaml packages: openmpi: require: - any_of: ["@4.1.5", "%c,cxx,fortran=gcc"] message: "in this example only 4.1.5 can build with other compilers" ``any_of`` is a list of specs. One of those specs must be satisfied and it is also allowed for the concretized spec to match more than one. In the above example, that means you could build ``openmpi@4.1.5%gcc``, ``openmpi@4.1.5%clang`` or ``openmpi@3.9%gcc``, but not ``openmpi@3.9%clang``. If a custom message is provided, and the requirement is not satisfiable, Spack will print the custom error message: .. code-block:: spec $ spack spec openmpi@3.9%clang ==> Error: in this example only 4.1.5 can build with other compilers We could express a similar requirement using the ``when`` attribute: .. code-block:: yaml packages: openmpi: require: - any_of: ["%c,cxx,fortran=gcc"] when: "@:4.1.4" message: "in this example only 4.1.5 can build with other compilers" In the example above, if the version turns out to be 4.1.4 or less, we require the compiler to be GCC. For readability, Spack also allows a ``spec`` key accepting a string when there is only a single constraint: .. code-block:: yaml packages: openmpi: require: - spec: "%c,cxx,fortran=gcc" when: "@:4.1.4" message: "in this example only 4.1.5 can build with other compilers" This code snippet and the one before it are semantically equivalent. Finally, instead of ``any_of`` you can use ``one_of`` which also takes a list of specs. The final concretized spec must match one and only one of them: .. code-block:: yaml packages: mpich: require: - one_of: ["+cuda", "+rocm"] In the example above, that means you could build ``mpich+cuda`` or ``mpich+rocm`` but not ``mpich+cuda+rocm``. .. note:: For ``any_of`` and ``one_of``, the order of specs indicates a preference: items that appear earlier in the list are preferred (note that these preferences can be ignored in favor of others). .. note:: When using a conditional requirement, Spack is allowed to actively avoid the triggering condition (the ``when=...`` spec) if that leads to a concrete spec with better scores in the optimization criteria. To check the current optimization criteria and their priorities you can run ``spack solve zlib``. Setting default requirements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can also set default requirements for all packages under ``all`` like this: .. code-block:: yaml packages: all: require: "%[when=%c]c=clang %[when=%cxx]cxx=clang" which means every spec will be required to use ``clang`` as the compiler for C and C++ code. .. warning:: The simpler config ``require: %clang`` will fail to build any package that does not include compiled code, because those packages cannot depend on ``clang`` (alias for ``llvm+clang``). In most contexts, default requirements must use either conditional dependencies or a :ref:`toolchain ` that combines conditional dependencies. Requirements on variants for all packages are possible too, but note that they are only enforced for those packages that define these variants, otherwise they are disregarded. For example: .. code-block:: yaml packages: all: require: - "+shared" - "+cuda" will just enforce ``+shared`` on ``zlib``, which has a boolean ``shared`` variant but no ``cuda`` variant. Constraints in a single spec literal are always considered as a whole, so in a case like: .. code-block:: yaml packages: all: require: "+shared +cuda" the default requirement will be enforced only if a package has both a ``cuda`` and a ``shared`` variant, and will never be partially enforced. Finally, ``all`` represents a *default set of requirements* - if there are specific package requirements, then the default requirements under ``all`` are disregarded. For example, with a configuration like this: .. code-block:: yaml packages: all: require: - "build_type=Debug" - "%[when=%c]c=clang %[when=%cxx]cxx=clang" cmake: require: - "build_type=Debug" - "%c,cxx=gcc" Spack requires ``cmake`` to use ``gcc`` and all other nodes (including ``cmake`` dependencies) to use ``clang``. If enforcing ``build_type=Debug`` is needed also on ``cmake``, it must be repeated in the specific ``cmake`` requirements. .. _setting-requirements-on-virtual-specs: Setting requirements on virtual specs ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A requirement on a virtual spec applies whenever that virtual is present in the DAG. This can be useful for fixing which virtual provider you want to use: .. code-block:: yaml packages: mpi: require: "mvapich2 %c,cxx,fortran=gcc" With the configuration above the only allowed ``mpi`` provider is ``mvapich2`` built with ``gcc``/``g++``/``gfortran``. Requirements on the virtual spec and on the specific provider are both applied, if present. For instance with a configuration like: .. code-block:: yaml packages: mpi: require: "mvapich2 %c,cxx,fortran=gcc" mvapich2: require: "~cuda" you will use ``mvapich2~cuda %c,cxx,fortran=gcc`` as an ``mpi`` provider. .. _package-strong-preferences: Conflicts and strong preferences ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If the semantic of requirements is too strong, you can also express "strong preferences" and "conflicts" from configuration files: .. code-block:: yaml packages: all: prefer: - "%c,cxx=clang" conflict: - "+shared" The ``prefer`` and ``conflict`` sections can be used whenever a ``require`` section is allowed. The argument is always a list of constraints, and each constraint can be either a simple string, or a more complex object: .. code-block:: yaml packages: all: conflict: - spec: "%c,cxx=clang" when: "target=x86_64_v3" message: "reason why clang cannot be used" The ``spec`` attribute is mandatory, while both ``when`` and ``message`` are optional. .. note:: Requirements allow for expressing both "strong preferences" and "conflicts". The syntax for doing so, though, may not be immediately clear. For instance, if we want to prevent any package from using ``%clang``, we can set: .. code-block:: yaml packages: all: require: - one_of: ["%clang", "@:"] Since only one of the requirements must hold, and ``@:`` is always true, the rule above is equivalent to a conflict. For "strong preferences" the same construction works, with the ``any_of`` policy instead of the ``one_of`` policy. .. _package-preferences: Package Preferences ------------------- In some cases package requirements can be too strong, and package preferences are the better option. Package preferences do not impose constraints on packages for particular versions or variants values, they rather only set defaults. The concretizer is free to change them if it must, due to other constraints, and also prefers reusing installed packages over building new ones that are a better match for preferences. .. seealso:: FAQ: :ref:`Why does Spack pick particular versions and variants? ` The ``target`` and ``providers`` preferences can only be set globally under the ``all`` section of ``packages.yaml``: .. code-block:: yaml packages: all: target: [x86_64_v3] providers: mpi: [mvapich2, mpich, openmpi] These preferences override Spack's default and effectively reorder priorities when looking for the best compiler, target or virtual package provider. Each preference takes an ordered list of spec constraints, with earlier entries in the list being preferred over later entries. In the example above all packages prefer to target the ``x86_64_v3`` microarchitecture and to use ``mvapich2`` if they depend on ``mpi``. The ``variants`` and ``version`` preferences can be set under package specific sections of the ``packages.yaml`` file: .. code-block:: yaml packages: opencv: variants: +debug gperftools: version: [2.2, 2.4, 2.3] In this case, the preference for ``opencv`` is to build with debug options, while ``gperftools`` prefers version 2.2 over 2.4. Any preference can be overwritten on the command line if explicitly requested. Preferences cannot overcome explicit constraints, as they only set a preferred ordering among homogeneous attribute values. Going back to the example, if ``gperftools@2.3:`` was requested, then Spack will install version 2.4 since the most preferred version 2.2 is prohibited by the version constraint. .. _package_permissions: Package Permissions ------------------- Spack can be configured to assign permissions to the files installed by a package. In the ``packages.yaml`` file under ``permissions``, the attributes ``read``, ``write``, and ``group`` control the package permissions. These attributes can be set per-package, or for all packages under ``all``. If permissions are set under ``all`` and for a specific package, the package-specific settings take precedence. The ``read`` and ``write`` attributes take one of ``user``, ``group``, and ``world``. .. code-block:: yaml packages: all: permissions: write: group group: spack my_app: permissions: read: group group: my_team The permissions settings describe the broadest level of access to installations of the specified packages. The execute permissions of the file are set to the same level as read permissions for those files that are executable. The default setting for ``read`` is ``world``, and for ``write`` is ``user``. In the example above, installations of ``my_app`` will be installed with user and group permissions but no world permissions, and owned by the group ``my_team``. All other packages will be installed with user and group write privileges, and world read privileges. Those packages will be owned by the group ``spack``. The ``group`` attribute assigns a Unix-style group to a package. All files installed by the package will be owned by the assigned group, and the sticky group bit will be set on the install prefix and all directories inside the install prefix. This will ensure that even manually placed files within the install prefix are owned by the assigned group. If no group is assigned, Spack will allow the OS default behavior to go as expected. .. _assigning-package-attributes: Assigning Package Attributes ---------------------------- You can assign class-level attributes in the configuration: .. code-block:: yaml packages: mpileaks: package_attributes: # Override existing attributes url: http://www.somewhereelse.com/mpileaks-1.0.tar.gz # ... or add new ones x: 1 Attributes set this way will be accessible to any method executed in the package.py file (e.g. the ``install()`` method). Values for these attributes may be any value parseable by yaml. These can only be applied to specific packages, not "all" or virtual packages. ================================================ FILE: lib/spack/docs/packaging_guide_advanced.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Advanced topics in Spack packaging, covering packages with multiple build systems, making packages discoverable with spack external find, and specifying ABI compatibility. .. list-table:: :widths: 25 25 25 25 :header-rows: 0 :width: 100% * - :doc:`1. Creation ` - :doc:`2. Build ` - :doc:`3. Testing ` - **4. Advanced** Packaging Guide: advanced topics ================================ This section of the packaging guide covers a few advanced topics. .. _multiple_build_systems: Multiple build systems ---------------------- Packages may use different build systems over time or across platforms. Spack is designed to handle this seamlessly within a single ``package.py`` file. Let's assume we work with ``curl`` and that the package is built using Autotools so far: .. code-block:: python from spack_repo.builtin.build_systems.autotools import AutotoolsPackage class Curl(AutotoolsPackage): depends_on("zlib-api") def configure_args(self): return [f"--with-zlib={self.spec['zlib-api'].prefix}"] To add CMake as a further build system we need to: 1. Add another base to the ``Curl`` package class (in our case ``cmake.CMakePackage``), 2. Explicitly declare which build systems are supported using the ``build_system`` directive, 3. Move the :doc:`build instructions ` in *separate builder classes*. .. code-block:: python from spack_repo.builtin.build_systems import autotools, cmake class Curl(cmake.CMakePackage, autotools.AutotoolsPackage): build_system("autotools", "cmake", default="cmake") depends_on("zlib-api") class AutotoolsBuilder(autotools.AutotoolsBuilder): def configure_args(self): return [f"--with-zlib={self.spec['zlib-api'].prefix}"] class CMakeBuilder(cmake.CMakeBuilder): def cmake_args(self): return [self.define_from_variant("USE_NGHTTP2", "nghttp2")] In general, with multiple build systems there is a clear split between the :doc:`package metadata ` and the :doc:`build instructions `: 1. The directives such as ``depends_on``, ``variant``, ``patch`` go into the package class 2. The build phase functions like ``configure``, ``build`` and ``install``, and helper functions such as ``cmake_args`` or ``configure_args`` go into the builder classes When ``curl`` is concretized, we can select its build system using the ``build_system`` variant, which is available for every package: .. code-block:: spec $ spack install curl build_system=cmake Override "phases" of a build system ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Sometimes package recipes need to override entire :ref:`phases ` of a build system. Let's assume this happens for ``cp2k``: .. code-block:: python from spack.package import * from spack_repo.builtin.build_systems import autotools class Cp2k(autotools.AutotoolsPackage): def install(self, spec: Spec, prefix: str) -> None: # ...existing code... pass If we want to add CMake as another build system we need to remember that the signature of phases changes when moving from the ``Package`` to the ``Builder`` class: .. code-block:: python from spack.package import * from spack_repo.builtin.build_systems import autotools, cmake class Cp2k(autotools.AutotoolsPackage, cmake.CMakePackage): build_system("autotools", "cmake", default="cmake") class AutotoolsBuilder(autotools.AutotoolsBuilder): def install(self, pkg: Cp2k, spec: Spec, prefix: str) -> None: # ...existing code... pass The ``install`` method now takes the ``Package`` instance as the first argument, since ``self`` refers to the builder class. Add dependencies conditional on a build system ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Many build dependencies are conditional on which build system is chosen. An effective way to handle this is to use a ``with when("build_system=...")`` block to specify dependencies that are only relevant for a specific build system: .. code-block:: python from spack.package import * from spack_repo.builtin.build_systems import cmake, autotools class Cp2k(cmake.CMakePackage, autotools.AutotoolsPackage): build_system("cmake", "autotools", default="cmake") # Runtime dependencies depends_on("ncurses") depends_on("libxml2") # Lowerbounds for cmake only apply when using cmake as the build system with when("build_system=cmake"): depends_on("cmake@3.18:", when="@2.0:", type="build") depends_on("cmake@3:", type="build") # Specify extra build dependencies used only in the configure script with when("build_system=autotools"): depends_on("perl", type="build") depends_on("pkgconfig", type="build") Transition from one build system to another ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Packages that transition from one build system to another can be modeled using :ref:`conditional variant values `: .. code-block:: python from spack.package import * from spack_repo.builtin.build_systems import cmake, autotools class Cp2k(cmake.CMakePackage, autotools.AutotoolsPackage): build_system( conditional("cmake", when="@0.64:"), conditional("autotools", when="@:0.63"), default="cmake", ) In the example, the directive imposes a change from ``Autotools`` to ``CMake`` going from ``v0.63`` to ``v0.64``. Inherit from a package with multiple build systems ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Customizing a package supporting multiple build systems is straightforward. If we need to only customize the metadata, we can just define the derived package class. For instance, let's assume we want to add a new version to the ``silo`` package: .. code-block:: python from spack_repo.builtin.packages.silo.package import Silo as BuiltinSilo class Silo(BuiltinSilo): # Version not in builtin.silo version("special_version") If we don't define any builder, Spack will reuse the custom builder from ``builtin.silo`` by default. If we need to customize the builder too, we just have to inherit from it, like any other Python class: .. code-block:: python from spack_repo.builtin.packages.silo.package import CMakeBuilder as SiloCMakeBuilder class CMakeBuilder(SiloCMakeBuilder): def cmake_args(self): return [self.define_from_variant("USE_NGHTTP2", "nghttp2")] .. _make-package-findable: Making a package discoverable with ``spack external find`` ---------------------------------------------------------- The simplest way to make a package discoverable with :ref:`spack external find ` is to: 1. Define the executables associated with the package. 2. Implement a method to determine the versions of these executables. Minimal detection ^^^^^^^^^^^^^^^^^ The first step is fairly simple, as it requires only to specify a package-level ``executables`` attribute: .. code-block:: python class Foo(Package): # Each string provided here is treated as a regular expression, and # would match for example "foo", "foobar", and "bazfoo". executables = ["foo"] This attribute must be a list of strings. Each string is a regular expression (e.g. "gcc" would match "gcc", "gcc-8.3", "my-weird-gcc", etc.) to determine a set of system executables that might be part of this package. Note that to match only executables named "gcc" the regular expression ``"^gcc$"`` must be used. Finally, to determine the version of each executable the ``determine_version`` method must be implemented: .. code-block:: python @classmethod def determine_version(cls, exe): """Return either the version of the executable passed as argument or ``None`` if the version cannot be determined. Args: exe (str): absolute path to the executable being examined """ This method receives as input the path to a single executable and must return as output its version as a string. If the version cannot be determined, or if the executable turns out to be a false positive, the value ``None`` must be returned, which ensures that the executable is discarded as a candidate. Implementing the two steps above is mandatory, and gives the package the basic ability to detect if a spec is present on the system at a given version. .. note:: Any executable for which the ``determine_version`` method returns ``None`` will be discarded and won't appear in later stages of the workflow described below. Additional functionality ^^^^^^^^^^^^^^^^^^^^^^^^ Besides the two mandatory steps described above, there are also optional methods that can be implemented to either increase the amount of details being detected or improve the robustness of the detection logic in a package. Variants and custom attributes """""""""""""""""""""""""""""" The ``determine_variants`` method can be optionally implemented in a package to detect additional details of the spec: .. code-block:: python @classmethod def determine_variants(cls, exes, version_str): """Return either a variant string, a tuple of a variant string and a dictionary of extra attributes that will be recorded in packages.yaml or a list of those items. Args: exes (list of str): list of executables (absolute paths) that live in the same prefix and share the same version version_str (str): version associated with the list of executables, as detected by ``determine_version`` """ This method takes as input a list of executables that live in the same prefix and share the same version string, and returns either: 1. A variant string 2. A tuple of a variant string and a dictionary of extra attributes 3. A list of items matching either 1 or 2 (if multiple specs are detected from the set of executables) If extra attributes are returned, they will be recorded in ``packages.yaml`` and be available for later reuse. As an example, the ``gcc`` package will record by default the different compilers found and an entry in ``packages.yaml`` would look like: .. code-block:: yaml packages: gcc: externals: - spec: "gcc@9.0.1 languages=c,c++,fortran" prefix: /usr extra_attributes: compilers: c: /usr/bin/x86_64-linux-gnu-gcc-9 c++: /usr/bin/x86_64-linux-gnu-g++-9 fortran: /usr/bin/x86_64-linux-gnu-gfortran-9 This allows us, for instance, to keep track of executables that would be named differently if built by Spack (e.g. ``x86_64-linux-gnu-gcc-9`` instead of just ``gcc``). .. TODO: we need to gather some more experience on overriding "prefix" and other special keywords in extra attributes, but as soon as we are confident that this is the way to go we should document the process. See https://github.com/spack/spack/pull/16526#issuecomment-653783204 Filter matching executables """"""""""""""""""""""""""" Sometimes defining the appropriate regex for the ``executables`` attribute might prove to be difficult, especially if one has to deal with corner cases or exclude "red herrings". To help keep the regular expressions as simple as possible, each package can optionally implement a ``filter_detected_exes`` method: .. code-block:: python @classmethod def filter_detected_exes(cls, prefix, exes_in_prefix): """Return a filtered list of the executables in prefix""" which takes as input a prefix and a list of matching executables and returns a filtered list of said executables. Using this method has the advantage of allowing custom logic for filtering, and does not restrict the user to regular expressions only. Consider the case of detecting the GNU C++ compiler. If we try to search for executables that match ``g++``, that would have the unwanted side effect of selecting also ``clang++`` - which is a C++ compiler provided by another package - if present on the system. Trying to select executables that contain ``g++`` but not ``clang`` would be quite complicated to do using only regular expressions. Employing the ``filter_detected_exes`` method it becomes: .. code-block:: python class Gcc(Package): executables = ["g++"] @classmethod def filter_detected_exes(cls, prefix, exes_in_prefix): return [x for x in exes_in_prefix if "clang" not in x] Another possibility that this method opens is to apply certain filtering logic when specific conditions are met (e.g. take some decisions on an OS and not on another). Validate detection ^^^^^^^^^^^^^^^^^^ To increase detection robustness, packagers may also implement a method to validate the detected Spec objects: .. code-block:: python @classmethod def validate_detected_spec(cls, spec, extra_attributes): """Validate a detected spec. Raise an exception if validation fails.""" This method receives a detected spec along with its extra attributes and can be used to check that certain conditions are met by the spec. Packagers can either use assertions or raise an ``InvalidSpecDetected`` exception when the check fails. If the conditions are not honored the spec will be discarded and any message associated with the assertion or the exception will be logged as the reason for discarding it. As an example, a package that wants to check that the ``compilers`` attribute is in the extra attributes can implement this method like this: .. code-block:: python @classmethod def validate_detected_spec(cls, spec, extra_attributes): """Check that "compilers" is in the extra attributes.""" msg = "the extra attribute 'compilers' must be set for the detected spec '{0}'".format(spec) assert "compilers" in extra_attributes, msg or like this: .. code-block:: python @classmethod def validate_detected_spec(cls, spec, extra_attributes): """Check that "compilers" is in the extra attributes.""" if "compilers" not in extra_attributes: msg = "the extra attribute 'compilers' must be set for the detected spec '{0}'".format( spec ) raise InvalidSpecDetected(msg) .. _determine_spec_details: Custom detection workflow ^^^^^^^^^^^^^^^^^^^^^^^^^ In the rare case when the mechanisms described so far don't fit the detection of a package, the implementation of all the methods above can be disregarded and instead a custom ``determine_spec_details`` method can be implemented directly in the package class (note that the definition of the ``executables`` attribute is still required): .. code-block:: python @classmethod def determine_spec_details(cls, prefix, exes_in_prefix): # exes_in_prefix = a set of paths, each path is an executable # prefix = a prefix that is common to each path in exes_in_prefix # return None or [] if none of the exes represent an instance of # the package. Return one or more Specs for each instance of the # package which is thought to be installed in the provided prefix ... This method takes as input a set of discovered executables (which match those specified by the user) as well as a common prefix shared by all of those executables. The function must return one or more :py:class:`spack.package.Spec` associated with the executables (it can also return ``None`` to indicate that no provided executables are associated with the package). As an example, consider a made-up package called ``foo-package`` which builds an executable called ``foo``. ``FooPackage`` would appear as follows: .. code-block:: python class FooPackage(Package): homepage = "..." url = "..." version(...) # Each string provided here is treated as a regular expression, and # would match for example "foo", "foobar", and "bazfoo". executables = ["foo"] @classmethod def determine_spec_details(cls, prefix, exes_in_prefix): candidates = [x for x in exes_in_prefix if os.path.basename(x) == "foo"] if not candidates: return # This implementation is lazy and only checks the first candidate exe_path = candidates[0] exe = Executable(exe_path) output = exe("--version", output=str, error=str) version_str = ... # parse output for version string return Spec.from_detection("foo-package@{0}".format(version_str)) Add detection tests to packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To ensure that software is detected correctly for multiple configurations and on different systems users can write a ``detection_test.yaml`` file and put it in the package directory alongside the ``package.py`` file. This YAML file contains enough information for Spack to mock an environment and try to check if the detection logic yields the results that are expected. As a general rule, attributes at the top-level of ``detection_test.yaml`` represent search mechanisms and they each map to a list of tests that should confirm the validity of the package's detection logic. The detection tests can be run with the following command: .. code-block:: console $ spack audit externals Errors that have been detected are reported to screen. Tests for PATH inspections """""""""""""""""""""""""" Detection tests insisting on ``PATH`` inspections are listed under the ``paths`` attribute: .. code-block:: yaml paths: - layout: - executables: - "bin/clang-3.9" - "bin/clang++-3.9" script: | echo "clang version 3.9.1-19ubuntu1 (tags/RELEASE_391/rc2)" echo "Target: x86_64-pc-linux-gnu" echo "Thread model: posix" echo "InstalledDir: /usr/bin" platforms: ["linux", "darwin"] results: - spec: "llvm@3.9.1 +clang~lld~lldb" If the ``platforms`` attribute is present, tests are run only if the current host matches one of the listed platforms. Each test is performed by first creating a temporary directory structure as specified in the corresponding ``layout`` and by then running package detection and checking that the outcome matches the expected ``results``. The exact details on how to specify both the ``layout`` and the ``results`` are reported in the table below: .. list-table:: Test based on PATH inspections :header-rows: 1 * - Option Name - Description - Allowed Values - Required Field * - ``layout`` - Specifies the filesystem tree used for the test - List of objects - Yes * - ``layout:[0]:executables`` - Relative paths for the mock executables to be created - List of strings - Yes * - ``layout:[0]:script`` - Mock logic for the executable - Any valid shell script - Yes * - ``results`` - List of expected results - List of objects (empty if no result is expected) - Yes * - ``results:[0]:spec`` - A spec that is expected from detection - Any valid spec - Yes * - ``results:[0]:extra_attributes`` - Extra attributes expected on the associated Spec - Nested dictionary with string as keys, and regular expressions as leaf values - No Reuse tests from other packages """"""""""""""""""""""""""""""" When using a custom repository, it is possible to customize a package that already exists in ``builtin`` and reuse its external tests. To do so, just write a ``detection_test.yaml`` alongside the customized ``package.py`` with an ``includes`` attribute. For instance the ``detection_test.yaml`` for ``myrepo.llvm`` might look like: .. code-block:: yaml includes: - "builtin.llvm" This YAML file instructs Spack to run the detection tests defined in ``builtin.llvm`` in addition to those locally defined in the file. .. _abi_compatibility: Specifying ABI Compatibility ---------------------------- .. warning:: The ``can_splice`` directive is experimental, and may be replaced by a higher-level interface in future versions of Spack. Packages can include ABI-compatibility information using the ``can_splice`` directive. For example, if ``Foo`` version 1.1 can always replace version 1.0, then the package could have: .. code-block:: python can_splice("foo@1.0", when="@1.1") For virtual packages, packages can also specify ABI compatibility with other packages providing the same virtual. For example, ``zlib-ng`` could specify: .. code-block:: python can_splice("zlib@1.3.1", when="@2.2+compat") Some packages have ABI-compatibility that is dependent on matching variant values, either for all variants or for some set of ABI-relevant variants. In those cases, it is not necessary to specify the full combinatorial explosion. The ``match_variants`` keyword can cover all single-value variants. .. code-block:: python # any value for bar as long as they're the same can_splice("foo@1.1", when="@1.2", match_variants=["bar"]) # any variant values if all single-value variants match can_splice("foo@1.2", when="@1.3", match_variants="*") The concretizer will use ABI compatibility to determine automatic splices when :ref:`automatic splicing` is enabled. Customizing Views ----------------- .. warning:: This is advanced functionality documented for completeness, and rarely needs customization. Spack environments manage a view of their packages, which is a single directory that merges all installed packages through symlinks, so users can easily access them. The methods of ``PackageViewMixin`` can be overridden to customize how packages are added to views. Sometimes it's impossible to get an application to work just through symlinking its executables, and patching is necessary. For example, Python scripts in a ``bin`` directory may have a shebang that points to the Python interpreter in Python's install prefix and not to the Python interpreter in the view. However, it's more convenient to have the shebang point to the Python interpreter in the view, since that interpreter can locate other Python packages in the view without ``PYTHONPATH`` being set. Therefore, Python extension packages (those inheriting from ``PythonPackage``) override ``add_files_to_view`` in order to rewrite shebang lines. ================================================ FILE: lib/spack/docs/packaging_guide_build.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to customizing the build process in Spack, covering installation procedures, build systems, and how to control the build with spec objects and environment variables. .. list-table:: :widths: 25 25 25 25 :header-rows: 0 :width: 100% * - :doc:`1. Creation ` - **2. Build** - :doc:`3. Testing ` - :doc:`4. Advanced ` Packaging Guide: customizing the build ====================================== In the first part of the packaging guide, we covered the basic structure of a package, how to specify dependencies, and how to define variants. In the second part, we will cover the installation procedure, build systems, and how to customize the build process. .. _installation_procedure: Overview of the installation procedure -------------------------------------- Whenever Spack installs software, it goes through a series of predefined steps: .. image:: images/installation_pipeline.png :scale: 60 % :align: center All these steps are influenced by the metadata in each ``package.py`` and by the current Spack configuration. Since build systems are different from one another, the execution of the last block in the figure is further expanded in a build system specific way. An example for ``CMake`` is, for instance: .. image:: images/builder_phases.png :align: center :scale: 60 % The predefined steps for each build system are called "phases". In general, the name and order in which the phases will be executed can be obtained by either reading the API docs at :py:mod:`~.spack_repo.builtin.build_systems`, or using the ``spack info`` command: .. code-block:: console :emphasize-lines: 13,14 $ spack info --phases m4 AutotoolsPackage: m4 Homepage: https://www.gnu.org/software/m4/m4.html Safe versions: 1.4.17 ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz Variants: Name Default Description sigsegv on Build the libsigsegv dependency Installation Phases: autoreconf configure build install Build Dependencies: libsigsegv ... An extensive list of available build systems and phases is provided in :ref:`installation_process`. Controlling the build process ----------------------------- As we have seen in the first part of the packaging guide, the usual workflow for creating a package is to start with ``spack create ``, which generates a ``package.py`` file for you with a boilerplate package class. This typically includes a package base class (e.g. ``AutotoolsPackage`` or ``CMakePackage``), a URL, and one or more versions. After you have added required dependencies and variants, you can start customizing the build process. There are various ways to do this, depending on the build system and the package itself. From simplest to most complex, the following are the most common ways to customize the build process: 1. **Implementing build system helper methods and properties**. Most build systems provide a set of helper methods that can be overridden to customize the build process without overriding entire phases. For example, for ``AutotoolsPackage`` you can specify the command line arguments for ``./configure`` by implementing ``configure_args``: .. code-block:: python class MyPkg(AutotoolsPackage): def configure_args(self): # FIXME: Add arguments other than --prefix # FIXME: If not needed delete this function args = [] return args Similarly for ``CMakePackage`` you can influence how ``cmake`` is invoked by implementing ``cmake_args``: .. code-block:: python class MyPkg(CMakePackage): def cmake_args(self): # FIXME: Add arguments other than # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE # FIXME: If not needed delete this function args = [] return args The exact methods and properties available depend on the build system you are using. See :doc:`build_systems` for a complete list of available build systems and their specific helper functions and properties. 2. **Setting environment variables**. Some build systems require specific environment variables to be set before the build starts. You can set these variables by overriding the ``setup_build_environment`` method in your package class: .. code-block:: python def setup_build_environment(self, env): env.set("MY_ENV_VAR", "value") This is useful for setting paths or other variables that the build system needs to find dependencies or configure itself correctly. See :ref:`setup-environment`. 3. **Complementing the build system with pre- or post-build steps**. In some cases, you may need to run additional commands before or after the build system phases. This is useful for installing additional files missed by the build system, or for running custom scripts. .. code-block:: python @run_after("install") def install_missing_files(self): install_tree("extra_files", self.prefix.bin) See :ref:`before_after_build_phases`. 4. **Overriding entire build phases**. If the default implementation of a build phase does not fit your needs, you can override the entire phase. See :ref:`overriding-phases` for examples. In any of the functions above, you can 1. **Make instructions dynamic**. Build instructions typically depend on the package's variants, version and its dependencies. For example, you can use .. code-block:: python if self.spec.satisfies("+variant_name"): ... to check if a variant is enabled, or .. code-block:: python self.spec["dependency_name"].prefix to get the prefix of a dependency. See :ref:`spec-objects` for more details on how to use specs in your package. 2. **Use Spack's Python Package API**. The ``from spack.package import *`` statement at the top of a ``package.py`` file allows you to access Spack's utilities and helper functions, such as ``which``, ``install_tree``, ``filter_file`` and others. See :ref:`python-package-api` for more details. .. _installation_process: What are build systems? ----------------------- Every package in Spack has an associated build system. For most packages, this will be a well-known system for which Spack provides a base class, like ``CMakePackage`` or ``AutotoolsPackage``. Even for packages that have no formal build process (e.g., just copying files), Spack still associates them with a generic build system class. Build systems have the following responsibilities: 1. **Define and implement the build phases**. Each build system defines a set of phases that are executed in a specific order. For example, ``AutotoolsPackage`` has the following phases: ``autoreconf``, ``configure``, ``build``, and ``install``. These phases are Python methods with a sensible default implementation that can be overridden by the package author. 2. **Add dependencies and variants**. Build systems can define dependencies and variants that are specific to the build system. For example, ``CMakePackage`` adds a ``cmake`` as a build dependency, and defines ``build_type`` as a variant (which maps to the ``CMAKE_BUILD_TYPE`` CMake variable). All build systems also define a special variant ``build_system``, which is useful in case of :ref:`multiple build systems `. 3. **Provide helper methods**. Build systems often provide helper functions and properties that the package author can use to customize the build configuration, without having to override entire phases. For example: * The ``CMakePackage`` lets users implement the ``cmake_args`` method to specify additional arguments for the ``cmake`` command * The ``MakefilePackage`` lets users set ``build_targets`` and ``install_targets`` properties to specify the targets to build and install. There are typically also helper functions to map variants to CMake or Autotools options: * The ``CMakePackage`` provides the ``self.define_from_variant("VAR_NAME", "variant_name")`` method to generate the appropriate ``-DVAR_NAME:BOOL=ON/OFF`` arguments for the ``cmake`` command. * The ``AutotoolsPackage`` provides helper functions like ``self.with_or_without("foo")`` to generate the appropriate ``--with-foo`` or ``--without-foo`` arguments for the ``./configure`` script. Here is a table of the most common build systems available in Spack: .. list-table:: :widths: 40 60 :header-rows: 1 * - Package Class - Description * - :doc:`AutotoolsPackage ` - For packages that use GNU Autotools (autoconf, automake, libtool). * - :doc:`CMakePackage ` - For packages that use CMake. * - :doc:`MakefilePackage ` - For packages that use plain Makefiles. * - :doc:`MesonPackage ` - For packages that use the Meson build system. * - :doc:`PythonPackage ` - For Python packages (setuptools, pip, etc.). * - :doc:`BundlePackage ` - For installing a collection of other packages. * - :doc:`Package ` - Generic package for custom builds, provides only an ``install`` phase. All build systems are defined in the ``spack_repo.builtin.build_systems`` module, which is part of the Spack builtin package repository. To use a particular build system, you need to import it in your ``package.py`` file, and then derive your package class from the appropriate base class: .. code-block:: python from spack_repo.builtin.build_systems.cmake import CMakePackage class MyPkg(CMakePackage): pass For a complete list of build systems and their specific helper functions and properties, see the :doc:`build_systems` documentation. .. _spec-objects: Configuring the build with spec objects --------------------------------------- Configuring a build is typically the first step in the build process. In many build systems it involves passing the right command line arguments to the configure script, and in some build systems it is a matter of setting the right environment variables. In this section we will use an Autotools package as an example, where we just need to implement the ``configure_args`` helper function. In general, whenever you implement helper functions of a build system or complement or override its build phases, you often need to make decisions based on the package's configuration. Spack is unique in that it allows you to write a *single* ``package.py`` for all configurations of a package. The central object in Spack that encodes the package's configuration is the **concrete spec**, which is available as ``self.spec`` in the package class. This is the object you need to query to make decisions about how to configure the build. Querying ``self.spec`` ^^^^^^^^^^^^^^^^^^^^^^ **Variants**. In the previous section of the packaging guide, we've seen :ref:`how to define variants `. As a packager, you are responsible for implementing the logic that translates the selected variant values into build instructions the build system can understand. If you want to pass a flag to the configure script only if the package is built with a specific variant, you can do so like this: .. code-block:: python variant("foo", default=False, description="Enable foo feature") def configure_args(self): args = [] if self.spec.satisfies("+foo"): args.append("--enable-foo") else: args.append("--disable-foo") return args For multi-valued variants, you can use the ``key=value`` syntax to test whether a specific value is selected: .. code-block:: python variant("threads", default="none", values=("pthreads", "openmp", "none"), multi=False, ...) def configure_args(self): args = [] if self.spec.satisfies("threads=pthreads"): args.append("--enable-threads=pthreads") elif self.spec.satisfies("threads=openmp"): args.append("--enable-threads=openmp") elif self.spec.satisfies("threads=none"): args.append("--disable-threads") return args Even if *multiple* values are selected, you can still use ``key=value`` to test for specific values: .. code-block:: python variant("languages", default="c,c++", values=("c", "c++", "fortran"), multi=True, ...) def configure_args(self): args = [] if self.spec.satisfies("languages=c"): args.append("--enable-c") if self.spec.satisfies("languages=c++"): args.append("--enable-c++") if self.spec.satisfies("languages=fortran"): args.append("--enable-fortran") return args Notice that many build systems provide helper functions to make the above code more concise. See :ref:`the Autotools docs ` and :ref:`the CMake docs `. Other than testing for certain variant values, you can also obtain the variant value directly with ``self.spec.variants["variant_name"].value``. This is useful when you want to pass the variant value as a command line argument to the build system. The type of this value depends on the variant type: * For boolean variants this is :data:`True` or :data:`False`. * For single-valued variants this is a :class:`str` value. * For multi-valued variants it is a tuple of :class:`str` values. An example of using this is shown below: .. code-block:: python variant( "cxxstd", default="11", values=("11", "14", "17", "20", "23"), multi=False, description="C++ standard", ) def configure_args(self): return [f"--with-cxxstd={self.spec.variants['cxxstd'].value}"] **Versions**. Similarly, versions are often used to dynamically change the build configuration: .. code-block:: python def configure_args(self): args = [] if self.spec.satisfies("@1.2:"): args.append("--enable-new-feature") return args This adds a flag only if the package is on version 1.2 or higher. **Dependencies**. You can also use the ``self.spec.satisfies`` method to test whether a dependency is present or not, and whether it is built with a specific variant or version. The ``^`` character is used to refer to packages that are required at runtime as well as build dependencies. More precisely, it includes all direct dependencies of ``build`` type and transitive dependencies of ``link`` or ``run`` type. .. code-block:: python if self.spec.satisfies("^python@3.8:"): args.append("--min-python-version=3.8") Here we test whether the package has a (possibly transitive) dependency on Python version 3.8 or higher. The ``%`` character is used to refer to direct dependencies only. A typical use case is when you want to test the compiler used to build the package. .. code-block:: python if self.spec.satisfies("%c=gcc@8:"): args.append("--enable-profile-guided-optimization") This example adds a flag when the C compiler is from GCC version 8 or higher. The ``%c=gcc`` syntax technically means that ``gcc`` is the provider for the ``c`` language virtual. .. tip:: Historically, many packages have been written using ``^dep`` to refer to a dependency. Modern Spack packages should consider using ``%dep`` instead, which is more precise: it can only match direct dependencies, which are listed in the ``depends_on`` statements. **Target specific configuration**. Spack always makes the special ``platform``, ``os`` and ``target`` variants available in the spec. These variants can be used to test the target platform, operating system and CPU microarchitecture the package. The following example shows how we can add a configure option only if the package is built for Apple Silicon: .. code-block:: python if self.spec.satisfies("platform=darwin target=aarch64:"): args.append("--enable-apple-silicon") Notice that ``target=aarch64:`` is a range which matches the whole family of ``aarch64`` microarchitectures, including ``m1``, ``m2``, and so on. You can use ranges starting at a specific microarchitecture as well, for example: .. code-block:: python if self.spec.satisfies("target=haswell:"): args.append("--enable-haswell") .. note:: The ``spec`` object encodes the *target* platform, os and architecture the package is being built for. This is different from the *host* platform (typically accessed via ``sys.platform``) which is the platform where Spack is running. When writing package recipes, you should always use the ``spec`` object to query the target platform, os and architecture. To see what targets are available in your Spack installation, you can use the following command: .. command-output:: spack arch --known-targets Referring to a dependency's prefix, libraries, and headers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Very often you need to inform the build system about the location of a dependency. The most common way to do this is to pass the dependency's prefix as a configure argument and let the build system detect the libraries and headers from there. To do this, you can obtain the **dependency's spec** by name: .. code-block:: python libxml2 = self.spec["libxml2"] The ``libxml2`` variable is itself a spec object, and we can refer to its properties: .. code-block:: python def configure_args(self): return [ f"--with-libxml2={self.spec['libxml2'].prefix}", ] Apart from the :ref:`prefix `, you can also access other attributes of the dependency, such as ``libs`` or ``headers``. See :ref:`custom-attributes` for how dependencies define these attributes. These attributes are typically only required if the package is unable to locate the libraries and headers itself, or if you want to be more specific about which libraries or headers to use. A more advanced example where we explicitly pass libraries and headers to the configure script is shown below. .. code-block:: python def configure_args(self): return [ f"--with-libxml2={self.spec['libxml2'].prefix}", f"--with-libxml2-libs={self.spec['libxml2'].libs.ld_flags}", f"--with-libxml2-include={self.spec['libxml2'].headers.include_flags}", ] The ``libs`` attribute is a :class:`~spack.package.LibraryList` object that can be used to get a list of libraries by path, but also to get the appropriate linker flags. Similarly, the ``headers`` attribute is a :class:`~spack.package.HeaderList`, which also has methods to get the relevant include flags. .. _blas_lapack_scalapack: **Virtual dependencies**. You can also refer to the prefix, libraries and headers of :ref:`virtual dependencies `. For example, suppose we have a package that depends on ``blas`` and ``lapack``. We can get the provider's (e.g. OpenBLAS or Intel MKL) prefixes like this: .. code-block:: python class MyPkg(AutotoolPackage): depends_on("blas") depends_on("lapack") def configure_args(self): return [ f"--with-blas={self.spec['blas'].prefix}", f"--with-lapack={self.spec['lapack'].prefix}", ] Many build systems struggle to locate the ``blas`` and ``lapack`` libraries during configure, either because they do not know the exact names of the libraries, or because the libraries are not in typical locations --- they may not even know whether ``blas`` and ``lapack`` are a single or separate libraries. In those cases, the build system could use some help, for which we give a few examples below: 1. Space separated list of full paths .. code-block:: python lapack_blas = spec["lapack"].libs + spec["blas"].libs args.append(f"--with-blas-lapack-lib={lapack_blas.joined()}") 2. Names of libraries and directories which contain them .. code-block:: python lapack_blas = spec["lapack"].libs + spec["blas"].libs args.extend( [ f"-DMATH_LIBRARY_NAMES={';'.join(lapack_blas.names)}", f"-DMATH_LIBRARY_DIRS={';'.join(lapack_blas.directories)}", ] ) 3. Search and link flags .. code-block:: python lapack_blas = spec["lapack"].libs + spec["blas"].libs args.append(f"-DMATH_LIBS={lapack_blas.ld_flags}") .. _before_after_build_phases: Before and after build phases ----------------------------- Typically the default implementation of the build system's phases is sufficient for most packages. However, in some cases you may need to complement the default implementation with some custom instructions. For example, some packages do not install all the files they should, and you want to fix this by simply copying the missing files after the normal install phase is done. Instead of overriding the entire phase, you can use ``@run_before`` and ``@run_after`` to run custom code before or after a specific phase: .. code-block:: python class MyPackage(CMakePackage): ... variant("extras", default=False, description="Install extra files") @run_before("cmake") def run_before_cmake_is_invoked(self) -> None: with open("custom_file.txt", "w") as f: f.write("This file is created before cmake is invoked.") @run_after("install", when="+extras") def custom_post_install_phase(self) -> None: # install missing files not covered by the build system install_tree("extras", self.prefix.share.extras) Then ``when="+extras"`` will ensure that the custom post-install phase is only run conditionally. The function body should contain the actual instructions you want to run before or after the build phase, which can involve :ref:`running executables ` and creating or copying files to the ``prefix`` directory using convenience functions from :ref:`Spack's Python Package API `. .. _overriding-phases: Overriding a build phase ------------------------ If a build phase does not do what you need, and you cannot achieve your goal either by implementing the helper methods of the build system, or by using the ``@run_before`` or ``@run_after`` decorators (see :ref:`before_after_build_phases`), you can override the entire build phase. The most common scenario is when a package simply does not have a well-defined build system. For example, the installation procedure may just be copying files or running a shell script. In that case, you can use the generic ``Package`` class, which defines only a single ``install()`` phase, to be overridden by the package author: .. code-block:: python from spack.package import * from spack_repo.builtin.build_systems.generic import Package class MyPkg(Package): # Override the install phase def install(self, spec: Spec, prefix: Prefix) -> None: install_tree("my_files", prefix.bin) Whichever build system is used, **every build phase function has the same set of arguments**. The arguments are: ``self`` This is the package object, which extends ``CMakePackage``. For API docs on Package objects, see :py:class:`Package `. ``spec`` This is the concrete spec object created by Spack from an abstract spec supplied by the user. It describes what should be installed. It will be of type :py:class:`Spec `. ``prefix`` This is where your package should install its files. It acts like a string, but it's actually its :ref:`own special type `. The function body should contain the actual build instructions, which typically involves: 1. Invoking the build system's commands such as ``make``, ``ninja``, ``python``, et cetera. See :ref:`running_build_executables` for how to do this. 2. Copying files to the ``prefix`` directory, which is where Spack expects the package to be installed. This can be done using Spack's built-in functions like ``install_tree()`` or ``install()``. See the :ref:`Spack's Python Package API ` for all convenience functions that can be used in the package class. The arguments ``spec`` and ``prefix`` are passed only for convenience, as they always correspond to ``self.spec`` and ``self.spec.prefix`` respectively, as we have already seen in :ref:`the previous section `. .. warning:: When working with :ref:`multiple build systems ` in a single package, the arguments for build phase functions are slightly different. .. _running_build_executables: Running build executables ------------------------- When you :ref:`override a build phase `, or when you write a :ref:`build phase hook `, you typically need to invoke executables like ``make``, ``cmake``, or ``python`` to kick off the build process. Spack makes some of these executables available as global functions, making it easy to run them in your package class: .. code-block:: python from spack.package import * from spack_repo.builtin.build_systems.generic import Package class MyPkg(Package): depends_on("make", type="build") depends_on("python", type="build") def install(self, spec: Spec, prefix: Prefix) -> None: python("generate-makefile.py", "--output=Makefile") make() make("install") The ``python()`` and ``make()`` functions in this example invoke the ``python3`` and ``make`` executables, respectively. Naturally, you may wonder where these variables come from, since they are not imported from anywhere --- your editor may even underline them in red because they are not defined in the package module. The answer lies in the ``python`` and ``make`` dependencies, which implement the :meth:`~spack.package.PackageBase.setup_dependent_package` method in their package classes. This sets up Python variables that can be used in the package class of dependents. There is a good reason that it's the *dependency* that sets up these variables, rather than the package itself. For example, the ``make`` package ensures sensible default arguments for the ``make`` executable, such as the ``-j`` flag to enable parallel builds. This means that you do not have to worry about these technical details in your package class; you can just use ``make("my_target")`` and Spack will take care of the rest. See the section about :ref:`parallel builds ` for more details. Not all dependencies set up such variables for dependent packages, in which case you have two further options: 1. Use the ``command`` attribute of the dependency. This is a good option, since it refers to an executable provided by a specific dependency. .. code-block:: python def install(self, spec: Spec, prefix: Prefix) -> None: cython = self.spec["py-cython"].command cython("setup.py", "build_ext", "--inplace") 2. Use the ``which`` function (from the ``spack.package`` module). Do note that this function relies on the order of the ``PATH`` environment variable, which may be less reliable than the first option. .. code-block:: python def install(self, spec: Spec, prefix: Prefix) -> None: cython = which("cython", required=True) cython("setup.py", "build_ext", "--inplace") All executables in Spack are instances of :class:`~spack.package.Executable`, see its API docs for more details. .. _attribute_parallel: Package-level parallelism ------------------------- Many build tools support parallel builds, including ``make`` and ``ninja``, as well as certain Python build tools. As mentioned in :ref:`the previous section `, the ``gmake`` and ``ninja`` packages make their executables available as global functions, which you can use in your package class. They automatically add the ``-j `` when invoked, where ```` is a sensible default for the number of jobs to run in parallel. This exact number :ref:`is determined ` depends on various factors, such as the ``spack install`` command line arguments, configuration options and available CPUs on the system. As a packager, you rarely need to pass the ``-j`` flag when calling ``make()`` or ``ninja()``; it is better to rely on the defaults. In certain cases however, you may need to override the default number of jobs for a specific package. If a package does not build properly in parallel, you can simply define ``parallel = False`` in your package class. For example: .. code-block:: python :emphasize-lines: 4 class ExamplePackage(MakefilePackage): """Example package that does not build in parallel.""" parallel = False This ensures that any ``make`` or ``ninja`` invocation will *not* set the ``-j `` option, and the build will run sequentially. You can also disable parallel builds only for specific make invocation: .. code-block:: python :emphasize-lines: 5 class Libelf(MakefilePackage): ... def install(self, spec: Spec, prefix: Prefix) -> None: make("install", parallel=False) In this case, the ``build`` phase will still execute in parallel, but the ``install`` phase will run sequentially. For packages whose build systems do not run ``make`` or ``ninja``, but have other executables or scripts that support parallel builds, you can control parallelism using the ``make_jobs`` global. This global variable is an integer that specifies the number of jobs to run in parallel during the build process. .. code-block:: python :emphasize-lines: 6 class Xios(Package): def install(self, spec: Spec, prefix: Prefix) -> None: make_xios = Executable("./make_xios") make_xios( "--with-feature", f"--jobs={make_jobs}", ) .. _python-package-api: Spack's Python Package API -------------------------- Whenever you implement :ref:`overriding phases ` or :ref:`before and after build phases `, you typically need to modify files, work with paths and run executables. Spack provides a number of convenience functions and classes of its own to make your life even easier, complementing the Python standard library. All of the functionality in this section is made available by importing the :mod:`spack.package` module. .. code-block:: python from spack.package import * This is already part of the boilerplate for packages created with ``spack create``. .. _file-filtering: File filtering functions ^^^^^^^^^^^^^^^^^^^^^^^^ :py:func:`filter_file(regex, repl, *filenames, **kwargs) ` Works like ``sed`` but with Python regular expression syntax. Takes a regular expression, a replacement, and a set of files. ``repl`` can be a raw string or a callable function. If it is a raw string, it can contain ``\1``, ``\2``, etc. to refer to capture groups in the regular expression. If it is a callable, it is passed the Python ``MatchObject`` and should return a suitable replacement string for the particular match. Examples: #. Filtering a Makefile to force it to use Spack's compiler wrappers: .. code-block:: python filter_file(r"^\s*CC\s*=.*", "CC = " + spack_cc, "Makefile") filter_file(r"^\s*CXX\s*=.*", "CXX = " + spack_cxx, "Makefile") filter_file(r"^\s*F77\s*=.*", "F77 = " + spack_f77, "Makefile") filter_file(r"^\s*FC\s*=.*", "FC = " + spack_fc, "Makefile") #. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``: .. code-block:: python filter_file(r"#!/usr/bin/perl", "#!/usr/bin/env perl", prefix.bin.bib2xhtml) #. Switching the compilers used by ``mpich``'s MPI wrapper scripts from ``cc``, etc. to the compilers used by the Spack build: .. code-block:: python filter_file("CC='cc'", "CC='%s'" % self.compiler.cc, prefix.bin.mpicc) filter_file("CXX='c++'", "CXX='%s'" % self.compiler.cxx, prefix.bin.mpicxx) :py:func:`change_sed_delimiter(old_delim, new_delim, *filenames) ` Some packages, like TAU, have a build system that can't install into directories with, e.g. "@" in the name, because they use hard-coded ``sed`` commands in their build. ``change_sed_delimiter`` finds all ``sed`` search/replace commands and changes the delimiter. E.g., if the file contains commands that look like ``s///``, you can use this to change them to ``s@@@``. Example of changing ``s///`` to ``s@@@`` in TAU: .. code-block:: python change_sed_delimiter("@", ";", "configure") change_sed_delimiter("@", ";", "utils/FixMakefile") change_sed_delimiter("@", ";", "utils/FixMakefile.sed.default") File functions ^^^^^^^^^^^^^^ :py:func:`ancestor(dir, n=1) ` Get the n\ :sup:`th` ancestor of the directory ``dir``. :py:func:`can_access(path) ` True if we can read and write to the file at ``path``. Same as native Python ``os.access(file_name, os.R_OK|os.W_OK)``. :py:func:`install(src, dest) ` Install a file to a particular location. For example, install a header into the ``include`` directory under the install ``prefix``: .. code-block:: python install("my-header.h", prefix.include) :py:func:`join_path(*paths) ` An alias for ``os.path.join``. This joins paths using the OS path separator. :py:func:`mkdirp(*paths) ` Create each of the directories in ``paths``, creating any parent directories if they do not exist. :py:func:`working_dir(dirname, kwargs) ` This is a Python `Context Manager `_ that makes it easier to work with subdirectories in builds. You use this with the Python ``with`` statement to change into a working directory, and when the with block is done, you change back to the original directory. Think of it as a safe ``pushd`` / ``popd`` combination, where ``popd`` is guaranteed to be called at the end, even if exceptions are thrown. Example usage: #. The ``libdwarf`` build first runs ``configure`` and ``make`` in a subdirectory called ``libdwarf``. It then implements the installation code itself. This is natural with ``working_dir``: .. code-block:: python with working_dir("libdwarf"): configure("--prefix=" + prefix, "--enable-shared") make() install("libdwarf.a", prefix.lib) #. Many CMake builds require that you build "out of source", that is, in a subdirectory. You can handle creating and ``cd``'ing to the subdirectory like the LLVM package does: .. code-block:: python with working_dir("spack-build", create=True): cmake( "..", "-DLLVM_REQUIRES_RTTI=1", "-DPYTHON_EXECUTABLE=/usr/bin/python", "-DPYTHON_INCLUDE_DIR=/usr/include/python2.6", "-DPYTHON_LIBRARY=/usr/lib64/libpython2.6.so", *std_cmake_args ) make() make("install") The ``create=True`` keyword argument causes the command to create the directory if it does not exist. :py:func:`touch(path) ` Create an empty file at ``path``. .. _multimethods: Multimethods and the ``@when`` decorator ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``@when`` annotation lets packages declare multiple versions of a method that will be called depending on the package's spec. This can be useful to handle cases where configure options are entirely different depending on the version of the package, or when the package is built for different platforms. .. code-block:: python class SomePackage(Package): ... @when("@:1") def configure_args(self): return ["--old-flag"] @when("@2:") def configure_args(self): return ["--new-flag"] You can write multiple ``@when`` specs that satisfy the package's spec, for example: .. code-block:: python class SomePackage(Package): ... depends_on("mpi") def setup_mpi(self): # the default, called when no @when specs match pass @when("^mpi@3:") def setup_mpi(self): # this will be called when mpi is version 3 or higher pass @when("^mpi@2:") def setup_mpi(self): # this will be called when mpi is version 2 or higher pass @when("^mpi@1:") def setup_mpi(self): # this will be called when mpi is version 1 or higher pass In situations like this, the first matching spec, in declaration order, will be called. If no ``@when`` spec matches, the default method (the one without the ``@when`` decorator) will be called. .. warning:: The default method (without the ``@when`` decorator) should come first in the declaration order. If not, it will erase all ``@when`` methods that precede it in the class. This is a limitation of decorators in Python. .. _prefix-objects: Prefix objects ^^^^^^^^^^^^^^ You can find the installation directory of package in Spack by using the ``self.prefix`` attribute of the package object. In :ref:`overriding-phases`, we saw that the ``install()`` method has a ``prefix`` argument, which is the same as ``self.prefix``. This variable behaves like a string, but it is actually an instance of the :py:class:`Prefix ` class, which provides some additional functionality to make it easier to work with file paths in Spack. In particular, you can use the ``.`` operator to join paths together, creating nested directory structures: ====================== ======================= Prefix Attribute Location ====================== ======================= ``prefix.bin`` ``$prefix/bin`` ``prefix.lib64`` ``$prefix/lib64`` ``prefix.share.man`` ``$prefix/share/man`` ``prefix.foo.bar.baz`` ``$prefix/foo/bar/baz`` ====================== ======================= Of course, this only works if your file or directory is a valid Python variable name. If your file or directory contains dashes or dots, use ``join`` instead: .. code-block:: python prefix.lib.join("libz.a") .. _environment-variables: The build environment --------------------- In Spack the term **build environment** is used somewhat interchangeably to refer to two things: 1. The set of *environment variables* during the build process 2. The *process* in which the build is executed Spack creates a separate process for each package build, and every build has its own environment variables. Changes in the build environment do not affect the Spack process itself, and they are not visible to other builds. Spack manages the build environment in the following ways: 1. It cleans the environment variables that may interfere with the build process (e.g. ``CFLAGS``, ``LD_LIBRARY_PATH``, etc.). 2. It sets a couple of variables for its own use, prefixed with ``SPACK_*``. 3. It sets a number of standard environment variables like ``PATH`` to make dependencies available during the build. 4. It sets custom, package specific environment variables defined in the package class of dependencies. For this guide, all that matters is to have a rough understanding of which environments you are supposed to set in your package, and which ones are set by Spack automatically. The following variables are considered "standard" and are managed by Spack: ===================== ==================================================== ``PATH`` Set to point to ``/bin`` directories of dependencies ``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake ``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies ===================== ==================================================== Other typical environment variables such as ``CC``, ``CXX`` and ``FC`` are set by the ``compiler-wrapper`` package. In your package, all you need to specify is language dependencies: .. code-block:: python class MyPackage(Package): depends_on("c", type="build") # ensures CC is set depends_on("cxx", type="build") # ensures CXX is set depends_on("fortran", type="build") # ensures FC is set The ``compiler-wrapper`` package is an "injected" dependency by the compiler package (which provides the ``c``, ``cxx``, and ``fortran`` virtuals). It takes care of setting the ``CC``, ``CXX``, and ``FC`` environment variables to the appropriate compiler executables, so you do not need to set them manually in your package. For other compiler related environment variables such as ``CFLAGS`` and ``CXXFLAGS``, see :ref:`compiler flags `. This requires a section of its own, because there are multiple ways to deal with compiler flags, and they can come from different sources. .. _setup-environment: Package specific environment variables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack provides a few methods to help package authors set environment variables programmatically. In total there are four such methods, distinguishing between the build and run environments, and between the package itself and its dependents: 1. :meth:`setup_build_environment(env, spec) ` 2. :meth:`setup_dependent_build_environment(env, dependent_spec) ` 3. :meth:`setup_run_environment(env) ` 4. :meth:`setup_dependent_run_environment(env, dependent_spec) ` All these methods take an ``env`` argument, which is an instance of the :class:`EnvironmentModifications ` class. The ``setup_build_environment`` method is for certain build systems (e.g. ``PythonPackage``) roughly equivalent to the ``configure_args`` or ``cmake_args`` methods. It allows you to set environment variables that are needed during the build of the package itself, and can be used to inform the build system about the package's configuration and where to find dependencies: .. code-block:: python class MyPackage(PythonPackage): def setup_build_environment(self, env: EnvironmentModifications) -> None: env.set("ENABLE_MY_FEATURE", self.spec.satisfies("+my_feature")) env.set("HDF5_DIR", self.spec["hdf5"].prefix) The ``setup_dependent_build_environment`` method is similar, but it is called for packages that depend on this package. This is often helpful to avoid repetitive configuration in dependent packages. As an example, a package like ``qt`` may want ``QTDIR`` to be set in the build environment of packages that depend on it. This can be done by overriding the ``setup_dependent_build_environment`` method: .. code-block:: python class Qt(Package): def setup_dependent_build_environment( self, env: EnvironmentModifications, dependent_spec: Spec ) -> None: env.set("QTDIR", self.prefix) The ``setup_run_environment`` and ``setup_dependent_run_environment`` are the counterparts for the run environment, primarily used in commands like ``spack load`` and ``spack env activate``. Do note however that these runtime environment variables are *also* relevant during the build process, since Spack effectively creates the runtime environment of build dependencies as part of the build process. For example, if a package ``my-pkg`` depends on ``autoconf`` as a build dependency, and ``autoconf`` needs ``perl`` at runtime, then ``perl``'s runtime environment will be set up during the build of ``my-pkg``. The following diagram will give you an idea when each of these methods is called in a build context: .. image:: images/setup_env.png :align: center Notice that ``setup_dependent_run_environment`` is called once for each dependent package, whereas ``setup_run_environment`` is called only once for the package itself. This means that the former should only be used if the environment variables depend on the dependent package, whereas the latter should be used if the environment variables depend only on the package itself. .. _setting-package-module-variables: Setting package module variables -------------------------------- Apart from modifying environment variables of the dependent package, you can also define Python variables to be used by the dependent. This is done by implementing :meth:`setup_dependent_package `. An example of this can be found in the ``Python`` package: .. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/python/package.py :pyobject: Python.setup_dependent_package :linenos: This allows Python packages to directly use these variables: .. code-block:: python def install(self, spec, prefix): ... install("script.py", python_platlib) .. note:: We recommend using ``setup_dependent_package`` sparingly, as it is not always clear where global variables are coming from when editing a ``package.py`` file. .. _compiler_flags: Compiler flags -------------- Setting compiler flags is a common task, but there are some subtleties that you should be aware of. Compiler flags can be set in three different places: 1. The end user, who can set flags directly from the command line with ``spack install pkg cflags=-O3`` variants or :doc:`compiler configuration `. In either case, these flags become part of the :ref:`concrete spec `. 2. The package author, who defines flags in the package class. 3. The build system itself, which typically has defaults like ``CFLAGS ?= -O2 -g`` or presets like ``CMAKE_BUILD_TYPE=Release``. The main challenge for packagers is to ensure that these flags are combined and applied correctly. .. warning:: A common pitfall when dealing with compiler flags in ``MakefilePackage`` and ``AutotoolsPackage`` is that the user and package author specified flags override the build system defaults. This can inadvertently lead to unoptimized builds. For example, suppose a user requests ``spack install pkg cflags=-Wno-unused`` and the build system defaults to ``CFLAGS=-O2 -g``. If the package takes the user request literally and sets ``CFLAGS=-Wextra`` as an environment variable, then the user-specified flags may *override* the build system defaults, and the build would not be optimized: the ``-O2`` flag would be lost. Whether environment variables like ``CFLAGS`` lead to this problem depends on the build system, and may differ from package to package. Because of this pitfall, Spack tries to work around the build system and defaults to **injecting compiler flags** through the compiler wrappers. This means that the build system is unaware of the extra compiler flags added by Spack. It also means that package authors typically do not need to deal with user-specified compiler flags when writing their package classes. However, there are two cases in which you may need to deal with compiler flags in your package class explicitly: 1. You need to pass default compiler flags to make a build work. This is typical for packages that do not have a configure phase, and requires *you* to set the appropriate flags per compiler. 2. The build system *needs to be aware* of the user-specified compiler flags to prevent a build failure. This is less common, but there are examples of packages that fail to build when ``-O3`` is used for a specific source file. In these cases, you can implement the :meth:`flag_handler ` method in your package class. This method has a curious return type, but once you understand it, it is quite powerful. Here is a simple example: .. code-block:: python class MyPackage(MakefilePackage): def flag_handler(self, name: str, flags: List[str]): if name in ("cflags", "cxxflags"): # Add optimization flags for C/C++ flags.append("-O3") if name == "fflags" and self.spec.satisfies("%fortran=gcc@14:"): # Add a specific flag for Fortran when using GCC 14 or higher flags.append("-fallow-argument-mismatch") # Pass these flags to the compiler wrappers return (flags, None, None) There are multiple things to unpack in this example, so let's go through them step by step. The ``flag_handler`` method is called by Spack once for each of the compiler flags supported in Spack. The ``name`` argument The ``name`` parameter is a string that indicates which compiler flag is being processed. It can be one of the following: * ``cppflags``: C preprocessor flags (e.g. ``-DMY_DEFINE=1``) * ``cflags``: C compilation flags * ``cxxflags``: C++ compilation flags * ``fflags``: Fortran compilation flags * ``ldflags``: Compiler flags for linking (e.g. ``-Wl,-Bstatic``) * ``ldlibs``: Libraries to link against (e.g. ``-lfoo``) The ``flags`` argument The ``flags`` parameter is a list that already contains the user-specified flags, and you can modify it as needed. Return value The return value determines *how* the flags are applied in the build process. It is a triplet that contains the list of flags: * ``(flags, None, None)``: inject the flags through the Spack **compiler wrappers**. This is the default behavior, and it means that the flags are applied directly to the compiler commands without the build system needing to know about them. * ``(None, flags, None)``: set these flags in **environment variables** like ``CFLAGS``, ``CXXFLAGS``, etc. This requires the build system to use these environment variables. * ``(None, None, flags)``: pass these flags **"on the command line"** to the build system. This requires the build system to support passing flags in this way. An example of a build system that supports this is ``CMakePackage``, and Spack will invoke ``cmake -DCMAKE_C_FLAGS=...`` and similar for the other flags. Spack also allows you to refer to common compiler flags in a more generic way, using the ``self.compiler`` object. This includes flags to set the C and C++ standard, as well as the compiler specific OpenMP flags, etc. .. code-block:: python class MyPackage(MakefilePackage): def flag_handler(self, name: str, flags: List[str]): if name == "cflags": # Set the C standard to C11 flags.append(self.compiler.c11_flag) elif name == "cxxflags": # Set the C++ standard to C++17 flags.append(self.compiler.cxx17_flag) return (flags, None, None) If you just want to influence how the flags are passed *without setting additional flags* in your package, Spack provides the following shortcut. To ensure that flags are always set as *environment variables*, you can use: .. code-block:: python from spack.package import * # for env_flags class MyPackage(MakefilePackage): flag_handler = env_flags # Use environment variables for all flags To ensure that flags are always *passed to the build system*, you can use: .. code-block:: python from spack.package import * # for build_system_flags class MyPackage(MakefilePackage): flag_handler = build_system_flags # Pass flags to the build system .. _compiler-wrappers: Compiler wrappers and flags --------------------------- As mentioned in the :ref:`build environment ` section, any package that depends on a language virtual (``c``, ``cxx``, or ``fortran``) not only gets a specific compiler package like ``gcc`` or ``llvm`` as a dependency, but also automatically gets the ``compiler-wrapper`` package injected as a dependency. The ``compiler-wrapper`` package has several responsibilities: * It sets the ``CC``, ``CXX``, and ``FC`` environment variables in the :ref:`build environment `. These variables point to a wrapper executable in the ``compiler-wrapper``'s bin directory, which is a shell script that ultimately invokes the actual, underlying compiler executable. * It ensures that three kinds of compiler flags are passed to the compiler when it is invoked: 1. Flags requested by the user and package author (see :ref:`compiler flags `) 2. Flags needed to locate headers and libraries (during the build as well as at runtime) 3. Target specific flags, like ``-march=x86-64-v3``, translated from the spec's ``target=`` variant. Automatic search flags ^^^^^^^^^^^^^^^^^^^^^^ The flags to locate headers and libraries are the following: * Compile-time library search paths: ``-L$dep_prefix/lib``, ``-L$dep_prefix/lib64`` * Runtime library search paths (RPATHs): ``-Wl,-rpath,$dep_prefix/lib``, ``-Wl,-rpath,$dep_prefix/lib64`` * Include search paths: ``-I$dep_prefix/include`` These flags are added automatically for *each* link-type dependency (and their transitive dependencies) of the package. The exact format of these flags is determined by the compiler being used. These automatic flags are particularly useful in build systems such as ``AutotoolsPackage``, ``MakefilePackage`` and certain ``PythonPackage`` packages that also contain C/C++ code. Typically configure scripts and Makefiles just work out of the box: the right headers are included and the right libraries are linked to. For example, consider a ``libdwarf`` package that just depends on ``libelf`` and specifies it is written in C: .. code-block:: python from spack.package import * from spack_repo.builtin.build_systems.autotools import AutotoolsPackage class Libdwarf(AutotoolsPackage): url = "..." version("1.0", sha256="...") depends_on("c") depends_on("libelf") You may not even have to implement :ref:`helper methods ` like ``configure_args`` to make it work. In the ``configure`` stage Spack by default simply :ref:`runs ` ``configure(f"--prefix={prefix}")``. The configure script picks up the compiler wrapper from the ``CC`` environment variable, and continues to run tests to find the ``libelf`` headers and libraries. Because the compiler wrapper is set up to automatically include the ``-I/include`` and ``-L/lib`` flags, the configure script succeeds and uses the correct ``libelf.h`` header and the ``libelf.so`` library out of the box. .. _handling_rpaths: Runtime library search paths ---------------------------- Spack heavily makes use of `RPATHs `_ on Linux and macOS to make executables directly runnable after installation. Executables are able to find their needed libraries *without* any of the infamous environment variables such as ``LD_LIBRARY_PATH`` on Linux or ``DYLD_LIBRARY_PATH`` on macOS. The :ref:`compiler wrapper ` is the main component that ensures that all binaries built by Spack have the correct RPATHs set. As a package author, you rarely need to worry about RPATHs: the relevant compiler flags are automatically injected through the compiler wrappers, and the build system is blissfully unaware of them. This works for most packages and build systems, with the notable exception of CMake, which has its own RPATH handling. CMake has its own RPATH handling, and distinguishes between build and install RPATHs. By default, during the build it registers RPATHs to all libraries it links to, so that just-built executables can be run during the build itself. Upon installation, these RPATHs are cleared, unless the user defines the install RPATHs. If you use the ``CMakePackage``, Spack automatically sets the ``CMAKE_INSTALL_RPATH_USE_LINK_PATH`` and ``CMAKE_INSTALL_RPATH`` defines to ensure that the install RPATHs are set correctly. For packages that do not fit ``CMakePackage`` but still run ``cmake`` as part of the build, it is recommended to look at :meth:`spack_repo.builtin.build_systems.cmake.CMakeBuilder.std_args` on how to set the install RPATHs correctly. MPI support in Spack --------------------- .. note:: The MPI support section is somewhat outdated and will be updated in the future. .. (This is just a comment not rendered in the docs) An attempt to update this section showed that Spack's handling of MPI has various issues. 1. MPI provider packages tend to set self.spec.mpicc in setup_dependent_package, which is wrong because that function is called for every dependent, meaning that mpi's spec is mutated repeatedly with possibly different values if the dependent_spec is used. 2. The suggestion to fix this was to make the "interface" such that a package class defines properties like `mpicc`, and dependents would do `self["mpi"].mpicc` to get the package attribute instead of the spec attribute. 3. While (2) is cleaner, it simply does not work for all MPI providers, because not all strictly adhere to the interface. The `msmpi` package notably does not have mpicc wrappers, and currently sets `self.spec.mpicc` in `setup_dependent_package` to the C compiler of the dependent, which again is wrong because there are many dependents. It is common for high-performance computing software/packages to use the Message Passing Interface ( ``MPI``). As a result of concretization, a given package can be built using different implementations of MPI such as ``OpenMPI``, ``MPICH`` or ``IntelMPI``. That is, when your package declares that it ``depends_on("mpi")``, it can be built with any of these ``mpi`` implementations. In some scenarios, to configure a package, one has to provide it with appropriate MPI compiler wrappers such as ``mpicc``, ``mpic++``. However, different implementations of ``MPI`` may have different names for those wrappers. Spack provides an idiomatic way to use MPI compilers in your package. To use MPI wrappers to compile your whole build, do this in your ``install()`` method: .. code-block:: python env["CC"] = spec["mpi"].mpicc env["CXX"] = spec["mpi"].mpicxx env["F77"] = spec["mpi"].mpif77 env["FC"] = spec["mpi"].mpifc That's all. A longer explanation of why this works is below. We don't try to force any particular build method on packagers. The decision to use MPI wrappers depends on the way the package is written, on common practice, and on "what works". Loosely, there are three types of MPI builds: 1. Some build systems work well without the wrappers and can treat MPI as an external library, where the person doing the build has to supply includes/libs/etc. This is fairly uncommon. 2. Others really want the wrappers and assume you're using an MPI "compiler" -- i.e., they have no mechanism to add MPI includes/libraries/etc. 3. CMake's ``FindMPI`` needs the compiler wrappers, but it uses them to extract ``-I`` / ``-L`` / ``-D`` arguments, then treats MPI like a regular library. Note that some CMake builds fall into case 2 because they either don't know about or don't like CMake's ``FindMPI`` support -- they just assume an MPI compiler. Also, some Autotools builds fall into case 3 (e.g., `here is an autotools version of CMake's FindMPI `_). Given all of this, we leave the use of the wrappers up to the packager. Spack will support all three ways of building MPI packages. Packaging Conventions ^^^^^^^^^^^^^^^^^^^^^ As mentioned above, in the ``install()`` method, ``CC``, ``CXX``, ``F77``, and ``FC`` point to Spack's wrappers around the chosen compiler. Spack's wrappers are not the MPI compiler wrappers, though they do automatically add ``-I``, ``-L``, and ``-Wl,-rpath`` args for dependencies in a similar way. The MPI wrappers are a bit different in that they also add ``-l`` arguments for the MPI libraries, and some add special ``-D`` arguments to trigger build options in MPI programs. For case 1 above, you generally don't need to do more than patch your Makefile or add configure args as you normally would. For case 3, you don't need to do much of anything, as Spack puts the MPI compiler wrappers in the PATH, and the build will find them and interrogate them. For case 2, things are a bit more complicated, as you'll need to tell the build to use the MPI compiler wrappers instead of Spack's compiler wrappers. All it takes is some lines like this: .. code-block:: python env["CC"] = spec["mpi"].mpicc env["CXX"] = spec["mpi"].mpicxx env["F77"] = spec["mpi"].mpif77 env["FC"] = spec["mpi"].mpifc Or, if you pass CC, CXX, etc. directly to your build with, e.g., ``--with-cc=``, you'll want to substitute ``spec["mpi"].mpicc`` in there instead, e.g.: .. code-block:: python configure("--prefix=%s" % prefix, "--with-cc=%s" % spec["mpi"].mpicc) Now, you may think that doing this will lose the includes, library paths, and RPATHs that Spack's compiler wrappers get you, but we've actually set things up so that the MPI compiler wrappers use Spack's compiler wrappers when run from within Spack. So using the MPI wrappers should really be as simple as the code above. ``spec["mpi"]`` ^^^^^^^^^^^^^^^^^^^^^ Okay, so how does all this work? If your package has a virtual dependency like ``mpi``, then referring to ``spec["mpi"]`` within ``install()`` will get you the concrete ``mpi`` implementation in your dependency DAG. That is a spec object just like the one passed to install, only the MPI implementations all set some additional properties on it to help you out. E.g., in ``openmpi``, you'll find this: .. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/openmpi/package.py :pyobject: Openmpi.setup_dependent_package That code allows the ``openmpi`` package to associate an ``mpicc`` property with the ``openmpi`` spec in the DAG, so that dependents can access it. ``mvapich2`` and ``mpich`` do similar things. So, no matter what MPI you're using, ``spec["mpi"].mpicc`` gets you the location of the MPI compilers. This allows us to have a fairly simple polymorphic interface for information about virtual dependencies like MPI. Wrapping wrappers ^^^^^^^^^^^^^^^^^^^^^ Spack likes to use its own compiler wrappers to make it easy to add ``RPATHs`` to builds, and to try hard to ensure that your builds use the right dependencies. This doesn't play nicely by default with MPI, so we have to do a couple of tricks. 1. If we build MPI with Spack's wrappers, ``mpicc`` and friends will be installed with hard-coded paths to Spack's wrappers, and using them from outside of Spack will fail because they only work within Spack. To fix this, we patch ``mpicc`` and friends to use the regular compilers. Look at the filter_compilers method in ``mpich``, ``openmpi``, or ``mvapich2`` for details. 2. We still want to use the Spack compiler wrappers when Spack is calling ``mpicc``. Luckily, wrappers in all mainstream MPI implementations provide environment variables that allow us to dynamically set the compiler to be used by ``mpicc``, ``mpicxx``, etc. Spack's build environment sets ``MPICC``, ``MPICXX``, etc. for MPICH derivatives and ``OMPI_CC``, ``OMPI_CXX``, etc. for OpenMPI. This makes the MPI compiler wrappers use the Spack compiler wrappers so that your dependencies still get proper RPATHs even if you use the MPI wrappers. MPI on Cray machines ^^^^^^^^^^^^^^^^^^^^^ The Cray programming environment notably uses its own compiler wrappers, which function like MPI wrappers. On Cray systems, the ``CC``, ``cc``, and ``ftn`` wrappers ARE the MPI compiler wrappers, and it's assumed that you'll use them for all of your builds. So on Cray we don't bother with ``mpicc``, ``mpicxx``, etc., Spack MPI implementations set ``spec["mpi"].mpicc`` to point to Spack's wrappers, which wrap the Cray wrappers, which wrap the regular compilers and include MPI flags. That may seem complicated, but for packagers, that means the same code for using MPI wrappers will work, even on a Cray: .. code-block:: python env["CC"] = spec["mpi"].mpicc This is because on Cray, ``spec["mpi"].mpicc`` is just ``spack_cc``. .. _packaging-workflow: Packaging workflow and commands ------------------------------- When you are building packages, you will likely not get things completely right the first time. After having :doc:`created a package `, the edit-install loop is a common workflow to get the package building correctly: .. code-block:: console $ spack edit mypackage $ spack install --verbose mypackage Whenever a build fails, Spack retains the build directory for you to inspect. The location of the build directory is printed in the build output, but you can also find it with the ``spack locate`` command, or navigate to it directly using ``spack cd``: .. code-block:: console $ spack locate mypackage /tmp/spack-stage/spack-stage-mypackage-1-2-3-abcdef $ spack cd mypackage $ pwd /tmp/spack-stage/spack-stage-mypackage-1-2-3-abcdef Inspecting the build environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Once you have navigated to the build directory after a failed build, you may also want to manually run build commands to troubleshoot the issue. This requires you to have all environment variables exactly set up as they are in the :ref:`build environment `. The command .. code-block:: console $ spack build-env mypackage -- /bin/sh is a convenient way to start a subshell with the build environment variables set up. Keeping the stage directory on success ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Sometimes a build completes successfully, but you encounter issues only when you try to run the installed package. In such cases, it can be useful to keep the build directory area to find out what went wrong. By default, ``spack install`` will delete the staging area once a package has been successfully built and installed. Use ``--keep-stage`` to leave the build directory intact: .. code-block:: console $ spack install --keep-stage This allows you to inspect the build directory and potentially debug the build. Once done, you could remove all sources and build directories with: .. code-block:: console $ spack clean --stage Keeping the install prefix on failure ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Conversely, if a build fails but *has* installed some files, you may want to keep the install prefix to diagnose the issue. By default, ``spack install`` deletes the install directory if anything fails during build. The ``--keep-prefix`` option allows you to keep the install prefix regardless of the build outcome. .. code-block:: console $ spack install --keep-prefix .. _cmd-spack-graph: Understanding the DAG ^^^^^^^^^^^^^^^^^^^^^ Sometimes when you are packaging software, it is useful to have a better understanding of the dependency graph of a package. The ``spack spec `` command gives you a good overview of dependencies right on the command line, but the tree structure may not be entirely clear. The ``spack graph `` command can help you visualize the dependency graph better. By default it generates an ASCII rendering of a spec's dependency graph, which can be complementary to the output of ``spack spec``. Much more powerful is the set of flags ``spack graph --color --dot ...``, which turns the dependency graph into `Dot `_ format. Tools such as `Graphviz `_ can render this. For example, you can generate a PDF of the dependency graph of a package with the following command: .. code-block:: console $ spack graph --dot hdf5 | dot -Tpdf > hdf5.pdf There are several online tools that can render Dot files directly in your browser as well. Another useful flag is ``spack graph --deptype=...`` which can reduce the size of the graph, by filtering out certain types of dependencies. For example, supplying ``--deptype=link`` will limit to link type dependencies only. The default is ``--deptype=all``, which is equivalent to ``--deptype=build,link,run,test``. Options for ``deptype`` include: * Any combination of ``build``, ``link``, ``run``, and ``test`` separated by commas. * ``all`` for all types of dependencies. ================================================ FILE: lib/spack/docs/packaging_guide_creation.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide for developers and administrators on how to package software for Spack, covering the structure of a package, creating and editing packages, and defining dependencies and variants. .. list-table:: :widths: 25 25 25 25 :header-rows: 0 :width: 100% * - **1. Creation** - :doc:`2. Build ` - :doc:`3. Testing ` - :doc:`4. Advanced ` Packaging Guide: defining a package =================================== This packaging guide is intended for developers or administrators who want to package software so that Spack can install it. It assumes that you have at least some familiarity with Python, and that you've read the :ref:`basic usage guide `, especially the part about :ref:`specs `. There are two key parts of Spack: #. **Specs**: expressions for describing builds of software, and #. **Packages**: Python modules that describe how to build and test software according to a spec. Specs allow a user to describe a *particular* build in a way that a package author can understand. Packages allow the packager to encapsulate the build logic for different versions, compilers, options, platforms, and dependency combinations in one place. Essentially, a package translates a spec into build logic. It also allows the packager to write spec-specific tests of the installed software. Packages in Spack are written in pure Python, so you can do anything in Spack that you can do in Python. Python was chosen as the implementation language for two reasons. First, Python is ubiquitous in the scientific software community. Second, it has many powerful features to help make package writing easy. .. _setting-up-for-package-development: Setting up for package development ---------------------------------- For developing new packages or working with existing ones, it's helpful to have the ``spack/spack-packages`` repository in a convenient location like your home directory, rather than the default ``~/.spack/package_repos//``. If you plan to contribute changes back to Spack, we recommend creating a fork of the `packages repository `_. See `GitHub's fork documentation `_ for details. Once you have a fork, clone it: .. code-block:: console $ git clone --depth=100 git@github.com:YOUR-USERNAME/spack-packages.git ~/spack-packages $ cd ~/spack-packages $ git remote add --track develop upstream git@github.com:spack/spack-packages.git Then configure Spack to use your local repository: .. code-block:: console $ spack repo set --destination ~/spack-packages builtin Before starting work, it's useful to create a new branch in your local repository. .. code-block:: console $ git checkout -b add-my-package Lastly, verify that Spack is picking up the right repository by checking the location of a known package, like ``zlib``: .. code-block:: console $ spack location --package-dir zlib /home/your-username/spack-packages/repos/spack_repo/builtin/packages/zlib With this setup, you can conveniently access the package files, and contribute changes back to Spack. Structure of a package ---------------------- A Spack package is a Python module ``package.py`` stored in a package repository. It contains a package class and sometimes a builder class that define its metadata and build behavior. The typical structure of a package is as follows: .. code-block:: python # spack_repo/builtin/packages/example/package.py # import of package / builder classes from spack_repo.builtin.build_systems.cmake import CMakePackage # import Package API from spack.package import * class Example(CMakePackage): """Example package""" # package description # Metadata and Directives homepage = "https://example.com" url = "https://example.com/example/v2.4.0.tar.gz" maintainers("github_user1", "github_user2") license("UNKNOWN", checked_by="github_user1") # version directives listed in order with the latest first version("2.4.0", sha256="845ccd79ed915fa2dedf3b2abde3fffe7f9f5673cc51be88e47e6432bd1408be") version("2.3.0", sha256="cd3274e0abcbc2dfb678d87595e9d3ab1c6954d7921d57a88a23cf4981af46c9") # variant directives expose build options variant("feature", default=False, description="Enable a specific feature") variant("codec", default=False, description="Build the CODEC executables") # dependency directives declare required software depends_on("cxx", type="build") depends_on("libfoo", when="+feature") # Build Instructions def cmake_args(self): return [ self.define_from_variant("BUILD_CODEC", "codec"), self.define("EXAMPLE_OPTIMIZED", False), self.define("BUILD_THIRDPARTY", False), ] The package class is named after the package, and can roughly be divided into two parts: * **metadata and directives**: attributes and directives that describe the package, such as its homepage, maintainers, license, variants, and dependencies. This is the declarative part of the package. * **build instructions**: methods that define how to build and install the package, such as `cmake_args()`. This is the imperative part of the package. In this part of the packaging guide we will cover the **metadata and directives** in detail. In the :doc:`second part `, we will cover the **build instructions**, including how to write custom build logic for different build systems. Package Names and the Package Directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Packages are referred to by their **package names**, whether it's on the command line or in a package recipe. Package names can contain lowercase letters, numbers, and dashes. Every package lives as a ``package.py`` file in a **package directory** inside a :ref:`package repository `. Usually the package name coincides with the directory name on the filesystem: the ``libelf`` package corresponds to the ``libelf/package.py`` file. .. note:: **Package name to directory mapping**. There is a one to one mapping between package names and package directories. Usually the mapping is trivial: the package name is the same as the directory name. However, there are a few exceptions to this rule: 1. Hyphens in package names are replaced by underscores in directory names. For example, the package name ``py-numpy`` maps to ``py_numpy/package.py``. 2. Names starting with numbers get an underscore prefix. For example, the package name ``7zip`` maps to ``_7zip/package.py``. 3. Package names that are reserved keywords in Python are also prefixed with an underscore. For example, the package name ``pass`` maps to ``_pass/package.py``. This ensures that every package directory is a valid Python module name. Package class names ^^^^^^^^^^^^^^^^^^^ Spack loads ``package.py`` files dynamically, and it needs to find a special class name in the file for the load to succeed. The **package class** is formed by converting words separated by ``-`` in the package name to CamelCase. If the package name starts with a number, we prefix the class name with ``_``. Here are some examples: ================= ================= Package Name Class Name ================= ================= ``foo-bar`` ``FooBar`` ``3proxy`` ``_3proxy`` ================= ================= In general, you won't have to remember this naming convention because :ref:`cmd-spack-create` and :ref:`cmd-spack-edit` handle the details for you. .. _creating-and-editing-packages: Creating and editing packages ----------------------------- Spack has various commands that help you create and edit packages. Spack can create the boilerplate for new packages and open them in your editor for you to fill in. It can also help you edit existing packages, so you don't have to navigate to the package directory manually. .. _controlling-the-editor: Controlling the editor ^^^^^^^^^^^^^^^^^^^^^^ When Spack needs to open an editor for you (e.g., for commands like :ref:`cmd-spack-create` or :ref:`cmd-spack-edit`), it looks at several environment variables to figure out what to use. The order of precedence is: * ``SPACK_EDITOR``: highest precedence, in case you want something specific for Spack; * ``VISUAL``: standard environment variable for full-screen editors like ``vim`` or ``emacs``; * ``EDITOR``: older environment variable for your editor. You can set any of these to the command you want to run, e.g., in ``bash`` you might run one of these: .. code-block:: console $ export VISUAL=vim $ export EDITOR="emacs -nw" $ export SPACK_EDITOR=nano If Spack finds none of these variables set, it will look for ``vim``, ``vi``, ``emacs``, ``nano``, and ``notepad``, in that order. .. _cmd-spack-create: Creating new packages ^^^^^^^^^^^^^^^^^^^^^ To create a new package, Spack provides a command that generates a ``package.py`` file in an existing repository, with a boilerplate package template. Here's an example: .. code-block:: console $ spack create https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 Spack examines the tarball URL and tries to figure out the name of the package to be created. If the name contains uppercase letters, these are automatically converted to lowercase. If the name contains underscores or periods, these are automatically converted to dashes. Spack also searches for *additional* versions located in the same directory on the website. Spack prompts you to tell you how many versions it found and asks you how many you would like to download and checksum: .. code-block:: console $ spack create https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 ==> This looks like a URL for gmp ==> Found 16 versions of gmp: 6.1.2 https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 6.1.1 https://gmplib.org/download/gmp/gmp-6.1.1.tar.bz2 6.1.0 https://gmplib.org/download/gmp/gmp-6.1.0.tar.bz2 ... 5.0.0 https://gmplib.org/download/gmp/gmp-5.0.0.tar.bz2 How many would you like to checksum? (default is 1, q to abort) Spack will automatically download the number of tarballs you specify (starting with the most recent) and checksum each of them. You do not *have* to download all of the versions up front. You can always choose to download just one tarball initially, and run :ref:`cmd-spack-checksum` later if you need more versions. Spack automatically creates a directory in the appropriate repository, generates a boilerplate template for your package, and opens up the new ``package.py`` in your favorite ``$EDITOR`` (see :ref:`controlling-the-editor` for details): .. code-block:: python :linenos: # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # ---------------------------------------------------------------------------- # If you submit this package back to Spack as a pull request, # please first remove this boilerplate and all FIXME comments. # # This is a template package file for Spack. We've put "FIXME" # next to all the things you'll want to change. Once you've handled # them, you can save this file and test your package like this: # # spack install gmp # # You can edit this file again by typing: # # spack edit gmp # # See the Spack documentation for more information on packaging. # ---------------------------------------------------------------------------- import spack_repo.builtin.build_systems.autotools from spack.package import * class Gmp(AutotoolsPackage): """FIXME: Put a proper description of your package here.""" # FIXME: Add a proper url for your package's homepage here. homepage = "https://www.example.com" url = "https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2" # FIXME: Add a list of GitHub accounts to # notify when the package is updated. # maintainers("github_user1", "github_user2") # FIXME: Add the SPDX identifier of the project's license below. # See https://spdx.org/licenses/ for a list. Upon manually verifying # the license, set checked_by to your Github username. license("UNKNOWN", checked_by="github_user1") version("6.2.1", sha256="eae9326beb4158c386e39a356818031bd28f3124cf915f8c5b1dc4c7a36b4d7c") # FIXME: Add dependencies if required. # depends_on("foo") def configure_args(self): # FIXME: Add arguments other than --prefix # FIXME: If not needed delete the function args = [] return args The tedious stuff (creating the class, checksumming archives) has been done for you. Spack correctly detected that ``gmp`` uses the ``autotools`` build system, so it created a new ``Gmp`` package that subclasses the ``AutotoolsPackage`` base class. The default installation procedure for a package subclassing the ``AutotoolsPackage`` is to go through the typical process of: .. code-block:: bash ./configure --prefix=/path/to/installation/directory make make check make install For most Autotools packages, this is sufficient. If you need to add additional arguments to the ``./configure`` call, add them via the ``configure_args`` function. In the generated package, the download ``url`` attribute is already set. All the things you still need to change are marked with ``FIXME`` labels. You can delete the commented instructions between the Spack license and the first import statement after reading them. The remaining tasks to complete are as follows: #. Add a description. Immediately inside the package class is a *docstring* in triple-quotes (``"""``). It is used to generate the description shown when users run ``spack info``. #. Change the ``homepage`` to a useful URL. The ``homepage`` is displayed when users run ``spack info`` so that they can learn more about your package. #. Add a comma-separated list of maintainers. Add a list of GitHub accounts of people who want to be notified any time the package is modified. See :ref:`maintainers`. #. Change the ``license`` to the correct license. The ``license`` is displayed when users run ``spack info`` so that they can learn more about your package. See :ref:`package_license`. #. Add ``depends_on()`` calls for the package's dependencies. ``depends_on`` tells Spack that other packages need to be built and installed before this one. See :ref:`dependencies`. #. Get the installation working. Your new package may require specific flags during ``configure``. These can be added via ``configure_args``. If no arguments are needed at this time, change the implementation to ``return []``. Specifics will differ depending on the package and its build system. :ref:`installation_process` is covered in detail later. Further package creation options """""""""""""""""""""""""""""""" If you do not have a tarball URL, you can still use ``spack create`` to generate the boilerplate for a package. .. code-block:: console $ spack create --name intel This will create a simple ``intel`` package with an ``install()`` method that you can craft to install your package. Likewise, you can force the build system to be used with ``--template`` and, in case it's needed, you can overwrite a package already in the repository with ``--force``: .. code-block:: console $ spack create --name gmp https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 $ spack create --force --template autotools https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 A complete list of available build system templates can be found by running ``spack create --help``. .. _cmd-spack-edit: Editing existing packages ^^^^^^^^^^^^^^^^^^^^^^^^^ One of the easiest ways to learn how to write packages is to look at existing ones. You can open an existing package in your editor using the ``spack edit`` command: .. code-block:: console $ spack edit gmp If you used ``spack create`` to create a package, you can get back to it later with ``spack edit``. The ``spack edit`` command saves you the trouble of figuring out the package location and navigating to it. If needed, you can still find the package location using the ``spack location`` command: .. code-block:: console $ spack location --package-dir gmp ~/spack-packages/repos/spack_repo/builtin/packages/gmp/ and with shell support enabled, you can also enter to the package directory: .. code-block:: console $ spack cd --package-dir gmp If you want to edit multiple packages at once, you can run .. code-block:: console $ spack edit without specifying a package name, which will open the directory containing all the packages in your editor. Finally, the commands ``spack location --repo`` and ``spack cd --repo`` help you navigate to the root of the package repository. Source code and versions ------------------------ Spack packages are designed to be built from source code. Typically every package version has a corresponding source code archive, which Spack downloads and verifies before building the package. .. _versions-and-fetching: Versions and URLs ^^^^^^^^^^^^^^^^^ The most straightforward way to add new versions to your package is to add a line like this in the package class: .. code-block:: python class Foo(Package): url = "http://example.com/foo-8.2.1.tar.gz" version("8.2.1", sha256="85f477fdd6f8194ab6a0e7afd1cb34eae46c775278d5db9d7ebc9ddaf50c23b1") version("8.2.0", sha256="427b2e244e73385515b8ad4f75358139d44a4c792d9b26ddffe2582835cedd8c") version("8.1.2", sha256="67630a20f92ace137e68b67f13010487a03e4f036cdd328e199db85d24a434a4") .. note:: By convention, we list versions in descending order, from newest to oldest. .. note:: :ref:`Bundle packages ` do not have source code so there is nothing to fetch. Consequently, their version directives consist solely of the version name (e.g., ``version("202309")``). Notice how you only have to specify the URL once, in the ``url`` field. Spack is smart enough to extrapolate the URL for each version based on the version number and download version ``8.2.0`` of the ``Foo`` package above from ``http://example.com/foo-8.2.0.tar.gz``. If the URL is particularly complicated or changes based on the release, you can override the default URL generation algorithm by defining your own :py:meth:`~spack.package.PackageBase.url_for_version` function. For example, the download URL for OpenMPI contains the ``major.minor`` version in one spot and the ``major.minor.patch`` version in another: .. code-block:: text https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2 In order to handle this, you can define a ``url_for_version()`` function like so: .. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/openmpi/package.py :pyobject: Openmpi.url_for_version With the use of this ``url_for_version()``, Spack knows to download OpenMPI ``2.1.1`` from .. code-block:: text http://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2 but download OpenMPI ``1.10.7`` from .. code-block:: text http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.7.tar.bz2 You'll notice that OpenMPI's ``url_for_version()`` function makes use of a special ``Version`` function called ``up_to()``. When you call ``version.up_to(2)`` on a version like ``1.10.0``, it returns ``1.10``. ``version.up_to(1)`` would return ``1``. This can be very useful for packages that place all ``X.Y.*`` versions in a single directory and then places all ``X.Y.Z`` versions in a sub-directory. There are a few ``Version`` properties you should be aware of. We generally prefer numeric versions to be separated by dots for uniformity, but not all tarballs are named that way. For example, ``icu4c`` separates its major and minor versions with underscores, like ``icu4c-57_1-src.tgz``. The value ``57_1`` can be obtained with the use of the ``version.underscored`` property. There are other separator properties as well: =================== ====== Property Result =================== ====== version.dotted 1.2.3 version.dashed 1-2-3 version.underscored 1_2_3 version.joined 123 =================== ====== .. note:: Python properties don't need parentheses. ``version.dashed`` is correct. ``version.dashed()`` is incorrect. In addition, these version properties can be combined with ``up_to()``. For example: .. code-block:: pycon >>> version = Version("1.2.3") >>> version.up_to(2).dashed Version("1-2") >>> version.underscored.up_to(2) Version("1_2") As you can see, order is not important. Just keep in mind that ``up_to()`` and the other version properties return ``Version`` objects, not strings. If a URL cannot be derived systematically, or there is a special URL for one of its versions, you can add an explicit URL for a particular version: .. code-block:: python version( "8.2.1", sha256="91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca", url="http://example.com/foo-8.2.1-special-version.tar.gz", ) When you supply a custom URL for a version, Spack uses that URL *verbatim* and does not perform extrapolation. The order of precedence of these methods is: #. package-level ``url`` #. ``url_for_version()`` #. version-specific ``url`` so if your package contains a ``url_for_version()``, it can be overridden by a version-specific ``url``. If your package does not contain a package-level ``url`` or ``url_for_version()``, Spack can determine which URL to download from even if only some of the versions specify their own ``url``. Spack will use the nearest URL *before* the requested version. This is useful for packages that have an easy to extrapolate URL, but keep changing their URL format every few releases. With this method, you only need to specify the ``url`` when the URL changes. .. _checksum-verification: Checksum verification ^^^^^^^^^^^^^^^^^^^^^ In the above example we see that each version is associated with a ``sha256`` checksum. Spack uses these checksums to verify that downloaded source code has not been modified, corrupted or compromised. Therefore, Spack requires that all URL downloads have a checksum, and refuses to install packages when checksum verification fails. .. note:: While this requirement can be disabled for development with ``spack install --no-checksum``, it is **not recommended**. .. warning:: **Trusted Downloads.** It is critical from a security and reproducibility standpoint that Spack be able to verify the downloaded source. This is accomplished using a hash. For URL downloads, Spack supports multiple cryptographic hash algorithms, including ``sha256`` (recommended), ``sha384`` and ``sha512``. See :ref:`version urls ` for more information. For repository downloads, which we will cover in more detail later, this is done by specifying a **full commit hash** (e.g., :ref:`git `, :ref:`hg `). .. _cmd-spack-checksum: Automatically adding new versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``spack checksum`` command can be used to automate the process of adding new versions to a package, assuming the package's download URLs follow a consistent pattern. ``spack checksum`` """""""""""""""""" Using ``spack checksum`` is straightforward: .. code-block:: console $ spack checksum libelf ==> Found 16 versions of libelf. 0.8.13 http://www.mr511.de/software/libelf-0.8.13.tar.gz 0.8.12 http://www.mr511.de/software/libelf-0.8.12.tar.gz 0.8.11 http://www.mr511.de/software/libelf-0.8.11.tar.gz 0.8.10 http://www.mr511.de/software/libelf-0.8.10.tar.gz 0.8.9 http://www.mr511.de/software/libelf-0.8.9.tar.gz 0.8.8 http://www.mr511.de/software/libelf-0.8.8.tar.gz 0.8.7 http://www.mr511.de/software/libelf-0.8.7.tar.gz 0.8.6 http://www.mr511.de/software/libelf-0.8.6.tar.gz 0.8.5 http://www.mr511.de/software/libelf-0.8.5.tar.gz ... 0.5.2 http://www.mr511.de/software/libelf-0.5.2.tar.gz How many would you like to checksum? (default is 1, q to abort) This does the same thing that ``spack create`` does, but it allows you to go back and add new versions easily as you need them (e.g., as they're released). It fetches the tarballs you ask for and prints out a list of ``version`` commands ready to copy/paste into your package file: .. code-block:: console ==> Checksummed new versions of libelf: version("0.8.13", sha256="ec6ddbe4b1ac220244230b040fd6a5a102a96337603e703885848ff64cb582a5") version("0.8.12", sha256="46db404a287b3d17210b4183cbc7055d7b8bbcb15957daeb51f2dc06002ca8a3") version("0.8.11", sha256="e5be0f5d199ad11fbc74e59a8e120cc8b6fbcadaf1827c4e8e6a133ceaadbc4c") version("0.8.10", sha256="f1708dd17a476a7abaf6c395723e0745ba8f6b196115513b6d8922d4b5bfbab4") .. note:: ``spack checksum`` assumes that Spack can extrapolate new URLs from an existing URL in the package, and that Spack can find similar URLs on a webpage. If that's not possible, e.g., if the package's developers don't name their source archive consistently, you'll need to manually add ``version`` calls yourself. By default, Spack will search for new versions by scraping the parent URL component of the source archive you gave it in the ``url`` attribute. So, if the sources are at ``http://example.com/downloads/foo-1.0.tar.gz``, Spack computes a *list URL* from it ``http://example.com/downloads/``, and scans that for links to other versions of the package. If you need to search another path for download links, you can supply some extra attributes that control how your package finds new versions. See the documentation on :ref:`attribute_list_url` and :ref:`attribute_list_depth`. .. _git_version_provenance: Git Version Provenance ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Checksummed assets are preferred but there are a few notable exceptions such as git branches and tags i.e ``pkg@develop``. These versions do not naturally have source provenance because they refer to a range of commits (branches) or can be changed outside the spack packaging infrastructure (tags). Without source provenance we cannot have full provenance. Spack has a reserved variant to allow users to complete provenance for these cases: ``pkg@develop commit=``. The ``commit`` variant must be supplied using the full 40 character commit SHA. Using a partial commit SHA or assigning the ``commit`` variant to a version that is not using a branch or tag reference will lead to an error during concretization. Spack will attempt to establish git version provenance by looking up commit SHA's for branch and tag based versions during concretization. There are 3 sources that it uses. In order, they are 1. The local cached downloads (already cached source code for the version needing provenance) 2. Source mirrors (compressed archives of the source code) 3. The git url provided in the package definition If Spack is unable to determine what the commit should be during concretization a warning will be issued. Users may also specify which commit SHA they want with the spec since it is simply a variant. In this case, or in the case of develop specs (see :ref:`cmd-spack-develop`), Spack will skip attempts to assign the commit SHA automatically. .. note:: Users wanting to track the latest commits from the internet should utilize ``spack clean --downloads`` prior to concretization to clean out cached downloads that will short-circuit internet queries. Disabling source mirrors or ensuring they don't contain branch/tag based versions will also be necessary. Above all else, the most robust way to ensure binaries have their desired commits is to provide the SHAs via user-specs or config i.e. ``commit=``. .. _attribute_list_url: ``list_url`` """""""""""" This optional attribute can be set to tell Spack where to scan for links to other versions of the package. For example, the following package has a ``list_url`` attribute that points to a page listing all available versions of the package: .. code-block:: python :linenos: class Example(Package): homepage = "http://www.example.com" url = "http://www.example.com/libexample-1.2.3.tar.gz" list_url = "http://www.example.com/downloads/all-versions.html" .. _attribute_list_depth: ``list_depth`` """""""""""""" Many packages have a listing of available versions on a single webpage, but not all do. For example, ``mpich`` has a tarball URL that looks like this: .. code-block:: python url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" But its downloads are a few clicks away from ``http://www.mpich.org/static/downloads/``. So, we need to add a ``list_url`` *and* a ``list_depth`` attribute: .. code-block:: python :linenos: class Mpich(Package): homepage = "http://www.mpich.org" url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" list_url = "http://www.mpich.org/static/downloads/" list_depth = 1 By default, Spack only looks at the top-level page available at ``list_url``. ``list_depth = 1`` tells it to follow up to 1 level of links from the top-level page. Note that here, this implies 1 level of subdirectories, as the ``mpich`` website is structured much like a filesystem. But ``list_depth`` really refers to link depth when spidering the page. .. _mirrors-of-the-main-url: Mirrors of the main URL ^^^^^^^^^^^^^^^^^^^^^^^ Spack supports listing mirrors of the main URL in a package by defining the ``urls`` attribute: .. code-block:: python class Foo(Package): urls = ["http://example.com/foo-1.0.tar.gz", "http://mirror.com/foo-1.0.tar.gz"] instead of just a single ``url``. This attribute is a list of possible URLs that will be tried in order when fetching packages. Notice that either one of ``url`` or ``urls`` can be present in a package, but not both at the same time. A well-known case of packages that can be fetched from multiple mirrors is that of GNU. For that, Spack goes a step further and defines a mixin class that takes care of all of the plumbing and requires packagers to just define a proper ``gnu_mirror_path`` attribute: .. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/autoconf/package.py :lines: 9-18 .. _preferred_versions: Preferring versions over others ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When users install a package without constraining the versions, Spack will typically pick the latest version available. Usually this is the desired behavior, but as a packager you may know that the latest version is not mature enough or has known issues that make it unsuitable for production use. In this case, you can mark an older version as preferred using the ``preferred=True`` argument in the ``version`` directive, so that Spack will default to the latest *preferred* version. .. code-block:: python class Foo(Package): version("2.0.0", sha256="...") version("1.2.3", sha256="...", preferred=True) See the section on :ref:`version ordering ` for more details and exceptions on how the latest version is computed. .. _deprecate: Deprecating old versions ^^^^^^^^^^^^^^^^^^^^^^^^ There are many reasons to remove old versions of software: #. Security vulnerabilities (most serious reason) #. No longer available for download (right to be forgotten) #. Maintainer/developer inability/unwillingness to support old versions #. Changing build systems that increase package complexity #. Changing dependencies/patches/resources/flags that increase package complexity #. Package or version rename At the same time, there are many reasons to keep old versions of software: #. Reproducibility #. Requirements for older packages (e.g., some packages still rely on Qt 3) In general, you should not remove old versions from a ``package.py`` directly. Instead, you should first deprecate them using the following syntax: .. code-block:: python version("1.2.3", sha256="...", deprecated=True) This has two effects. First, ``spack info`` will no longer advertise that version. Second, commands like ``spack install`` that fetch the package will require user approval: .. code-block:: spec $ spack install openssl@1.0.1e ==> Warning: openssl@1.0.1e is deprecated and may be removed in a future Spack release. ==> Fetch anyway? [y/N] If you use ``spack install --deprecated``, this check can be skipped. This also applies to package recipes that are renamed or removed. You should first deprecate all versions before removing a package. If you need to rename it, you can deprecate the old package and create a new package at the same time. Version deprecations should always last at least one release cycle of the builtin package repository before the version is completely removed. No version should be removed without such a deprecation process. This gives users a chance to complain about the deprecation in case the old version is needed for some application. If you require a deprecated version of a package, simply submit a PR to remove ``deprecated=True`` from the package. However, you may be asked to help maintain this version of the package if the current maintainers are unwilling to support this older version. .. _version-comparison: Version ordering ^^^^^^^^^^^^^^^^ Without :ref:`version constraints `, :ref:`preferences ` and :ref:`deprecations `, Spack will always pick *the latest* version as defined in the package. What latest means is determined by the version comparison rules defined in Spack, *not* the order in which versions are listed in the package file. Spack imposes a generic total ordering on the set of versions, independently from the package they are associated with. Most Spack versions are numeric, a tuple of integers; for example, ``0.1``, ``6.96``, or ``1.2.3.1``. In this very basic case, version comparison is lexicographical on the numeric components: ``1.2 < 1.2.1 < 1.2.2 < 1.10``. Other separators for components are also possible, for example ``2025-03-01 < 2025-06``. Spack can also support string components such as ``1.1.1a`` and ``1.y.0``. String components are considered less than numeric components, so ``1.y.0 < 1.0``. This is for consistency with `RPM `_. String components do not have to be separated by dots or any other delimiter. So, the contrived version ``1y0`` is identical to ``1.y.0``. Pre-release suffixes also contain string parts, but they are handled in a special way. For example ``1.2.3alpha1`` is parsed as a pre-release of the version ``1.2.3``. This allows Spack to order it before the actual release: ``1.2.3alpha1 < 1.2.3``. Spack supports alpha, beta and release candidate suffixes: ``1.2alpha1 < 1.2beta1 < 1.2rc1 < 1.2``. Any suffix not recognized as a pre-release is treated as an ordinary string component, so ``1.2 < 1.2-mysuffix``. Finally, there are a few special string components that are considered "infinity versions". They include ``develop``, ``main``, ``master``, ``head``, ``trunk``, and ``stable``, in descending order. For example: ``1.2 < develop``. These are useful for specifying the most recent development version of a package (often a moving target like a git branch), without assigning a specific version number. Infinity versions are not automatically used when determining the latest version of a package unless explicitly required by another package or user. More formally, the order on versions is defined as follows. A version string is split into a list of components based on delimiters such as ``.``, ``-``, ``_``, and string boundaries. The components are split into the **release** and a possible **pre-release** (if the last component is numeric and the second to last is a string ``alpha``, ``beta`` or ``rc``). The release components are ordered lexicographically, with comparison between different types of components as follows: #. The following special strings are considered larger than any other numeric or non-numeric version component, and satisfy the following order between themselves: ``develop > main > master > head > trunk > stable``. #. Numbers are ordered numerically, are less than special strings, and larger than other non-numeric components. #. All other non-numeric components are less than numeric components, and are ordered alphabetically. Finally, if the release components are equal, the pre-release components are used to break the tie. The logic behind this sort order is two-fold: #. Non-numeric versions are usually used for special cases while developing or debugging a piece of software. Keeping most of them less than numeric versions ensures that Spack chooses numeric versions by default whenever possible. #. The most-recent development version of a package will usually be newer than any released numeric versions. This allows the ``@develop`` version to satisfy dependencies like ``depends_on(abc, when="@x.y.z:")`` .. _vcs-fetch: Fetching from code repositories ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For some packages, source code is provided in a Version Control System (VCS) repository rather than in a tarball. Spack can fetch packages from VCS repositories. Currently, Spack supports fetching with :ref:`Git `, :ref:`Mercurial (hg) `, :ref:`Subversion (svn) `, and :ref:`CVS (cvs) `. In all cases, the destination is the standard stage source path. To fetch a package from a source repository, Spack needs to know which VCS to use and where to download from. Much like with ``url``, package authors can specify a class-level ``git``, ``hg``, ``svn``, or ``cvs`` attribute containing the correct download location. Many packages developed with Git have both a Git repository as well as release tarballs available for download. Packages can define both a class-level tarball URL and VCS. For example: .. code-block:: python class Trilinos(CMakePackage): homepage = "https://trilinos.org/" url = "https://github.com/trilinos/Trilinos/archive/trilinos-release-12-12-1.tar.gz" git = "https://github.com/trilinos/Trilinos.git" version("develop", branch="develop") version("master", branch="master") version("12.12.1", sha256="87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7") version("12.10.1", sha256="0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f") version("12.8.1", sha256="a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478") If a package contains both a ``url`` and ``git`` class-level attribute, Spack decides which to use based on the arguments to the ``version()`` directive. Versions containing a specific branch, tag, commit or revision are assumed to be for VCS download methods, while versions containing a checksum are assumed to be for URL download methods. Like ``url``, if a specific version downloads from a different repository than the default repo, it can be overridden with a version-specific argument. .. note:: In order to reduce ambiguity, each package can only have a single VCS top-level attribute in addition to ``url``. In the rare case that a package uses multiple VCS, a fetch strategy can be specified for each version. For example, the ``rockstar`` package contains: .. code-block:: python class Rockstar(MakefilePackage): homepage = "https://bitbucket.org/gfcstanford/rockstar" version("develop", git="https://bitbucket.org/gfcstanford/rockstar.git") version("yt", hg="https://bitbucket.org/MatthewTurk/rockstar") .. _git-fetch: Git """"""" Git fetching supports the following parameters to the ``version`` directive: * ``git``: URL of the git repository, if different than the class-level ``git``. * ``branch``: Name of a :ref:`branch ` to fetch. * ``tag``: Name of a :ref:`tag ` to fetch. * ``commit``: SHA hash (or prefix) of a :ref:`commit ` to fetch. * ``submodules``: Also fetch :ref:`submodules ` recursively when checking out this repository. * ``submodules_delete``: A list of submodules to forcibly delete from the repository after fetching. Useful if a version in the repository has submodules that have disappeared/are no longer accessible. * ``get_full_repo``: Ensure the full git history is checked out with all remote branch information. Normally (``get_full_repo=False``, the default), the git option ``--depth 1`` will be used if the version of git and the specified transport protocol support it, and ``--single-branch`` will be used if the version of git supports it. * ``git_sparse_paths``: Only clone the provided :ref:`relative paths `. The destination directory for the clone is the standard stage source path. .. note:: ``tag`` and ``branch`` should not be combined in the version parameters. We strongly recommend that all ``tag`` entries be paired with ``commit``. .. warning:: **Trusted Downloads.** It is critical from a security and reproducibility standpoint that Spack be able to verify the downloaded source. Providing the full ``commit`` SHA hash allows for Spack to preserve provenance for all binaries since git commits are guaranteed to be unique points in the git history. Whereas, the mutable nature of branches and tags cannot provide such a guarantee. A git download *is trusted* only if the full commit SHA is specified. Therefore, it is *the* recommended way to securely download from a Git repository. .. _git-default-branch: Default branch A version with only a name results in fetching a repository's default branch: .. code-block:: python class Example(Package): git = "https://github.com/example-project/example.git" version("develop") Aside from use of HTTPS, there is no way to verify that the repository has not been compromised. Furthermore, the commit you get when you install the package likely won't be the same commit that was used when the package was first written. There is also the risk that the default branch may change. .. warning:: This download method is **untrusted**, and is **not recommended**. It is better to specify a branch name (see :ref:`below `). .. _git-branches: Branches To fetch a particular branch, use the ``branch`` parameter, preferably with the same name as the version. For example, .. code-block:: python version("main", branch="main") version("experimental", branch="experimental") Branches are moving targets, which means the commit you get when you install the package likely won't be the one used when the package was first written. .. note:: Common branch names are special in terms of how Spack determines the latest version of a package. See "infinity versions" in :ref:`version ordering ` for more information. .. warning:: This download method is **untrusted**, and is **not recommended** for production installations. .. _git-tags: Tags To fetch from a particular tag, use ``tag`` instead: .. code-block:: python version("1.0.1", tag="v1.0.1") While tags are generally more stable than branches, Git allows tags to be moved. Many developers use tags to denote rolling releases, and may move the tag when a bug is fixed. .. warning:: This download method is **untrusted**, and is **not recommended**. If you must use a ``tag``, it is recommended to combine it with the ``commit`` option (see :ref:`below `). .. _git-commits: Commits To fetch a particular commit, use the ``commit`` argument: .. code-block:: python version("2014-10-08", commit="1e6ef73d93a28240f954513bc4c2ed46178fa32b") version("1.0.4", tag="v1.0.4", commit="420136f6f1f26050d95138e27cf8bc905bc5e7f52") It may be useful to provide a saner version for commits like this, e.g., you might use the date as the version, as done in the first example above. Or, if you know the commit at which a release was cut, you can use the release version. It is up to the package author to decide which of these options makes the most sense. .. warning:: A git download is *trusted only if* the **full commit sha** is specified. .. hint:: **Avoid using the commit hash as the version.** It is not recommended to use the commit hash as the version itself, since it won't sort properly for :ref:`version ordering ` purposes. .. _git-submodules: Submodules You can supply ``submodules=True`` to cause Spack to fetch submodules recursively along with the repository. .. code-block:: python version("1.1.0", commit="907d5f40d653a73955387067799913397807adf3", submodules=True) If a package needs more fine-grained control over submodules, define ``submodules`` to be a callable function that takes the package instance as its only argument. The function needs to return a list of submodules to be fetched. .. code-block:: python def submodules(package): submodules = [] if "+variant-1" in package.spec: submodules.append("submodule_for_variant_1") if "+variant-2" in package.spec: submodules.append("submodule_for_variant_2") return submodules class MyPackage(Package): version("1.1.0", commit="907d5f40d653a73955387067799913397807adf3", submodules=submodules) For more information about git submodules see the man page of git: ``man git-submodule``. .. _git-sparse-checkout: Sparse-Checkout If you only want to clone a subset of the contents of a git repository, you can supply ``git_sparse_paths`` at the package or version level to utilize git's sparse-checkout feature. The paths can be specified through an attribute, property or callable function. This option is useful for large repositories containing separate features that can be built independently. .. note:: This leverages a newer feature in git that requires version ``2.25.0`` or greater. If ``git_sparse_paths`` is supplied to a git version that is too old then a warning will be issued before standard cloning operations are performed. .. note:: Paths to directories result in the cloning of *all* of their contents, including the contents of their subdirectories. The ``git_sparse_paths`` attribute needs to provide a list of relative paths within the repository. If using a property -- a function decorated with ``@property`` -- or an argument that is a callable function, the function needs to return a list of paths. For example, using the attribute approach: .. code-block:: python class MyPackage(package): # using an attribute git_sparse_paths = ["doe", "rae"] version("1.0.0") version("1.1.0") results in the files from the top level directory of the repository and the contents of the ``doe`` and ``rae`` relative paths within the repository to be cloned. Alternatively, you can provide the paths to the version directive argument using a callable function whose return value is a list for paths. For example: .. code-block:: python def sparse_path_function(package): paths = ["doe", "rae", "me/file.cpp"] if package.spec.version > Version("1.2.0"): paths.extend(["fae"]) return paths class MyPackage(Package): version("1.1.5", git_sparse_paths=sparse_path_function) version("1.2.0", git_sparse_paths=sparse_path_function) version("1.2.5", git_sparse_paths=sparse_path_function) version("1.1.5", git_sparse_paths=sparse_path_function) results in the cloning of the files from the top level directory of the repository, the contents of the ``doe`` and ``rae`` relative paths, *and* the ``me/file.cpp`` file. If the package version is greater than ``1.2.0`` then the contents of the ``fae`` relative path will also be cloned. .. note:: The version directives in the examples above are simplified to emphasize use of this feature. Trusted downloads require a hash, such as a :ref:`sha256 ` or :ref:`commit `. .. _github-fetch: GitHub """""" If a project is hosted on GitHub, *any* valid Git branch, tag, or hash may be downloaded as a tarball. This is accomplished simply by constructing an appropriate URL. Spack can checksum any package downloaded this way, thereby producing a trusted download. For example, the following downloads a particular hash, and then applies a checksum. .. code-block:: python version( "1.9.5.1.1", sha256="8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be", url="https://www.github.com/jswhit/pyproj/tarball/0be612cc9f972e38b50a90c946a9b353e2ab140f", ) Alternatively, you could provide the GitHub ``url`` for one version as a property and Spack will extrapolate the URL for other versions as described in :ref:`Versions and URLs `. .. _hg-fetch: Mercurial """"""""" Fetching with Mercurial works much like :ref:`Git `, but you use the ``hg`` parameter. The destination directory is still the standard stage source path. .. _hg-default-branch: Default branch Add the ``hg`` attribute with no ``revision`` passed to ``version``: .. code-block:: python class Example(Package): hg = "https://bitbucket.org/example-project/example" version("develop") As with Git's default fetching strategy, there is no way to verify the integrity of the download. .. warning:: This download method is **untrusted**, and is **not recommended**. .. _hg-revisions: Revisions To fetch a particular revision, use the ``revision`` parameter: .. code-block:: python version("1.0", revision="v1.0") Unlike ``git``, which has special parameters for different types of revisions, you can use ``revision`` for branches, tags, and **commits** when you fetch with Mercurial. .. warning:: Like Git, fetching specific branches or tags is an **untrusted** download method, and is **not recommended**. The recommended fetch strategy is to specify a particular commit hash as the revision. .. _svn-fetch: Subversion """""""""" To fetch with subversion, use the ``svn`` and ``revision`` parameters. The destination directory will be the standard stage source path. Fetching the head Simply add an ``svn`` parameter to the package: .. code-block:: python class Example(Package): svn = "https://outreach.scidac.gov/svn/example/trunk" version("develop") .. warning:: This download method is **untrusted**, and is **not recommended** for the same reasons as mentioned above. .. _svn-revisions: Fetching a revision To fetch a particular revision, add a ``revision`` argument to the version directive: .. code-block:: python version("develop", revision=128) Unfortunately, Subversion has no commit hashing scheme like Git and Mercurial do, so there is no way to guarantee that the download you get is the same as the download used when the package was created. Use at your own risk. .. warning:: This download method is **untrusted**, and is **not recommended**. Subversion branches are handled as part of the directory structure, so you can check out a branch or tag by changing the URL. If you want to package multiple branches, simply add a ``svn`` argument to each version directive. .. _cvs-fetch: CVS """"""" CVS (Concurrent Versions System) is an old centralized version control system. It is a predecessor of Subversion. To fetch with CVS, use the ``cvs``, branch, and ``date`` parameters. The destination directory will be the standard stage source path. .. _cvs-head: Fetching the head Simply add a ``cvs`` parameter to the package: .. code-block:: python class Example(Package): cvs = ":pserver:outreach.scidac.gov/cvsroot%module=modulename" version("1.1.2.4") CVS repository locations are described using an older syntax that is different from today's ubiquitous URL syntax. ``:pserver:`` denotes the transport method. CVS servers can host multiple repositories (called "modules") at the same location, and one needs to specify both the server location and the module name to access. Spack combines both into one string using the ``%module=modulename`` suffix shown above. .. warning:: This download method is **untrusted**. .. _cvs-date: Fetching a date Versions in CVS are commonly specified by date. To fetch a particular branch or date, add a ``branch`` and/or ``date`` argument to the version directive: .. code-block:: python version("2021.4.22", branch="branchname", date="2021-04-22") Unfortunately, CVS does not identify repository-wide commits via a revision or hash like Subversion, Git, or Mercurial do. This makes it impossible to specify an exact commit to check out. .. warning:: This download method is **untrusted**. CVS has more features, but since CVS is rarely used these days, Spack does not support all of them. Sources that are not archives ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack normally expands archives (e.g., ``*.tar.gz`` and ``*.zip``) automatically into a standard stage source directory (``self.stage.source_path``) after downloading them. If you want to skip this step (e.g., for self-extracting executables and other custom archive types), you can add ``expand=False`` to a ``version`` directive. .. code-block:: python version( "8.2.1", sha256="a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4", url="http://example.com/foo-8.2.1-special-version.sh", expand=False, ) When ``expand`` is set to ``False``, Spack sets the current working directory to the directory containing the downloaded archive before it calls your ``install`` method. Within ``install``, the path to the downloaded archive is available as ``self.stage.archive_file``. Here is an example snippet for packages distributed as self-extracting archives. The example sets permissions on the downloaded file to make it executable, then runs it with some arguments. .. code-block:: python def install(self, spec, prefix): set_executable(self.stage.archive_file) installer = Executable(self.stage.archive_file) installer("--prefix=%s" % prefix, "arg1", "arg2", "etc.") Extra Resources ^^^^^^^^^^^^^^^ Some packages (most notably compilers) provide optional features if additional resources are expanded within their source tree before building. In Spack it is possible to describe such a need with the ``resource`` directive: .. code-block:: python resource( name="cargo", git="https://github.com/rust-lang/cargo.git", tag="0.10.0", destination="cargo", ) The arguments are similar to those of the ``versions`` directive. The keyword ``destination`` is relative to the source root of the package and should point to where the resource is to be expanded. Download caching ^^^^^^^^^^^^^^^^ Spack maintains a cache (described :ref:`here `) which saves files retrieved during package installations to avoid re-downloading in the case that a package is installed with a different specification (but the same version) or reinstalled on account of a change in the hashing scheme. In rare cases, it may be necessary to avoid caching for a particular version by adding ``no_cache=True`` as an option to the ``version()`` directive. Example situations would be a "snapshot"-like Version Control System (VCS) tag, a VCS branch such as ``v6-16-00-patches``, or a URL specifying a regularly updated snapshot tarball. .. _version_constraints: Specifying version constraints ------------------------------ Many Spack directives allow limiting versions to support features such as :ref:`backward and forward compatibility `. These constraints on :ref:`package specs ` are defined using the ``@`` syntax. (See :ref:`version-specifier` for more information.) For example, the following: .. code-block:: python depends_on("foo") depends_on("python@3") conflicts("^foo@1.2.3:", when="@:4.5") illustrates, in order, three of four forms of version range constraints: implicit, lower bound and upper bound. The fourth form provides lower *and* upper bounds on the version. In this example, the implicit range is used to indicate that the package :ref:`depends on ` *any* ``python`` *with* ``3`` *as the major version number* (e.g., ``3.13.5``). The other two range constraints are shown in the :ref:`conflict ` with the dependency package ``foo``. The conflict with ``foo`` *at version* ``1.2.3`` *or newer* is **triggered** for builds of the package at *any version up to and including* ``4.5``. For an example of the fourth form, suppose the dependency in this example had been ``python@3.6:3``. In this case, the package would depend on *any version of* ``python`` *from* ``3.6`` *on so long as the major version number is* ``3``. While you can constrain the spec to a single version -- using the ``@=`` form of ``specifier`` -- **ranges are preferred** even if they would only match a single version currently defined in the package. Using ranges helps avoid overly constrained dependencies, patches, and conflicts. They also come in handy when, for example, users define versions in :ref:`packages-config` that include custom suffixes. For example, if the package defines the version ``1.2.3``, we know from :ref:`version-comparison`, that a user-defined version ``1.2.3-custom`` will satisfy the version constraint ``@1.2.3``. .. warning:: Specific ``@=`` versions should only be used in **exceptional cases**, such as when the package has a versioning scheme that omits the zero in the first patch release. For example, suppose a package defines versions: ``3.1``, ``3.1.1`` and ``3.1.2``. Then the specifier ``@=3.1`` is the correct way to select only ``3.1``, whereas ``@3.1`` would be satisfied by all three versions. .. _variants: Variants -------- Many software packages can be configured to enable optional features, which often come at the expense of additional dependencies or longer build times. To be flexible enough and support a wide variety of use cases, Spack allows you to expose to the end-user the ability to choose which features should be activated in a package at the time it is installed. The mechanism to be employed is the :py:func:`~spack.package.variant` directive. Boolean variants ^^^^^^^^^^^^^^^^ In their simplest form, variants are boolean options specified at the package level: .. code-block:: python class Hdf5(AutotoolsPackage): ... variant("shared", default=True, description="Builds a shared version of the library") with a default value and a description of their meaning in the package. With this variant defined, users can now run ``spack install hdf5 +shared`` and ``spack install hdf5 ~shared`` to enable or disable the ``shared`` feature, respectively. See also the :ref:`basic-variants` for the spec syntax of variants. Of course, merely defining a variant in a package does not automatically enable or disable any features in the build system. As a packager, you are responsible for translating variants to build system flags or environment variables, to influence the build process. We will see this in action in the next part of the packaging guide, where we talk about :ref:`configuring the build with spec objects `. Other than influencing the build process, variants are often used to specify optional :ref:`dependencies of a package `. For example, a package may depend on another package only if a certain variant is enabled: .. code-block:: python class Hdf5(AutotoolsPackage): ... variant("szip", default=False, description="Enable szip support") depends_on("szip", when="+szip") In this case, ``szip`` is modeled as an optional dependency of ``hdf5``, and users can run ``spack install hdf5 +szip`` to enable it. Single-valued variants ^^^^^^^^^^^^^^^^^^^^^^ Other than boolean variants, Spack supports single- and multi-valued variants that can take one or more *string* values. To define a *single-valued* variant, simply pass a tuple of possible values to the ``variant`` directive, together with ``multi=False``: .. code-block:: python class Blis(Package): ... variant( "threads", default="none", values=("pthreads", "openmp", "none"), multi=False, description="Multithreading support", ) This allows users to ``spack install blis threads=openmp``. In the example above the argument ``multi=False`` indicates that only a **single value** can be selected at a time. This constraint is enforced by the solver, and an error is emitted if a user specifies two or more values at the same time: .. code-block:: spec $ spack spec blis threads=openmp,pthreads Input spec -------------------------------- blis threads=openmp,pthreads Concretized -------------------------------- ==> Error: multiple values are not allowed for variant "threads" .. hint:: In the example above, the value ``threads=none`` is a variant value like any other, and means that *no value is selected*. In Spack, all variants have to have a value, so ``none`` was chosen as a *convention* to indicate that no value is selected. Multi-valued variants ^^^^^^^^^^^^^^^^^^^^^ Like single-valued variants, multi-valued variants take one or more *string* values, but allow users to select multiple values at the same time. To define a *multi-valued* variant, simply pass ``multi=True`` instead: .. code-block:: python class Gcc(AutotoolsPackage): ... variant( "languages", default="c,c++,fortran", values=("ada", "brig", "c", "c++", "fortran", "objc"), multi=True, description="Compilers and runtime libraries to build", ) This allows users to run ``spack install languages=c,c++``, where the values are separated by commas. Advanced validation of multi-valued variants """""""""""""""""""""""""""""""""""""""""""" As noted above, the value ``none`` is a value like any other, which raises the question: what if a variant allows multiple values to be selected, *or* none at all? Naively, one might think that this can be achieved by simply creating a multi-valued variant that includes the value ``none``: .. code-block:: python class Adios(AutotoolsPackage): ... variant( "staging", values=("dataspaces", "flexpath", "none"), multi=True, description="Enable dataspaces and/or flexpath staging transports", ) but this does not prevent users from selecting the nonsensical option ``staging=dataspaces,none``. In these cases, more advanced validation logic is required to prevent ``none`` from being selected along with any other value. Spack provides two validator functions to help with this, which can be passed to the ``values=`` argument of the ``variant`` directive. The first validator function is :py:func:`~spack.package.any_combination_of`, which can be used as follows: .. code-block:: python class Adios(AutotoolsPackage): ... variant( "staging", values=any_combination_of("flexpath", "dataspaces"), description="Enable dataspaces and/or flexpath staging transports", ) This solves the issue by allowing the user to select either any combination of the values ``flexpath`` and ``dataspaces``, or ``none``. In other words, users can specify ``staging=none`` to select nothing, or any of ``staging=dataspaces``, ``staging=flexpath``, and ``staging=dataspaces,flexpath``. The second validator function :py:func:`~spack.package.disjoint_sets` generalizes this idea further: .. code-block:: python class Mvapich2(AutotoolsPackage): ... variant( "process_managers", description="List of the process managers to activate", values=disjoint_sets(("auto",), ("slurm",), ("hydra", "gforker", "remshell")) .prohibit_empty_set() .with_error("'slurm' or 'auto' cannot be activated along with other process managers") .with_default("auto") .with_non_feature_values("auto"), ) In this case, examples of valid options are ``process_managers=auto``, ``process_managers=slurm``, and ``process_managers=hydra,remshell``, whereas ``process_managers=slurm,hydra`` is invalid, as it picks values from two different sets. Both validator functions return a :py:class:`~spack.variant.DisjointSetsOfValues` object, which defines chaining methods to further customize the behavior of the variant. .. _variant-conditional-values: Conditional Possible Values ^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are cases where a variant may take multiple values, and the list of allowed values expands over time. Consider, for instance, the C++ standard with which we might compile Boost, which can take one of multiple possible values with the latest standards only available for more recent versions. To model a similar situation we can use *conditional possible values* in the variant declaration: .. code-block:: python variant( "cxxstd", default="98", values=( "98", "11", "14", # C++17 is not supported by Boost < 1.63.0. conditional("17", when="@1.63.0:"), # C++20/2a is not supported by Boost < 1.73.0 conditional("2a", "2b", when="@1.73.0:"), ), multi=False, description="Use the specified C++ standard when building.", ) The snippet above allows ``98``, ``11`` and ``14`` as unconditional possible values for the ``cxxstd`` variant, while ``17`` requires a version greater than or equal to ``1.63.0`` and both ``2a`` and ``2b`` require a version greater than or equal to ``1.73.0``. Conditional Variants ^^^^^^^^^^^^^^^^^^^^ As new versions of packages are released, optional features may be added and removed. Sometimes, features are only available for a particular platform or architecture. To reduce the visual clutter in specs, packages can define variants *conditionally* using a ``when`` clause. The variant will only be present on specs that satisfy this condition. For example, the following package defines a variant ``bar`` that exists only when it is at version 2.0 or higher, and a variant ``baz`` that exists only on the Darwin platform: .. code-block:: python class Foo(Package): ... variant("bar", default=False, when="@2.0:", ...) variant("baz", default=True, when="platform=darwin", ...) Do note that conditional variants can also be a source of confusion. In Spack, the absence of a variant is different from it being disabled. For example, a user might run ``spack install foo ~bar``, expecting it to allow version 1.0 (which does not have the ``bar`` feature) and version 2.0 (with the feature disabled). However, the constraint ``~bar`` tells Spack that the ``bar`` variant *must exist* and be disabled. This forces Spack to select version 2.0 or higher, where the variant is defined. Sticky Variants ^^^^^^^^^^^^^^^ The variant directive can be marked as ``sticky`` by setting the corresponding argument to ``True``: .. code-block:: python variant("bar", default=False, sticky=True) A ``sticky`` variant differs from a regular one in that it is always set to either: #. An explicit value appearing in a spec literal or #. Its default value The concretizer thus is not free to pick an alternate value to work around conflicts, but will error out instead. Setting this property on a variant is useful in cases where the variant allows some dangerous or controversial options (e.g., using unsupported versions of a compiler for a library) and the packager wants to ensure that allowing these options is done on purpose by the user, rather than automatically by the solver. Overriding Variants ^^^^^^^^^^^^^^^^^^^ Packages may override variants for several reasons, most often to change the default from a variant defined in a parent class or to change the conditions under which a variant is present on the spec. When a variant is defined multiple times, whether in the same package file or in a subclass and a superclass, the last definition is used for all attributes **except** for the ``when`` clauses. The ``when`` clauses are accumulated through all invocations, and the variant is present on the spec if any of the accumulated conditions are satisfied. For example, consider the following package: .. code-block:: python class Foo(Package): ... variant("bar", default=False, when="@1.0", description="help1") variant("bar", default=True, when="platform=darwin", description="help2") ... This package ``foo`` has a variant ``bar`` when the spec satisfies either ``@1.0`` or ``platform=darwin``, but not for other platforms at other versions. The default for this variant, when it is present, is always ``True``, regardless of which condition of the variant is satisfied. This allows packages to override variants in packages or build system classes from which they inherit, by modifying the variant values without modifying the ``when`` clause. It also allows a package to implement ``or`` semantics for a variant ``when`` clause by duplicating the variant definition. .. _dependencies: Dependencies ------------ We've covered how to build a simple package, but what if one package relies on another package to build? How do you express that in a package file? And how do you refer to the other package in the build script for your own package? Spack makes this relatively easy. Let's take a look at the ``libdwarf`` package to see how it's done: .. code-block:: python :emphasize-lines: 9 :linenos: class Libdwarf(Package): homepage = "http://www.prevanders.net/dwarf.html" url = "http://www.prevanders.net/libdwarf-20130729.tar.gz" list_url = homepage version("20130729", sha256="092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6") ... depends_on("libelf") def install(self, spec, prefix): ... ``depends_on()`` ^^^^^^^^^^^^^^^^ The highlighted ``depends_on("libelf")`` call tells Spack that it needs to build and install the ``libelf`` package before it builds ``libdwarf``. This means that in your ``install()`` method, you are guaranteed that ``libelf`` has been built and installed successfully, so you can rely on it for your ``libdwarf`` build. .. _dependency_specs: Dependency specs ^^^^^^^^^^^^^^^^ ``depends_on`` doesn't just take the name of another package. It can take a full spec as well. This means that you can restrict the versions or other configuration options of ``libelf`` that ``libdwarf`` will build with. For example, suppose that in the ``libdwarf`` package you write: .. code-block:: python depends_on("libelf@0.8") Now ``libdwarf`` will require ``libelf`` in the range ``0.8``, which includes patch versions ``0.8.1``, ``0.8.2``, etc. Apart from version restrictions, you can also specify variants if this package requires optional features of the dependency. .. code-block:: python depends_on("libelf@0.8 +parser +pic") Both users *and* package authors use the same spec syntax to refer to different package configurations. Users use the spec syntax on the command line to find installed packages or to install packages with particular constraints, and package authors can use specs to describe relationships between packages. .. _version_compatibility: Specifying backward and forward compatibility ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Packages are often compatible with a range of versions of their dependencies. This is typically referred to as backward and forward compatibility. Spack allows you to specify this in the ``depends_on`` directive using version ranges. **Backward compatibility** means that the package requires at least a certain version of its dependency: .. code-block:: python depends_on("python@3.10:") In this case, the package requires Python 3.10 or newer, as specified in the project's :file:`pyproject.toml`. Commonly, packages drop support for older versions of a dependency as they release new versions. In Spack you can conveniently add every backward compatibility rule as a separate line: .. code-block:: python # backward compatibility with Python depends_on("python@3.8:") depends_on("python@3.9:", when="@1.2:") depends_on("python@3.10:", when="@1.4:") This means that in general we need Python 3.8 or newer; from version 1.2 onwards we need Python 3.9 or newer; from version 1.4 onwards we need Python 3.10 or newer. Notice that it's fine to have overlapping ranges in the ``when`` clauses. **Forward compatibility** means that the package requires at most a certain version of its dependency. Forward compatibility rules are necessary when there are breaking changes in the dependency that the package cannot handle. In Spack we often add forward compatibility bounds only at the time a new, breaking version of a dependency is released. As with backward compatibility, it is typical to see a list of forward compatibility bounds in a package file as separate lines: .. code-block:: python # forward compatibility with Python depends_on("python@:3.12", when="@:1.10") depends_on("python@:3.13", when="@:1.12") Notice how the ``:`` now appears before the version number both in the dependency and in the ``when`` clause. This tells Spack that in general we need Python 3.13 or older up to version ``1.12.x``, and up to version ``1.10.x`` we need Python 3.12 or older. Said differently, forward compatibility with Python 3.13 was added in version 1.11, while version 1.13 added forward compatibility with Python 3.14. Notice that a version range ``@:3.12`` includes *any* patch version number ``3.12.x``, which is often useful when specifying forward compatibility bounds. So far we have seen open-ended version ranges, which is by far the most common use case. It is also possible to specify both a lower and an upper bound on the version of a dependency, like this: .. code-block:: python depends_on("python@3.10:3.12") There is short syntax to specify that a package is compatible with say any ``3.x`` version: .. code-block:: python depends_on("python@3") The above is equivalent to ``depends_on("python@3:3")``, which means at least Python version 3 and at most any ``3.x.y`` version. In very rare cases, you may need to specify an exact version, for example if you need to distinguish between ``3.2`` and ``3.2.1``: .. code-block:: python depends_on("pkg@=3.2") But in general, you should try to use version ranges as much as possible, so that custom suffixes are included too. The above example can be rewritten in terms of ranges as follows: .. code-block:: python depends_on("pkg@3.2:3.2.0") A spec can contain a version list of ranges and individual versions separated by commas. For example, if you need Boost 1.59.0 or newer, but there are known issues with 1.64.0, 1.65.0, and 1.66.0, you can say: .. code-block:: python depends_on("boost@1.59.0:1.63,1.65.1,1.67.0:") or, if those particular versions are excluded due to bugs rather than removed and reintroduced features: .. code-block:: python depends_on("boost@1.59.0:") conflicts("^boost@1.64.0,1.65.0,1.66.0") Always specify version ranges with an open-world assumption: - all "ground truths" about exclusions and inclusions (e.g., versions with features added or removed) must satisfy the range, and - no potential but unknown versions are excluded from the range. This practice avoids overconstraining version ranges, which can lead to concretization errors, and ensures that every version in a package is *meaningful* and not just *incidental* (i.e., based on the version you happened to test). In the above example, the project has presumably documented (with pyproject.toml, CMakeLists.txt, or release notes) that ``@:1.58`` are incompatible, and it is known from testing that ``@1.67`` is compatible. It is *not* known whether future versions ``@1.68:`` are incompatible, so they must be included by the range. If and when future versions are known incompatible, the version range should be constrained with an upper bound. .. _dependency-types: Dependency types ^^^^^^^^^^^^^^^^ Not all dependencies are created equal, and Spack allows you to specify exactly what kind of a dependency you need. For example: .. code-block:: python depends_on("cmake", type="build") depends_on("py-numpy", type=("build", "run")) depends_on("libelf", type=("build", "link")) depends_on("py-pytest", type="test") The following dependency types are available: * **build**: the dependency will be added to the ``PATH`` and ``PYTHONPATH`` at build-time. * **link**: the dependency will be added to Spack's compiler wrappers, automatically injecting the appropriate linker flags, including ``-I``, ``-L``, and RPATH/RUNPATH handling. * **run**: the dependency will be added to the ``PATH`` and ``PYTHONPATH`` at run-time. This is true for both ``spack load`` and the module files Spack writes. * **test**: the dependency will be added to the ``PATH`` and ``PYTHONPATH`` at build-time. The only difference between "build" and "test" is that test dependencies are only built if the user requests unit tests with ``spack install --test``. One of the advantages of the ``build`` dependency type is that although the dependency needs to be installed in order for the package to be built, it can be uninstalled without concern afterwards. ``link`` and ``run`` disallow this because uninstalling the dependency would break the package. ``build``, ``link``, and ``run`` dependencies all affect the hash of Spack packages (along with ``sha256`` sums of patches and archives used to build the package, and a `canonical hash `_ of the ``package.py`` recipes). ``test`` dependencies do not affect the package hash, as they are only used to construct a test environment *after* building and installing a given package installation. Older versions of Spack did not include build dependencies in the hash, but this has been `fixed `_ as of |Spack v0.18|_. .. |Spack v0.18| replace:: Spack ``v0.18`` .. _Spack v0.18: https://github.com/spack/spack/releases/tag/v0.18.0 If the dependency type is not specified, Spack uses a default of ``("build", "link")``. This is the common case for compiler languages. Non-compiled packages like Python modules commonly use ``("build", "run")``. This means that the compiler wrappers don't need to inject the dependency's ``prefix/lib`` directory, but the package needs to be in ``PATH`` and ``PYTHONPATH`` during the build process and later when a user wants to run the package. Conditional dependencies ^^^^^^^^^^^^^^^^^^^^^^^^ You may have a package that only requires a dependency under certain conditions. For example, you may have a package with optional MPI support. You would then provide a variant to reflect that the feature is optional and specify the MPI dependency only applies when MPI support is enabled. In that case, you could say something like: .. code-block:: python variant("mpi", default=False, description="Enable MPI support") depends_on("mpi", when="+mpi") Suppose that, starting from version 3, the above package also has optional `Trilinos` support. Furthermore, you want to ensure that when `Trilinos` support is enabled, the package can be built both with and without MPI. Further suppose you require a version of `Trilinos` no older than 12.6. In that case, the `trilinos` variant and dependency directives would be: .. code-block:: python variant("trilinos", default=False, description="Enable Trilinos support") depends_on("trilinos@12.6:", when="@3: +trilinos") depends_on("trilinos@12.6: +mpi", when="@3: +trilinos +mpi") Alternatively, you could use the `when` context manager to equivalently specify the `trilinos` variant dependencies as follows: .. code-block:: python with when("@3: +trilinos"): depends_on("trilinos@12.6:") depends_on("trilinos +mpi", when="+mpi") The argument to ``when`` in either case can include any Spec constraints that are supported on the command line using the same :ref:`syntax `. .. note:: If a dependency isn't typically used, you can save time by making it conditional since Spack will not build the dependency unless it is required for the Spec. .. _dependency_dependency_patching: Dependency patching ^^^^^^^^^^^^^^^^^^^ Some packages maintain special patches on their dependencies, either to add new features or to fix bugs. This typically makes a package harder to maintain, and we encourage developers to upstream (contribute back) their changes rather than maintaining patches. However, in some cases it's not possible to upstream. Maybe the dependency's developers don't accept changes, or maybe they just haven't had time to integrate them. For times like these, Spack's ``depends_on`` directive can optionally take a patch or list of patches: .. code-block:: python class SpecialTool(Package): ... depends_on("binutils", patches="special-binutils-feature.patch") ... Here, the ``special-tool`` package requires a special feature in ``binutils``, so it provides an extra ``patches=`` keyword argument. This is similar to the `patch directive `_, with one small difference. Here, ``special-tool`` is responsible for the patch, so it should live in ``special-tool``'s directory in the package repository, not the ``binutils`` directory. If you need something more sophisticated, you can nest a ``patch()`` directive inside ``depends_on``: .. code-block:: python class SpecialTool(Package): ... depends_on( "binutils", patches=patch( "special-binutils-feature.patch", level=3, when="@:1.3" # condition on binutils ), when="@2.0:", # condition on special-tool ) ... Note that there are two optional ``when`` conditions here -- one on the ``patch`` directive and the other on ``depends_on``. The condition in the ``patch`` directive applies to ``binutils`` (the package being patched), while the condition in ``depends_on`` applies to ``special-tool``. See `patch directive `_ for details on all the arguments the ``patch`` directive can take. Finally, if you need *multiple* patches on a dependency, you can provide a list for ``patches``, e.g.: .. code-block:: python class SpecialTool(Package): ... depends_on( "binutils", patches=[ "binutils-bugfix1.patch", "binutils-bugfix2.patch", patch( "https://example.com/special-binutils-feature.patch", sha256="252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866", when="@:1.3", ), ], when="@2.0:", ) ... As with ``patch`` directives, patches are applied in the order they appear in the package file (or in this case, in the list). .. note:: You may wonder whether dependency patching will interfere with other packages that depend on ``binutils``. It won't. As described in :ref:`patching`, Patching a package adds the ``sha256`` of the patch to the package's spec, which means it will have a *different* unique hash than other versions without the patch. The patched version coexists with unpatched versions, and Spack's support for :ref:`handling_rpaths` guarantees that each installation finds the right version. If two packages depend on ``binutils`` patched *the same* way, they can both use a single installation of ``binutils``. .. _virtual-dependencies: Virtual dependencies -------------------- In some cases, more than one package can satisfy another package's dependency. One way this can happen is if a package depends on a particular *interface*, but there are multiple *implementations* of the interface, and the package could be built with any of them. A *very* common interface in HPC is the `Message Passing Interface (MPI) `_, which is used in many large-scale parallel applications. MPI has several different implementations (e.g., `MPICH `_, `OpenMPI `_, and `MVAPICH `_) and scientific applications can be built with any one of them. Many package managers handle interfaces like this by requiring many variations of the package recipe for each implementation of MPI, e.g., ``foo``, ``foo-mvapich``, ``foo-mpich``. In Spack every package is defined in a single ``package.py`` file, and avoids the combinatorial explosion through *virtual dependencies*. ``provides`` ^^^^^^^^^^^^ In Spack, ``mpi`` is handled as a *virtual package*. A package like ``mpileaks`` can depend on the virtual ``mpi`` just like any other package, by supplying a ``depends_on`` call in the package definition. For example: .. code-block:: python :linenos: :emphasize-lines: 7 class Mpileaks(Package): homepage = "https://github.com/hpc/mpileaks" url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" version("1.0", sha256="768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d") depends_on("mpi") depends_on("adept-utils") depends_on("callpath") Here, ``callpath`` and ``adept-utils`` are concrete packages, but there is no actual package for ``mpi``, so we say it is a *virtual* package. The syntax of ``depends_on`` is the same for both. If we look inside the package file of an MPI implementation, say MPICH, we'll see something like this: .. code-block:: python class Mpich(Package): provides("mpi") ... The ``provides("mpi")`` call tells Spack that the ``mpich`` package can be used to satisfy the dependency of any package that ``depends_on("mpi")``. Providing multiple virtuals simultaneously ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Packages can provide more than one virtual dependency. Sometimes, due to implementation details, there are subsets of those virtuals that need to be provided together by the same package. A well-known example is ``openblas``, which provides both the ``lapack`` and ``blas`` API in a single ``libopenblas`` library. A package that needs ``lapack`` and ``blas`` must either use ``openblas`` to provide both, or not use ``openblas`` at all. It cannot pick one or the other. To express this constraint in a package, the two virtual dependencies must be listed in the same ``provides`` directive: .. code-block:: python provides("blas", "lapack") This makes it impossible to select ``openblas`` as a provider for one of the two virtual dependencies and not for the other. If you try to, Spack will report an error: .. code-block:: spec $ spack spec netlib-scalapack ^[virtuals=lapack] openblas ^[virtuals=blas] atlas ==> Error: concretization failed for the following reasons: 1. Package 'openblas' needs to provide both 'lapack' and 'blas' together, but provides only 'lapack' Versioned Interfaces ^^^^^^^^^^^^^^^^^^^^ Just as you can pass a spec to ``depends_on``, so can you pass a spec to ``provides`` to add constraints. This allows Spack to support the notion of *versioned interfaces*. The MPI standard has gone through many revisions, each with new functions added, and each revision of the standard has a version number. Some packages may require a recent implementation that supports MPI-3 functions, but some MPI versions may only provide up to MPI-2. Others may need MPI 2.1 or higher. You can indicate this by adding a version constraint to the spec passed to ``provides``: .. code-block:: python provides("mpi@:2") Suppose that the above ``provides`` call is in the ``mpich2`` package. This says that ``mpich2`` provides MPI support *up to* version 2, but if a package ``depends_on("mpi@3")``, then Spack will *not* build that package with ``mpich2``. Currently, names and versions are the only spec components supported for virtual packages. ``provides when`` ^^^^^^^^^^^^^^^^^ The same package may provide different versions of an interface depending on *its* version. Above, we simplified the ``provides`` call in ``mpich`` to make the explanation easier. In reality, this is how ``mpich`` calls ``provides``: .. code-block:: python provides("mpi@:3", when="@3:") provides("mpi@:1", when="@1:") The ``when`` argument to ``provides`` allows you to specify optional constraints on the *providing* package, or the *provider*. The provider only provides the declared virtual spec when *it* matches the constraints in the ``when`` clause. Here, when ``mpich`` is at version 3 or higher, it provides MPI up to version 3. When ``mpich`` is at version 1 or higher, it provides the MPI virtual package at version 1. The ``when`` qualifier ensures that Spack selects a suitably high version of ``mpich`` to satisfy some other package that ``depends_on`` a particular version of MPI. It will also prevent a user from building with too low a version of ``mpich``. For example, suppose the package ``foo`` declares this: .. code-block:: python class Foo(Package): ... depends_on("mpi@2") Suppose a user invokes ``spack install`` like this: .. code-block:: spec $ spack install foo ^mpich@1.0 Spack will fail with a constraint violation, because the version of MPICH requested is too low for the ``mpi`` requirement in ``foo``. .. _language-dependencies: Language and compiler dependencies ---------------------------------- Whenever you use ``spack create`` to create a new package, Spack scans the package's source code and heuristically adds *language dependencies*, which look like this: .. code-block:: python depends_on("c", type="build") depends_on("cxx", type="build") depends_on("fortran", type="build") The languages ``c``, ``cxx`` and ``fortran`` are **virtuals provided by compiler packages**, such as ``gcc``, ``llvm``, or ``intel-oneapi-compilers``. When you concretize a package that depends on ``c``, Spack will select a compiler for it that provides the ``c`` virtual package. Typically one compiler will be used to provide all languages, but Spack is allowed to create a mixed toolchain. For example, the ``c`` compiler could be ``clang`` from the ``llvm`` package, whereas the ``fortran`` compiler could be ``gfortran`` from the ``gcc``. This means that language dependencies translate to one or more compiler packages as build dependencies. .. _packaging_conflicts: Conflicts --------- Sometimes packages have known bugs, or limitations, that would prevent them from concretizing or building usable software. Spack makes it possible to express such constraints with the ``conflicts`` directive, which takes a spec that is known to cause a conflict and optional ``when`` and ``msg`` arguments. The ``when`` argument is a spec that triggers the conflict. The ``msg`` argument allows you to provide a custom error message that Spack prints when the spec to be installed satisfies the conflict spec and ``when`` trigger. Adding the following to a package: .. code-block:: python conflicts( "%intel-oneapi-compilers@:2024", when="@:1.2", msg="known bug when using Intel oneAPI compilers through v2024", ) expresses that the current package *cannot be built* with Intel oneAPI compilers *up through any version* ``2024`` *when trying to install the package with a version up to* ``1.2``. If the ``when`` argument is omitted, then the conflict is *always triggered* for specs satisfying the conflict spec. For example, .. code-block:: python conflicts("+cuda+rocm", msg="Cannot build with both cuda and rocm enabled") means the package cannot be installed with both variants enabled. Similarly, a conflict can be based on where the build is being performed. For example, .. code-block:: python for os in ["ventura", "monterey", "bigsur"]: conflicts(f"platform=darwin os={os}", msg=f"{os} is not supported") means the package cannot be built on a Mac running Ventura, Monterey, or Big Sur. .. note:: These examples illustrate a few of the types of constraints that can be specified. Conflict and ``when`` specs can constrain the compiler, :ref:`version `, :ref:`variants `, :ref:`architecture `, :ref:`dependencies `, and more. See :ref:`sec-specs` for more information. .. _packaging_requires: Requires -------- Sometimes packages can be built only with specific options. In those cases the ``requires`` directive can be used. It allows for complex conditions involving more than a single spec through the ability to specify multiple required specs before keyword arguments. The same optional ``when`` and ``msg`` arguments as ``conflicts`` are supported (see :ref:`packaging_conflicts`). The directive also supports a ``policy`` argument for determining how the multiple required specs apply. Values for ``policy`` may be either ``any_of`` or ``one_of`` (default) and have the same semantics described for their equivalents in :ref:`package-requirements`. .. hint:: We recommend that the ``policy`` argument be explicitly specified when multiple specs are used with the directive. For example, suppose a package can only be built with Apple Clang on Darwin. This requirement would be specified as: .. code-block:: python requires( "%apple-clang", when="platform=darwin", msg="builds only with Apple Clang compiler on Darwin", ) Similarly, suppose a package only builds for the ``x86_64`` target: .. code-block:: python requires("target=x86_64:", msg="package is only available on x86_64") Or the package must be built with a GCC or Clang that supports C++ 20, which you could ensure by adding the following: .. code-block:: python requires( "%gcc@10:", "%clang@16:", policy="one_of", msg="builds only with a GCC or Clang that support C++ 20", ) .. note:: These examples show only a few of the constraints that can be specified. Required and ``when`` specs can constrain the compiler, :ref:`version `, :ref:`variants `, :ref:`architecture `, :ref:`dependencies `, and more. See :ref:`sec-specs` for more information. .. _patching: Patches ------- Depending on the host architecture, package version, known bugs, or other issues, you may need to patch your software to get it to build correctly. Like many other package systems, Spack allows you to store patches alongside your package files and apply them to source code after it's downloaded. ``patch`` ^^^^^^^^^ You can specify patches in your package file with the ``patch()`` directive. The first argument can be either the filename or URL of the patch file to be applied to your source. .. note:: Use of a URL is preferred over maintaining patch files in the package repository. This helps reduce the size of the package repository, which can become an issue for those with limited space (or allocations). Filename patch """""""""""""" You can supply the name of the patch file. For example, a simple conditional ``patch`` based on a file for the ``mvapich2`` package looks like: .. code-block:: python class Mvapich2(Package): ... patch("ad_lustre_rwcontig_open_source.patch", when="@1.9:") This patch will only be applied when attempting to install the package at version ``1.9`` or newer. When a filename is provided, the patch needs to live within the Spack source tree. The above patch file lives with the package file within the package repository directory structure in the following location: .. code-block:: none spack_repo/builtin/packages/ mvapich2/ package.py ad_lustre_rwcontig_open_source.patch URL patch file """""""""""""" If you supply a URL instead of a filename you have two options: patch file URL or commit patch file URL. In either case, you must supply a checksum. Spack requires the ``sha256`` hash so that different patches applied to the same package will have unique identifiers. Patches will be fetched from their URLs, checked, and applied to your source code. .. note:: To ensure consistency, a ``sha256`` checksum must be provided for the patch. You can use the GNU utils ``sha256sum`` or the macOS ``shasum -a 256`` commands to generate a checksum for a patch file. Here is an example of specifying the unconditional use of a patch file URL: .. code-block:: python patch( "http://www.nwchem-sw.org/images/Tddft_mxvec20.patch", sha256="252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866", ) Sometimes you can specify the patch file associated with a repository commit. For example, GitHub allows you to reference the commit in the name of the patch file through a URL in the form ``https://github.com///commit/.patch``. Below is an example of specifying a conditional commit patch: .. code-block:: python patch( "https://github.com/ornladios/ADIOS/commit/17aee8aeed64612cd8cfa0b949147091a5525bbe.patch?full_index=1", sha256="aea47e56013b57c2d5d36e23e0ae6010541c3333a84003784437768c2e350b05", when="@1.12.0: +mpi", ) In this case the patch is only processed when attempting to install version ``1.12.0`` or higher of the package when the package's ``mpi`` variant is enabled. .. note: Be sure to append ``?full_index=1`` to the GitHub URL to ensure the patch file consistently contains the complete, stable hash information for reproducible patching. Use the resulting URL to get the patch file contents that you then run through the appropriate utility to get the corresponding ``sha256`` value. Compressed patches """""""""""""""""" Spack can also handle compressed patches. If you use these, Spack needs a little more help. Specifically, it needs *two* checksums: the ``sha256`` of the patch and ``archive_sha256`` for the compressed archive. ``archive_sha256`` helps Spack ensure that the downloaded file is not corrupted or malicious, before running it through a tool like ``tar`` or ``zip``. The ``sha256`` of the patch is still required so that it can be included in specs. Providing it in the package file ensures that Spack won't have to download and decompress patches it won't end up using at install time. Both the archive and patch checksum are checked when patch archives are downloaded. .. code-block:: python patch( "http://www.nwchem-sw.org/images/Tddft_mxvec20.patch.gz", sha256="252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866", archive_sha256="4e8092a161ec6c3a1b5253176fcf33ce7ba23ee2ff27c75dbced589dabacd06e", ) ``patch`` keyword arguments are described below. ``sha256``, ``archive_sha256`` """""""""""""""""""""""""""""" Hashes of downloaded patch and compressed archive, respectively. Only needed for patches fetched from URLs. ``when`` """""""" If supplied, this is a spec that tells Spack when to apply the patch. If the installed package spec matches this spec, the patch will be applied. In our example above, the patch is applied when ``mvapich`` is at version ``1.9`` or higher. ``level`` """"""""" This tells Spack how to run the ``patch`` command. By default, the level is 1 and Spack runs ``patch -p 1``. If level is 2, Spack will run ``patch -p 2``, and so on. A lot of people are confused by the level, so here's a primer. If you look in your patch file, you may see something like this: .. code-block:: diff :linenos: --- a/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 +++ b/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 @@ -8,7 +8,7 @@ * Copyright (C) 2008 Sun Microsystems, Lustre group \*/ -#define _XOPEN_SOURCE 600 +//#define _XOPEN_SOURCE 600 #include #include #include "ad_lustre.h" Lines 1-2 show paths with synthetic ``a/`` and ``b/`` prefixes. These are placeholders for the two ``mvapich2`` source directories that ``diff`` compared when it created the patch file. This is git's default behavior when creating patch files, but other programs may behave differently. ``-p1`` strips off the first level of the prefix in both paths, allowing the patch to be applied from the root of an expanded ``mvapich2`` archive. If you set level to ``2``, it would strip off ``src``, and so on. It's generally easier to just structure your patch file so that it applies cleanly with ``-p1``, but if you're using a patch you didn't create yourself, ``level`` can be handy. ``working_dir`` """"""""""""""" This tells Spack where to run the ``patch`` command. By default, the working directory is the source path of the stage (``.``). However, sometimes patches are made with respect to a subdirectory and this is where the working directory comes in handy. Internally, the working directory is given to ``patch`` via the ``-d`` option. Let's take the example patch from above and assume for some reason, it can only be downloaded in the following form: .. code-block:: diff :linenos: --- a/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 +++ b/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 @@ -8,7 +8,7 @@ * Copyright (C) 2008 Sun Microsystems, Lustre group \*/ -#define _XOPEN_SOURCE 600 +//#define _XOPEN_SOURCE 600 #include #include #include "ad_lustre.h" Hence, the patch needs to be applied in the ``src/mpi`` subdirectory, and the ``working_dir="src/mpi"`` option would exactly do that. Patch functions ^^^^^^^^^^^^^^^^^^^^^ In addition to supplying patch files, you can write a custom function to patch a package's source. For example, the ``py-pyside2`` package contains some custom code for tweaking the way the PySide build handles include files: .. _pyside-patch: .. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/py_pyside2/package.py :pyobject: PyPyside2.patch :linenos: A ``patch`` function, if present, will be run after patch files are applied and before ``install()`` is run. You could put this logic in ``install()``, but putting it in a patch function gives you some benefits. First, Spack ensures that the ``patch()`` function is run once per code checkout. That means that if you run install, hit ctrl-C, and run install again, the code in the patch function is only run once. .. _patch_dependency_patching: Dependency patching ^^^^^^^^^^^^^^^^^^^ So far we've covered how the ``patch`` directive can be used by a package to patch *its own* source code. Packages can *also* specify patches to be applied to their dependencies, if they require special modifications. As with all packages in Spack, a patched dependency library can coexist with other versions of that library. See the `section on depends_on `_ for more details. .. _patch_inspecting_patches: Inspecting patches ^^^^^^^^^^^^^^^^^^^ If you want to better understand the patches that Spack applies to your packages, you can do that using ``spack spec``, ``spack find``, and other query commands. Let's look at ``m4``. If you run ``spack spec m4``, you can see the patches that would be applied to ``m4``: .. code-block:: spec $ spack spec m4 Input spec -------------------------------- m4 Concretized -------------------------------- m4@1.4.18%apple-clang@9.0.0 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv arch=darwin-highsierra-x86_64 ^libsigsegv@2.11%apple-clang@9.0.0 arch=darwin-highsierra-x86_64 You can also see patches that have been applied to installed packages with ``spack find -v``: .. code-block:: spec $ spack find -v m4 ==> 1 installed package -- darwin-highsierra-x86_64 / apple-clang@9.0.0 ----------------- m4@1.4.18 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv .. _cmd-spack-resource: In both cases above, you can see that the patches' sha256 hashes are stored on the spec as a variant. As mentioned above, this means that you can have multiple, differently-patched versions of a package installed at once. You can look up a patch by its sha256 hash (or a short version of it) using the ``spack resource show`` command .. code-block:: console $ spack resource show 3877ab54 3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 path: .../spack_repo/builtin/packages/m4/gnulib-pgi.patch applies to: builtin.m4 ``spack resource show`` looks up downloadable resources from package files by hash and prints out information about them. Above, we see that the ``3877ab54`` patch applies to the ``m4`` package. The output also tells us where to find the patch. Things get more interesting if you want to know about dependency patches. For example, when ``dealii`` is built with ``boost@1.68.0``, it has to patch boost to work correctly. If you didn't know this, you might wonder where the extra boost patches are coming from: .. code-block:: console $ spack spec dealii ^boost@1.68.0 ^hdf5+fortran | grep "\^boost" ^boost@1.68.0 ^boost@1.68.0%apple-clang@9.0.0+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199,b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=darwin-highsierra-x86_64 $ spack resource show b37164268 b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f path: .../spack_repo/builtin/packages/dealii/boost_1.68.0.patch applies to: builtin.boost patched by: builtin.dealii Here you can see that the patch is applied to ``boost`` by ``dealii``, and that it lives in ``dealii``'s directory in Spack's ``builtin`` package repository. .. _packaging_extensions: Extensions ---------- Spack's support for package extensions is documented extensively in :ref:`extensions`. This section documents how to make your own extendable packages and extensions. To support extensions, a package needs to set its ``extendable`` property to ``True``, e.g.: .. code-block:: python class Python(Package): ... extendable = True ... To make a package into an extension, simply add an ``extends`` call in the package definition, and pass it the name of an extendable package: .. code-block:: python class PyNumpy(Package): ... extends("python") ... This accomplishes a few things. Firstly, the Python package can set special variables such as ``PYTHONPATH`` for all extensions when the run or build environment is set up. Secondly, filesystem views can ensure that extensions are put in the same prefix as their extendee. This ensures that Python in a view can always locate its Python packages, even without environment variables set. A package can only extend one other package at a time. To support packages that may extend one of a list of other packages, Spack supports multiple ``extends`` directives as long as at most one of them is selected as a dependency during concretization. For example, a lua package could extend either ``lua`` or ``lua-luajit``, but not both: .. code-block:: python class LuaLpeg(Package): ... variant("use_lua", default=True) extends("lua", when="+use_lua") extends("lua-luajit", when="~use_lua") ... Now, a user can install, and activate, the ``lua-lpeg`` package for either lua or ``lua-luajit``. Adding additional constraints ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Some packages produce a Python extension, but require a minimum version of Python to work correctly. In those cases, a ``depends_on()`` declaration should be made in addition to the ``extends()`` declaration: .. code-block:: python class Icebin(Package): extends("python", when="+python") depends_on("python@3.12:", when="+python") Many packages produce Python extensions for *some* variants, but not others: they should extend ``python`` only if the appropriate variant(s) are selected. This may be accomplished with conditional ``extends()`` declarations: .. code-block:: python class FooLib(Package): variant("python", default=True, description="Build the Python extension Module") extends("python", when="+python") ... Mixins for common metadata -------------------------- Spack's package repository contains a number of mixin classes that can be used to simplify package definitions and to share common metadata and behavior across multiple packages. For instance, packages that depend on ``cuda`` typically need variants such as ``+cuda`` and ``cuda_arch``, and conflicts to specify compatibility between architectures, compilers and CUDA versions. To avoid duplicating this metadata in every package that requires CUDA, Spack provides a mixin class called ``CudaPackage`` that can be used to inherit this common metadata and behavior. Other mixin classes such as ``GNUMirrorPackage`` do not add variants or conflicts, but configure the usual GNU mirror URLs for downloading source code. The following table lists the full list of mixin classes available in Spack's builtin package repository. +----------------------------------------------------------------------------+----------------------------------+ | **API docs** | **Description** | +============================================================================+==================================+ | :class:`~spack_repo.builtin.build_systems.cuda.CudaPackage` | A helper class for packages that | | | use CUDA | +----------------------------------------------------------------------------+----------------------------------+ | :class:`~spack_repo.builtin.build_systems.rocm.ROCmPackage` | A helper class for packages that | | | use ROCm | +----------------------------------------------------------------------------+----------------------------------+ | :class:`~spack_repo.builtin.build_systems.gnu.GNUMirrorPackage` | A helper class for GNU packages | | | | +----------------------------------------------------------------------------+----------------------------------+ | :class:`~spack_repo.builtin.build_systems.python.PythonExtension` | A helper class for Python | | | extensions | +----------------------------------------------------------------------------+----------------------------------+ | :class:`~spack_repo.builtin.build_systems.sourceforge.SourceforgePackage` | A helper class for packages | | | from sourceforge.org | +----------------------------------------------------------------------------+----------------------------------+ | :class:`~spack_repo.builtin.build_systems.sourceware.SourcewarePackage` | A helper class for packages | | | from sourceware.org | +----------------------------------------------------------------------------+----------------------------------+ | :class:`~spack_repo.builtin.build_systems.xorg.XorgPackage` | A helper class for x.org | | | packages | +----------------------------------------------------------------------------+----------------------------------+ These mixins should be used as additional base classes for your package, in addition to the base class that you would normally use (e.g. ``MakefilePackage``, ``AutotoolsPackage``, etc.): .. code-block:: python class Cp2k(MakefilePackage, CudaPackage): pass In the example above ``Cp2k`` inherits the variants and conflicts defined by ``CudaPackage``. .. _maintainers: Maintainers ----------- Each package in Spack may have one or more GitHub accounts for people who want to be notified whenever the package is modified. The list also provides contacts for people needing help with build errors. Adding maintainers is easy. After familiarizing yourself with the responsibilities of the :ref:`Package Maintainers ` role, you simply need to declare their GitHub accounts in the ``maintainers`` directive: .. code-block:: python maintainers("github_user1", "github_user2") .. warning:: Please do not add accounts without consent of the owner. The final list of maintainers includes accounts declared in the package's base classes. .. _package_license: License Information ------------------- Most of the software in Spack is open source, and most open source software is released under one or more `common open source licenses `_. Specifying the license that a package is released under in a project's ``package.py`` is good practice. To specify a license, find the `SPDX identifier `_ for a project and then add it using the license directive: .. code-block:: python license("") For example, the SPDX ID for the Apache Software License, version 2.0 is ``Apache-2.0``, so you'd write: .. code-block:: python license("Apache-2.0") Or, for a dual-licensed package like Spack, you would use an `SPDX Expression `_ with both of its licenses: .. code-block:: python license("Apache-2.0 OR MIT") Note that specifying a license without a ``when=`` clause makes it apply to all versions and variants of the package, which might not actually be the case. For example, a project might have switched licenses at some point or have certain build configurations that include files that are licensed differently. Spack itself used to be under the ``LGPL-2.1`` license, until it was relicensed in version ``0.12`` in 2018. You can specify when a ``license()`` directive applies using a ``when=`` clause, just like other directives. For example, to specify that a specific license identifier should only apply to versions up to ``0.11``, but another license should apply for later versions, you could write: .. code-block:: python license("LGPL-2.1", when="@:0.11") license("Apache-2.0 OR MIT", when="@0.12:") Note that unlike for most other directives, the ``when=`` constraints in the ``license()`` directive can't intersect. Spack needs to be able to resolve exactly one license identifier expression for any given version. To specify *multiple* licenses, use SPDX expressions and operators as above. The operators you probably care most about are: * ``OR``: user chooses one license to adhere to; and * ``AND``: user has to adhere to all the licenses. You may also care about `license exceptions `_ that use the ``WITH`` operator, e.g. ``Apache-2.0 WITH LLVM-exception``. Many of the licenses that are currently in the spack repositories have been automatically determined. While this is great for bulk adding license information and is most likely correct, there are sometimes edge cases that require manual intervention. To determine which licenses are validated and which are not, there is the ``checked_by`` parameter in the license directive: .. code-block:: python license("", when="", checked_by="") When you have validated a package license, either when doing so explicitly or as part of packaging a new package, please set the ``checked_by`` parameter to your Github username to signal that the license has been manually verified. .. _license: Proprietary software -------------------- In order to install proprietary software, Spack needs to know a few more details about a package. The following class attributes should be defined. ``license_required`` ^^^^^^^^^^^^^^^^^^^^ Boolean. If set to ``True``, this software requires a license. If set to ``False``, all of the following attributes will be ignored. Defaults to ``False``. ``license_comment`` ^^^^^^^^^^^^^^^^^^^ String. Contains the symbol used by the license manager to denote a comment. Defaults to ``#``. ``license_files`` ^^^^^^^^^^^^^^^^^ List of strings. These are files that the software searches for when looking for a license. All file paths must be relative to the installation directory. More complex packages like Intel may require multiple licenses for individual components. Defaults to the empty list. ``license_vars`` ^^^^^^^^^^^^^^^^ List of strings. Environment variables that can be set to tell the software where to look for a license if it is not in the usual location. Defaults to the empty list. ``license_url`` ^^^^^^^^^^^^^^^ String. A URL pointing to license setup instructions for the software. Defaults to the empty string. For example, let's take a look at the Arm Forge package. .. code-block:: python # Licensing license_required = True license_comment = "#" license_files = ["licences/Licence"] license_vars = [ "ALLINEA_LICENSE_DIR", "ALLINEA_LICENCE_DIR", "ALLINEA_LICENSE_FILE", "ALLINEA_LICENCE_FILE", ] license_url = "https://developer.arm.com/documentation/101169/latest/Use-Arm-Licence-Server" Arm Forge requires a license. Its license manager uses the ``#`` symbol to denote a comment. It expects the license file to be named ``License`` and to be located in a ``licenses`` directory in the installation prefix. If you would like the installation file to be located elsewhere, simply set ``ALLINEA_LICENSE_DIR`` or one of the other license variables after installation. For further instructions on installation and licensing, see the URL provided. If your package requires the license to install, you can reference the location of this global license using ``self.global_license_file``. After installation, symlinks for all of the files given in ``license_files`` will be created, pointing to this global license. If you install a different version or variant of the package, Spack will automatically detect and reuse the already existing global license. If the software you are trying to package doesn't rely on license files, Spack will print a warning message, letting the user know that they need to set an environment variable or pointing them to installation documentation. Grouping directives ------------------- We have seen various directives such as ``depends_on``, ``conflicts``, and ``requires``. Very often, these directives share a common argument, which you becomes repetitive and verbose to write. .. _group_when_spec: Grouping with ``when()`` ^^^^^^^^^^^^^^^^^^^^^^^^ Spack provides a context manager called ``when()`` that allows you to group directives by a common constraint or condition. .. code-block:: python class Gcc(AutotoolsPackage): with when("+nvptx"): depends_on("cuda") conflicts("@:6", msg="NVPTX only supported in gcc 7 and above") conflicts("languages=ada") conflicts("languages=brig") conflicts("languages=go") The snippet above is equivalent to the more verbose: .. code-block:: python class Gcc(AutotoolsPackage): depends_on("cuda", when="+nvptx") conflicts("@:6", when="+nvptx", msg="NVPTX only supported in gcc 7 and above") conflicts("languages=ada", when="+nvptx") conflicts("languages=brig", when="+nvptx") conflicts("languages=go", when="+nvptx") Constraints from the ``when`` block are composable with ``when`` arguments in directives inside the block. For instance, .. code-block:: python with when("+elpa"): depends_on("elpa+openmp", when="+openmp") is equivalent to: .. code-block:: python depends_on("elpa+openmp", when="+openmp+elpa") Constraints from nested context managers are also combined together, but they are rarely needed, and are not recommended. .. _default_args: Grouping with ``default_args()`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ More generally, if directives have a common set of default arguments, you can group them together in a ``with default_args()`` block: .. code-block:: python class PyExample(PythonPackage): with default_args(type=("build", "run")): depends_on("py-foo") depends_on("py-foo@2:", when="@2:") depends_on("py-bar") depends_on("py-bz") The above is short for: .. code-block:: python class PyExample(PythonPackage): depends_on("py-foo", type=("build", "run")) depends_on("py-foo@2:", when="@2:", type=("build", "run")) depends_on("py-bar", type=("build", "run")) depends_on("py-bz", type=("build", "run")) .. note:: The ``with when()`` context manager is composable, while ``with default_args()`` merely overrides the default. For example: .. code-block:: python with default_args(when="+feature"): depends_on("foo") depends_on("bar") depends_on("baz", when="+baz") is equivalent to: .. code-block:: python depends_on("foo", when="+feature") depends_on("bar", when="+feature") depends_on("baz", when="+baz") # Note: not when="+feature+baz" .. _custom-attributes: ``home``, ``command``, ``headers``, and ``libs`` ------------------------------------------------ Often a package will need to provide attributes for dependents to query various details about what it provides. While any number of custom defined attributes can be implemented by a package, the four specific attributes described below are always available on every package with default implementations and the ability to customize with alternate implementations in the case of virtual packages provided: =========== =========================================== ===================== Attribute Purpose Default =========== =========================================== ===================== ``home`` The installation path for the package ``spec.prefix`` ``command`` An executable command for the package | ``spec.name`` found in | ``.home.bin`` ``headers`` A list of headers provided by the package | All headers searched | recursively in ``.home.include`` ``libs`` A list of libraries provided by the package | ``lib{spec.name}`` searched | recursively in ``.home`` starting | with ``lib``, ``lib64``, then the | rest of ``.home`` =========== =========================================== ===================== Each of these can be customized by implementing the relevant attribute as a ``@property`` in the package's class: .. code-block:: python :linenos: class Foo(Package): ... @property def libs(self): # The library provided by Foo is libMyFoo.so return find_libraries("libMyFoo", root=self.home, recursive=True) A package may also provide custom implementations of each attribute for the virtual packages it provides, by implementing the ``_`` property in its package class. The implementation used is the first one found from: #. Specialized virtual: ``Package._`` #. Generic package: ``Package.`` #. Default The use of customized attributes is demonstrated in the next example. Example: Customized attributes for virtual packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Consider a package ``foo`` that can optionally provide two virtual packages ``bar`` and ``baz``. When both are enabled, the installation tree appears as follows: .. code-block:: console include/foo.h include/bar/bar.h lib64/libFoo.so lib64/libFooBar.so baz/include/baz/baz.h baz/lib/libFooBaz.so The install tree shows that ``foo`` provides the header ``include/foo.h`` and library ``lib64/libFoo.so`` in its install prefix. The virtual package ``bar`` provides the header ``include/bar/bar.h`` and library ``lib64/libFooBar.so``, also in ``foo``'s install prefix. The ``baz`` package, however, is provided in the ``baz`` subdirectory of ``foo``'s prefix with the ``include/baz/baz.h`` header and ``lib/libFooBaz.so`` library. Such a package could implement the optional attributes as follows: .. code-block:: python :linenos: class Foo(Package): ... variant("bar", default=False, description="Enable the Foo implementation of bar") variant("baz", default=False, description="Enable the Foo implementation of baz") ... provides("bar", when="+bar") provides("baz", when="+baz") ... # Just the foo headers @property def headers(self): return find_headers("foo", root=self.home.include, recursive=False) # Just the foo libraries @property def libs(self): return find_libraries("libFoo", root=self.home, recursive=True) # The header provided by the bar virtual package @property def bar_headers(self): return find_headers("bar/bar.h", root=self.home.include, recursive=False) # The library provided by the bar virtual package @property def bar_libs(self): return find_libraries("libFooBar", root=self.home, recursive=True) # The baz virtual package home @property def baz_home(self): return self.prefix.baz # The header provided by the baz virtual package @property def baz_headers(self): return find_headers("baz/baz", root=self.baz_home.include, recursive=False) # The library provided by the baz virtual package @property def baz_libs(self): return find_libraries("libFooBaz", root=self.baz_home, recursive=True) Now consider another package, ``foo-app``, depending on all three: .. code-block:: python :linenos: class FooApp(CMakePackage): ... depends_on("foo") depends_on("bar") depends_on("baz") The resulting spec objects for its dependencies shows the result of the above attribute implementations: .. code-block:: pycon # The core headers and libraries of the foo package >>> spec["foo"] foo@1.0/ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6 >>> spec["foo"].prefix "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6" # home defaults to the package install prefix without an explicit implementation >>> spec["foo"].home "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6" # foo headers from the foo prefix >>> spec["foo"].headers HeaderList([ "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include/foo.h", ]) # foo include directories from the foo prefix >>> spec["foo"].headers.directories ["/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include"] # foo libraries from the foo prefix >>> spec["foo"].libs LibraryList([ "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64/libFoo.so", ]) # foo library directories from the foo prefix >>> spec["foo"].libs.directories ["/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64"] .. code-block:: pycon # The virtual bar package in the same prefix as foo # bar resolves to the foo package >>> spec["bar"] foo@1.0/ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6 >>> spec["bar"].prefix "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6" # home defaults to the foo prefix without either a Foo.bar_home # or Foo.home implementation >>> spec["bar"].home "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6" # bar header in the foo prefix >>> spec["bar"].headers HeaderList([ "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include/bar/bar.h" ]) # bar include dirs from the foo prefix >>> spec["bar"].headers.directories ["/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include"] # bar library from the foo prefix >>> spec["bar"].libs LibraryList([ "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64/libFooBar.so" ]) # bar library directories from the foo prefix >>> spec["bar"].libs.directories ["/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64"] .. code-block:: pycon # The virtual baz package in a subdirectory of foo's prefix # baz resolves to the foo package >>> spec["baz"] foo@1.0/ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6 >>> spec["baz"].prefix "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6" # baz_home implementation provides the subdirectory inside the foo prefix >>> spec["baz"].home "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz" # baz headers in the baz subdirectory of the foo prefix >>> spec["baz"].headers HeaderList([ "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/include/baz/baz.h" ]) # baz include directories in the baz subdirectory of the foo prefix >>> spec["baz"].headers.directories [ "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/include" ] # baz libraries in the baz subdirectory of the foo prefix >>> spec["baz"].libs LibraryList([ "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/lib/libFooBaz.so" ]) # baz library directories in the baz subdirectory of the foo prefix >>> spec["baz"].libs.directories [ "/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/lib" ] Style guidelines for packages ----------------------------- The following guidelines are provided, in the interests of making Spack packages work in a consistent manner: Variant Names ^^^^^^^^^^^^^ Spack packages with variants similar to already-existing Spack packages should use the same name for their variants. Standard variant names are: ======= ======== ======================== Name Default Description ======= ======== ======================== shared True Build shared libraries mpi True Use MPI python False Build Python extension ======= ======== ======================== If specified in this table, the corresponding default is recommended. The semantics of the ``shared`` variant are important. When a package is built ``~shared``, the package guarantees that no shared libraries are built. When a package is built ``+shared``, the package guarantees that shared libraries are built, but it makes no guarantee about whether static libraries are built. Version definitions ^^^^^^^^^^^^^^^^^^^ Spack packages should list supported versions with the newest first. Using ``home`` vs ``prefix`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``home`` and ``prefix`` are both attributes that can be queried on a package's dependencies, often when passing configure arguments pointing to the location of a dependency. The difference is that while ``prefix`` is the location on disk where a concrete package resides, ``home`` is the `logical` location that a package resides, which may be different than ``prefix`` in the case of virtual packages or other special circumstances. For most use cases inside a package, its dependency locations can be accessed via either ``self.spec["foo"].home`` or ``self.spec["foo"].prefix``. Specific packages that should be consumed by dependents via ``.home`` instead of ``.prefix`` should be noted in their respective documentation. See :ref:`custom-attributes` for more details and an example implementing a custom ``home`` attribute. ================================================ FILE: lib/spack/docs/packaging_guide_testing.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to adding tests to Spack packages to ensure correct installation and functionality. .. list-table:: :widths: 25 25 25 25 :header-rows: 0 :width: 100% * - :doc:`1. Creation ` - :doc:`2. Build ` - **3. Testing** - :doc:`4. Advanced ` Packaging Guide: testing installations ====================================== In this part of the packaging guide we will cover how to ensure your package builds correctly by adding tests to it. .. _checking_an_installation: Checking an installation ------------------------ A package that *appears* to install successfully does not mean it is actually installed correctly or will continue to work indefinitely. There are a number of possible points of failure so Spack provides features for checking the software along the way. Failures can occur during and after the installation process. The build may start, but the software may not end up fully installed. The installed software may not work at all, or may not work as expected. The software may work after being installed, but due to changes on the system, may stop working days, weeks, or months after being installed. This section describes Spack's support for checks that can be performed during and after its installation. The former checks are referred to as ``build-time tests`` and the latter as ``stand-alone (or smoke) tests``. .. _build_time-tests: Build-time tests ^^^^^^^^^^^^^^^^ Spack infers the status of a build based on the contents of the install prefix. Success is assumed if anything (e.g., a file or directory) is written after ``install()`` completes. Otherwise, the build is assumed to have failed. However, the presence of install prefix contents is not a sufficient indicator of success so Spack supports the addition of tests that can be performed during `spack install` processing. Consider a simple Autotools build using the following commands: .. code-block:: console $ ./configure --prefix=/path/to/installation/prefix $ make $ make install Standard Autotools and CMake do not write anything to the prefix from the ``configure`` and ``make`` commands. Files are only written from the ``make install`` after the build completes. .. note:: If you want to learn more about ``Autotools`` and ``CMake`` packages in Spack, refer to :ref:`AutotoolsPackage ` and :ref:`CMakePackage `, respectively. What can you do to check that the build is progressing satisfactorily? If there are specific files and/or directories expected of a successful installation, you can add basic, fast ``sanity checks``. You can also add checks to be performed after one or more installation phases. .. note:: Build-time tests are performed when the ``--test`` option is passed to ``spack install``. .. warning:: Build-time test failures result in a failed installation of the software. .. _sanity-checks: Adding sanity checks """""""""""""""""""" Unfortunately, many builds of scientific software modify the installation prefix **before** ``make install``. Builds like this can falsely report success when an error occurs before the installation is complete. Simple sanity checks can be used to identify files and/or directories that are required of a successful installation. Spack checks for the presence of the files and directories after ``install()`` runs. If any of the listed files or directories are missing, then the build will fail and the install prefix will be removed. If they all exist, then Spack considers the build successful from a sanity check perspective and keeps the prefix in place. For example, the sanity checks for the ``reframe`` package below specify that eight paths must exist within the installation prefix after the ``install`` method completes. .. code-block:: python class Reframe(Package): ... # sanity check sanity_check_is_file = [join_path("bin", "reframe")] sanity_check_is_dir = [ "bin", "config", "docs", "reframe", "tutorials", "unittests", "cscs-checks", ] When you run ``spack install`` with tests enabled, Spack will ensure that a successfully installed package has the required files and/or directories. For example, running: .. code-block:: spec $ spack install --test=root reframe results in Spack checking that the installation created the following **file**: * ``self.prefix.bin.reframe`` and the following **directories**: * ``self.prefix.bin`` * ``self.prefix.config`` * ``self.prefix.docs`` * ``self.prefix.reframe`` * ``self.prefix.tutorials`` * ``self.prefix.unittests`` * ``self.prefix.cscs-checks`` If **any** of these paths are missing, then Spack considers the installation to have failed. .. note:: You **MUST** use ``sanity_check_is_file`` to specify required files and ``sanity_check_is_dir`` for required directories. .. _install_phase-tests: Adding installation phase tests """"""""""""""""""""""""""""""" Sometimes packages appear to build "correctly" only to have runtime behavior issues discovered at a later stage, such as after a full software stack relying on them has been built. Checks can be performed at different phases of the package installation to possibly avoid these types of problems. Some checks are built-in to different build systems, while others will need to be added to the package. Built-in installation phase tests are provided by packages inheriting from select :ref:`build systems `, where naming conventions are used to identify typical test identifiers for those systems. In general, you won't need to add anything to your package to take advantage of these tests if your software's build system complies with the convention; otherwise, you'll want or need to override the post-phase method to perform other checks. .. list-table:: Built-in installation phase tests :header-rows: 1 * - Build System Class - Post-Build Phase Method (Runs) - Post-Install Phase Method (Runs) * - :ref:`AutotoolsPackage ` - ``check`` (``make test``, ``make check``) - ``installcheck`` (``make installcheck``) * - :ref:`CachedCMakePackage ` - ``check`` (``make check``, ``make test``) - Not applicable * - :ref:`CMakePackage ` - ``check`` (``make check``, ``make test``) - Not applicable * - :ref:`MakefilePackage ` - ``check`` (``make test``, ``make check``) - ``installcheck`` (``make installcheck``) * - :ref:`MesonPackage ` - ``check`` (``make test``, ``make check``) - Not applicable * - :ref:`PerlPackage ` - ``check`` (``make test``) - Not applicable * - :ref:`PythonPackage ` - Not applicable - ``test_imports`` (module imports) * - :ref:`QMakePackage ` - ``check`` (``make check``) - Not applicable * - :ref:`SConsPackage ` - ``build_test`` (must be overridden) - Not applicable * - :ref:`SIPPackage ` - Not applicable - ``test_imports`` (module imports) * - :ref:`WafPackage ` - ``build_test`` (must be overridden) - ``install_test`` (must be overridden) For example, the ``Libelf`` package inherits from ``AutotoolsPackage`` and its ``Makefile`` has a standard ``check`` target. So Spack will automatically run ``make check`` after the ``build`` phase when it is installed using the ``--test`` option, such as: .. code-block:: spec $ spack install --test=root libelf In addition to overriding any built-in build system installation phase tests, you can write your own install phase tests. You will need to use two decorators for each phase test method: * ``run_after`` * ``on_package_attributes`` The first decorator tells Spack when in the installation process to run your test method installation process; namely *after* the provided installation phase. The second decorator tells Spack to only run the checks when the ``--test`` option is provided on the command line. .. note:: Be sure to place the directives above your test method in the order ``run_after`` *then* ``on_package_attributes``. .. note:: You also want to be sure the package supports the phase you use in the ``run_after`` directive. For example, ``PackageBase`` only supports the ``install`` phase while the ``AutotoolsPackage`` and ``MakefilePackage`` support both ``install`` and ``build`` phases. Assuming both ``build`` and ``install`` phases are available, you can add additional checks to be performed after each of those phases based on the skeleton provided below. .. code-block:: python class YourMakefilePackage(MakefilePackage): ... @run_after("build") @on_package_attributes(run_tests=True) def check_build(self): # Add your custom post-build phase tests pass @run_after("install") @on_package_attributes(run_tests=True) def check_install(self): # Add your custom post-install phase tests pass .. note:: You could also schedule work to be done **before** a given phase using the ``run_before`` decorator. By way of a concrete example, the ``reframe`` package mentioned previously has a simple installation phase check that runs the installed executable. The check is implemented as follows: .. code-block:: python class Reframe(Package): ... # check if we can run reframe @run_after("install") @on_package_attributes(run_tests=True) def check_list(self): with working_dir(self.stage.source_path): reframe = Executable(self.prefix.bin.reframe) reframe("-l") Checking build-time test results """""""""""""""""""""""""""""""" Checking the results of these tests after running ``spack install --test`` can be done by viewing the spec's ``install-time-test-log.txt`` file whose location will depend on whether the spec installed successfully. A successful installation results in the build and stage logs being copied to the ``.spack`` subdirectory of the spec's prefix. For example, .. code-block:: spec $ spack install --test=root zlib@1.2.13 ... [+] /home/user/spack/opt/spack/linux-rhel8-broadwell/gcc-10.3.1/zlib-1.2.13-tehu6cbsujufa2tb6pu3xvc6echjstv6 $ cat /home/user/spack/opt/spack/linux-rhel8-broadwell/gcc-10.3.1/zlib-1.2.13-tehu6cbsujufa2tb6pu3xvc6echjstv6/.spack/install-time-test-log.txt If the installation fails due to build-time test failures, then both logs will be left in the build stage directory as illustrated below: .. code-block:: spec $ spack install --test=root zlib@1.2.13 ... See build log for details: /var/tmp/user/spack-stage/spack-stage-zlib-1.2.13-lxfsivs4htfdewxe7hbi2b3tekj4make/spack-build-out.txt $ cat /var/tmp/user/spack-stage/spack-stage-zlib-1.2.13-lxfsivs4htfdewxe7hbi2b3tekj4make/install-time-test-log.txt .. _cmd-spack-test: Stand-alone tests ^^^^^^^^^^^^^^^^^ While build-time tests are integrated with the installation process, stand-alone tests are expected to run days, weeks, even months after the software is installed. The goal is to provide a mechanism for gaining confidence that packages work as installed **and** *continue* to work as the underlying software evolves. Packages can add and inherit stand-alone tests. The ``spack test`` command is used for stand-alone testing. .. admonition:: Stand-alone test methods should complete within a few minutes. Execution speed is important since these tests are intended to quickly assess whether installed specs work on the system. Spack cannot spare resources for more extensive testing of packages included in CI stacks. Consequently, stand-alone tests should run relatively quickly -- as in on the order of at most a few minutes -- while testing at least key aspects of the installed software. Save more extensive testing for other tools. Tests are defined in the package using methods with names beginning ``test_``. This allows Spack to support multiple independent checks, or parts. Files needed for testing, such as source, data, and expected outputs, may be saved from the build and/or stored with the package in the repository. Regardless of origin, these files are automatically copied to the spec's test stage directory prior to execution of the test method(s). Spack also provides helper functions to facilitate common processing. .. tip:: **The status of stand-alone tests can be used to guide follow-up testing efforts.** Passing stand-alone tests justifies performing more thorough testing, such as running extensive unit or regression tests or tests that run at scale, when available. These tests are outside of the scope of Spack packaging. Failing stand-alone tests indicate problems with the installation and, therefore, no reason to proceed with more resource-intensive tests until the failures have been investigated. .. _configure-test-stage: Configuring the test stage directory """""""""""""""""""""""""""""""""""" Stand-alone tests utilize a test stage directory to build, run, and track tests in the same way Spack uses a build stage directory to install software. The default test stage root directory, ``$HOME/.spack/test``, is defined in :ref:`config.yaml `. This location is customizable by adding or changing the ``test_stage`` path such that: .. code-block:: yaml config: test_stage: /path/to/test/stage Packages can use the ``self.test_suite.stage`` property to access the path. .. admonition:: Each spec being tested has its own test stage directory. The ``config:test_stage`` option is the path to the root of a **test suite**'s stage directories. Other package properties that provide paths to spec-specific subdirectories and files are described in :ref:`accessing-files`. .. _adding-standalone-tests: Adding stand-alone tests """""""""""""""""""""""" Test recipes are defined in the package using methods with names beginning ``test_``. This allows for the implementation of multiple independent tests. Each method has access to the information Spack tracks on the package, such as options, compilers, and dependencies, supporting the customization of tests to the build. Standard Python ``assert`` statements and other error reporting mechanisms can be used. These exceptions are automatically caught and reported as test failures. Each test method is an *implicit test part* named by the method. Its purpose is the method's docstring. Providing a meaningful purpose for the test gives context that can aid debugging. Spack outputs both the name and purpose at the start of test execution so it's also important that the docstring/purpose be brief. .. tip:: We recommend naming test methods so it is clear *what* is being tested. For example, if a test method is building and/or running an executable called ``example``, then call the method ``test_example``. This, together with a similarly meaningful test purpose, will aid test comprehension, debugging, and maintainability. Stand-alone tests run in an environment that provides access to information on the installed software, such as build options, dependencies, and compilers. Build options and dependencies are accessed using the same spec checks used by build recipes. Examples of checking :ref:`variant settings ` and :ref:`spec constraints ` can be found at the provided links. .. admonition:: Spack automatically sets up the test stage directory and environment. Spack automatically creates the test stage directory and copies relevant files *prior to* running tests. It can also ensure build dependencies are available **if** necessary. The path to the test stage is configurable (see :ref:`configure-test-stage`). Files that Spack knows to copy are those saved from the build (see :ref:`cache_extra_test_sources`) and those added to the package repository (see :ref:`cache_custom_files`). Spack will use the value of the ``test_requires_compiler`` property to determine whether it needs to also set up build dependencies (see :ref:`test-build-tests`). The ``MyPackage`` package below provides two basic test examples: ``test_example`` and ``test_example2``. The first runs the installed ``example`` and ensures its output contains an expected string. The second runs ``example2`` without checking output so is only concerned with confirming the executable runs successfully. If the installed spec is not expected to have ``example2``, then the check at the top of the method will raise a special ``SkipTest`` exception, which is captured to facilitate reporting skipped test parts to tools like CDash. .. code-block:: python class MyPackage(Package): ... def test_example(self): """ensure installed example works""" expected = "Done." example = which(self.prefix.bin.example) # Capture stdout and stderr from running the Executable # and check that the expected output was produced. out = example(output=str.split, error=str.split) assert expected in out, f"Expected '{expected}' in the output" def test_example2(self): """run installed example2""" if self.spec.satisfies("@:1.0"): # Raise SkipTest to ensure flagging the test as skipped for # test reporting purposes. raise SkipTest("Test is only available for v1.1 on") example2 = which(self.prefix.bin.example2) example2() Output showing the identification of each test part after running the tests is illustrated below. .. code-block:: console $ spack test run --alias mypackage mypackage@2.0 ==> Spack test mypackage ... $ spack test results -l mypackage ==> Results for test suite 'mypackage': ... ==> [2024-03-10-16:03:56.625439] test: test_example: ensure installed example works ... PASSED: MyPackage::test_example ==> [2024-03-10-16:03:56.625439] test: test_example2: run installed example2 ... PASSED: MyPackage::test_example2 .. admonition:: Do NOT implement tests that must run in the installation prefix. Use of the package spec's installation prefix for building and running tests is **strongly discouraged**. Doing so causes permission errors for shared spack instances *and* facilities that install the software in read-only file systems or directories. Instead, start these test methods by explicitly copying the needed files from the installation prefix to the test stage directory. Note the test stage directory is the current directory when the test is executed with the ``spack test run`` command. .. admonition:: Test methods for library packages should build test executables. Stand-alone tests for library packages *should* build test executables that utilize the *installed* library. Doing so ensures the tests follow a similar build process that users of the library would follow. For more information on how to do this, see :ref:`test-build-tests`. .. tip:: If you want to see more examples from packages with stand-alone tests, run ``spack pkg grep "def\stest" | sed "s/\/package.py.*//g" | sort -u`` from the command line to get a list of the packages. .. _adding-standalone-test-parts: Adding stand-alone test parts """"""""""""""""""""""""""""" Sometimes dependencies between steps of a test lend themselves to being broken into parts. Tracking the pass/fail status of each part can aid debugging. Spack provides a ``test_part`` context manager for use within test methods. Each test part is independently run, tracked, and reported. Test parts are executed in the order they appear. If one fails, subsequent test parts are still performed even if they would also fail. This allows tools like CDash to track and report the status of test parts across runs. The pass/fail status of the enclosing test is derived from the statuses of the embedded test parts. .. admonition:: Test method and test part names **must** be unique. Test results reporting requires that test methods and embedded test parts within a package have unique names. The signature for ``test_part`` is: .. code-block:: python def test_part(pkg, test_name, purpose, work_dir=".", verbose=False): ... where each argument has the following meaning: * ``pkg`` is an instance of the package for the spec under test. * ``test_name`` is the name of the test part, which must start with ``test_``. * ``purpose`` is a brief description used as a heading for the test part. Output from the test is written to a test log file allowing the test name and purpose to be searched for test part confirmation and debugging. * ``work_dir`` is the path to the directory in which the test will run. The default of ``None``, or ``"."``, corresponds to the spec's test stage (i.e., ``self.test_suite.test_dir_for_spec(self.spec)``). .. admonition:: Start test part names with the name of the enclosing test. We **highly recommend** starting the names of test parts with the name of the enclosing test. Doing so helps with the comprehension, readability and debugging of test results. Suppose ``MyPackage`` installs multiple executables that need to run in a specific order since the outputs from one are inputs of others. Further suppose we want to add an integration test that runs the executables in order. We can accomplish this goal by implementing a stand-alone test method consisting of test parts for each executable as follows: .. code-block:: python class MyPackage(Package): ... def test_series(self): """run setup, perform, and report""" with test_part(self, "test_series_setup", purpose="setup operation"): exe = which(self.prefix.bin.setup) exe() with test_part(self, "test_series_run", purpose="perform operation"): exe = which(self.prefix.bin.run) exe() with test_part(self, "test_series_report", purpose="generate report"): exe = which(self.prefix.bin.report) exe() The result is ``test_series`` runs the following executable in order: ``setup``, ``run``, and ``report``. In this case no options are passed to any of the executables and no outputs from running them are checked. Consequently, the implementation could be simplified with a for-loop as follows: .. code-block:: python class MyPackage(Package): ... def test_series(self): """execute series setup, run, and report""" for exe, reason in [ ("setup", "setup operation"), ("run", "perform operation"), ("report", "generate report"), ]: with test_part(self, f"test_series_{exe}", purpose=reason): exe = which(self.prefix.bin.join(exe)) exe() In both cases, since we're using a context manager, each test part in ``test_series`` will execute regardless of the status of the other test parts. Now let's look at the output from running the stand-alone tests where the second test part, ``test_series_run``, fails. .. code-block:: console $ spack test run --alias mypackage mypackage@1.0 ==> Spack test mypackage ... $ spack test results -l mypackage ==> Results for test suite 'mypackage': ... ==> [2024-03-10-16:03:56.625204] test: test_series: execute series setup, run, and report ==> [2024-03-10-16:03:56.625439] test: test_series_setup: setup operation ... PASSED: MyPackage::test_series_setup ==> [2024-03-10-16:03:56.625555] test: test_series_run: perform operation ... FAILED: MyPackage::test_series_run ==> [2024-03-10-16:03:57.003456] test: test_series_report: generate report ... FAILED: MyPackage::test_series_report FAILED: MyPackage::test_series ... Since test parts depended on the success of previous parts, we see that the failure of one results in the failure of subsequent checks and the overall result of the test method, ``test_series``, is failure. .. tip:: If you want to see more examples from packages using ``test_part``, run ``spack pkg grep "test_part(" | sed "s/\/package.py.*//g" | sort -u`` from the command line to get a list of the packages. .. _test-build-tests: Building and running test executables """"""""""""""""""""""""""""""""""""" .. admonition:: Reuse build-time sources and (small) input data sets when possible. We **highly recommend** reusing build-time test sources and pared down input files for testing installed software. These files are easier to keep synchronized with software capabilities when they reside within the software's repository. More information on saving files from the installation process can be found at :ref:`cache_extra_test_sources`. If that is not possible, you can add test-related files to the package repository (see :ref:`cache_custom_files`). It will be important to remember to maintain them so they work across listed or supported versions of the package. Packages that build libraries are good examples of cases where you'll want to build test executables from the installed software before running them. Doing so requires you to let Spack know it needs to load the package's compiler configuration. This is accomplished by setting the package's ``test_requires_compiler`` property to ``True``. .. admonition:: ``test_requires_compiler = True`` is required to build test executables. Setting the property to ``True`` ensures access to the compiler through canonical environment variables (e.g., ``CC``, ``CXX``, ``FC``, ``F77``). It also gives access to build dependencies like ``cmake`` through their ``spec objects`` (e.g., ``self.spec["cmake"].prefix.bin.cmake`` for the path or ``self.spec["cmake"].command`` for the ``Executable`` instance). Be sure to add the property at the top of the package class under other properties like the ``homepage``. The example below, which ignores how ``cxx-example.cpp`` is acquired, illustrates the basic process of compiling a test executable using the installed library before running it. .. code-block:: python class MyLibrary(Package): ... test_requires_compiler = True ... def test_cxx_example(self): """build and run cxx-example""" exe = "cxx-example" ... cxx = which(os.environ["CXX"]) cxx(f"-L{self.prefix.lib}", f"-I{self.prefix.include}", f"{exe}.cpp", "-o", exe) cxx_example = which(exe) cxx_example() Typically the files used to build and/or run test executables are either cached from the installation (see :ref:`cache_extra_test_sources`) or added to the package repository (see :ref:`cache_custom_files`). There is nothing preventing the use of both. .. _cache_extra_test_sources: Saving build- and install-time files """""""""""""""""""""""""""""""""""" You can use the ``cache_extra_test_sources`` helper routine to copy directories and/or files from the source build stage directory to the package's installation directory. Spack will automatically copy these files for you when it sets up the test stage directory and before it begins running the tests. The signature for ``cache_extra_test_sources`` is: .. code-block:: python def cache_extra_test_sources(pkg, srcs): ... where each argument has the following meaning: * ``pkg`` is an instance of the package for the spec under test. * ``srcs`` is a string *or* a list of strings corresponding to the paths of subdirectories and/or files needed for stand-alone testing. .. warning:: Paths provided in the ``srcs`` argument **must be relative** to the staged source directory. They will be copied to the equivalent relative location under the test stage directory prior to test execution. Contents of subdirectories and files are copied to a special test cache subdirectory of the installation prefix. They are automatically copied to the appropriate relative paths under the test stage directory prior to executing stand-alone tests. .. tip:: *Perform test-related conversions once when copying files.* If one or more of the copied files needs to be modified to reference the installed software, it is recommended that those changes be made to the cached files **once** in the post-``install`` copy method **after** the call to ``cache_extra_test_sources``. This will reduce the amount of unnecessary work in the test method **and** avoid problems running stand-alone tests in shared instances and facility deployments. The ``filter_file`` function can be quite useful for such changes (see :ref:`file-filtering`). Below is a basic example of a test that relies on files from the installation. This package method reuses the contents of the ``examples`` subdirectory, which is assumed to have all of the files necessary to allow ``make`` to compile and link ``foo.c`` and ``bar.c`` against the package's installed library. .. code-block:: python class MyLibPackage(MakefilePackage): ... @run_after("install") def copy_test_files(self): cache_extra_test_sources(self, "examples") def test_example(self): """build and run the examples""" examples_dir = self.test_suite.current_test_cache_dir.examples with working_dir(examples_dir): make = which("make") make() for program in ["foo", "bar"]: with test_part(self, f"test_example_{program}", purpose=f"ensure {program} runs"): exe = Executable(program) exe() In this case, ``copy_test_files`` copies the associated files from the build stage to the package's test cache directory under the installation prefix. Running ``spack test run`` for the package results in Spack copying the directory and its contents to the test stage directory. The ``working_dir`` context manager ensures the commands within it are executed from the ``examples_dir``. The test builds the software using ``make`` before running each executable, ``foo`` and ``bar``, as independent test parts. .. note:: The method name ``copy_test_files`` here is for illustration purposes. You are free to use a name that is better suited to your package. The key to copying files for stand-alone testing at build time is use of the ``run_after`` directive, which ensures the associated files are copied **after** the provided build stage (``install``) when the installation prefix **and** files are available. The test method uses the path contained in the package's ``self.test_suite.current_test_cache_dir`` property for the root directory of the copied files. In this case, that's the ``examples`` subdirectory. .. tip:: If you want to see more examples from packages that cache build files, run ``spack pkg grep cache_extra_test_sources | sed "s/\/package.py.*//g" | sort -u`` from the command line to get a list of the packages. .. _cache_custom_files: Adding custom files """"""""""""""""""" Sometimes it is helpful or necessary to include custom files for building and/or checking the results of tests as part of the package. Examples of the types of files that might be useful are: - test source files - test input files - test build scripts - expected test outputs While obtaining such files from the software repository is preferred (see :ref:`cache_extra_test_sources`), there are circumstances where doing so is not feasible such as when the software is not being actively maintained. When test files cannot be obtained from the repository or there is a need to supplement files that can, Spack supports the inclusion of additional files under the ``test`` subdirectory of the package in the Spack repository. The following example assumes a ``custom-example.c`` is saved in ``MyLibrary`` package's ``test`` subdirectory. It also assumes the program simply needs to be compiled and linked against the installed ``MyLibrary`` software. .. code-block:: python class MyLibrary(Package): ... test_requires_compiler = True ... def test_custom_example(self): """build and run custom-example""" src_dir = self.test_suite.current_test_data_dir exe = "custom-example" with working_dir(src_dir): cc = which(os.environ["CC"]) cc(f"-L{self.prefix.lib}", f"-I{self.prefix.include}", f"{exe}.cpp", "-o", exe) custom_example = Executable(exe) custom_example() In this case, ``spack test run`` for the package results in Spack copying the contents of the ``test`` subdirectory to the test stage directory path in ``self.test_suite.current_test_data_dir`` before calling ``test_custom_example``. Use of the ``working_dir`` context manager ensures the commands to build and run the program are performed from within the appropriate subdirectory of the test stage. .. _expected_test_output_from_file: Reading expected output from a file """"""""""""""""""""""""""""""""""" The helper function ``get_escaped_text_output`` is available for packages to retrieve properly formatted text from a file potentially containing special characters. The signature for ``get_escaped_text_output`` is: .. code-block:: python def get_escaped_text_output(filename): ... where ``filename`` is the path to the file containing the expected output. The path provided to ``filename`` for one of the copied custom files (:ref:`custom file `) is in the path rooted at ``self.test_suite.current_test_data_dir``. The example below shows how to reference both the custom database (``packages.db``) and expected output (``dump.out``) files Spack copies to the test stage: .. code-block:: python import re class Sqlite(AutotoolsPackage): ... def test_example(self): """check example table dump""" test_data_dir = self.test_suite.current_test_data_dir db_filename = test_data_dir.join("packages.db") ... expected = get_escaped_text_output(test_data_dir.join("dump.out")) sqlite3 = which(self.prefix.bin.sqlite3) out = sqlite3(db_filename, ".dump", output=str.split, error=str.split) for exp in expected: assert re.search(exp, out), f"Expected '{exp}' in output" If the files were instead cached from installing the software, the paths to the two files would be found under the ``self.test_suite.current_test_cache_dir`` directory as shown below: .. code-block:: python def test_example(self): """check example table dump""" test_cache_dir = self.test_suite.current_test_cache_dir db_filename = test_cache_dir.join("packages.db") ... expected = get_escaped_text_output(test_cache_dir.join("dump.out")) ... Alternatively, if both files had been installed by the software into the ``share/tests`` subdirectory of the installation prefix, the paths to the two files would be referenced as follows: .. code-block:: python def test_example(self): """check example table dump""" db_filename = self.prefix.share.tests.join("packages.db") ... expected = get_escaped_text_output(self.prefix.share.tests.join("dump.out")) ... .. _check_outputs: Comparing expected to actual outputs """""""""""""""""""""""""""""""""""" The ``check_outputs`` helper routine is available for packages to ensure multiple expected outputs from running an executable are contained within the actual outputs. The signature for ``check_outputs`` is: .. code-block:: python def check_outputs(expected, actual): ... where each argument has the expected type and meaning: * ``expected`` is a string or list of strings containing the expected (raw) output. * ``actual`` is a string containing the actual output from executing the command. Invoking the method is the equivalent of: .. code-block:: python errors = [] for check in expected: if not re.search(check, actual): errors.append(f"Expected '{check}' in output '{actual}'") if errors: raise RuntimeError("\n ".join(errors)) .. tip:: If you want to see more examples from packages that use this helper, run ``spack pkg grep check_outputs | sed "s/\/package.py.*//g" | sort -u`` from the command line to get a list of the packages. .. _accessing-files: Finding package- and test-related files """"""""""""""""""""""""""""""""""""""""" You may need to access files from one or more locations when writing stand-alone tests. This can happen if the software's repository does not include test source files or includes them but has no way to build the executables using the installed headers and libraries. In these cases you may need to reference the files relative to one or more root directories. The table below lists relevant path properties and provides additional examples of their use. See :ref:`expected_test_output_from_file` for examples of accessing files saved from the software repository, package repository, and installation. .. list-table:: Directory-to-property mapping :header-rows: 1 * - Root Directory - Package Property - Example(s) * - Package (Spec) Installation - ``self.prefix`` - ``self.prefix.include``, ``self.prefix.lib`` * - Dependency Installation - ``self.spec[""].prefix`` - ``self.spec["trilinos"].prefix.include`` * - Test Suite Stage - ``self.test_suite.stage`` - ``join_path(self.test_suite.stage, "results.txt")`` * - Spec's Test Stage - ``self.test_suite.test_dir_for_spec()`` - ``self.test_suite.test_dir_for_spec(self.spec)`` * - Current Spec's Build-time Files - ``self.test_suite.current_test_cache_dir`` - ``join_path(self.test_suite.current_test_cache_dir.examples, "foo.c")`` * - Current Spec's Custom Test Files - ``self.test_suite.current_test_data_dir`` - ``join_path(self.test_suite.current_test_data_dir, "hello.f90")`` .. _inheriting-tests: Inheriting stand-alone tests """""""""""""""""""""""""""" Stand-alone tests defined in parent (e.g., :ref:`build-systems`) and virtual (e.g., :ref:`virtual-dependencies`) packages are executed by packages that inherit from or provide interface implementations for those packages, respectively. The table below summarizes the stand-alone tests that will be executed along with those implemented in the package itself. .. list-table:: Inherited/provided stand-alone tests :header-rows: 1 * - Parent/Provider Package - Stand-alone Tests * - `C `_ - Compiles ``hello.c`` and runs it * - `Cxx `_ - Compiles and runs several ``hello`` programs * - `Fortran `_ - Compiles and runs ``hello`` programs (``F`` and ``f90``) * - `Mpi `_ - Compiles and runs ``mpi_hello`` (``c``, ``fortran``) * - :ref:`PythonPackage ` - Imports modules listed in the ``self.import_modules`` property with defaults derived from the tarball * - :ref:`SipPackage ` - Imports modules listed in the ``self.import_modules`` property with defaults derived from the tarball These tests are very basic so it is important that package developers and maintainers provide additional stand-alone tests customized to the package. .. warning:: Any package that implements a test method with the same name as an inherited method will override the inherited method. If that is not the goal and you are not explicitly calling and adding functionality to the inherited method for the test, then make sure that all test methods and embedded test parts have unique test names. One example of a package that adds its own stand-alone tests to those "inherited" by the virtual package it provides an implementation for is the `OpenMPI package `_. Below are snippets from running and viewing the stand-alone test results for ``openmpi``: .. code-block:: console $ spack test run --alias openmpi openmpi@4.1.4 ==> Spack test openmpi ==> Testing package openmpi-4.1.4-ubmrigj ============================== 1 passed of 1 spec ============================== $ spack test results -l openmpi ==> Results for test suite 'openmpi': ==> test specs: ==> openmpi-4.1.4-ubmrigj PASSED ==> Testing package openmpi-4.1.4-ubmrigj ==> [2023-03-10-16:03:56.160361] Installing $spack/opt/spack/linux-rhel7-broadwell/gcc-8.3.1/openmpi-4.1.4-ubmrigjrqcafh3hffqcx7yz2nc5jstra/.spack/test to $test_stage/xez37ekynfbi4e7h4zdndfemzufftnym/openmpi-4.1.4-ubmrigj/cache/openmpi ==> [2023-03-10-16:03:56.625204] test: test_bin: test installed binaries ==> [2023-03-10-16:03:56.625439] test: test_bin_mpirun: run and check output of mpirun ==> [2023-03-10-16:03:56.629807] '$spack/opt/spack/linux-rhel7-broadwell/gcc-8.3.1/openmpi-4.1.4-ubmrigjrqcafh3hffqcx7yz2nc5jstra/bin/mpirun' '-n' '1' 'ls' '..' openmpi-4.1.4-ubmrigj repo openmpi-4.1.4-ubmrigj-test-out.txt test_suite.lock PASSED: test_bin_mpirun ... ==> [2023-03-10-16:04:01.486977] test: test_version_oshcc: ensure version of oshcc is 8.3.1 SKIPPED: test_version_oshcc: oshcc is not installed ... ==> [2023-03-10-16:04:02.215227] Completed testing ==> [2023-03-10-16:04:02.215597] ======================== SUMMARY: openmpi-4.1.4-ubmrigj ======================== Openmpi::test_bin_mpirun .. PASSED Openmpi::test_bin_ompi_info .. PASSED Openmpi::test_bin_oshmem_info .. SKIPPED Openmpi::test_bin_oshrun .. SKIPPED Openmpi::test_bin_shmemrun .. SKIPPED Openmpi::test_bin .. PASSED ... ============================== 1 passed of 1 spec ============================== .. _cmd-spack-test-list: ``spack test list`` """"""""""""""""""" Packages available for install testing can be found using the ``spack test list`` command. The command outputs all installed packages that have defined stand-alone test methods. Alternatively you can use the ``--all`` option to get a list of all packages that have stand-alone test methods even if the packages are not installed. For more information, refer to :ref:`spack-test-list`. .. _cmd-spack-test-run: ``spack test run`` """""""""""""""""" Install tests can be run for one or more installed packages using the ``spack test run`` command. A ``test suite`` is created for all of the provided specs. The command accepts the same arguments provided to ``spack install`` (see :ref:`sec-specs`). If no specs are provided the command tests all specs in the active environment or all specs installed in the Spack instance if no environment is active. Test suites can be named using the ``--alias`` option. Unaliased test suites use the content hash of their specs as their name. Some of the more commonly used debugging options are: - ``--fail-fast`` stops testing each package after the first failure - ``--fail-first`` stops testing packages after the first failure Test output is written to a text log file by default, though ``junit`` and ``cdash`` are outputs available through the ``--log-format`` option. For more information, refer to :ref:`spack-test-run`. .. _cmd-spack-test-results: ``spack test results`` """""""""""""""""""""" The ``spack test results`` command shows results for all completed test suites by default. The alias or content hash can be provided to limit reporting to the corresponding test suite. The ``--logs`` option includes the output generated by the associated test(s) to facilitate debugging. The ``--failed`` option limits results shown to that of the failed tests, if any, of matching packages. For more information, refer to :ref:`spack-test-results`. .. _cmd-spack-test-find: ``spack test find`` """"""""""""""""""" The ``spack test find`` command lists the aliases or content hashes of all test suites whose results are available. For more information, refer to :ref:`spack-test-find`. .. _cmd-spack-test-remove: ``spack test remove`` """"""""""""""""""""" The ``spack test remove`` command removes test suites to declutter the test stage directory. You are prompted to confirm the removal of each test suite **unless** you use the ``--yes-to-all`` option. For more information, refer to :ref:`spack-test-remove`. ================================================ FILE: lib/spack/docs/pipelines.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to generate and run automated build pipelines in Spack for CI instances, enabling the building and deployment of binaries and reporting to CDash. .. _pipelines: CI Pipelines ============ Spack provides commands that support generating and running automated build pipelines in CI instances. At the highest level, it works like this: provide a Spack environment describing the set of packages you care about, and include a description of how those packages should be mapped to Gitlab runners. Spack can then generate a ``.gitlab-ci.yml`` file containing job descriptions for all your packages that can be run by a properly configured CI instance. When run, the generated pipeline will build and deploy binaries, and it can optionally report to a CDash instance regarding the health of the builds as they evolve over time. Getting started with pipelines ------------------------------ To get started with automated build pipelines, a Gitlab instance with version ``>= 12.9`` (more about Gitlab CI `here `_) with at least one `runner `_ configured is required. This can be done quickly by setting up a local Gitlab instance. It is possible to set up pipelines on gitlab.com, but the builds there are limited to 60 minutes and generic hardware. It is also possible to `hook up `_ Gitlab to Google Kubernetes Engine (`GKE `_) or Amazon Elastic Kubernetes Service (`EKS `_), though those topics are outside the scope of this document. After setting up a Gitlab instance for running CI, the basic steps for setting up a build pipeline are as follows: #. Create a repository in the Gitlab instance with CI and a runner enabled. #. Add a ``spack.yaml`` at the root containing your pipeline environment #. Add a ``.gitlab-ci.yml`` at the root containing two jobs (one to generate the pipeline dynamically, and one to run the generated jobs). #. Push a commit containing the ``spack.yaml`` and ``.gitlab-ci.yml`` mentioned above to the GitLab repository See the :ref:`functional_example` section for a minimal working example. See also the :ref:`custom_Workflow` section for a link to an example of a custom workflow based on Spack pipelines. Spack's pipelines are now making use of the `trigger `_ syntax to run dynamically generated `child pipelines `_. Note that the use of dynamic child pipelines requires running Gitlab version ``>= 12.9``. .. _functional_example: Functional Example ------------------ The simplest fully functional standalone example of a working pipeline can be examined live at this example `project `_ on gitlab.com. Here's the ``.gitlab-ci.yml`` file from that example that builds and runs the pipeline: .. code-block:: yaml stages: ["generate", "build"] variables: SPACK_REPOSITORY: "https://github.com/spack/spack.git" SPACK_REF: "develop-2024-10-06" SPACK_USER_CONFIG_PATH: ${CI_PROJECT_DIR} SPACK_BACKTRACE: 1 generate-pipeline: tags: - saas-linux-small-amd64 stage: generate image: name: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01 script: - git clone ${SPACK_REPOSITORY} - cd spack && git checkout ${SPACK_REF} && cd ../ - . "./spack/share/spack/setup-env.sh" - spack --version - spack env activate --without-view . - spack -d -v --color=always ci generate --check-index-only --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" artifacts: paths: - "${CI_PROJECT_DIR}/jobs_scratch_dir" build-pipeline: stage: build trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml job: generate-pipeline strategy: depend needs: - artifacts: true job: generate-pipeline The key thing to note above is that there are two jobs: The first job to run, ``generate-pipeline``, runs the ``spack ci generate`` command to generate a dynamic child pipeline and write it to a YAML file, which is then picked up by the second job, ``build-jobs``, and used to trigger the downstream pipeline. And here's the Spack environment built by the pipeline represented as a ``spack.yaml`` file: .. code-block:: yaml spack: view: false concretizer: unify: true reuse: false definitions: - pkgs: - zlib - bzip2 ~debug - compiler: - "%gcc" specs: - matrix: - - $pkgs - - $compiler ci: target: gitlab pipeline-gen: - any-job: tags: - saas-linux-small-amd64 image: name: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01 before_script: - git clone ${SPACK_REPOSITORY} - cd spack && git checkout ${SPACK_REF} && cd ../ - . "./spack/share/spack/setup-env.sh" - spack --version - export SPACK_USER_CONFIG_PATH=${CI_PROJECT_DIR} - spack config blame mirrors .. note:: The use of ``reuse: false`` in Spack environments used for pipelines is almost always what you want, as without it your pipelines will not rebuild packages even if package hashes have changed. This is due to the concretizer strongly preferring known hashes when ``reuse: true``. The ``ci`` section in the above environment file contains the bare minimum configuration required for ``spack ci generate`` to create a working pipeline. The ``target: gitlab`` tells Spack that the desired pipeline output is for GitLab. However, this isn't strictly required, as currently, GitLab is the only possible output format for pipelines. The ``pipeline-gen`` section contains the key information needed to specify attributes for the generated jobs. Notice that it contains a list which has only a single element in this case. In real pipelines, it will almost certainly have more elements, and in those cases, order is important: Spack starts at the bottom of the list and works upwards when applying attributes. But in this simple case, we use only the special key ``any-job`` to indicate that Spack should apply the specified attributes (``tags``, ``image``, and ``before_script``) to any job it generates. This includes jobs for building/pushing all packages, a ``rebuild-index`` job at the end of the pipeline, as well as any ``noop`` jobs that might be needed by GitLab when no rebuilds are required. Something to note is that in this simple case, we rely on Spack to generate a reasonable script for the package build jobs (it just creates a script that invokes ``spack ci rebuild``). Another thing to note is the use of the ``SPACK_USER_CONFIG_DIR`` environment variable in any generated jobs. The purpose of this is to make Spack aware of one final file in the example, the one that contains the mirror configuration. This file, ``mirrors.yaml`` looks like this: .. code-block:: yaml mirrors: buildcache-destination: url: oci://registry.gitlab.com/spack/pipeline-quickstart binary: true access_pair: id_variable: CI_REGISTRY_USER secret_variable: CI_REGISTRY_PASSWORD Note the name of the mirror is ``buildcache-destination``, which is required as of Spack 0.23 (see below for more information). The mirror URL simply points to the container registry associated with the project, while ``id_variable`` and ``secret_variable`` refer to environment variables containing the access credentials for the mirror. When Spack builds packages for this example project, they will be pushed to the project container registry, where they will be available for subsequent jobs to install as dependencies or for other pipelines to use to build runnable container images. Spack commands supporting pipelines ----------------------------------- Spack provides a ``ci`` command with a few sub-commands supporting Spack CI pipelines. These commands are covered in more detail in this section. .. _cmd-spack-ci: ``spack ci`` ^^^^^^^^^^^^ Super-command for functionality related to generating pipelines and executing pipeline jobs. .. _cmd-spack-ci-generate: ``spack ci generate`` ^^^^^^^^^^^^^^^^^^^^^ Throughout this documentation, references to the "mirror" mean the target mirror which is checked for the presence of up-to-date specs, and where any scheduled jobs should push built binary packages. When running ``spack ci generate`` it is required to configure a mirror named ``buildcache-destination`` to be used as the target mirror. It is permitted to configure any number of other mirrors as sources for your pipelines, but only the ``buildcache-destination`` mirror will be used as the destination mirror. Concretizes the specs in the active environment, stages them (as described in :ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk. During concretization of the environment, ``spack ci generate`` also writes a ``spack.lock`` file which is then provided to generated child jobs and made available in all generated job artifacts to aid in reproducing failed builds in a local environment. This means there are two artifacts that need to be exported in your pipeline generation job (defined in your ``.gitlab-ci.yml``). The first is the output yaml file of ``spack ci generate``, and the other is the directory containing the concrete environment files. In the :ref:`functional_example` section, we only mentioned one path in the ``artifacts`` ``paths`` list because we used ``--artifacts-root`` as the top level directory containing both the generated pipeline yaml and the concrete environment. Using ``--prune-dag`` or ``--no-prune-dag`` configures whether or not jobs are generated for specs that are already up to date on the mirror. If enabling DAG pruning using ``--prune-dag``, more information may be required in your ``spack.yaml`` file, see the :ref:`noop_jobs` section below regarding ``noop-job``. The optional ``--check-index-only`` argument can be used to speed up pipeline generation by telling Spack to consider only remote build cache indices when checking the remote mirror to determine if each spec in the DAG is up to date or not. The default behavior is for Spack to fetch the index and check it, but if the spec is not found in the index, it also performs a direct check for the spec on the mirror. If the remote build cache index is out of date, which can easily happen if it is not updated frequently, this behavior ensures that Spack has a way to know for certain about the status of any concrete spec on the remote mirror, but can slow down pipeline generation significantly. The optional ``--output-file`` argument should be an absolute path (including file name) to the generated pipeline, and if not given, the default is ``./.gitlab-ci.yml``. While optional, the ``--artifacts-root`` argument is used to determine where the concretized environment directory should be located. This directory will be created by ``spack ci generate`` and will contain the ``spack.yaml`` and generated ``spack.lock`` which are then passed to all child jobs as an artifact. This directory will also be the root directory for all artifacts generated by jobs in the pipeline. .. _cmd-spack-ci-rebuild: ``spack ci rebuild`` ^^^^^^^^^^^^^^^^^^^^ The purpose of ``spack ci rebuild`` is to take an assigned spec and ensure a binary of a successful build exists on the target mirror. If the binary does not already exist, it is built from source and pushed to the mirror. The associated stand-alone tests are optionally run against the new build. Additionally, files for reproducing the build outside the CI environment are created to facilitate debugging. If a binary for the spec does not exist on the target mirror, an install shell script, ``install.sh``, is created and saved in the current working directory. The script is run in a job to install the spec from source. The resulting binary package is pushed to the mirror. If ``cdash`` is configured for the environment, the build results will be uploaded to the site. Environment variables and values in the ``ci::pipeline-gen`` section of the ``spack.yaml`` environment file provide inputs to this process. The two main sources of environment variables are variables written into ``.gitlab-ci.yml`` by ``spack ci generate`` and the GitLab CI runtime. Several key CI pipeline variables are described in :ref:`ci_environment_variables`. If the ``--tests`` option is provided, stand-alone tests are performed but only if the build was successful *and* the package does not appear in the list of ``broken-tests-packages``. A shell script, ``test.sh``, is created and run to perform the tests. On completion, test logs are exported as job artifacts for review and to facilitate debugging. If ``cdash`` is configured, test results are also uploaded to the site. A snippet from an example ``spack.yaml`` file illustrating use of this option *and* specification of a package with broken tests is given below. The inclusion of a spec for building ``gptune`` is not shown here. Note that ``--tests`` is passed to ``spack ci rebuild`` as part of the ``build-job`` script. .. code-block:: yaml ci: pipeline-gen: - build-job: script: - . "./share/spack/setup-env.sh" - spack --version - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture.platform}-{architecture.target}/{name}-{version}-{hash}'" - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) broken-tests-packages: - gptune In this case, even if ``gptune`` is successfully built from source, the pipeline will *not* run its stand-alone tests since the package is listed under ``broken-tests-packages``. Spack's cloud pipelines provide actual, up-to-date examples of the CI/CD configuration and environment files used by Spack. You can find them under Spack's `stacks `_ repository directory. .. _cmd-spack-ci-rebuild-index: ``spack ci rebuild-index`` ^^^^^^^^^^^^^^^^^^^^^^^^^^ This is a convenience command to rebuild the build cache index associated with the mirror in the active, GitLab-enabled environment (specifying the mirror URL or name is not required). .. _cmd-spack-ci-reproduce-build: ``spack ci reproduce-build`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Given the URL to a GitLab pipeline rebuild job, downloads and unzips the artifacts into a local directory (which can be specified with the optional ``--working-dir`` argument), then finds the target job in the generated pipeline to extract details about how it was run. Assuming the job used a docker image, the command prints a ``docker run`` command line and some basic instructions on how to reproduce the build locally. Note that jobs failing in the pipeline will print messages giving the arguments you can pass to ``spack ci reproduce-build`` in order to reproduce a particular build locally. Job Types ------------------------------------ Rebuild (build) ^^^^^^^^^^^^^^^ Rebuild jobs, denoted as ``build-job``'s in the ``pipeline-gen`` list, are jobs associated with concrete specs that have been marked for rebuild. By default, a simple script for doing rebuild is generated but may be modified as needed. The default script does three main steps: change directories to the pipelines concrete environment, activate the concrete environment, and run the ``spack ci rebuild`` command: .. code-block:: bash cd ${concrete_environment_dir} spack env activate --without-view . spack ci rebuild .. _rebuild_index: Update Index (reindex) ^^^^^^^^^^^^^^^^^^^^^^ By default, while a pipeline job may rebuild a package, create a build cache entry, and push it to the mirror, it does not automatically re-generate the mirror's build cache index afterward. Because the index is not needed by the default rebuild jobs in the pipeline, not updating the index at the end of each job avoids possible race conditions between simultaneous jobs, and it avoids the computational expense of regenerating the index. This potentially saves minutes per job, depending on the number of binary packages in the mirror. As a result, the default is that the mirror's build cache index may not correctly reflect the mirror's contents at the end of a pipeline. To make sure the build cache index is up to date at the end of your pipeline, Spack generates a job to update the build cache index of the target mirror at the end of each pipeline by default. You can disable this behavior by adding ``rebuild-index: False`` inside the ``ci`` section of your Spack environment. Reindex jobs do not allow modifying the ``script`` attribute since it is automatically generated using the target mirror listed in the ``mirrors::mirror`` configuration. Signing (signing) ^^^^^^^^^^^^^^^^^ This job is run after all of the rebuild jobs are completed and is intended to be used to sign the package binaries built by a protected CI run. Signing jobs are generated only if a signing job ``script`` is specified and the Spack CI job type is protected. Note, if an ``any-job`` section contains a script, this will not implicitly create a ``signing`` job; a signing job may only exist if it is explicitly specified in the configuration with a ``script`` attribute. Specifying a signing job without a script does not create a signing job, and the job configuration attributes will be ignored. Signing jobs are always assigned the runner tags ``aws``, ``protected``, and ``notary``. .. _noop_jobs: No Op (noop) ^^^^^^^^^^^^ If no specs in an environment need to be rebuilt during a given pipeline run (meaning all are already up to date on the mirror), a single successful job (a NO-OP) is still generated to avoid an empty pipeline (which GitLab considers to be an error). The ``noop-job*`` sections can be added to your ``spack.yaml`` where you can provide ``tags`` and ``image`` or ``variables`` for the generated NO-OP job. This section also supports providing ``before_script``, ``script``, and ``after_script``, in case you want to take some custom actions in the case of an empty pipeline. Following is an example of this section added to a ``spack.yaml``: .. code-block:: yaml spack: ci: pipeline-gen: - noop-job: tags: ["custom", "tag"] image: name: "some.image.registry/custom-image:latest" entrypoint: ["/bin/bash"] script:: - echo "Custom message in a custom script" The example above illustrates how you can provide the attributes used to run the NO-OP job in the case of an empty pipeline. The only field for the NO-OP job that might be generated for you is ``script``, but that will only happen if you do not provide one yourself. Notice in this example the ``script`` uses the ``::`` notation to prescribe override behavior. Without this, the ``echo`` command would have been prepended to the automatically generated script rather than replacing it. ci.yaml ------------------------------------ Here's an example of a Spack configuration file describing a build pipeline: .. code-block:: yaml spack: ci: target: gitlab rebuild_index: true broken-specs-url: https://broken.specs.url broken-tests-packages: - gptune pipeline-gen: - submapping: - match: - os=ubuntu24.04 build-job: tags: - spack-kube image: spack/ubuntu-noble - match: - os=almalinux9 build-job: tags: - spack-kube image: spack/almalinux9 cdash: build-group: Release Testing url: https://cdash.spack.io project: Spack site: Spack AWS Gitlab Instance The ``ci`` config section is used to configure how the pipeline workload should be generated, mainly how the jobs for building specs should be assigned to the configured runners on your instance. The main section for configuring pipelines is ``pipeline-gen``, which is a list of job attribute sections that are merged, using the same rules as Spack configs (:ref:`config-scope-precedence`), from the bottom up. The order sections are applied is to be consistent with how Spack orders scope precedence when merging lists. There are two main section types: ``-job`` sections and ``submapping`` sections. Job Attribute Sections ^^^^^^^^^^^^^^^^^^^^^^ Each type of job may have attributes added or removed via sections in the ``pipeline-gen`` list. Job type specific attributes may be specified using the keys ``-job`` to add attributes to all jobs of type ```` or ``-job-remove`` to remove attributes of type ````. Each section may only contain one type of job attribute specification, i.e., ``build-job`` and ``noop-job`` may not coexist but ``build-job`` and ``build-job-remove`` may. .. note:: The ``*-remove`` specifications are applied before the additive attribute specification. For example, in the case where both ``build-job`` and ``build-job-remove`` are listed in the same ``pipeline-gen`` section, the value will still exist in the merged build-job after applying the section. All of the attributes specified are forwarded to the generated CI jobs, however special treatment is applied to the attributes ``tags``, ``image``, ``variables``, ``script``, ``before_script``, and ``after_script`` as they are components recognized explicitly by the Spack CI generator. For the ``tags`` attribute, Spack will remove reserved tags (:ref:`reserved_tags`) from all jobs specified in the config. In some cases, such as for ``signing`` jobs, reserved tags will be added back based on the type of CI that is being run. Once a runner has been chosen to build a release spec, the ``build-job*`` sections provide information determining details of the job in the context of the runner. At least one of the ``build-job*`` sections must contain a ``tags`` key, which is a list containing at least one tag used to select the runner from among the runners known to the GitLab instance. For Docker executor type runners, the ``image`` key is used to specify the Docker image used to build the release spec (and could also appear as a dictionary with a ``name`` specifying the image name, as well as an ``entrypoint`` to override whatever the default for that image is). For other types of runners the ``variables`` key will be useful to pass any information on to the runner that it needs to do its work (e.g. scheduler parameters, etc.). Any ``variables`` provided here will be added, verbatim, to each job. The ``build-job`` section also allows users to supply custom ``script``, ``before_script``, and ``after_script`` sections to be applied to every job scheduled on that runner. This allows users to do any custom preparation or cleanup tasks that fit their particular workflow, as well as completely customize the rebuilding of a spec if they so choose. Spack will not generate a ``before_script`` or ``after_script`` for jobs, but if you do not provide a custom ``script``, Spack will generate one for you that assumes the concrete environment directory is located within your ``--artifacts-root`` (or if not provided, within your ``$CI_PROJECT_DIR``), activates that environment for you, and invokes ``spack ci rebuild``. Sections that specify scripts (``script``, ``before_script``, ``after_script``) are all read as lists of commands or lists of lists of commands. It is recommended to write scripts as lists of lists if scripts will be composed via merging. The default behavior of merging lists will remove duplicate commands and potentially apply unwanted reordering, whereas merging lists of lists will preserve the local ordering and never removes duplicate commands. When writing commands to the CI target script, all lists are expanded and flattened into a single list. Submapping Sections ^^^^^^^^^^^^^^^^^^^ A special case of attribute specification is the ``submapping`` section which may be used to apply job attributes to build jobs based on the package spec associated with the rebuild job. Submapping is specified as a list of spec ``match`` lists associated with ``build-job``/``build-job-remove`` sections. There are two options for ``match_behavior``: either ``first`` or ``merge`` may be specified. In either case, the ``submapping`` list is processed from the bottom up, and then each ``match`` list is searched for a string that satisfies the check ``spec.satisfies({match_item})`` for each concrete spec. In the case of ``match_behavior: first``, the first ``match`` section in the list of ``submappings`` that contains a string that satisfies the spec will apply its ``build-job*`` attributes to the rebuild job associated with that spec. This is the default behavior and will be the method if no ``match_behavior`` is specified. In the case of ``merge`` match, all of the ``match`` sections in the list of ``submappings`` that contain a string that satisfies the spec will have the associated ``build-job*`` attributes applied to the rebuild job associated with that spec. Again, the attributes will be merged starting from the bottom match going up to the top match. In the case that no match is found in a submapping section, no additional attributes will be applied. Dynamic Mapping Sections ^^^^^^^^^^^^^^^^^^^^^^^^ For large scale CI where cost optimization is required, dynamic mapping allows for the use of real-time mapping schemes served by a web service. This type of mapping does not support the ``-remove`` type behavior, but it does follow the rest of the merge rules for configurations. The dynamic mapping service needs to implement a single REST API interface for getting requests ``GET [:PORT][/PATH]?spec=``. example request. .. code-block:: text https://my-dyn-mapping.spack.io/allocation?spec=zlib-ng@2.1.6 +compat+opt+shared+pic+new_strategies arch=linux-ubuntu20.04-x86_64_v3%gcc@12.0.0 With an example response that updates kubernetes request variables, overrides the max retries for GitLab, and prepends a note about the modifications made by the my-dyn-mapping.spack.io service. .. code-block:: text 200 OK { "variables": { "KUBERNETES_CPU_REQUEST": "500m", "KUBERNETES_MEMORY_REQUEST": "2G", }, "retry": { "max:": "1"} "script+:": [ "echo \"Job modified by my-dyn-mapping.spack.io\"" ] } The ci.yaml configuration section takes the URL endpoint as well as a number of options to configure how responses are handled. It is possible to specify a list of allowed and ignored configuration attributes under ``allow`` and ``ignore`` respectively. It is also possible to configure required attributes under ``required`` section. Options to configure the client timeout and SSL verification using the ``timeout`` and ``verify_ssl`` options. By default, the ``timeout`` is set to the option in ``config:timeout`` and ``verify_ssl`` is set to the option in ``config:verify_ssl``. Passing header parameters to the request can be achieved through the ``header`` section. The values of the variables passed to the header may be environment variables that are expanded at runtime, such as a private token configured on the runner. Here is an example configuration pointing to ``my-dyn-mapping.spack.io/allocation``. .. code-block:: yaml ci: pipeline-gen: - dynamic-mapping: endpoint: my-dyn-mapping.spack.io/allocation timeout: 10 verify_ssl: true header: PRIVATE_TOKEN: ${MY_PRIVATE_TOKEN} MY_CONFIG: "fuzz_allocation:false" allow: - variables ignore: - script require: [] Broken Specs URL ^^^^^^^^^^^^^^^^ The optional ``broken-specs-url`` key tells Spack to check against a list of specs that are known to be currently broken in ``develop``. If any such specs are found, the ``spack ci generate`` command will fail with an error message informing the user what broken specs were encountered. This allows the pipeline to fail early and avoid wasting compute resources attempting to build packages that will not succeed. CDash ^^^^^^ The optional ``cdash`` section provides information that will be used by the ``spack ci generate`` command (invoked by ``spack ci start``) for reporting to CDash. All the jobs generated from this environment will belong to a "build group" within CDash that can be tracked over time. As the release progresses, this build group may have jobs added or removed. The URL, project, and site are used to specify the CDash instance to which build results should be reported. Take a look at the `schema `_ for the ``ci`` section of the Spack environment file, to see precisely what syntax is allowed there. .. _reserved_tags: Reserved Tags ^^^^^^^^^^^^^ Spack has a subset of tags (``public``, ``protected``, and ``notary``) that it reserves for classifying runners that may require special permissions or access. The tags ``public`` and ``protected`` are used to distinguish between runners that use public permissions and runners with protected permissions. The ``notary`` tag is a special tag that is used to indicate runners that have access to the highly protected information used for signing binaries using the ``signing`` job. .. _staging_algorithm: Summary of ``.gitlab-ci.yml`` generation algorithm ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ All specs yielded by the matrix (or all the specs in the environment) have their dependencies computed, and the entire resulting set of specs are staged together before being run through the ``ci/pipeline-gen`` entries, where each staged spec is assigned a runner. "Staging" is the name given to the process of figuring out in what order the specs should be built, taking into consideration Gitlab CI rules about jobs/stages. In the staging process, the goal is to maximize the number of jobs in any stage of the pipeline, while ensuring that the jobs in any stage only depend on jobs in previous stages (since those jobs are guaranteed to have completed already). As a runner is determined for a job, the information in the merged ``any-job*`` and ``build-job*`` sections is used to populate various parts of the job description that will be used by the target CI pipelines. Once all the jobs have been assigned a runner, the ``.gitlab-ci.yml`` is written to disk. The short example provided above would result in the ``readline``, ``ncurses``, and ``pkgconf`` packages getting staged and built on the runner chosen by the ``spack-k8s`` tag. In this example, Spack assumes the runner is a Docker executor type runner, and thus certain jobs will be run in the ``centos7`` container and others in the ``ubuntu-18.04`` container. The resulting ``.gitlab-ci.yml`` will contain 6 jobs in three stages. Once the jobs have been generated, the presence of a ``SPACK_CDASH_AUTH_TOKEN`` environment variable during the ``spack ci generate`` command would result in all of the jobs being put in a build group on CDash called "Release Testing" (that group will be created if it didn't already exist). .. _ci_artifacts: CI Artifacts Directory Layout ----------------------------- When running the CI build using the command ``spack ci rebuild`` a number of directories are created for storing data generated during the CI job. The default root directory for artifacts is ``job_scratch_root``. This can be overridden by passing the argument ``--artifacts-root`` to the ``spack ci generate`` command or by setting the ``SPACK_ARTIFACTS_ROOT`` environment variable in the build job scripts. The top-level directories under the artifact root are ``concrete_environment``, ``logs``, ``reproduction``, ``tests``, and ``user_data``. Spack does not restrict what is written to any of these directories nor does it require user specified files be written to any specific directory. ``concrete_environment`` ^^^^^^^^^^^^^^^^^^^^^^^^ The directory ``concrete_environment`` is used to communicate the ``spack ci generate`` processed ``spack.yaml`` and the concrete ``spack.lock`` for the CI environment. ``logs`` ^^^^^^^^ The directory ``logs`` contains the Spack build log, ``spack-build-out.txt``, and the Spack build environment modification file, ``spack-build-mod-env.txt``. Additionally, all files specified by the packages ``Builder`` property ``archive_files`` are also copied here (i.e., ``CMakeCache.txt`` in ``CMakeBuilder``). ``reproduction`` ^^^^^^^^^^^^^^^^ The directory ``reproduction`` is used to store the files needed by the ``spack ci reproduce-build`` command. This includes ``repro.json``, copies of all of the files in ``concrete_environment``, the concrete spec JSON file for the current spec being built, and all of the files written in the artifacts root directory. The ``repro.json`` file is not versioned and is only designed to work with the version that Spack CI was run with. An example of what a ``repro.json`` may look like is here. .. code-block:: json { "job_name": "adios2@2.9.2 /feaevuj %gcc@11.4.0 arch=linux-ubuntu20.04-x86_64_v3 E4S ROCm External", "job_spec_json": "adios2.json", "ci_project_dir": "/builds/spack/spack" } ``tests`` ^^^^^^^^^ The directory ``tests`` is used to store output from running ``spack test ``. This may or may not have data in it depending on the package that was built and the availability of tests. ``user_data`` ^^^^^^^^^^^^^ The directory ``user_data`` is used to store everything else that shouldn't be copied to the ``reproduction`` directory. Users may use this to store additional logs or metrics or other types of files generated by the build job. Using a custom Spack in your pipeline ------------------------------------- If your runners will not have a version of Spack ready to invoke, or if for some other reason you want to use a custom version of Spack to run your pipelines, this section provides an example of how you could take advantage of user-provided pipeline scripts to accomplish this fairly simply. First, consider specifying the source and version of Spack you want to use with variables, either written directly into your ``.gitlab-ci.yml``, or provided by CI variables defined in the GitLab UI or from some upstream pipeline. Let's say you choose the variable names ``SPACK_REPO`` and ``SPACK_REF`` to refer to the particular fork of Spack and branch you want for running your pipeline. You can then refer to those in a custom shell script invoked both from your pipeline generation job and your rebuild jobs. Here's the ``generate-pipeline`` job from the top of this document, updated to clone and source a custom Spack: .. code-block:: yaml generate-pipeline: tags: - before_script: - git clone ${SPACK_REPO} - pushd spack && git checkout ${SPACK_REF} && popd - . "./spack/share/spack/setup-env.sh" script: - spack env activate --without-view . - spack ci generate --check-index-only --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/pipeline.yml" after_script: - rm -rf ./spack artifacts: paths: - "${CI_PROJECT_DIR}/jobs_scratch_dir" That takes care of getting the desired version of Spack when your pipeline is generated by ``spack ci generate``. You also want your generated rebuild jobs (all of them) to clone that version of Spack, so next you would update your ``spack.yaml`` from above as follows: .. code-block:: yaml spack: # ... ci: pipeline-gen: - build-job: tags: - spack-kube image: spack/ubuntu-noble before_script: - git clone ${SPACK_REPO} - pushd spack && git checkout ${SPACK_REF} && popd - . "./spack/share/spack/setup-env.sh" script: - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR} - spack -d ci rebuild after_script: - rm -rf ./spack Now all of the generated rebuild jobs will use the same shell script to clone Spack before running their actual workload. Now imagine you have long pipelines with many specs to be built, and you are pointing to a Spack repository and branch that has a tendency to change frequently, such as the main repo and its ``develop`` branch. If each child job checks out the ``develop`` branch, that could result in some jobs running with one SHA of Spack, while later jobs run with another. To help avoid this issue, the pipeline generation process saves global variables called ``SPACK_VERSION`` and ``SPACK_CHECKOUT_VERSION`` that capture the version of Spack used to generate the pipeline. While the ``SPACK_VERSION`` variable simply contains the human-readable value produced by ``spack -V`` at pipeline generation time, the ``SPACK_CHECKOUT_VERSION`` variable can be used in a ``git checkout`` command to make sure all child jobs checkout the same version of Spack used to generate the pipeline. To take advantage of this, you could simply replace ``git checkout ${SPACK_REF}`` in the example ``spack.yaml`` above with ``git checkout ${SPACK_CHECKOUT_VERSION}``. On the other hand, if you're pointing to a Spack repository and branch under your control, there may be no benefit in using the captured ``SPACK_CHECKOUT_VERSION``, and you can instead just clone using the variables you define (``SPACK_REPO`` and ``SPACK_REF`` in the example above). .. _custom_workflow: Custom Workflow --------------- There are many ways to take advantage of Spack CI pipelines to achieve custom workflows for building packages or other resources. One example of a custom pipelines workflow is the Spack tutorial container `repo `_. This project uses GitHub (for source control), GitLab (for automated Spack CI pipelines), and DockerHub automated builds to build Docker images (complete with fully populated binary mirror) used by instructors and participants of a Spack tutorial. Take a look at the repo to see how it is accomplished using Spack CI pipelines, and see the following markdown files at the root of the repository for descriptions and documentation describing the workflow: ``DESCRIPTION.md``, ``DOCKERHUB_SETUP.md``, ``GITLAB_SETUP.md``, and ``UPDATING.md``. .. _ci_environment_variables: Environment variables affecting pipeline operation -------------------------------------------------- Certain secrets and some other information should be provided to the pipeline infrastructure via environment variables, usually for reasons of security, but in some cases to support other pipeline use cases such as PR testing. The environment variables used by the pipeline infrastructure are described here. ``AWS_ACCESS_KEY_ID`` ^^^^^^^^^^^^^^^^^^^^^ Optional. Only needed when binary mirror is an S3 bucket. ``AWS_SECRET_ACCESS_KEY`` ^^^^^^^^^^^^^^^^^^^^^^^^^ Optional. Only needed when binary mirror is an S3 bucket. ``S3_ENDPOINT_URL`` ^^^^^^^^^^^^^^^^^^^ Optional. Only needed when binary mirror is an S3 bucket that is *not* on AWS. ``CDASH_AUTH_TOKEN`` ^^^^^^^^^^^^^^^^^^^^ Optional. Only needed to report build groups to CDash. ``SPACK_SIGNING_KEY`` ^^^^^^^^^^^^^^^^^^^^^ Optional. Only needed if you want ``spack ci rebuild`` to trust the key you store in this variable, in which case, it will subsequently be used to sign and verify binary packages (when installing or creating build caches). You could also have already trusted a key Spack knows about, or if no key is present anywhere, Spack will install specs using ``--no-check-signature`` and create build caches using ``-u`` (for unsigned binaries). ================================================ FILE: lib/spack/docs/repositories.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Learn how to set up and manage package repositories in Spack, enabling you to maintain custom packages and override built-in ones. .. _repositories: Package Repositories (repos.yaml) ================================= Spack comes with thousands of built-in package recipes. As of Spack v1.0, these are hosted in a separate Git repository at `spack/spack-packages `_. A **package repository** is a directory that Spack searches when it needs to find a package by name. You may need to maintain packages for restricted, proprietary, or experimental software separately from the built-in repository. Spack allows you to configure local and remote repositories using either the ``repos.yaml`` configuration file or the ``spack repo`` command. This document describes how to set up and manage these package repositories. Structure of an Individual Package Repository --------------------------------------------- An individual Spack package repository is a directory structured as follows: .. code-block:: text /path/to/repos/ # the top-level dir is added to the Python search path spack_repo/ # every package repository is part of the spack_repo Python module myrepo/ # directory for the 'myrepo' repository (matches namespace) repo.yaml # configuration file for this package repository packages/ # directory containing package directories hdf5/ # directory for the hdf5 package package.py # the package recipe file mpich/ # directory for the mpich package package.py # the package recipe file mpich-1.9-bugfix.patch # example patch file trilinos/ package.py ... * ``repo.yaml``. This file contains metadata for this specific repository, for example: .. code-block:: yaml repo: namespace: myrepo api: v2.0 It defines primarily: * ``namespace``. A unique identifier for this repository (e.g., ``myrepo``, ``projectx``). See the :ref:`Namespaces ` section for more details. * ``api``. The version of the Spack Package API this repository adheres to (e.g., ``v2.0``). Spack itself defines what range of API versions it supports, and will error if it encounters a repository with an unsupported API version. * ``packages/``. This directory contains subdirectories for each package in the repository. Each package directory contains a ``package.py`` file and any patches or other files needed to build the package. Package repositories allow you to: 1. Maintain your own packages separately from Spack's built-in set. 2. Share your packages (e.g., by hosting them on a shared file system or in a Git repository) without committing them to the main ``spack/spack-packages`` repository. 3. Override built-in Spack packages with your own implementations. Packages in a separate repository can also *depend on* built-in Spack packages, allowing you to leverage existing recipes without re-implementing them. Package Names ^^^^^^^^^^^^^ Package names are defined by the directory names under ``packages/``. In the example above, the package names are ``hdf5``, ``mpich``, and ``trilinos``. Package names can only contain lowercase characters ``a-z``, digits ``0-9`` and hyphens ``-``. .. note:: Package names are **derived** from the directory names under ``packages/``. Package directories are required to be valid Python module names, which means they cannot contain hyphens or start with a digit. This means that a package named ``my-package`` would be stored in a directory named ``my_package/``, and a package named ``7zip`` would be stored in a directory named ``_7zip/`` with an underscore prefix to make it a valid Python module name. The mapping between package names and directory names is one-to-one. Use ``spack list`` to see how Spack resolves the package names from the directory names. Configuring Repositories with ``repos.yaml`` -------------------------------------------- Spack uses ``repos.yaml`` files found in its :ref:`configuration scopes ` (e.g., ``~/.spack/``, ``etc/spack/``) to discover and prioritize package repositories. Note that this ``repos.yaml`` (plural) configuration file is distinct from the ``repo.yaml`` (singular) file within each individual package repository. Spack supports two main types of repository configurations: Local Repositories (Path-based) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can point Spack to a repository on your local filesystem: .. code-block:: yaml :caption: ``~/.spack/repos.yaml`` :name: code-example-local-repo repos: my_local_packages: /path/to/my_repository_root Here, ``/path/to/my_repository_root`` should be the directory containing that repository's ``repo.yaml`` and ``packages/`` subdirectory. Git-based Repositories ^^^^^^^^^^^^^^^^^^^^^^ Spack can clone and use repositories directly from Git URLs: .. code-block:: yaml repos: my_remote_repo: https://github.com/myorg/spack-custom-pkgs.git Automatic Cloning """"""""""""""""" When Spack first encounters a Git-based repository configuration, it automatically clones it. By default, these repositories are cloned into a subdirectory within ``~/.spack/package_repos/``, named with a hash of the repository URL. To change directories to the package repository, you can use ``spack cd --repo [name]``. To find where a repository is cloned, you can use ``spack location --repo [name]`` or ``spack repo list``. The ``name`` argument is optional; if omitted, Spack will use the first package repository in configuration order. Customizing Clone Location """""""""""""""""""""""""" The default clone location (``~/.spack/package_repos/``) might not be convenient for package maintainers who want to make changes to packages. You can specify a custom local directory for Spack to clone a Git repository into, or to use if the repository is already cloned there. This is done using the ``destination`` key in ``repos.yaml`` or via the ``spack repo set --destination`` command (see :ref:`cmd-spack-repo-set-destination`). For example, to use ``~/custom_packages_clone`` for ``my_remote_repo``: .. code-block:: yaml :caption: ``~/.spack/repos.yaml`` :name: code-example-location repos: my_remote_repo: git: https://github.com/myorg/spack-custom-pkgs.git destination: ~/custom_packages_clone If the ``git`` URL is defined in a lower-precedence configuration (like Spack's defaults for ``builtin``), you only need to specify the ``destination`` in your user-level ``repos.yaml``. Spack can make the configuration changes for you using ``spack repo set --destination ~/spack-packages builtin``, or you can directly edit your ``repos.yaml`` file: .. code-block:: yaml :caption: ``~/.spack/repos.yaml`` :name: code-example-builtin repos: builtin: destination: ~/spack-packages Updating and pinning """""""""""""""""""" Repos can be pinned to a git branch, tag, or commit. .. code-block:: yaml :caption: ``~/.spack/repos.yaml`` :name: code-example-branch repos: builtin: branch: releases/v2025.07 # tag: v2025.07.0 # commit: 6427933daecef74b981d1f773731aeace3b06ede The ``spack repo update`` command will update the repo on disk to match the current state of the config. If the repo is pinned to a commit or tag, it will ensure the repo on disk reflects that commit or tag. If the repo is pinned to a branch or unpinned, ``spack repo update`` will pull the most recent state of the branch (the default branch if unpinned). Git repositories need a package repo index """""""""""""""""""""""""""""""""""""""""" A single Git repository can contain one or more Spack package repositories. To enable Spack to discover these, the root of the Git repository should contain a ``spack-repo-index.yaml`` file. This file lists the relative paths to package repository roots within the git repo. For example, assume a Git repository at ``https://example.com/my_org/my_pkgs.git`` has the following structure .. code-block:: text my_pkgs.git/ spack-repo-index.yaml # metadata file at the root of the Git repo ... spack_pkgs/ spack_repo/ my_org/ comp_sci_packages/ # package repository for computer science packages repo.yaml packages/ hdf5/ package.py mpich/ package.py physics_packages/ # package repository for physics packages repo.yaml packages/ gromacs/ package.py The ``spack-repo-index.yaml`` in the root of ``https://example.com/my_org/my_pkgs.git`` should look like this: .. code-block:: yaml :caption: ``my_pkgs.git/spack-repo-index.yaml`` :name: code-example-repo-index repo_index: paths: - spack_pkgs/spack_repo/my_org/comp_sci_packages - spack_pkgs/spack_repo/my_org/physics_packages If ``my_pkgs.git`` is configured in ``repos.yaml`` as follows: .. code-block:: yaml :caption: ``~/.spack/repos.yaml`` :name: code-example-git-repo repos: example_mono_repo: https://example.com/my_org/my_pkgs.git Spack will clone ``my_pkgs.git`` and look for ``spack-repo-index.yaml``. It will then register two separate repositories based on the paths found (e.g., ``/spack_pkgs/spack_repo/my_org/comp_sci_packages`` and ``/spack_pkgs/spack_repo/my_org/physics_packages``), each with its own namespace defined in its respective ``repo.yaml`` file. Thus, one ``repos.yaml`` entry for a Git mono-repo can lead to *multiple repositories* being available to Spack. If you want only one of the package repositories from a Git mono-repo, you can override the paths in your user-level ``repos.yaml``. For example, if you only want the computer science packages: .. code-block:: yaml :caption: ``~/.spack/repos.yaml`` :name: code-example-specific-repo repos: example_mono_repo: git: https://example.com/my_org/my_pkgs.git paths: - spack_pkgs/spack_repo/my_org/comp_sci_packages The ``spack repo add`` command can help you set up these configurations easily. The ``builtin`` Repository ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack's extensive collection of built-in packages resides at `spack/spack-packages `_. By default, Spack is configured to use this as a Git-based repository. The default configuration in ``$spack/etc/spack/defaults/repos.yaml`` looks something like this: .. code-block:: yaml repos: builtin: git: https://github.com/spack/spack-packages.git .. _namespaces: Namespaces ---------- Every repository in Spack has an associated **namespace** defined in the ``namespace:`` key of its top-level ``repo.yaml`` file. For example, the built-in repository (from ``spack/spack-packages``) has its namespace defined as ``builtin``: .. code-block:: yaml :caption: ``repo.yaml`` of ``spack/spack-packages`` :name: code-example-repo-yaml repo: namespace: builtin api: v2.0 # Or newer Spack records the repository namespace of each installed package. For example, if you install the ``mpich`` package from the ``builtin`` repo, Spack records its fully qualified name as ``builtin.mpich``. This accomplishes two things: 1. You can have packages with the same name from different namespaces installed simultaneously. 2. You can easily determine which repository a package came from after it is installed (more :ref:`below `). .. note:: The ``namespace`` defined in the package repository's ``repo.yaml`` is the **authoritative source** for the namespace. It is *not* derived from the local configuration in ``repos.yaml``. This means that the namespace is determined by the repository maintainer, not by the user or local configuration. Nested Namespaces for Organizations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ As we have already seen in the Git-based package repositories example above, you can create nested namespaces by using periods in the namespace name. For example, a repository for packages related to computation at LLNL might have the namespace ``llnl.comp``, while one for physical and life sciences could be ``llnl.pls``. On the file system, this requires a directory structure like this: .. code-block:: text /path/to/repos/ spack_repo/ llnl/ comp/ repo.yaml # Contains namespace: llnl.comp packages/ mpich/ package.py pls/ repo.yaml # Contains namespace: llnl.pls packages/ hdf5/ package.py Uniqueness ^^^^^^^^^^ Spack cannot ensure global uniqueness of all namespaces, but it will prevent you from registering two repositories with the same namespace *at the same time* in your current configuration. If you try to add a repository that has the same namespace as an already registered one, Spack will print a warning and may ignore the new addition or apply specific override logic depending on the configuration. .. _namespace-example: Namespace Example ^^^^^^^^^^^^^^^^^ Suppose LLNL maintains its own version of ``mpich`` (in a repository with namespace ``llnl.comp``), separate from Spack's built-in ``mpich`` package (namespace ``builtin``). If you've installed both, ``spack find`` alone might be ambiguous: .. code-block:: console $ spack find ==> 2 installed packages. -- linux-rhel6-x86_64 / gcc@4.4.7 ------------- mpich@3.2 mpich@3.2 Using ``spack find -N`` displays packages with their namespaces: .. code-block:: console $ spack find -N ==> 2 installed packages. -- linux-rhel6-x86_64 / gcc@4.4.7 ------------- builtin.mpich@3.2 llnl.comp.mpich@3.2 Now you can distinguish them. Packages differing only by namespace will have different hashes: .. code-block:: console $ spack find -lN ==> 2 installed packages. -- linux-rhel6-x86_64 / gcc@4.4.7 ------------- c35p3gc builtin.mpich@3.2 itoqmox llnl.comp.mpich@3.2 All Spack commands that take a package :ref:`spec ` also accept a fully qualified spec with a namespace, allowing you to be specific: .. code-block:: spec $ spack uninstall llnl.comp.mpich Search Order and Overriding Packages ------------------------------------- When Spack resolves an unqualified package name (e.g., ``mpich`` in ``spack install mpich``), it searches the configured repositories in the order they appear in the *merged* ``repos.yaml`` configuration (from highest to lowest precedence scope, and top to bottom within each file). The first repository found that provides the package will be used. For Git-based mono-repos, the individual repositories listed in its ``spack-repo-index.yaml`` are effectively inserted into this search order based on the mono-repo's position. This search order allows you to override built-in packages. If you have your own ``mpich`` in a repository ``my_custom_repo``, and ``my_custom_repo`` is listed before ``builtin`` in your ``repos.yaml``, Spack will use your version of ``mpich`` by default. Suppose your effective (merged) ``repos.yaml`` implies the following order: 1. ``proto`` (local repo at ``~/my_spack_repos/spack_repo/proto_repo``) 2. ``llnl`` (local repo at ``/usr/local/repos/spack_repo/llnl_repo``) 3. ``builtin`` (Spack's default packages from ``spack/spack-packages``) And the packages are: +--------------+------------------------------------------------+-----------------------------+ | Namespace | Source | Packages | +==============+================================================+=============================+ | ``proto`` | ``~/my_spack_repos/spack_repo/proto_repo`` | ``mpich`` | +--------------+------------------------------------------------+-----------------------------+ | ``llnl`` | ``/usr/local/repos/spack_repo/llnl_repo`` | ``hdf5`` | +--------------+------------------------------------------------+-----------------------------+ | ``builtin`` | `spack/spack-packages` (Git) | ``mpich``, ``hdf5``, others | +--------------+------------------------------------------------+-----------------------------+ If ``hdf5`` depends on ``mpich``: * ``spack install hdf5`` will install ``llnl.hdf5 ^proto.mpich``. Spack finds ``hdf5`` first in ``llnl``. For its dependency ``mpich``, Spack searches again from the top, finding ``mpich`` first in ``proto``. You can force a particular repository's package using a fully qualified name: * ``spack install hdf5 ^builtin.mpich`` will install ``llnl.hdf5 ^builtin.mpich``. * ``spack install builtin.hdf5 ^builtin.mpich`` will install ``builtin.hdf5 ^builtin.mpich``. To see which repositories will be used for a build *before* installing, use ``spack spec -N``: .. code-block:: spec $ spack spec -N hdf5 llnl.hdf5@1.10.0 ^proto.mpich@3.2 ^builtin.zlib@1.2.8 .. warning:: While you *can* use a fully qualified package name in a ``depends_on`` directive within a ``package.py`` file (e.g., ``depends_on("proto.hdf5")``), this is **strongly discouraged**. It makes the package non-portable and tightly coupled to a specific repository configuration, hindering sharing and composition of repositories. A package will fail to load if the hardcoded namespace's repository is not registered. .. _cmd-spack-repo: The ``spack repo`` Command -------------------------- Spack provides commands to manage your repository configurations. .. _cmd-spack-repo-list: ``spack repo list`` ^^^^^^^^^^^^^^^^^^^^^^ This command shows all repositories Spack currently knows about, including their namespace, API version, and resolved path (local path or clone directory for Git repos). .. code-block:: console $ spack repo list [+] my_local v2.0 /path/to/spack_repo/my_local_packages [+] comp_sci_packages v2.0 ~/.spack/package_repos//spack_pkgs/spack_repo/comp_sci_packages [+] physics_packages v2.0 ~/.spack/package_repos//spack_pkgs/spack_repo/physics_packages # From the same git repo [+] builtin v2.0 ~/.spack/package_repos//repos/spack_repo/builtin Spack shows a green ``[+]`` next to each repository that is available for use. It shows a red ``[-]`` to indicate that package repositories cannot be used due to an error (e.g., unsupported API version, missing ``repo.yaml``, etc.). It can also show just a gray ``-`` if it is a Git-based package repository that has not been cloned yet. Note that for Git-based package repositories, ``spack repo list`` will show entries for *each* individual package repository registered via ``spack-repo-index.yaml``. This contrasts with ``spack config get repos``, which shows the raw configuration from ``repos.yaml`` files, including just the Git URL for a mono-repo: .. code-block:: console $ spack config get repos repos: my_local_packages: /path/to/spack_repo/my_local_packages example_mono_repo: https://example.com/my_org/my_pkgs.git # contains two package repositories builtin: git: https://github.com/spack/spack-packages.git # destination: /my/custom/path (if set by user) .. _cmd-spack-repo-create: ``spack repo create`` ^^^^^^^^^^^^^^^^^^^^^ To create the directory structure for a new, empty local repository: .. code-block:: console $ spack repo create ~/my_spack_projects myorg.projectx ==> Created repo with namespace 'myorg.projectx'. ==> To register it with spack, run this command: spack repo add ~/my_spack_projects/spack_repo/myorg/projectx This command creates the following structure: .. code-block:: text ~/my_spack_projects/ spack_repo/ myorg/ projectx/ repo.yaml # Contains namespace: myorg.projectx packages/ # Empty directory for new package.py files The ```` is where the ``spack_repo/`` hierarchy will be created. The ```` can be simple (e.g., ``myrepo``) or nested (e.g., ``myorg.projectx``), and Spack will create the corresponding directory structure. .. _cmd-spack-repo-add: ``spack repo add`` ^^^^^^^^^^^^^^^^^^ To register package repositories from local paths or a remote Git repositories with Spack: * **For a local path:** Provide the path to the repository's root directory (the one containing ``repo.yaml`` and ``packages/``). .. code-block:: console $ spack repo add ~/my_spack_projects/spack_repo/myorg/projectx ==> Added repo to config with name 'myorg.projectx'. * **For a Git repository:** Provide the Git URL. .. code-block:: console $ spack repo add --name my_pkgs https://github.com/spack/spack-packages.git ~/my_pkgs Cloning into '/home/user/my_pkgs'... ==> Added repo to config with name 'my_pkgs'. Notice that for Git-based package repositories, you need to specify a configuration name explicitly, which is the key used in your ``repos.yaml`` configuration file. The example also shows providing a custom destination path ``~/my_pkgs``. You can omit this if you want Spack to use the default clone location (e.g., ``~/.spack/package_repos/``). After adding, packages from this repository should appear in ``spack list`` and be installable. .. _cmd-spack-repo-remove: ``spack repo remove`` ^^^^^^^^^^^^^^^^^^^^^ To unregister a repository, use its configuration name (the key in ``repos.yaml``) or its local path. By configuration name (e.g., ``projectx`` from the add example): .. code-block:: console $ spack repo remove projectx ==> Removed repository 'projectx'. By path (for a local repo): .. code-block:: console $ spack repo remove ~/my_spack_projects/spack_repo/myorg/projectx ==> Removed repository '/home/user/my_spack_projects/spack_repo/myorg/projectx'. This command removes the corresponding entry from your ``repos.yaml`` configuration. It does *not* delete the local repository files or any cloned Git repositories. .. _cmd-spack-repo-set-destination: ``spack repo set`` ^^^^^^^^^^^^^^^^^^ For Git-based repositories, this command allows you to specify a custom local directory where Spack should clone the repository, or use an existing clone. The ```` is the key used in your ``repos.yaml`` file for that Git repository (e.g., ``builtin``, ``my_remote_repo``). .. code-block:: console $ spack repo set --destination /my/custom/path/for/spack-packages builtin ==> Updated repo 'builtin' This updates your user-level ``repos.yaml``, adding or modifying the ``destination:`` key for the specified repository configuration name. .. code-block:: yaml :caption: ``~/.spack/repos.yaml`` after ``spack repo set`` :name: code-example-specific-destination repos: builtin: destination: /my/custom/path/for/spack-packages # The 'git:' URL is typically inherited from Spack's default configuration for 'builtin' Spack will then use ``/my/custom/path/for/spack-packages`` for the ``builtin`` repository. If the directory doesn't exist, Spack will clone into it. If it exists and is a valid Git repository, Spack will use it. Repository Namespaces and Python -------------------------------- Package repositories in Spack (from ``api: v2.0`` or newer) are structured to integrate smoothly with Python's import system. They are effectively Python namespace packages under the top-level ``spack_repo`` namespace. The ``api: v2.0`` repository structure ensures that packages can be imported using a standard Python module path: ``spack_repo..packages..package``. For instance, the ``mpich`` package from the ``builtin`` repository corresponds to the Python module ``spack_repo.builtin.packages.mpich.package``. This allows you to easily extend or subclass package classes from other repositories in your own ``package.py`` files: .. code-block:: python # In a package file (e.g. my_custom_mpich/package.py) in your custom repo # Import the original Mpich class from the 'builtin' repository from spack_repo.builtin.packages.mpich.package import Mpich as BuiltinMpich class MyCustomMpich(BuiltinMpich): # Override versions, variants, or methods from BuiltinMpich version("3.5-custom", sha256="...") # Add a new variant variant("custom_feature", default=False, description="Enable my custom feature") def install(self, spec, prefix): if "+custom_feature" in spec: # Do custom things pass super().install(spec, prefix) # Call parent install method Spack manages Python's ``sys.path`` at runtime to make these imports discoverable across all registered repositories. This capability is powerful for creating derivative packages or slightly modifying existing ones without copying entire package files. ================================================ FILE: lib/spack/docs/requirements.txt ================================================ sphinx==9.1.0 sphinxcontrib-programoutput==0.19 sphinxcontrib-svg2pdfconverter==2.1.0 sphinx-copybutton==0.5.2 sphinx-last-updated-by-git==0.3.8 sphinx-sitemap==2.9.0 furo==2025.12.19 docutils==0.22.4 pygments==2.20.0 pytest==9.0.3 ================================================ FILE: lib/spack/docs/roles_and_responsibilities.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to distinguish the roles and responsibilities associated with managing the Spack Packages repository. .. _packaging-roles: Packaging Roles and Responsibilities ==================================== There are four roles related to `Spack Package `_ repository Pull Requests (PRs): #. :ref:`package-contributors`, #. :ref:`package-reviewers`, #. :ref:`package-maintainers`, and #. :ref:`committers`. One person can assume multiple roles (e.g., a Package Contributor may also be a Maintainer; a Package Reviewer may also be a Committer). This section defines and describes the responsibilities of each role. .. _package-contributors: Package Contributors -------------------- Contributors submit changes to packages through PRs `Spack Package `_ repository Pull Requests (PRs). As a Contributor, you are **expected** to test your changes on **at least one platform** outside of Spack’s Continuous Integration (CI) checks. .. note:: We also ask that you include the output from ``spack debug report`` from the platform you used to facilitate PR reviews. .. _package-reviewers: Package Reviewers ----------------- Anyone can review a PR so we encourage Spack’s community members to review and comment on those involving software in which they have expertise and/or interest. As a Package Reviewer, you are **expected** to assess changes in PRs to the best of your ability and knowledge with special consideration to the information contained in the :ref:`package-review-guide`. .. _package-maintainers: Maintainers (Package Owners) ---------------------------- Maintainers are individuals (technically GitHub accounts) who appear in a package’s :ref:`maintainers` directive. These are people who have agreed to be notified of and given the opportunity to review changes to packages. They are, from a Spack package perspective, `Code Owners `_ of the package, whether or not they “own” or work on the software that the package builds. As a Maintainer, you are **expected**, when available, to: * review PRs in a timely manner (reported in :ref:`committers`) to confirm that the changes made to the package are reasonable; * confirm that packages successfully build on at least one platform; and * attempt to confirm that any updated or included tests pass. See :ref:`build_success_reviews` for acceptable forms of build success confirmation. .. note:: If at least one maintainer approves a PR -– and there are no objections from others -– then the PR can be merged by any of the :ref:`committers`. .. _committers: Committers ---------- Committers are vetted individuals who are allowed to merge PRs into the ``develop`` branch. As a Committer, you are **expected** to: * ensure **at least one review** is performed prior to merging (GitHub rules enforce this); * encourage **at least one** :ref:`Package Maintainer ` (if any) to comment and/or review the PR; * allow Package Maintainers (if any) **up to one week** to comment or provide a review; * determine if the criteria defined in :ref:`package-review-guide` are met; and * **merge the reviewed PR** at their discretion. .. note:: If there are no :ref:`package-maintainers` or the Maintainers have not commented or reviewed the PR within the allotted time, you will need to conduct the review. .. tip:: The following criteria must be met in order to become a Committer: * cannot be an anonymous account; * must come from a known and trustworthy organization; * demonstrated record of contribution to Spack; * have an account on the Spack Slack workspace; * be approved by the Onboarding subcommittee; and * (proposed) be known to at least 3 members of the core development team. ================================================ FILE: lib/spack/docs/signing.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Understand the Spack package signing process, which ensures data integrity for official packages from automated CI pipelines through cryptographic signing. .. _signing: Spack Package Signing ===================== The goal of package signing in Spack is to provide data integrity assurances around official packages produced by the automated Spack CI pipelines. These assurances directly address the security of Spack's software supply chain by explaining why a security-conscious user can be reasonably justified in the belief that packages installed via Spack have an uninterrupted auditable trail back to change management decisions judged to be appropriate by the Spack maintainers. This is achieved through cryptographic signing of packages built by Spack CI pipelines based on code that has been transparently reviewed and approved on GitHub. This document describes the signing process for interested users. .. _risks: Risks, Impact and Threat Model ------------------------------ This document addresses the approach taken to safeguard Spack's reputation with regard to the integrity of the package data produced by Spack's CI pipelines. It does not address issues of data confidentiality (Spack is intended to be largely open source) or availability (efforts are described elsewhere). With that said, the main reputational risk can be broadly categorized as a loss of faith in the data integrity due to a breach of the private key used to sign packages. Remediation of a private key breach would require republishing the public key with a revocation certificate, generating a new signing key, an assessment and potential rebuild/resigning of all packages since the key was breached, and finally direct intervention by every spack user to update their copy of Spack's public keys used for local verification. The primary threat model used in mitigating the risks of these stated impacts is one of individual error not malicious intent or insider threat. The primary objective is to avoid the above impacts by making a private key breach nearly impossible due to oversight or configuration error. Obvious and straightforward measures are taken to mitigate issues of malicious interference in data integrity and insider threats but these attack vectors are not systematically addressed. It should be hard to exfiltrate the private key intentionally, and almost impossible to leak the key by accident. .. _overview: Pipeline Overview ----------------- Spack pipelines build software through progressive stages where packages in later stages nominally depend on packages built in earlier stages. For both technical and design reasons these dependencies are not implemented through the default GitLab artifacts mechanism; instead built packages are uploaded to AWS S3 mirrors (buckets) where they are retrieved by subsequent stages in the pipeline. Two broad categories of pipelines exist: Pull Request (PR) pipelines and Develop/Release pipelines. - PR pipelines are launched in response to pull requests made by trusted and untrusted users. Packages built on these pipelines upload code to quarantined AWS S3 locations which cache the built packages for the purposes of review and iteration on the changes proposed in the pull request. Packages built on PR pipelines can come from untrusted users so signing of these pipelines is not implemented. Jobs in these pipelines are executed via normal GitLab runners both within the AWS GitLab infrastructure and at affiliated institutions. - Develop and Release pipelines **sign** the packages they produce and carry strong integrity assurances that trace back to auditable change management decisions. These pipelines only run after members from a trusted group of reviewers verify that the proposed changes in a pull request are appropriate. Once the PR is merged, or a release is cut, a pipeline is run on protected GitLab runners which provide access to the required signing keys within the job. Intermediary keys are used to sign packages in each stage of the pipeline as they are built and a final job officially signs each package external to any specific package's build environment. An intermediate key exists in the AWS infrastructure and for each affiliated institution that maintains protected runners. The runners that execute these pipelines exclusively accept jobs from protected branches meaning the intermediate keys are never exposed to unreviewed code and the official keys are never exposed to any specific build environment. .. _key_architecture: Key Architecture ---------------- Spack's CI process uses public-key infrastructure (PKI) based on GNU Privacy Guard (gpg) keypairs to sign public releases of spack package metadata, also called specs. Two classes of GPG keys are involved in the process to reduce the impact of an individual private key compromise, these key classes are the *Intermediate CI Key* and *Reputational Key*. Each of these keys has signing sub-keys that are used exclusively for signing packages. This can be confusing so for the purpose of this explanation we will refer to Root and Signing keys. Each key has a private and a public component as well as one or more identities and zero or more signatures. Intermediate CI Key ------------------- The Intermediate key class is used to sign and verify packages between stages within a develop or release pipeline. An intermediate key exists for the AWS infrastructure as well as each affiliated institution that maintains protected runners. These intermediate keys are made available to the GitLab execution environment building the package so that the package's dependencies may be verified by the Signing Intermediate CI Public Key and the final package may be signed by the Signing Intermediate CI Private Key. +---------------------------------------------------------------------------------------------------------+ | **Intermediate CI Key (GPG)** | +==================================================+======================================================+ | Root Intermediate CI Private Key (RSA 4096) | Root Intermediate CI Public Key (RSA 4096) | +--------------------------------------------------+------------------------------------------------------+ | Signing Intermediate CI Private Key (RSA 4096) | Signing Intermediate CI Public Key (RSA 4096) | +--------------------------------------------------+------------------------------------------------------+ | Identity: "Intermediate CI Key " | +---------------------------------------------------------------------------------------------------------+ | Signatures: None | +---------------------------------------------------------------------------------------------------------+ The *Root intermediate CI Private Key*\ is stripped out of the GPG key and stored offline completely separate from Spack's infrastructure. This allows the core development team to append revocation certificates to the GPG key and issue new sub-keys for use in the pipeline. It is our expectation that this will happen on a semi-regular basis. A corollary of this is that *this key should not be used to verify package integrity outside the internal CI process.* Reputational Key ---------------- The Reputational Key is the public facing key used to sign complete groups of development and release packages. Only one key pair exists in this class of keys. In contrast to the Intermediate CI Key the Reputational Key *should* be used to verify package integrity. At the end of develop and release pipelines a final pipeline job pulls down all signed package metadata built by the pipeline, verifies they were signed with an Intermediate CI Key, then strips the Intermediate CI Key signature from the package and re-signs them with the Signing Reputational Private Key. The officially signed packages are then uploaded back to the AWS S3 mirror. Please note that separating use of the reputational key into this final job is done to prevent leakage of the key in a spack package. Because the Signing Reputational Private Key is never exposed to a build job it cannot accidentally end up in any built package. +---------------------------------------------------------------------------------------------------------+ | **Reputational Key (GPG)** | +==================================================+======================================================+ | Root Reputational Private Key (RSA 4096)# | Root Reputational Public Key (RSA 4096) | +--------------------------------------------------+------------------------------------------------------+ | Signing Reputational Private Key (RSA 4096) | Signing Reputational Public Key (RSA 4096) | +--------------------------------------------------+------------------------------------------------------+ | Identity: "Spack Project " | +---------------------------------------------------------------------------------------------------------+ | Signatures: Signed by core development team [#f1]_ | +---------------------------------------------------------------------------------------------------------+ The Root Reputational Private Key is stripped out of the GPG key and stored offline completely separate from Spack's infrastructure. This allows the core development team to append revocation certificates to the GPG key in the unlikely event that the Signing Reputation Private Key is compromised. In general it is the expectation that rotating this key will happen infrequently if at all. This should allow relatively transparent verification for the end-user community without needing deep familiarity with GnuPG or Public Key Infrastructure. .. _build_cache_signing: Build Cache Signing ------------------- For an in-depth description of the layout of a binary mirror, see the :ref:`documentation` covering binary caches. The key takeaway from that discussion that applies here is that the entry point to a binary package is its manifest. The manifest refers unambiguously to the spec metadata and compressed archive, which are stored as content-addressed blobs. The manifest files can either be signed or unsigned, but are always given a name ending with ``.spec.manifest.json`` regardless. The difference between signed and unsigned manifests is simply that the signed version is wrapped in a gpg cleartext signature, as illustrated below: .. code-block:: text -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 { "version": 3, "data": [ { "contentLength": 10731083, "mediaType": "application/vnd.spack.install.v2.tar+gzip", "compression": "gzip", "checksumAlgorithm": "sha256", "checksum": "0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210" }, { "contentLength": 1000, "mediaType": "application/vnd.spack.spec.v5+json", "compression": "gzip", "checksumAlgorithm": "sha256", "checksum": "fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041" } ] } -----BEGIN PGP SIGNATURE----- iQGzBAEBCgAdFiEEdbwFKBFJCcB24mB0GAEP+tc8mwcFAmf2rr4ACgkQGAEP+tc8 mwfefwv+KJs8MsQ5ovFaBdmyx5H/3k4rO4QHBzuSPOB6UaxErA9IyOB31iP6vNTU HzYpxz6F5dJCJWmmNEMN/0+vjhMHEOkqd7M1l5reVcxduTF2yc4tBZUO2gienEHL W0e+SnUznl1yc/aVpChUiahO2zToCsI8HZRNT4tu6iCnE/OpghqjsSdBOZHmSNDD 5wuuCxfDUyWI6ZlLclaaB7RdbCUUJf/iqi711J+wubvnDFhc6Ynwm1xai5laJ1bD ev3NrSb2AAroeNFVo4iECA0fZC1OZQYzaRmAEhBXtCideGJ5Zf2Cp9hmCwNK8Hq6 bNt94JP9LqC3FCCJJOMsPyOOhMSA5MU44zyyzloRwEQpHHLuFzVdbTHA3dmTc18n HxNLkZoEMYRc8zNr40g0yb2lCbc+P11TtL1E+5NlE34MX15mPewRCiIFTMwhCnE3 gFSKtW1MKustZE35/RUwd2mpJRf+mSRVCl1f1RiFjktLjz7vWQq7imIUSam0fPDr XD4aDogm =RrFX -----END PGP SIGNATURE----- If a user has trusted the public key associated with the private key used to sign the above manifest file, the signature can be verified with gpg, as follows: .. code-block:: console $ gpg --verify gcc-runtime-12.3.0-s2nqujezsce4x6uhtvxscu7jhewqzztx.spec.manifest.json When attempting to install a binary package that has been signed, spack will attempt to verify the signature with one of the trusted keys in its keyring, and will fail if unable to do so. While not recommended, it is possible to force installation of a signed package without verification by providing the ``--no-check-signature`` argument to ``spack install ...``. .. _internal_implementation: Internal Implementation ----------------------- The technical implementation of the pipeline signing process includes components defined in Amazon Web Services, the Kubernetes cluster, at affiliated institutions, and the GitLab/GitLab Runner deployment. We present the technical implementation in two interdependent sections. The first addresses how secrets are managed through the lifecycle of a develop or release pipeline. The second section describes how Gitlab Runner and pipelines are configured and managed to support secure automated signing. Secrets Management ^^^^^^^^^^^^^^^^^^ As stated above the Root Private Keys (intermediate and reputational) are stripped from the GPG keys and stored outside Spack's infrastructure. .. .. admonition:: TODO .. :class: warning .. - Explanation here about where and how access is handled for these keys. .. - Both Root private keys are protected with strong passwords .. - Who has access to these and how? Intermediate CI Key ^^^^^^^^^^^^^^^^^^^ Multiple intermediate CI signing keys exist, one Intermediate CI Key for jobs run in AWS, and one key for each affiliated institution (e.g. University of Oregon). Here we describe how the Intermediate CI Key is managed in AWS: The Intermediate CI Key (including the Signing Intermediate CI Private Key) is exported as an ASCII armored file and stored in a Kubernetes secret called ``spack-intermediate-ci-signing-key``. For convenience sake, this same secret contains an ASCII-armored export of just the *public* components of the Reputational Key. This secret also contains the *public* components of each of the affiliated institutions' Intermediate CI Key. These are potentially needed to verify dependent packages which may have been found in the public mirror or built by a protected job running on an affiliated institution's infrastructure in an earlier stage of the pipeline. Procedurally the ``spack-intermediate-ci-signing-key`` secret is used in the following way: 1. A ``large-arm-prot`` or ``large-x86-prot`` protected runner picks up a job tagged ``protected`` from a protected GitLab branch. (See :ref:`protected_runners`). 2. Based on its configuration, the runner creates a job Pod in the pipeline namespace and mounts the ``spack-intermediate-ci-signing-key`` Kubernetes secret into the build container 3. The Intermediate CI Key, affiliated institutions' public key and the Reputational Public Key are imported into a keyring by the ``spack gpg ...`` sub-command. This is initiated by the job's build script which is created by the generate job at the beginning of the pipeline. 4. Assuming the package has dependencies those spec manifests are verified using the keyring. 5. The package is built and the spec manifest is generated 6. The spec manifest is signed by the keyring and uploaded to the mirror's build cache. Reputational Key ^^^^^^^^^^^^^^^^ Because of the increased impact to end users in the case of a private key breach, the Reputational Key is managed separately from the Intermediate CI Keys and has additional controls. First, the Reputational Key was generated outside of Spack's infrastructure and has been signed by the core development team. The Reputational Key (along with the Signing Reputational Private Key) was then ASCII armor exported to a file. Unlike the Intermediate CI Key this exported file is not stored as a base64 encoded secret in Kubernetes. Instead\ *the key file itself*\ is encrypted and stored in Kubernetes as the ``spack-signing-key-encrypted`` secret in the pipeline namespace. The encryption of the exported Reputational Key (including the Signing Reputational Private Key) is handled by `AWS Key Management Store (KMS) data keys `__. The private key material is decrypted and imported at the time of signing into a memory mounted temporary directory holding the keychain. The signing job uses the `AWS Encryption SDK `__ (i.e. ``aws-encryption-cli``) to decrypt the Reputational Key. Permission to decrypt the key is granted to the job Pod through a Kubernetes service account specifically used for this, and only this, function. Finally, for convenience sake, this same secret contains an ASCII-armored export of the *public* components of the Intermediate CI Keys and the Reputational Key. This allows the signing script to verify that packages were built by the pipeline (both on AWS or at affiliated institutions), or signed previously as a part of a different pipeline. This is done *before* importing decrypting and importing the Signing Reputational Private Key material and officially signing the packages. Procedurally the ``spack-signing-key-encrypted`` secret is used in the following way: 1. The ``spack-package-signing-gitlab-runner`` protected runner picks up a job tagged ``notary`` from a protected GitLab branch (See :ref:`protected_runners`). 2. Based on its configuration, the runner creates a job pod in the pipeline namespace. The job is run in a stripped down purpose-built image ``ghcr.io/spack/notary:latest`` Docker image. The runner is configured to only allow running jobs with this image. 3. The runner also mounts the ``spack-signing-key-encrypted`` secret to a path on disk. Note that this becomes several files on disk, the public components of the Intermediate CI Keys, the public components of the Reputational CI, and an AWS KMS encrypted file containing the Signing Reputational Private Key. 4. In addition to the secret, the runner creates a tmpfs memory mounted directory where the GnuPG keyring will be created to verify, and then resign the package specs. 5. The job script syncs all spec manifest files from the build cache to a working directory in the job's execution environment. 6. The job script then runs the ``sign.sh`` script built into the Notary Docker image. 7. The ``sign.sh`` script imports the public components of the Reputational and Intermediate CI Keys and uses them to verify good signatures on the spec.manifest.json files. If any signed manifest does not verify, the job immediately fails. 8. Assuming all manifests are verified, the ``sign.sh`` script then unpacks the manifest json data from the signed file in preparation for being re-signed with the Reputational Key. 9. The private components of the Reputational Key are decrypted to standard out using ``aws-encryption-cli`` directly into a ``gpg --import ...`` statement which imports the key into the keyring mounted in-memory. 10. The private key is then used to sign each of the manifests and the keyring is removed from disk. 11. The re-signed manifests are resynced to the AWS S3 Mirror and the public signing of the packages for the develop or release pipeline that created them is complete. Non service-account access to the private components of the Reputational Key that are managed through access to the symmetric secret in KMS used to encrypt the data key (which in turn is used to encrypt the GnuPG key - See:\ `Encryption SDK Documentation `__). A small trusted subset of the core development team are the only individuals with access to this symmetric key. .. _protected_runners: Protected Runners and Reserved Tags ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack has a large number of Gitlab Runners operating in its build farm. These include runners deployed in the AWS Kubernetes cluster as well as runners deployed at affiliated institutions. The majority of runners are shared runners that operate across projects in `gitlab.spack.io `_. These runners pick up jobs primarily from the spack/spack project and execute them in PR pipelines. A small number of runners operating on AWS and at affiliated institutions are registered as specific *protected* runners on the spack/spack project. In addition to protected runners there are protected branches on the spack/spack project. These are the ``develop`` branch, any release branch (i.e. managed with the ``releases/v*`` wildcard) and any tag branch (managed with the ``v*`` wildcard). Finally, Spack's pipeline generation code reserves certain tags to make sure jobs are routed to the correct runners; these tags are ``public``, ``protected``, and ``notary``. Understanding how all this works together to protect secrets and provide integrity assurances can be a little confusing so lets break these down: Protected Branches Protected branches in Spack prevent anyone other than Maintainers in GitLab from pushing code. In the case of Spack, the only Maintainer level entity pushing code to protected branches is Spack bot. Protecting branches also marks them in such a way that Protected Runners will only run jobs from those branches Protected Runners Protected Runners only run jobs from protected branches. Because protected runners have access to secrets, it's critical that they not run jobs from untrusted code (i.e. PR branches). If they did, it would be possible for a PR branch to tag a job in such a way that a protected runner executed that job and mounted secrets into a code execution environment that had not been reviewed by Spack maintainers. Note however that in the absence of tagging used to route jobs, public runners *could* run jobs from protected branches. No secrets would be at risk of being breached because non-protected runners do not have access to those secrets; lack of secrets would, however, cause the jobs to fail. Reserved Tags To mitigate the issue of public runners picking up protected jobs Spack uses a small set of "reserved" job tags (Note that these are *job* tags not git tags). These tags are "public", "private", and "notary." The majority of jobs executed in Spack's GitLab instance are executed via a ``generate`` job. The generate job code systematically ensures that no user defined configuration sets these tags. Instead, the ``generate`` job sets these tags based on rules related to the branch where this pipeline originated. If the job is a part of a pipeline on a PR branch it sets the ``public`` tag. If the job is part of a pipeline on a protected branch it sets the ``protected`` tag. Finally if the job is the package signing job and it is running on a pipeline that is part of a protected branch then it sets the ``notary`` tag. Protected Runners are configured to only run jobs from protected branches. Only jobs running in pipelines on protected branches are tagged with ``protected`` or ``notary`` tags. This tightly couples jobs on protected branches to protected runners that provide access to the secrets required to sign the built packages. The secrets can **only** be accessed via: 1. Runners under direct control of the core development team. 2. Runners under direct control of trusted maintainers at affiliated institutions. 3. By code running the automated pipeline that has been reviewed by the Spack maintainers and judged to be appropriate. Other attempts (either through malicious intent or incompetence) can at worst grab jobs intended for protected runners which will cause those jobs to fail alerting both Spack maintainers and the core development team. .. [#f1] The Reputational Key has also cross signed core development team keys. ================================================ FILE: lib/spack/docs/spack.yaml ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # # These are requirements for building the documentation. You can run # these commands in this directory to install Sphinx and its plugins, # then build the docs: # # spack env activate . # spack install # make # spack: specs: # Sphinx - "py-sphinx@3.4:4.1.1,4.1.3:" - py-sphinxcontrib-programoutput - py-docutils@:0.16 - py-sphinx-design - py-sphinx-rtd-theme - py-pygments@:2.12 # VCS - git - mercurial - subversion # Plotting - graphviz concretizer: unify: true ================================================ FILE: lib/spack/docs/spec_syntax.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A detailed guide to the Spack spec syntax for describing package constraints, including versions, variants, and dependencies. .. _sec-specs: Spec Syntax =========== Spack has a specific syntax to describe package constraints. Each constraint is individually referred to as a *spec*. Spack uses specs to: 1. Refer to a particular build configuration of a package, or 2. Express requirements, or preferences, on packages via configuration files, or 3. Query installed packages, or build caches Specs are more than a package name and a version; you can use them to specify the compiler, compiler version, architecture, compile options, and dependency options for a build. In this section, we'll go over the full syntax of specs. Here is an example of using a complex spec to install a very specific configuration of ``mpileaks``: .. code-block:: spec $ spack install mpileaks@1.2:1.4 +debug ~qt target=x86_64_v3 %gcc@15 ^libelf@1.1 %clang@20 The figure below helps you get a sense of the various parts that compose this spec: .. figure:: images/spec_anatomy.svg :alt: Spack spec with annotations :width: 740 :height: 180 When installing this, you will get: * The ``mpileaks`` package at some version between ``1.2`` and ``1.4`` (inclusive), * with ``debug`` options enabled, and without ``qt`` support, * optimized for an ``x86_64_v3`` architecture, * built using ``gcc`` at version ``15``, * depending on ``libelf`` at version ``1.1``, built with ``clang`` at version ``20``. Most specs will not be as complicated as this one, but this is a good example of what is possible with specs. There are a few general rules that we can already infer from this first example: 1. Users can be as vague, or as specific, as they want about the details of building packages 2. The spec syntax is recursive, i.e. each dependency after ``%`` or ``^`` is a spec itself 3. Transitive dependencies come after the ``^`` sigil, and they always refer to the root package 4. Direct dependencies come after the ``%`` sigil, and they refer either to the root package, or to the last transitive dependency defined The flexibility the spec syntax offers in specifying the details of a build makes Spack good for beginners and experts alike. .. _software-model: Software Model -------------- To really understand what's going on above, we need to think about how software is structured. An executable or a library generally depends on other libraries in order to run. We can represent the relationship between a package and its dependencies as a graph. Here is a simplified dependency graph for ``mpileaks``: .. graphviz:: digraph { node[ fontname=Monaco, penwidth=2, fontsize=124, margin=.4, shape=box, fillcolor=lightblue, style="rounded,filled" ] mpileaks -> { mpich callpath } callpath -> { mpich dyninst } dyninst -> libdwarf -> libelf dyninst -> libelf } Each box above is a package, and each arrow represents a dependency on some other package. For example, we say that the package ``mpileaks`` *depends on* ``callpath`` and ``mpich``. ``mpileaks`` also depends *indirectly* on ``dyninst``, ``libdwarf``, and ``libelf``, in that these libraries are dependencies of ``callpath``. To install ``mpileaks``, Spack has to build all of these packages. Dependency graphs in Spack have to be acyclic, and the *depends on* relationship is directional, so this is a *directed, acyclic graph* or *DAG*. The package name identifier in the spec is the root of some dependency DAG, and the DAG itself is implicit. Spack knows the precise dependencies among packages, but users do not need to know the full DAG structure. Each ``^`` in the full spec refers to a *transitive* dependency of the root package. Each ``%`` refers to a *direct* dependency, either of the root, or of the last defined transitive dependency . Spack allows only a single configuration of each package, where that is needed for consistency. Above, both ``mpileaks`` and ``callpath`` depend on ``mpich``, but ``mpich`` appears only once in the DAG. You cannot build an ``mpileaks`` version that depends on one version of ``mpich`` *and* on a ``callpath`` version that depends on some *other* version of ``mpich``. In general, such a configuration would likely behave unexpectedly at runtime, and Spack enforces this to ensure a consistent runtime environment. The purpose of specs is to abstract this full DAG away from Spack users. A user who does not care about the DAG at all, can refer to ``mpileaks`` by simply writing: .. code-block:: spec mpileaks The spec becomes only slightly more complicated, if that user knows that ``mpileaks`` indirectly uses ``dyninst`` and wants a particular version of ``dyninst``: .. code-block:: spec mpileaks ^dyninst@8.1 Spack will fill in the rest of the details before installing the spec. The user only needs to know package names and minimal details about their relationship. You can put all the same modifiers on dependency specs that you would put on the root spec. That is, you can specify their versions, variants, and architectures just like any other spec. Specifiers are associated with the nearest package name to their left. .. _sec-virtual-dependencies: Virtual dependencies ^^^^^^^^^^^^^^^^^^^^ The dependency graph for ``mpileaks`` we saw above wasn't *quite* accurate. ``mpileaks`` uses MPI, which is an interface that has many different implementations. Above, we showed ``mpileaks`` and ``callpath`` depending on ``mpich``, which is one *particular* implementation of MPI. However, we could build either with another implementation, such as ``openmpi`` or ``mvapich``. Spack represents interfaces like this using *virtual dependencies*. The real dependency DAG for ``mpileaks`` looks like this: .. graphviz:: digraph { node[ fontname=Monaco, penwidth=2, fontsize=124, margin=.4, shape=box, fillcolor=lightblue, style="rounded,filled" ] mpi [color=red] mpileaks -> mpi mpileaks -> callpath -> mpi callpath -> dyninst dyninst -> libdwarf -> libelf dyninst -> libelf } Notice that ``mpich`` has now been replaced with ``mpi``. There is no *real* MPI package, but some packages *provide* the MPI interface, and these packages can be substituted in for ``mpi`` when ``mpileaks`` is built. Spack is unique in that its virtual packages can be versioned, just like regular packages. A particular version of a package may provide a particular version of a virtual package. A package can *depend on* a particular version of a virtual package. For instance, if an application needs MPI-2 functions, it can depend on ``mpi@2:`` to indicate that it needs some implementation that provides MPI-2 functions. Below are more details about the specifiers that you can add to specs. .. _version-specifier: Version specifier ----------------- A version specifier .. code-block:: spec pkg@specifier comes after a package name and starts with ``@``. It can be something abstract that matches multiple known versions or a specific version. The version specifier usually represents *a range of versions*: .. code-block:: spec # All versions between v1.0 and v1.5. # This includes any v1.5.x version @1.0:1.5 # All versions up to and including v3 # This would include v3.4 etc. @:3 # All versions above and including v4.2 @4.2: but can also be *a specific version*: .. code-block:: spec # Exactly version v3.2, will NOT match v3.2.1 etc. @=3.2 As a shorthand, ``@3`` is equivalent to the range ``@3:3`` and includes any version with major version ``3``. Versions are ordered lexicographically by their components. For more details on the order, see :ref:`the packaging guide `. Notice that you can distinguish between the specific version ``@=3.2`` and the range ``@3.2``. This is useful for packages that follow a versioning scheme that omits the zero patch version number: ``3.2``, ``3.2.1``, ``3.2.2``, etc. In general, it is preferable to use the range syntax ``@3.2``, because ranges also match versions with one-off suffixes, such as ``3.2-custom``. A version specifier can also be a list of ranges and specific versions, separated by commas. For example: .. code-block:: spec @1.0:1.5,=1.7.1 matches any version in the range ``1.0:1.5`` and the specific version ``1.7.1``. Git versions ^^^^^^^^^^^^ .. note:: Users wanting to just match specific commits for branch or tag based versions should assign the ``commit`` variant (``commit=<40 char sha>``). Spack reserves this variant specifically to track provenance of git based versions. Spack will attempt to compute this value for you automatically during concretization and raise a warning if it is unable to assign the commit. Further details can be found in :ref:`git_version_provenance`. For packages with a ``git`` attribute, ``git`` references may be specified instead of a numerical version (i.e., branches, tags, and commits). Spack will stage and build based off the ``git`` reference provided. Acceptable syntaxes for this are: .. code-block:: spec # commit hashes foo@abcdef1234abcdef1234abcdef1234abcdef1234 # 40 character hashes are automatically treated as git commits foo@git.abcdef1234abcdef1234abcdef1234abcdef1234 # branches and tags foo@git.develop # use the develop branch foo@git.0.19 # use the 0.19 tag Spack always needs to associate a Spack version with the git reference, which is used for version comparison. This Spack version is heuristically taken from the closest valid git tag among the ancestors of the git ref. Once a Spack version is associated with a git ref, it is always printed with the git ref. For example, if the commit ``@git.abcdefg`` is tagged ``0.19``, then the spec will be shown as ``@git.abcdefg=0.19``. If the git ref is not exactly a tag, then the distance to the nearest tag is also part of the resolved version. ``@git.abcdefg=0.19.git.8`` means that the commit is 8 commits away from the ``0.19`` tag. In cases where Spack cannot resolve a sensible version from a git ref, users can specify the Spack version to use for the git ref. This is done by appending ``=`` and the Spack version to the git ref. For example: .. code-block:: spec foo@git.my_ref=3.2 # use the my_ref tag or branch, but treat it as version 3.2 for version comparisons foo@git.abcdef1234abcdef1234abcdef1234abcdef1234=develop # use the given commit, but treat it as develop for version comparisons Details about how versions are compared and how Spack determines if one version is less than another are discussed in the developer guide. .. _basic-variants: Variants -------- Variants are named options associated with a particular package and are typically used to enable or disable certain features at build time. They are optional, as each package must provide default values for each variant it makes available. The variants available for a particular package are defined by the package author. ``spack info `` will provide information on what build variants are available. There are different types of variants. Boolean Variants ^^^^^^^^^^^^^^^^ Typically used to enable or disable a feature at compile time. For example, a package might have a ``debug`` variant that can be explicitly enabled with: .. code-block:: spec +debug and disabled with .. code-block:: spec ~debug Single-valued Variants ^^^^^^^^^^^^^^^^^^^^^^ Often used to set defaults. For example, a package might have a ``compression`` variant that determines the default compression algorithm, which users could set to: .. code-block:: spec compression=gzip or .. code-block:: spec compression=zstd Multi-valued Variants ^^^^^^^^^^^^^^^^^^^^^ A package might have a ``fabrics`` variant that determines which network fabrics to support. Users could activate multiple values at the same time. For instance: .. code-block:: spec fabrics=verbs,ofi enables both InfiniBand verbs and OpenFabrics interfaces. The values are separated by commas. The meaning of ``fabrics=verbs,ofi`` is to enable *at least* the specified fabrics, but other fabrics may be enabled as well. If the intent is to enable *only* the specified fabrics, then the: .. code-block:: spec fabrics:=verbs,ofi syntax should be used with the ``:=`` operator. Variant propagation to dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack allows variants to propagate their value to the package's dependencies by using ``++``, ``--``, and ``~~`` for boolean variants. For example, for a ``debug`` variant: .. code-block:: spec mpileaks ++debug # enabled debug will be propagated to dependencies mpileaks +debug # only mpileaks will have debug enabled To propagate the value of non-boolean variants Spack uses ``name==value``. For example, for the ``stackstart`` variant: .. code-block:: spec mpileaks stackstart==4 # variant will be propagated to dependencies mpileaks stackstart=4 # only mpileaks will have this variant value Spack also allows variants to be propagated from a package that does not have that variant. Compiler Flags -------------- Compiler flags are specified using the same syntax as non-boolean variants, but fulfill a different purpose. While the function of a variant is set by the package, compiler flags are used by the compiler wrappers to inject flags into the compile line of the build. Additionally, compiler flags can be inherited by dependencies by using ``==``. ``spack install libdwarf cppflags=="-g"`` will install both libdwarf and libelf with the ``-g`` flag injected into their compile line. Notice that the value of the compiler flags must be quoted if it contains any spaces. Any of ``cppflags=-O3``, ``cppflags="-O3"``, ``cppflags='-O3'``, and ``cppflags="-O3 -fPIC"`` are acceptable, but ``cppflags=-O3 -fPIC`` is not. Additionally, if the value of the compiler flags is not the last thing on the line, it must be followed by a space. The command ``spack install libelf cppflags="-O3"%intel`` will be interpreted as an attempt to set ``cppflags="-O3%intel"``. The six compiler flags are injected in the same order as implicit make commands in GNU Autotools. If all flags are set, the order is ``$cppflags $cflags|$cxxflags $ldflags $ldlibs`` for C and C++, and ``$fflags $cppflags $ldflags $ldlibs`` for Fortran. .. _architecture_specifiers: Architecture specifiers ----------------------- Each node in the dependency graph of a spec has an architecture attribute. This attribute is a triplet of platform, operating system, and processor. You can specify the elements either separately by using the reserved keywords ``platform``, ``os``, and ``target``: .. code-block:: spec $ spack install libelf platform=linux $ spack install libelf os=ubuntu18.04 $ spack install libelf target=broadwell Normally, users don't have to bother specifying the architecture if they are installing software for their current host, as in that case the values will be detected automatically. If you need fine-grained control over which packages use which targets (or over *all* packages' default target), see :ref:`package-preferences`. .. _support-for-microarchitectures: .. _cmd-spack-arch: Support for specific microarchitectures ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack knows how to detect and optimize for many specific microarchitectures and encodes this information in the ``target`` portion of the architecture specification. A complete list of the microarchitectures known to Spack can be obtained in the following way: .. command-output:: spack arch --known-targets When a spec is installed, Spack matches the compiler being used with the microarchitecture being targeted to inject appropriate optimization flags at compile time. Giving a command such as the following: .. code-block:: spec $ spack install zlib target=icelake %gcc@14 will produce compilation lines similar to: .. code-block:: console $ /usr/bin/gcc-14 -march=icelake-client -mtune=icelake-client -c ztest10532.c $ /usr/bin/gcc-14 -march=icelake-client -mtune=icelake-client -c -fPIC -O2 ztest10532. ... where the flags ``-march=icelake-client -mtune=icelake-client`` are injected by Spack based on the requested target and compiler. If Spack determines that the requested compiler cannot optimize for the requested target or cannot build binaries for that target at all, it will exit with a meaningful error message: .. code-block:: spec $ spack install zlib target=icelake %gcc@5 ==> Error: cannot produce optimized binary for micro-architecture "icelake" with gcc@5.5.0 [supported compiler versions are 8:] Conversely, if an older compiler is selected for a newer microarchitecture, Spack will optimize for the best match instead of failing: .. code-block:: spec $ spack arch linux-ubuntu18.04-broadwell $ spack spec zlib%gcc@4.8 Input spec -------------------------------- zlib%gcc@4.8 Concretized -------------------------------- zlib@1.2.11%gcc@4.8+optimize+pic+shared arch=linux-ubuntu18.04-haswell $ spack spec zlib%gcc@9.0.1 Input spec -------------------------------- zlib%gcc@9.0.1 Concretized -------------------------------- zlib@1.2.11%gcc@9.0.1+optimize+pic+shared arch=linux-ubuntu18.04-broadwell In the snippet above, for instance, the microarchitecture was demoted to ``haswell`` when compiling with ``gcc@4.8`` because support to optimize for ``broadwell`` starts from ``gcc@4.9:``. Finally, if Spack has no information to match the compiler and target, it will proceed with the installation but avoid injecting any microarchitecture-specific flags. .. _sec-dependencies: Dependencies ------------ Each node in a DAG can specify dependencies using either the ``%`` or the ``^`` sigil: * The ``%`` sigil identifies direct dependencies, which means there must be an edge connecting the dependency to the node they refer to. * The ``^`` sigil identifies transitive dependencies, which means the dependency just needs to be in the sub-DAG of the node they refer to. The order of transitive dependencies does not matter when writing a spec. For example, these two specs represent exactly the same configuration: .. code-block:: spec mpileaks ^callpath@1.0 ^libelf@0.8.3 mpileaks ^libelf@0.8.3 ^callpath@1.0 Direct dependencies specified with ``%`` apply either to the most recent transitive dependency (``^``), or, if none, to the root package in the spec. So in the spec: .. code-block:: spec root %dep1 ^transitive %dep2 %dep3 ``dep1`` is a direct dependency of ``root``, while both ``dep2`` and ``dep3`` are direct dependencies of ``transitive``. Constraining virtual packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When installing a package that depends on a virtual package, see :ref:`sec-virtual-dependencies`, you can opt to specify the particular provider you want to use, or you can let Spack pick. For example, if you just type this: .. code-block:: spec $ spack install mpileaks Then Spack will pick an ``mpi`` provider for you according to site policies. If you really want a particular version, say ``mpich``, then you could run this instead: .. code-block:: spec $ spack install mpileaks ^mpich This forces Spack to use some version of ``mpich`` for its implementation. As always, you can be even more specific and require a particular ``mpich`` version: .. code-block:: spec $ spack install mpileaks ^mpich@3 The ``mpileaks`` package in particular only needs MPI-1 commands, so any MPI implementation will do. If another package depends on ``mpi@2`` and you try to give it an insufficient MPI implementation (e.g., one that provides only ``mpi@:1``), then Spack will raise an error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. .. _explicit-binding-virtuals: Explicit binding of virtual dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are packages that provide more than just one virtual dependency. When interacting with them, users might want to utilize just a subset of what they could provide and use other providers for virtuals they need. It is possible to be more explicit and tell Spack which dependency should provide which virtual, using a special syntax: .. code-block:: spec $ spack spec strumpack ^mpi=intel-parallel-studio+mkl ^lapack=openblas Concretizing the spec above produces the following DAG: .. figure:: images/strumpack_virtuals.svg :width: 3044 :height: 1683 where ``intel-parallel-studio`` *could* provide ``mpi``, ``lapack``, and ``blas`` but is used only for the former. The ``lapack`` and ``blas`` dependencies are satisfied by ``openblas``. Dependency edge attributes ^^^^^^^^^^^^^^^^^^^^^^^^^^ Some specs require additional information about the relationship between a package and its dependency. This information lives on the edge between the two, and can be specified by following the dependency sigil with square-brackets ``[]``. Edge attributes are always specified as key-value pairs: .. code-block:: spec root ^[key=value] dep In the following sections we'll discuss the edge attributes that are currently allowed in the spec syntax. Virtuals """""""" Packages can provide, or depend on, multiple virtual packages. Users can select which virtuals to use from which dependency by specifying the ``virtuals`` edge attribute: .. code-block:: spec $ spack install mpich %[virtuals=c,cxx] clang %[virtuals=fortran] gcc The command above tells Spack to use ``clang`` to provide the ``c`` and ``cxx`` virtuals, and ``gcc`` to provide the ``fortran`` virtual. The special syntax we have seen in :ref:`explicit-binding-virtuals` is a more compact way to specify the ``virtuals`` edge attribute. For instance, an equivalent formulation of the command above is: .. code-block:: spec $ spack install mpich %c,cxx=clang %fortran=gcc Conditional dependencies """""""""""""""""""""""" Conditional dependencies allow dependency constraints to be applied only under certain conditions. We can express conditional constraints by specifying the ``when`` edge attribute: .. code-block:: spec $ spack install hdf5 ^[when=+mpi] mpich@3.1 This tells Spack that hdf5 should depend on ``mpich@3.1`` if it is configured with MPI support. Dependency propagation ^^^^^^^^^^^^^^^^^^^^^^ The dependency specifications on a node, can be propagated using a double percent ``%%`` sigil. This is particularly useful when specifying compilers. For instance, the following command: .. code-block:: spec $ spack install hdf5+cxx+fortran %%c,cxx=clang %%fortran=gfortran tells Spack to install ``hdf5`` using Clang as the C and C++ compiler, and GCC as the Fortran compiler. It also tells Spack to propagate the same choices, as :ref:`strong preferences `, to the runtime sub-DAG of ``hdf5``. Build tools are unaffected and can still prefer to use a different compiler. Specifying Specs by Hash ------------------------ Complicated specs can become cumbersome to enter on the command line, especially when many of the qualifications are necessary to distinguish between similar installs. To avoid this, when referencing an existing spec, Spack allows you to reference specs by their hash. We previously discussed the spec hash that Spack computes. In place of a spec in any command, substitute ``/`` where ```` is any amount from the beginning of a spec hash. For example, let's say that you accidentally installed two different ``mvapich2`` installations. If you want to uninstall one of them but don't know what the difference is, you can run: .. code-block:: spec $ spack find --long mvapich2 ==> 2 installed packages. -- linux-centos7-x86_64 / gcc@6.3.0 ---------- qmt35td mvapich2@2.2%gcc er3die3 mvapich2@2.2%gcc You can then uninstall the latter installation using: .. code-block:: spec $ spack uninstall /er3die3 Or, if you want to build with a specific installation as a dependency, you can use: .. code-block:: spec $ spack install trilinos ^/er3die3 If the given spec hash is sufficiently long as to be unique, Spack will replace the reference with the spec to which it refers. Otherwise, it will prompt for a more qualified hash. Note that this will not work to reinstall a dependency uninstalled by ``spack uninstall --force``. Specs on the command line ------------------------- The characters used in the spec syntax were chosen to work well with most shells. However, there are cases where the shell may interpret the spec before Spack gets a chance to parse it, leading to unexpected results. Here we document two such cases, and how to avoid them. Unix shells ^^^^^^^^^^^ On Unix-like systems, the shell may expand ``~foo`` to the home directory of a user named ``foo``, so Spack won't see it as a :ref:`disabled boolean variant ` ``foo``. To work around this without quoting, you can avoid whitespace between the package name and boolean variants: .. code-block:: spec mpileaks ~debug # shell may expand this to `mpileaks /home/debug` mpileaks~debug # use this instead Alternatively, you can use a hyphen ``-`` character to disable a variant, but be aware that this *requires* a space between the package name and the variant: .. code-block:: spec mpileaks-debug # wrong: refers to a package named "mpileaks-debug" mpileaks -debug # right: refers to a package named mpileaks with debug disabled As a last resort, ``debug=False`` can also be used to disable a boolean variant. Windows CMD ^^^^^^^^^^^ In Windows CMD, the caret ``^`` is an escape character, and needs itself escaping. Similarly, the equals ``=`` character has special meaning in CMD. To use the caret and equals characters in a spec, you can quote and escape them like this: .. code-block:: console C:\> spack install mpileaks "^^libelf" "foo=bar" These issues are not present in PowerShell. See GitHub issue `#42833 `_ and `#43348 `_ for more details. ================================================ FILE: lib/spack/docs/toolchains_yaml.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: Define named compiler sets (toolchains) in Spack to easily and consistently apply compiler choices for C, C++, and Fortran across different packages. .. _toolchains: Toolchains (toolchains.yaml) ============================= Toolchains let you group a set of compiler constraints under a single, user-defined name. This allows you to reference a complex set of compiler choices for C, C++, and Fortran, with a simple spec like ``%my_toolchain``. They are defined under the ``toolchains`` section of the configuration. .. seealso:: The sections :ref:`language-dependencies` and :ref:`explicit-binding-virtuals` provide more background on how Spack handles languages and compilers. Basic usage ----------- As an example, the following configuration file defines a toolchain named ``llvm_gfortran``: .. code-block:: yaml :caption: ``~/.spack/toolchains.yaml`` toolchains: llvm_gfortran: - spec: cflags=-O3 - spec: "%c=llvm" when: "%c" - spec: "%cxx=llvm" when: "%cxx" - spec: "%fortran=gcc" when: "%fortran" The ``when`` clause in each entry determines if that line's ``spec`` is applied. In this example, it means that ``llvm`` is used as a compiler for the C and C++ languages, and ``gcc`` for Fortran, *whenever the package uses those languages*. The spec ``cflags=-O3`` is *always* applied, because there is no ``when`` clause for that spec. The toolchain can be referenced using .. code-block:: spec $ spack install my-package %llvm_gfortran Toolchains are useful for three reasons: 1. **They reduce verbosity.** Instead of multiple constraints ``%c,cxx=clang %fortran=gcc``, you can simply write ``%llvm_gfortran``. 2. **They apply conditionally.** You can use ``my-package %llvm_gfortran`` even if ``my-package`` is not written in Fortran. 3. **They apply locally.** Toolchains are used at the level of a single spec. .. _pitfalls-without-toolchains: Pitfalls without toolchains --------------------------- The conditional nature of toolchains is important, because it helps you avoid two common pitfalls when specifying compilers. 1. Firstly, when you specify ``my-package %gcc``, your spec is **underconstrained**: Spack has to make ``my-package`` depend on ``gcc``, but the constraint does not rule out mixed compilers, such as ``gcc`` for C and ``llvm`` for C++. 2. Secondly, when you specify ``my-package %c,cxx,fortran=gcc`` to be more explicit, your spec might be **overconstrained**. You not only require ``gcc`` for all languages, but *also* that ``my-package`` uses *all* these languages. This will cause a concretization error if ``my-package`` is written in C and C++, but not Fortran. Combining toolchains -------------------- Different toolchains can be used independently or even in the same spec. Consider the following configuration: .. code-block:: yaml :caption: ``~/.spack/toolchains.yaml`` toolchains: llvm_gfortran: - spec: cflags=-O3 - spec: "%c=llvm" when: "%c" - spec: "%cxx=llvm" when: "%cxx" - spec: "%fortran=gcc" when: "%fortran" gcc_all: - spec: "%c=gcc" when: "%c" - spec: "%cxx=gcc" when: "%cxx" - spec: "%fortran=gcc" when: "%fortran" Now, you can use these toolchains in a single spec: .. code-block:: spec $ spack install hdf5+fortran%llvm_gfortran ^mpich %gcc_all This will result in: * An ``hdf5`` compiled with ``llvm`` for the C/C++ components, but with its Fortran components compiled with ``gfortran``, * Built against an MPICH installation compiled entirely with ``gcc`` for C, C++, and Fortran. Toolchains for other dependencies --------------------------------- While toolchains are typically used to define compiler presets, they can be used for other dependencies as well. A common use case is to define a toolchain that also picks a specific MPI implementation. In the following example, we define a toolchain that uses ``openmpi@5`` as an MPI provider, and ``llvm@19`` as the compiler for C and C++: .. code-block:: yaml :caption: ``~/.spack/toolchains.yaml`` toolchains: clang_openmpi: - spec: "%c=llvm@19" when: "%c" - spec: "%cxx=llvm@19" when: "%cxx" - spec: "%mpi=openmpi@5" when: "%mpi" The general pattern in toolchains configuration is to use a ``when`` condition that specifies a direct dependency on a *virtual* package, and a ``spec`` that :ref:`requires a specific provider for that virtual `. Notice that it's possible to achieve similar configuration with :doc:`packages.yaml `: .. code-block:: yaml :caption: ~/.spack/packages.yaml packages: c: require: [llvm@19] cxx: require: [llvm@19] mpi: require: [openmpi@5] The difference is that the toolchain can be applied **locally** in a spec, while the ``packages.yaml`` configuration is always global. This makes toolchains particularly useful in Spack environments. Toolchains in Spack environments -------------------------------- Toolchains can be used to simplify the construction of a list of specs for Spack environments using :ref:`spec matrices `, when the list includes packages with different language requirements: .. code-block:: yaml :caption: spack.yaml spack: specs: - matrix: - [kokkos, hdf5~cxx+fortran, py-scipy] - ["%llvm_gfortran"] Note that in this case, we can use a single matrix, and the user doesn't need to know exactly which package requires which language. Without toolchains, it would be difficult to enforce compilers directly, because: * ``kokkos`` depends on C and C++, but not Fortran * ``hdf5~cxx+fortran`` depends on C and Fortran, but not C++ * ``py-scipy`` depends on C, C++, and Fortran .. note:: Toolchains are currently limited to using only direct dependencies (``%``) in their definition. Transitive dependencies are not allowed. ================================================ FILE: lib/spack/docs/windows.rst ================================================ .. Copyright Spack Project Developers. See COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) .. meta:: :description lang=en: A guide to setting up and using Spack on Windows, including installing prerequisites and configuring the environment. .. _windows_support: Spack On Windows ================ Windows support for Spack is currently under development. While this work is still in an early stage, it is currently possible to set up Spack and perform a few operations on Windows. This section will guide you through the steps needed to install Spack and start running it on a fresh Windows machine. Step 1: Install prerequisites ----------------------------- To use Spack on Windows, you will need the following packages. Required: * Microsoft Visual Studio * Python * Git * 7z Optional: * Intel Fortran (needed for some packages) .. note:: Currently MSVC is the only compiler tested for C/C++ projects. Intel OneAPI provides Fortran support. Microsoft Visual Studio ^^^^^^^^^^^^^^^^^^^^^^^ Microsoft Visual Studio provides the only Windows C/C++ compiler that is currently supported by Spack. Spack additionally requires that the Windows SDK (including WGL) to be installed as part of your Visual Studio installation as it is required to build many packages from source. We require several specific components to be included in the Visual Studio installation. One is the C/C++ toolset, which can be selected as "Desktop development with C++" or "C++ build tools," depending on installation type (Professional, Build Tools, etc.) The other required component is "C++ CMake tools for Windows," which can be selected from among the optional packages. This provides CMake and Ninja for use during Spack configuration. If you already have Visual Studio installed, you can make sure these components are installed by rerunning the installer. Next to your installation, select "Modify" and look at the "Installation details" pane on the right. Intel Fortran ^^^^^^^^^^^^^ For Fortran-based packages on Windows, we strongly recommend Intel's oneAPI Fortran compilers. The suite is free to download from Intel's website, located at https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/fortran-compiler.html. The executable of choice for Spack will be Intel's Beta Compiler, ifx, which supports the classic compiler's (ifort's) frontend and runtime libraries by using LLVM. Python ^^^^^^ As Spack is a Python-based package, an installation of Python will be needed to run it. Python 3 can be downloaded and installed from the Windows Store, and will be automatically added to your ``PATH`` in this case. .. note:: Spack currently supports Python versions later than 3.2 inclusive. Git ^^^ A bash console and GUI can be downloaded from https://git-scm.com/downloads. If you are unfamiliar with Git, there are a myriad of resources online to help guide you through checking out repositories and switching development branches. When given the option of adjusting your ``PATH``, choose the ``Git from the command line and also from 3rd-party software`` option. This will automatically update your ``PATH`` variable to include the ``git`` command. Spack support on Windows is currently dependent on installing the Git for Windows project as the project providing Git support on Windows. This is additionally the recommended method for installing Git on Windows, a link to which can be found above. Spack requires the utilities vendored by this project. 7zip ^^^^ A tool for extracting ``.xz`` files is required for extracting source tarballs. The latest 7-Zip can be located at https://sourceforge.net/projects/sevenzip/. Step 2: Install and setup Spack ------------------------------- We are now ready to get the Spack environment set up on our machine. We begin by using Git to clone the Spack repo, hosted at https://github.com/spack/spack.git into a desired directory, for our purposes today, called ``spack_install``. In order to install Spack with Windows support, run the following one-liner in a Windows CMD prompt. .. code-block:: console $ git clone https://github.com/spack/spack.git .. note:: If you chose to install Spack into a directory on Windows that is set up to require Administrative Privileges, Spack will require elevated privileges to run. Administrative Privileges can be denoted either by default, such as ``C:\Program Files``, or administrator-applied administrative restrictions on a directory that Spack installs files to such as ``C:\Users`` Step 3: Run and configure Spack ------------------------------- On Windows, Spack supports both primary native shells, PowerShell and the traditional command prompt. To use Spack, pick your favorite shell, and run ``bin\spack_cmd.bat`` or ``share/spack/setup-env.ps1`` (you may need to Run as Administrator) from the top-level Spack directory. This will provide a Spack-enabled shell. If you receive a warning message that Python is not in your ``PATH`` (which may happen if you installed Python from the website and not the Windows Store), add the location of the Python executable to your ``PATH`` now. You can permanently add Python to your ``PATH`` variable by using the ``Edit the system environment variables`` utility in Windows Control Panel. To configure Spack, first run the following command inside the Spack console: .. code-block:: console $ spack compiler find This creates a ``.staging`` directory in our Spack prefix, along with a ``windows`` subdirectory containing a ``packages.yaml`` file. On a fresh Windows installation with the above packages installed, this command should only detect Microsoft Visual Studio and the Intel Fortran compiler will be integrated within the first version of MSVC present in the ``packages.yaml`` output. Spack provides a default ``config.yaml`` file for Windows that it will use unless overridden. This file is located at ``etc\spack\defaults\windows\config.yaml``. You can read more on how to do this and write your own configuration files in the :ref:`Configuration Files` section of our documentation. If you do this, pay particular attention to the ``build_stage`` block of the file as this specifies the directory that will temporarily hold the source code for the packages to be installed. This path name must be sufficiently short for compliance with CMD, otherwise you will see build errors during installation (particularly with CMake) tied to long path names. To allow Spack's use of external tools and dependencies already on your system, the external pieces of software must be described in the ``packages.yaml`` file. There are two methods to populate this file: The first and easiest choice is to use Spack to find installations on your system. In the Spack terminal, run the following commands: .. code-block:: console $ spack external find cmake $ spack external find ninja The ``spack external find `` will find executables on your system with the same name given. The command will store the items found in ``packages.yaml`` in the ``.staging\`` directory. Assuming that the command found CMake and Ninja executables in the previous step, continue to Step 4. If no executables were found, we may need to manually direct Spack towards the CMake and Ninja installations we set up with Visual Studio. Therefore, your ``packages.yaml`` file will look something like this, possibly with slight variations in the paths to CMake and Ninja: .. code-block:: yaml packages: cmake: externals: - spec: cmake@3.19 prefix: 'c:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake' buildable: false ninja: externals: - spec: ninja@1.8.2 prefix: 'c:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja' buildable: false You can also use a separate installation of CMake if you have one and prefer to use it. If you don't have a path to Ninja analogous to the above, then you can obtain it by running the Visual Studio Installer and following the instructions at the start of this section. Also note that YAML files use spaces for indentation and not tabs, so ensure that this is the case when editing one directly. .. note:: The use of Cygwin is not officially supported by Spack and is not tested. However, Spack will not prevent this, so if choosing to use Spack with Cygwin, know that no functionality is guaranteed. Step 4: Use Spack ----------------- Once the configuration is complete, it is time to give the installation a test. Install a basic package through the Spack console via: .. code-block:: spec $ spack install cpuinfo If in the previous step, you did not have CMake or Ninja installed, running the command above should install both packages. .. note:: Windows has a few idiosyncrasies when it comes to the Spack spec syntax and the use of certain shells See the Spack spec syntax doc for more information For developers -------------- The intent is to provide a Windows installer that will automatically set up Python, Git, and Spack, instead of requiring the user to do so manually. Instructions for creating the installer are at https://github.com/spack/spack/blob/develop/lib/spack/spack/cmd/installer/README.md ================================================ FILE: lib/spack/llnl/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import warnings import spack.error import spack.llnl warnings.warn( "The `llnl` module will be removed in Spack v1.1", category=spack.error.SpackAPIWarning, stacklevel=2, ) __path__ = spack.llnl.__path__ ================================================ FILE: lib/spack/spack/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import functools import os import re from typing import Optional import spack.paths import spack.util.git #: PEP440 canonical ... string __version__ = "1.2.0.dev0" spack_version = __version__ #: The current Package API version implemented by this version of Spack. The Package API defines #: the Python interface for packages as well as the layout of package repositories. The minor #: version is incremented when the package API is extended in a backwards-compatible way. The major #: version is incremented upon breaking changes. This version is changed independently from the #: Spack version. package_api_version = (2, 4) #: The minimum Package API version that this version of Spack is compatible with. This should #: always be a tuple of the form ``(major, 0)``, since compatibility with vX.Y implies #: compatibility with vX.0. min_package_api_version = (1, 0) def __try_int(v): try: return int(v) except ValueError: return v #: (major, minor, micro, dev release) tuple spack_version_info = tuple([__try_int(v) for v in __version__.split(".")]) @functools.lru_cache(maxsize=None) def get_spack_commit() -> Optional[str]: """Get the Spack git commit sha. Returns: (str or None) the commit sha if available, otherwise None """ git_path = os.path.join(spack.paths.prefix, ".git") if not os.path.exists(git_path): return None git = spack.util.git.git() if not git: return None rev = git( "-C", spack.paths.prefix, "rev-parse", "HEAD", output=str, error=os.devnull, fail_on_error=False, ) if git.returncode != 0: return None match = re.match(r"[a-f\d]{7,}$", rev) return match.group(0) if match else None def get_version() -> str: """Get a descriptive version of this instance of Spack. Outputs ``" ()"``. The commit sha is only added when available. """ commit = get_spack_commit() if commit: return f"{spack_version} ({commit})" return spack_version def get_short_version() -> str: """Short Spack version.""" return f"{spack_version_info[0]}.{spack_version_info[1]}" __all__ = [ "spack_version_info", "spack_version", "get_version", "get_spack_commit", "get_short_version", "package_api_version", "min_package_api_version", ] ================================================ FILE: lib/spack/spack/aliases.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Alias names to convert legacy compilers to builtin packages and vice-versa""" BUILTIN_TO_LEGACY_COMPILER = { "llvm": "clang", "intel-oneapi-compilers": "oneapi", "llvm-amdgpu": "rocmcc", "intel-oneapi-compilers-classic": "intel", "acfl": "arm", } LEGACY_COMPILER_TO_BUILTIN = { "clang": "llvm", "oneapi": "intel-oneapi-compilers", "rocmcc": "llvm-amdgpu", "intel": "intel-oneapi-compilers-classic", "arm": "acfl", } ================================================ FILE: lib/spack/spack/archspec.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Adapter for the archspec library.""" import spack.vendor.archspec.cpu import spack.spec def microarchitecture_flags(spec: spack.spec.Spec, language: str) -> str: """Get compiler flags for the spec's microarchitecture. Args: spec: The spec defining the target microarchitecture and compiler. language: The language (``"c"``, ``"cxx"``, ``"fortran"``) used to select the appropriate compiler from the spec. Example:: >>> spec.format("{target}") 'm1' >>> spec["c"].format("{name}{@version}") 'apple-clang@17.0.0' >>> microarchitecture_flags(spec, language="c") '-mcpu=apple-m1' """ target = spec.target if not spec.has_virtual_dependency(language): raise ValueError(f"The spec {spec.name} does not depend on {language}") elif target is None: raise ValueError(f"The spec {spec.name} does not have a target defined") compiler = spec.dependencies(virtuals=language)[0] return microarchitecture_flags_from_target(target, compiler) def microarchitecture_flags_from_target( target: spack.vendor.archspec.cpu.Microarchitecture, compiler: spack.spec.Spec ) -> str: """Get compiler flags for the spec's microarchitecture. Similar to :func:`microarchitecture_flags`, but takes a ``target`` and ``compiler`` directly instead of a spec. Args: target: The target microarchitecture. compiler: The spec defining the compiler. """ # Try to check if the current compiler comes with a version number or has an unexpected suffix. # If so, treat it as a compiler with a custom spec. version_number, _ = spack.vendor.archspec.cpu.version_components( compiler.version.dotted_numeric_string ) try: return target.optimization_flags(compiler.package.archspec_name(), version_number) except ValueError: return "" #: The host target family, like x86_64 or aarch64 HOST_TARGET_FAMILY = spack.vendor.archspec.cpu.host().family ================================================ FILE: lib/spack/spack/audit.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes and functions to register audit checks for various parts of Spack and run them on-demand. To register a new class of sanity checks (e.g. sanity checks for compilers.yaml), the first action required is to create a new AuditClass object: .. code-block:: python audit_cfgcmp = AuditClass( tag="CFG-COMPILER", description="Sanity checks on compilers.yaml", kwargs=() ) This object is to be used as a decorator to register functions that will perform each a single check: .. code-block:: python @audit_cfgcmp def _search_duplicate_compilers(error_cls): pass These functions need to take as argument the keywords declared when creating the decorator object plus an ``error_cls`` argument at the end, acting as a factory to create Error objects. It should return a (possibly empty) list of errors. Calls to each of these functions are triggered by the ``run`` method of the decorator object, that will forward the keyword arguments passed as input. """ import ast import collections import collections.abc import glob import inspect import io import itertools import os import pathlib import pickle import re import warnings from typing import Iterable, List, Set, Tuple from urllib.request import urlopen import spack.builder import spack.config import spack.enums import spack.fetch_strategy import spack.llnl.util.lang import spack.patch import spack.repo import spack.spec import spack.util.crypto import spack.util.spack_yaml as syaml import spack.variant from spack.llnl.string import plural #: Map an audit tag to a list of callables implementing checks CALLBACKS = {} #: Map a group of checks to the list of related audit tags GROUPS = collections.defaultdict(list) class Error: """Information on an error reported in a test.""" def __init__(self, summary, details): self.summary = summary self.details = tuple(details) def __str__(self): if self.details: return f"{self.summary}\n" + "\n".join(f" {detail}" for detail in self.details) return self.summary def __eq__(self, other): if self.summary != other.summary or self.details != other.details: return False return True def __hash__(self): value = (self.summary, self.details) return hash(value) class AuditClass(collections.abc.Sequence): def __init__(self, group, tag, description, kwargs): """Return an object that acts as a decorator to register functions associated with a specific class of sanity checks. Args: group (str): group in which this check is to be inserted tag (str): tag uniquely identifying the class of sanity checks description (str): description of the sanity checks performed by this tag kwargs (tuple of str): keyword arguments that each registered function needs to accept """ if tag in CALLBACKS: msg = 'audit class "{0}" already registered' raise ValueError(msg.format(tag)) self.group = group self.tag = tag self.description = description self.kwargs = kwargs self.callbacks = [] # Init the list of hooks CALLBACKS[self.tag] = self # Update the list of tags in the group GROUPS[self.group].append(self.tag) def __call__(self, func): self.callbacks.append(func) def __getitem__(self, item): return self.callbacks[item] def __len__(self): return len(self.callbacks) def run(self, **kwargs): msg = 'please pass "{0}" as keyword arguments' msg = msg.format(", ".join(self.kwargs)) assert set(self.kwargs) == set(kwargs), msg errors = [] kwargs["error_cls"] = Error for fn in self.callbacks: errors.extend(fn(**kwargs)) return errors def run_group(group, **kwargs): """Run the checks that are part of the group passed as argument. Args: group (str): group of checks to be run **kwargs: keyword arguments forwarded to the checks Returns: List of (tag, errors) that failed. """ reports = [] for check in GROUPS[group]: errors = run_check(check, **kwargs) reports.append((check, errors)) return reports def run_check(tag, **kwargs): """Run the checks associated with a single tag. Args: tag (str): tag of the check **kwargs: keyword arguments forwarded to the checks Returns: Errors occurred during the checks """ return CALLBACKS[tag].run(**kwargs) # TODO: For the generic check to be useful for end users, # TODO: we need to implement hooks like described in # TODO: https://github.com/spack/spack/pull/23053/files#r630265011 #: Generic checks relying on global state generic = AuditClass( group="generic", tag="GENERIC", description="Generic checks relying on global variables", kwargs=(), ) #: Sanity checks on compilers.yaml config_compiler = AuditClass( group="configs", tag="CFG-COMPILER", description="Sanity checks on compilers.yaml", kwargs=() ) @config_compiler def _search_duplicate_compilers(error_cls): """Report compilers with the same spec and two different definitions""" errors = [] compilers = list(sorted(spack.config.get("compilers"), key=lambda x: x["compiler"]["spec"])) for spec, group in itertools.groupby(compilers, key=lambda x: x["compiler"]["spec"]): group = list(group) if len(group) == 1: continue error_msg = "Compiler defined multiple times: {0}" try: details = [str(x._start_mark).strip() for x in group] except Exception: details = [] errors.append(error_cls(summary=error_msg.format(spec), details=details)) return errors #: Sanity checks on packages.yaml config_packages = AuditClass( group="configs", tag="CFG-PACKAGES", description="Sanity checks on packages.yaml", kwargs=() ) #: Sanity checks on packages.yaml config_repos = AuditClass( group="configs", tag="CFG-REPOS", description="Sanity checks on repositories", kwargs=() ) @config_packages def _search_duplicate_specs_in_externals(error_cls): """Search for duplicate specs declared as externals""" errors, externals = [], collections.defaultdict(list) packages_yaml = spack.config.get("packages") for name, pkg_config in packages_yaml.items(): # No externals can be declared under all if name == "all" or "externals" not in pkg_config: continue current_externals = pkg_config["externals"] for entry in current_externals: # Ask for the string representation of the spec to normalize # aspects of the spec that may be represented in multiple ways # e.g. +foo or foo=true key = str(spack.spec.Spec(entry["spec"])) externals[key].append(entry) for spec, entries in sorted(externals.items()): # If there's a single external for a spec we are fine if len(entries) < 2: continue # Otherwise wwe need to report an error error_msg = "Multiple externals share the same spec: {0}".format(spec) try: lines = [str(x._start_mark).strip() for x in entries] details = ( ["Please remove all but one of the following entries:"] + lines + ["as they might result in non-deterministic hashes"] ) except (TypeError, AttributeError): details = [] errors.append(error_cls(summary=error_msg, details=details)) return errors @config_packages def _avoid_mismatched_variants(error_cls): """Warns if variant preferences have mismatched types or names.""" errors = [] packages_yaml = spack.config.CONFIG.get_config("packages") for pkg_name in packages_yaml: # 'all:' must be more forgiving, since it is setting defaults for everything if pkg_name == "all" or "variants" not in packages_yaml[pkg_name]: continue preferences = packages_yaml[pkg_name]["variants"] if not isinstance(preferences, list): preferences = [preferences] for variants in preferences: current_spec = spack.spec.Spec(variants) pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) for variant in current_spec.variants.values(): # Variant does not exist at all if variant.name not in pkg_cls.variant_names(): summary = ( f"Setting a preference for the '{pkg_name}' package to the " f"non-existing variant '{variant.name}'" ) errors.append(_make_config_error(preferences, summary, error_cls=error_cls)) continue # Variant cannot accept this value try: spack.variant.prevalidate_variant_value(pkg_cls, variant, strict=True) except Exception: summary = ( f"Setting the variant '{variant.name}' of the '{pkg_name}' package " f"to the invalid value '{str(variant)}'" ) errors.append(_make_config_error(preferences, summary, error_cls=error_cls)) return errors @config_packages def _wrongly_named_spec(error_cls): """Warns if the wrong name is used for an external spec""" errors = [] packages_yaml = spack.config.CONFIG.get_config("packages") for pkg_name in packages_yaml: if pkg_name == "all": continue externals = packages_yaml[pkg_name].get("externals", []) is_virtual = spack.repo.PATH.is_virtual(pkg_name) for entry in externals: spec = spack.spec.Spec(entry["spec"]) regular_pkg_is_wrong = not is_virtual and pkg_name != spec.name virtual_pkg_is_wrong = is_virtual and not any( p.name == spec.name for p in spack.repo.PATH.providers_for(pkg_name) ) if regular_pkg_is_wrong or virtual_pkg_is_wrong: summary = f"Wrong external spec detected for '{pkg_name}': {spec}" errors.append(_make_config_error(entry, summary, error_cls=error_cls)) return errors @config_packages def _ensure_all_virtual_packages_have_default_providers(error_cls): """All virtual packages must have a default provider explicitly set.""" configuration = spack.config.create() defaults = configuration.get_config("packages", _merged_scope="defaults") default_providers = defaults["all"]["providers"] virtuals = spack.repo.PATH.provider_index.providers default_providers_filename = configuration.scopes["defaults"].get_section_filename("packages") return [ error_cls(f"'{virtual}' must have a default provider in {default_providers_filename}", []) for virtual in virtuals if virtual not in default_providers ] @config_repos def _ensure_no_folders_without_package_py(error_cls): """Check that we don't leave any folder without a package.py in repos""" errors = [] for repository in spack.repo.PATH.repos: missing = [] for entry in os.scandir(repository.packages_path): if not entry.is_dir() or entry.name == "__pycache__": continue package_py = pathlib.Path(entry.path) / spack.repo.package_file_name if not package_py.exists(): missing.append(entry.path) if missing: summary = ( f"The '{repository.namespace}' repository misses a package.py file" f" in the following folders" ) errors.append(error_cls(summary=summary, details=[f"{x}" for x in missing])) return errors def _make_config_error(config_data, summary, error_cls): s = io.StringIO() s.write("Occurring in the following file:\n") syaml.dump_config(config_data, stream=s, blame=True) return error_cls(summary=summary, details=[s.getvalue()]) #: Sanity checks on package directives package_directives = AuditClass( group="packages", tag="PKG-DIRECTIVES", description="Sanity checks on specs used in directives", kwargs=("pkgs",), ) package_attributes = AuditClass( group="packages", tag="PKG-ATTRIBUTES", description="Sanity checks on reserved attributes of packages", kwargs=("pkgs",), ) package_properties = AuditClass( group="packages", tag="PKG-PROPERTIES", description="Sanity checks on properties a package should maintain", kwargs=("pkgs",), ) #: Sanity checks on linting # This can take some time, so it's run separately from packages package_https_directives = AuditClass( group="packages-https", tag="PKG-HTTPS-DIRECTIVES", description="Sanity checks on https checks of package urls, etc.", kwargs=("pkgs",), ) @package_properties def _check_build_test_callbacks(pkgs, error_cls): """Ensure stand-alone test methods are not included in build-time callbacks. Test methods are for checking the installed software as stand-alone tests. They could also be called during the post-install phase of a build. """ errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) test_callbacks = getattr(pkg_cls, "build_time_test_callbacks", None) has_test_method = test_callbacks and any([m.startswith("test_") for m in test_callbacks]) if has_test_method: msg = f"Package {pkg_name} includes stand-alone test methods in build-time checks." callbacks = ", ".join(test_callbacks) instr = f"Remove the following from 'build_time_test_callbacks': {callbacks}" errors.append(error_cls(msg.format(pkg_name), [instr])) return errors @package_directives def _directives_can_be_evaluated(pkgs, error_cls): """Ensure that all directives in a package can be evaluated.""" errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) for attr in pkg_cls._dict_to_directives: try: getattr(pkg_cls, attr) except Exception as e: error_msg = f"Package '{pkg_name}' has invalid directive '{attr}'" details = [str(e)] errors.append(error_cls(error_msg, details)) return errors @package_directives def _check_patch_urls(pkgs, error_cls): """Ensure that patches fetched from GitHub and GitLab have stable sha256 hashes.""" github_patch_url_re = ( r"^https?://(?:patch-diff\.)?github(?:usercontent)?\.com/" r".+/.+/(?:commit|pull)/[a-fA-F0-9]+\.(?:patch|diff)" ) github_pull_commits_re = ( r"^https?://(?:patch-diff\.)?github(?:usercontent)?\.com/" r".+/.+/pull/\d+/commits/[a-fA-F0-9]+\.(?:patch|diff)" ) # Only .diff URLs have stable/full hashes: # https://forum.gitlab.com/t/patches-with-full-index/29313 gitlab_patch_url_re = ( r"^https?://(?:.+)?gitlab(?:.+)/" r".+/.+/-/(?:commit|merge_requests)/[a-fA-F0-9]+\.(?:patch|diff)" ) errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) for condition, patches in pkg_cls.patches.items(): for patch in patches: if not isinstance(patch, spack.patch.UrlPatch): continue if re.match(github_pull_commits_re, patch.url): url = re.sub(r"/pull/\d+/commits/", r"/commit/", patch.url) url = re.sub(r"^(.*)(? None: if not isinstance(node, self.path[self.depth]): return self.depth += 1 if self.depth == len(self.path): self.in_function = True super().generic_visit(node) if self.depth == len(self.path): self.in_function = False self.locals.clear() self.depth -= 1 def generic_visit(self, node: ast.AST) -> None: # Recurse into function definitions if self.depth < len(self.path): return self.descend_in_function_def(node) elif not self.in_function: return elif isinstance(node, ast.Global): for name in node.names: if name in self.magic_globals: self.references_to_globals.append((name, node.lineno)) elif isinstance(node, ast.Assign): # visit the rhs before lhs super().visit(node.value) for target in node.targets: super().visit(target) elif isinstance(node, ast.Name) and node.id in self.magic_globals: if isinstance(node.ctx, ast.Load) and node.id not in self.locals: self.references_to_globals.append((node.id, node.lineno)) elif isinstance(node.ctx, ast.Store): self.locals.add(node.id) else: super().generic_visit(node) @package_properties def _uses_deprecated_globals(pkgs, error_cls): """Ensure that packages do not use deprecated globals""" errors = [] for pkg_name in pkgs: # some packages scheduled to be removed in v0.23 are not worth fixing. pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) if all(v.get("deprecated", False) for v in pkg_cls.versions.values()): continue file = spack.repo.PATH.filename_for_package_name(pkg_name) tree = ast.parse(open(file, "rb").read()) visitor = DeprecatedMagicGlobals(("std_cmake_args", "std_meson_args", "std_pip_args")) visitor.visit(tree) if visitor.references_to_globals: errors.append( error_cls( f"Package '{pkg_name}' uses deprecated globals", [ f"{file}:{line} references '{name}'" for name, line in visitor.references_to_globals ], ) ) return errors @package_properties def _ensure_test_docstring(pkgs, error_cls): """Ensure stand-alone test methods have a docstring. The docstring of a test method is implicitly used as the description of the corresponding test part during test results reporting. """ doc_regex = r'\s+("""[^"]+""")' errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) methods = inspect.getmembers(pkg_cls, predicate=lambda x: inspect.isfunction(x)) method_names = [] for name, test_fn in methods: if not name.startswith("test_"): continue # Ensure the test method has a docstring source = inspect.getsource(test_fn) match = re.search(doc_regex, source) if match is None or len(match.group(0).replace('"', "").strip()) == 0: method_names.append(name) num_methods = len(method_names) if num_methods > 0: methods = plural(num_methods, "method", show_n=False) docstrings = plural(num_methods, "docstring", show_n=False) msg = f"Package {pkg_name} has test {methods} with empty or missing {docstrings}." names = ", ".join(method_names) instr = [ "Docstrings are used as descriptions in test outputs.", f"Add a concise summary to the following {methods} in '{pkg_cls.__module__}':", f"{names}", ] errors.append(error_cls(msg, instr)) return errors @package_properties def _ensure_test_implemented(pkgs, error_cls): """Ensure stand-alone test methods are implemented. The test method is also required to be non-empty. """ def skip(line): ln = line.strip() return ln.startswith("#") or "pass" in ln doc_regex = r'\s+("""[^"]+""")' errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) methods = inspect.getmembers(pkg_cls, predicate=lambda x: inspect.isfunction(x)) method_names = [] for name, test_fn in methods: if not name.startswith("test_"): continue source = inspect.getsource(test_fn) # Attempt to ensure the test method is implemented. impl = re.sub(doc_regex, r"", source).splitlines()[1:] lines = [ln.strip() for ln in impl if not skip(ln)] if not lines: method_names.append(name) num_methods = len(method_names) if num_methods > 0: methods = plural(num_methods, "method", show_n=False) msg = f"Package {pkg_name} has empty or missing test {methods}." names = ", ".join(method_names) instr = [ f"Implement or remove the following {methods} from '{pkg_cls.__module__}': {names}" ] errors.append(error_cls(msg, instr)) return errors @package_https_directives def _linting_package_file(pkgs, error_cls): """Check for correctness of links""" errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) homepage = pkg_cls.homepage if not homepage: continue # Does the homepage have http, and if so, does https work? if homepage.startswith("http://"): try: with urlopen(f"https://{homepage[7:]}") as response: if response.getcode() == 200: msg = 'Package "{0}" uses http but has a valid https endpoint.' errors.append(msg.format(pkg_cls.name)) except Exception as e: msg = 'Error with attempting https for "{0}": ' errors.append(error_cls(msg.format(pkg_cls.name), [str(e)])) continue return spack.llnl.util.lang.dedupe(errors) @package_directives def _variant_issues_in_directives(pkgs, error_cls): """Report unknown, wrong, or propagating variants in directives for this package""" errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) filename = spack.repo.PATH.filename_for_package_name(pkg_name) # Check the "conflicts" directive for trigger, conflicts in pkg_cls.conflicts.items(): errors.extend( _issues_in_directive_constraint( pkg_cls, spack.spec.Spec(trigger), directive="conflicts", error_cls=error_cls, filename=filename, requestor=pkg_name, ) ) for conflict, _ in conflicts: errors.extend( _issues_in_directive_constraint( pkg_cls, spack.spec.Spec(conflict), directive="conflicts", error_cls=error_cls, filename=filename, requestor=pkg_name, ) ) # Check "depends_on" directive for trigger, deps_by_name in pkg_cls.dependencies.items(): vrn = spack.spec.Spec(trigger) errors.extend( _issues_in_directive_constraint( pkg_cls, vrn, directive="depends_on", error_cls=error_cls, filename=filename, requestor=pkg_name, ) ) for dep_name, dep in deps_by_name.items(): if spack.repo.PATH.is_virtual(dep_name): continue try: dep_pkg_cls = spack.repo.PATH.get_pkg_class(dep_name) except spack.repo.UnknownPackageError: continue errors.extend( _issues_in_directive_constraint( dep_pkg_cls, dep.spec, directive="depends_on", error_cls=error_cls, filename=filename, requestor=pkg_name, ) ) # Check "provides" directive for when_spec in pkg_cls.provided: errors.extend( _issues_in_directive_constraint( pkg_cls, when_spec, directive="provides", error_cls=error_cls, filename=filename, requestor=pkg_name, ) ) # Check "resource" directive for vrn in pkg_cls.resources: errors.extend( _issues_in_directive_constraint( pkg_cls, vrn, directive="resource", error_cls=error_cls, filename=filename, requestor=pkg_name, ) ) return spack.llnl.util.lang.dedupe(errors) @package_directives def _issues_in_depends_on_directive(pkgs, error_cls): """Reports issues with 'depends_on' directives. Issues might be unknown dependencies, unknown variants or variant values, or declaration of nested dependencies. """ errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) filename = spack.repo.PATH.filename_for_package_name(pkg_name) for when, deps_by_name in pkg_cls.dependencies.items(): for dep_name, dep in deps_by_name.items(): def check_virtual_with_variants(spec, msg): if not spack.repo.PATH.is_virtual(spec.name) or not spec.variants: return error = error_cls( f"{pkg_name}: {msg}", [f"remove variants from '{spec}' in depends_on directive in {filename}"], ) errors.append(error) check_virtual_with_variants(dep.spec, "virtual dependency cannot have variants") check_virtual_with_variants(dep.spec, "virtual when= spec cannot have variants") # No need to analyze virtual packages if spack.repo.PATH.is_virtual(dep_name): continue # check for unknown dependencies try: dependency_pkg_cls = spack.repo.PATH.get_pkg_class(dep_name) except spack.repo.UnknownPackageError: # This dependency is completely missing, so report # and continue the analysis summary = f"{pkg_name}: unknown package '{dep_name}' in 'depends_on' directive" details = [f" in {filename}"] errors.append(error_cls(summary=summary, details=details)) continue # Check for self-referential specs similar to: # # depends_on("foo@X.Y", when="^foo+bar") # # That would allow clingo to choose whether to have foo@X.Y+bar in the graph. problematic_edges = [ x for x in when.edges_to_dependencies(dep_name) if not x.virtuals ] if problematic_edges and not dep.patches: summary = ( f"{pkg_name}: dependency on '{dep.spec}' when '{when}' is self-referential" ) details = [ ( f" please specify better using '^[virtuals=...] {dep_name}', or " f"substitute with an equivalent condition on '{pkg_name}'" ), f" in {filename}", ] errors.append(error_cls(summary=summary, details=details)) continue # check variants dependency_variants = dep.spec.variants for name, variant in dependency_variants.items(): try: spack.variant.prevalidate_variant_value( dependency_pkg_cls, variant, dep.spec, strict=True ) except Exception as e: summary = ( f"{pkg_name}: wrong variant used for dependency in 'depends_on()'" ) error_msg = str(e) if isinstance(e, KeyError): error_msg = ( f"variant {str(e).strip()} does not exist in package {dep_name}" f" in package '{dep_name}'" ) errors.append( error_cls(summary=summary, details=[error_msg, f"in {filename}"]) ) return errors @package_directives def _ensure_variant_defaults_are_parsable(pkgs, error_cls): """Ensures that variant defaults are present and parsable from cli""" def check_variant(pkg_cls, variant, vname): # bool is a subclass of int in python. Permitting a default that is an instance # of 'int' means both foo=false and foo=0 are accepted. Other falsish values are # not allowed, since they can't be parsed from CLI ('foo=') default_is_parsable = isinstance(variant.default, int) or variant.default if not default_is_parsable: msg = f"Variant '{vname}' of package '{pkg_cls.name}' has an unparsable default value" return [error_cls(msg, [])] try: vspec = variant.make_default() except spack.variant.MultipleValuesInExclusiveVariantError: msg = f"Can't create default value for variant '{vname}' in package '{pkg_cls.name}'" return [error_cls(msg, [])] try: variant.validate_or_raise(vspec, pkg_cls.name) except spack.variant.InvalidVariantValueError: msg = "Default value of variant '{vname}' in package '{pkg.name}' is invalid" question = "Is it among the allowed values?" return [error_cls(msg, [question])] return [] errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) for vname in pkg_cls.variant_names(): for _, variant_def in pkg_cls.variant_definitions(vname): errors.extend(check_variant(pkg_cls, variant_def, vname)) return errors @package_directives def _ensure_variants_have_descriptions(pkgs, error_cls): """Ensures that all variants have a description.""" errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) for name in pkg_cls.variant_names(): for when, variant in pkg_cls.variant_definitions(name): if not variant.description: msg = f"Variant '{name}' in package '{pkg_name}' is missing a description" errors.append(error_cls(msg, [])) return errors @package_directives def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls): """Report if version constraints used in directives are not satisfiable""" errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) filename = spack.repo.PATH.filename_for_package_name(pkg_name) dependencies_to_check = [] for _, deps_by_name in pkg_cls.dependencies.items(): for dep_name, dep in deps_by_name.items(): # Skip virtual dependencies for the time being, check on # their versions can be added later if spack.repo.PATH.is_virtual(dep_name): continue dependencies_to_check.append(dep.spec) host_architecture = spack.spec.ArchSpec.default_arch() for s in dependencies_to_check: dependency_pkg_cls = None try: dependency_pkg_cls = spack.repo.PATH.get_pkg_class(s.name) # Some packages have hacks that might cause failures on some platform # Allow to explicitly set conditions to skip version checks in that case skip_conditions = getattr(dependency_pkg_cls, "skip_version_audit", []) skip_version_check = False for condition in skip_conditions: if host_architecture.satisfies(spack.spec.Spec(condition).architecture): skip_version_check = True break assert skip_version_check or any( v.intersects(s.versions) for v in list(dependency_pkg_cls.versions) ) except Exception: summary = ( "{0}: dependency on {1} cannot be satisfied by known versions of {1.name}" ).format(pkg_name, s) details = ["happening in " + filename] if dependency_pkg_cls is not None: details.append( "known versions of {0.name} are {1}".format( s, ", ".join([str(x) for x in dependency_pkg_cls.versions]) ) ) errors.append(error_cls(summary=summary, details=details)) return errors def _issues_in_directive_constraint(pkg, constraint, *, directive, error_cls, filename, requestor): errors = [] errors.extend( _analyze_variants_in_directive( pkg, constraint, directive=directive, error_cls=error_cls, filename=filename, requestor=requestor, ) ) errors.extend( _analize_propagated_deps_in_directive( pkg, constraint, directive=directive, error_cls=error_cls, filename=filename, requestor=requestor, ) ) return errors def _analyze_variants_in_directive(pkg, constraint, *, directive, error_cls, filename, requestor): errors = [] variant_names = pkg.variant_names() summary = f"{requestor}: wrong variant in '{directive}' directive" for name, v in constraint.variants.items(): if name == "commit": # Automatic variant continue if name not in variant_names: msg = f"variant {name} does not exist in {pkg.name}" errors.append(error_cls(summary=summary, details=[msg, f"in {filename}"])) continue if v.propagate: propagation_summary = f"{requestor}: propagating variant in '{directive}' directive" msg = f"using {constraint} in a directive, which propagates the '{name}' variant" errors.append(error_cls(summary=propagation_summary, details=[msg, f"in {filename}"])) try: spack.variant.prevalidate_variant_value(pkg, v, constraint, strict=True) except ( spack.variant.InconsistentValidationError, spack.variant.MultipleValuesInExclusiveVariantError, spack.variant.InvalidVariantValueError, ) as e: msg = str(e).strip() errors.append(error_cls(summary=summary, details=[msg, f"in {filename}"])) return errors def _analize_propagated_deps_in_directive( pkg, constraint, *, directive, error_cls, filename, requestor ): errors = [] summary = f"{requestor}: dependency propagation ('%%') in '{directive}' directive" for edge in constraint.traverse_edges(): if edge.propagation != spack.enums.PropagationPolicy.NONE: msg = f"'{edge.spec}' contains a propagated dependency" errors.append(error_cls(summary=summary, details=[msg, f"in {filename}"])) return errors @package_directives def _named_specs_in_when_arguments(pkgs, error_cls): """Reports named specs in the 'when=' attribute of a directive. Note that 'conflicts' is the only directive allowing that. """ errors = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) def _refers_to_pkg(when): when_spec = spack.spec.Spec(when) return not when_spec.name or when_spec.name == pkg_name def _error_items(when_dict): for when, elts in when_dict.items(): if not _refers_to_pkg(when): yield when, elts, [f"using '{when}', should be '^{when}'"] def _extracts_errors(triggers, summary): _errors = [] for trigger in list(triggers): if not _refers_to_pkg(trigger): details = [f"using '{trigger}', should be '^{trigger}'"] _errors.append(error_cls(summary=summary, details=details)) return _errors for when, dnames, details in _error_items(pkg_cls.dependencies): errors.extend( error_cls(f"{pkg_name}: wrong 'when=' condition for '{dname}' dependency", details) for dname in dnames ) for when, variants_by_name in pkg_cls.variants.items(): for vname, variant in variants_by_name.items(): summary = f"{pkg_name}: wrong 'when=' condition for the '{vname}' variant" errors.extend(_extracts_errors([when], summary)) for when, providers, details in _error_items(pkg_cls.provided): errors.extend( error_cls(f"{pkg_name}: wrong 'when=' condition for '{provided}' virtual", details) for provided in providers ) for when, requirements, details in _error_items(pkg_cls.requirements): errors.append( error_cls(f"{pkg_name}: wrong 'when=' condition in 'requires' directive", details) ) for when, _, details in _error_items(pkg_cls.patches): errors.append( error_cls(f"{pkg_name}: wrong 'when=' condition in 'patch' directives", details) ) for when, _, details in _error_items(pkg_cls.resources): errors.append( error_cls(f"{pkg_name}: wrong 'when=' condition in 'resource' directives", details) ) return spack.llnl.util.lang.dedupe(errors) #: Sanity checks on package directives external_detection = AuditClass( group="externals", tag="PKG-EXTERNALS", description="Sanity checks for external software detection", kwargs=("pkgs", "debug_log"), ) def packages_with_detection_tests(): """Return the list of packages with a corresponding detection_test.yaml file.""" import spack.config import spack.util.path to_be_tested = [] for current_repo in spack.repo.PATH.repos: namespace = current_repo.namespace packages_dir = pathlib.PurePath(current_repo.packages_path) pattern = packages_dir / "**" / "detection_test.yaml" pkgs_with_tests = [ f"{namespace}.{str(pathlib.PurePath(x).parent.name)}" for x in glob.glob(str(pattern)) ] to_be_tested.extend(pkgs_with_tests) return to_be_tested @external_detection def _test_detection_by_executable(pkgs, debug_log, error_cls): """Test drive external detection for packages""" import spack.detection errors = [] # Filter the packages and retain only the ones with detection tests pkgs_with_tests = packages_with_detection_tests() selected_pkgs = [] for current_package in pkgs_with_tests: _, unqualified_name = spack.repo.partition_package_name(current_package) # Check for both unqualified name and qualified name if unqualified_name in pkgs or current_package in pkgs: selected_pkgs.append(current_package) selected_pkgs.sort() if not selected_pkgs: summary = "No detection test to run" details = [f' "{p}" has no detection test' for p in pkgs] warnings.warn("\n".join([summary] + details)) return errors for pkg_name in selected_pkgs: for idx, test_runner in enumerate( spack.detection.detection_tests(pkg_name, spack.repo.PATH) ): debug_log(f"[{__file__}]: running test {idx} for package {pkg_name}") specs = test_runner.execute() expected_specs = test_runner.expected_specs not_detected = set(expected_specs) - set(specs) if not_detected: summary = pkg_name + ": cannot detect some specs" details = [f'"{s}" was not detected [test_id={idx}]' for s in sorted(not_detected)] errors.append(error_cls(summary=summary, details=details)) not_expected = set(specs) - set(expected_specs) if not_expected: summary = pkg_name + ": detected unexpected specs" msg = '"{0}" was detected, but was not expected [test_id={1}]' details = [msg.format(s, idx) for s in sorted(not_expected)] errors.append(error_cls(summary=summary, details=details)) matched_detection = [] for candidate in expected_specs: try: idx = specs.index(candidate) matched_detection.append((candidate, specs[idx])) except (AttributeError, ValueError): pass def _compare_extra_attribute(_expected, _detected, *, _spec): result = [] # If they are string expected is a regex if isinstance(_expected, str) and isinstance(_detected, str): try: _regex = re.compile(_expected) except re.error: _summary = f'{pkg_name}: illegal regex in "{_spec}" extra attributes' _details = [f"{_expected} is not a valid regex"] return [error_cls(summary=_summary, details=_details)] if not _regex.match(_detected): _summary = ( f'{pkg_name}: error when trying to match "{_expected}" ' f"in extra attributes" ) _details = [f"{_detected} does not match the regex"] return [error_cls(summary=_summary, details=_details)] elif isinstance(_expected, dict) and isinstance(_detected, dict): _not_detected = set(_expected.keys()) - set(_detected.keys()) if _not_detected: _summary = f"{pkg_name}: cannot detect some attributes for spec {_spec}" _details = [ f'"{_expected}" was expected', f'"{_detected}" was detected', ] + [f'attribute "{s}" was not detected' for s in sorted(_not_detected)] result.append(error_cls(summary=_summary, details=_details)) _common = set(_expected.keys()) & set(_detected.keys()) for _key in _common: result.extend( _compare_extra_attribute(_expected[_key], _detected[_key], _spec=_spec) ) else: _summary = f'{pkg_name}: error when trying to detect "{_expected}"' _details = [f"{_detected} was detected instead"] return [error_cls(summary=_summary, details=_details)] return result for expected, detected in matched_detection: # We might not want to test all attributes, so avoid not_expected not_detected = set(expected.extra_attributes) - set(detected.extra_attributes) if not_detected: summary = f"{pkg_name}: cannot detect some attributes for spec {expected}" details = [ f'"{s}" was not detected [test_id={idx}]' for s in sorted(not_detected) ] errors.append(error_cls(summary=summary, details=details)) common = set(expected.extra_attributes) & set(detected.extra_attributes) for key in common: errors.extend( _compare_extra_attribute( expected.extra_attributes[key], detected.extra_attributes[key], _spec=expected, ) ) return errors ================================================ FILE: lib/spack/spack/binary_distribution.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import concurrent.futures import contextlib import copy import datetime import hashlib import io import itertools import json import os import pathlib import re import shutil import sys import tarfile import tempfile import textwrap import time import urllib.error import urllib.parse import urllib.request import warnings from collections import defaultdict from contextlib import closing from typing import IO, Callable, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union, cast import spack.caches import spack.config import spack.database import spack.deptypes as dt import spack.error import spack.hash_types as ht import spack.hooks import spack.hooks.sbang import spack.llnl.util.filesystem as fsys import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.oci.image import spack.oci.oci import spack.oci.opener import spack.paths import spack.platforms import spack.relocate as relocate import spack.spec import spack.stage import spack.store import spack.user_environment import spack.util.archive import spack.util.crypto import spack.util.file_cache as file_cache import spack.util.gpg import spack.util.parallel import spack.util.path import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.util.timer as timer import spack.util.url as url_util import spack.util.web as web_util from spack import traverse from spack.llnl.util.filesystem import mkdirp from spack.oci.image import ( Digest, ImageReference, default_config, default_manifest, ensure_valid_tag, ) from spack.oci.oci import ( copy_missing_layers_with_retry, get_manifest_and_config_with_retry, list_tags, upload_blob_with_retry, upload_manifest_with_retry, ) from spack.package_prefs import get_package_dir_permissions, get_package_group from spack.relocate_text import utf8_paths_to_single_binary_regex from spack.stage import Stage from spack.util.executable import which from .enums import InstallRecordStatus from .url_buildcache import ( CURRENT_BUILD_CACHE_LAYOUT_VERSION, BlobRecord, BuildcacheComponent, BuildcacheEntryError, BuildcacheManifest, InvalidMetadataFile, ListMirrorSpecsError, MirrorMetadata, URLBuildcacheEntry, get_entries_from_cache, get_url_buildcache_class, get_valid_spec_file, ) class BuildCacheDatabase(spack.database.Database): """A database for binary buildcaches. A database supports writing buildcache index files, in which case certain fields are not needed in each install record, and no locking is required. To use this feature, it provides ``lock_cfg=NO_LOCK``, and override the list of ``record_fields``. """ record_fields = ("spec", "ref_count", "in_buildcache") def __init__(self, root): super().__init__(root, lock_cfg=spack.database.NO_LOCK, layout=None) self._write_transaction_impl = spack.llnl.util.lang.nullcontext self._read_transaction_impl = spack.llnl.util.lang.nullcontext def _handle_old_db_versions_read(self, check, db, *, reindex: bool): if not self.is_readable(): raise spack.database.DatabaseNotReadableError( f"cannot read buildcache v{self.db_version} at {self.root}" ) return self._handle_current_version_read(check, db) class FetchCacheError(Exception): """Error thrown when fetching the cache failed, usually a composite error list.""" def __init__(self, errors): if not isinstance(errors, list): raise TypeError("Expected a list of errors") self.errors = errors if len(errors) > 1: msg = " Error {0}: {1}: {2}" self.message = "Multiple errors during fetching:\n" self.message += "\n".join( ( msg.format(i + 1, err.__class__.__name__, str(err)) for (i, err) in enumerate(errors) ) ) else: err = errors[0] self.message = "{0}: {1}".format(err.__class__.__name__, str(err)) super().__init__(self.message) class BinaryCacheIndex: """ The BinaryCacheIndex tracks what specs are available on (usually remote) binary caches. This index is "best effort", in the sense that whenever we don't find what we're looking for here, we will attempt to fetch it directly from configured mirrors anyway. Thus, it has the potential to speed things up, but cache misses shouldn't break any spack functionality. At the moment, everything in this class is initialized as lazily as possible, so that it avoids slowing anything in spack down until absolutely necessary. """ def __init__(self, cache_root: Optional[str] = None): self._index_cache_root: str = cache_root or binary_index_location() # the key associated with the serialized _local_index_cache self._index_contents_key = "contents.json" # a FileCache instance storing copies of remote binary cache indices self._index_file_cache: file_cache.FileCache = file_cache.FileCache(self._index_cache_root) self._index_file_cache_initialized = False # stores a map of mirror URL and version layout to index hash and cache key (index path) self._local_index_cache: dict[str, dict] = {} # hashes of remote indices already ingested into the concrete spec # cache (_mirrors_for_spec) self._specs_already_associated: Set[str] = set() # mapping from mirror urls to the time.time() of the last index fetch and a bool indicating # whether the fetch succeeded or not. self._last_fetch_times: Dict[MirrorMetadata, Tuple[float, bool]] = {} #: Dictionary mapping DAG hashes of specs to Spec objects self._known_specs: Dict[str, spack.spec.Spec] = {} #: Dictionary mapping DAG hashes of specs to a list of mirrors where they can be found self._mirrors_for_spec: Dict[str, Set[MirrorMetadata]] = defaultdict(set) def _init_local_index_cache(self): if not self._index_file_cache_initialized: cache_key = self._index_contents_key self._local_index_cache = {} with self._index_file_cache.read_transaction(cache_key) as cache_file: if cache_file is not None: self._local_index_cache = json.load(cache_file) self._index_file_cache_initialized = True def _write_local_index_cache(self): self._init_local_index_cache() cache_key = self._index_contents_key with self._index_file_cache.write_transaction(cache_key) as (old, new): json.dump(self._local_index_cache, new) def regenerate_spec_cache(self, clear_existing=False): """Populate the local cache of concrete specs (``_mirrors_for_spec``) from the locally cached buildcache index files. This is essentially a no-op if it has already been done, as we keep track of the index hashes for which we have already associated the built specs.""" self._init_local_index_cache() if clear_existing: self._specs_already_associated = set() self._mirrors_for_spec = defaultdict(set) self._known_specs = {} for mirror_metadata in self._local_index_cache: cache_entry = self._local_index_cache[mirror_metadata] cached_index_path = cache_entry["index_path"] cached_index_hash = cache_entry["index_hash"] if cached_index_hash not in self._specs_already_associated: self._associate_built_specs_with_mirror( cached_index_path, MirrorMetadata.from_string(mirror_metadata) ) self._specs_already_associated.add(cached_index_hash) def _associate_built_specs_with_mirror(self, cache_key, mirror_metadata: MirrorMetadata): with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: db = BuildCacheDatabase(tmpdir) with self._index_file_cache.read_transaction(cache_key) as f: if f is not None: try: db._read_from_stream(f) except spack.database.InvalidDatabaseVersionError as e: tty.warn( "you need a newer Spack version to read the buildcache index for the " f"following v{mirror_metadata.version} mirror: " f"'{mirror_metadata.url}'. {e.database_version_message}" ) return spec_list = [ s for s in db.query_local(installed=InstallRecordStatus.ANY) # todo, make it easier to get install records associated with specs if s.external or db._data[s.dag_hash()].in_buildcache ] for spec in spec_list: dag_hash = spec.dag_hash() mirrors = self._mirrors_for_spec[dag_hash] mirrors.add(mirror_metadata.strip_view()) if dag_hash not in self._known_specs: self._known_specs[dag_hash] = spec def get_all_built_specs(self) -> List[spack.spec.Spec]: """Returns a list of all concrete specs known to be available in a binary cache.""" return list(self._known_specs.values()) def find_built_spec(self, spec: spack.spec.Spec) -> List[MirrorMetadata]: """Returns a list of MirrorMetadata objects indicating which mirrors have the given concrete spec. This method does not trigger reading anything from remote mirrors, but rather just checks if the concrete spec is found within the cache. The cache can be updated by calling ``update()`` on the cache. Args: spec: Concrete spec to find """ return self.find_by_hash(spec.dag_hash()) def find_by_hash(self, dag_hash: str) -> List[MirrorMetadata]: """Same as find_built_spec but uses the hash of a spec. Args: dag_hash: hash of the spec to search """ return list(self._mirrors_for_spec.get(dag_hash, [])) def update_spec(self, spec: spack.spec.Spec, found_list: List[MirrorMetadata]) -> None: """Update the cache with a new list of mirrors for a given spec.""" spec_dag_hash = spec.dag_hash() if spec_dag_hash not in self._mirrors_for_spec: self._mirrors_for_spec[spec_dag_hash] = set(found_list) self._known_specs[spec_dag_hash] = spec else: current_list = self._mirrors_for_spec[spec_dag_hash] for new_entry in found_list: current_list.add(new_entry.strip_view()) def update(self, with_cooldown: bool = False) -> None: """Make sure local cache of buildcache index files is up to date. If the same mirrors are configured as the last time this was called and none of the remote buildcache indices have changed, calling this method will only result in fetching the index hash from each mirror to confirm it is the same as what is stored locally. Otherwise, the buildcache ``index.json`` and ``index.json.hash`` files are retrieved from each configured mirror and stored locally (both in memory and on disk under ``_index_cache_root``).""" self._init_local_index_cache() configured_mirrors = [ MirrorMetadata(m.fetch_url, layout_version, m.fetch_view) for m in spack.mirrors.mirror.MirrorCollection(binary=True).values() for layout_version in m.supported_layout_versions ] items_to_remove = [] spec_cache_clear_needed = False spec_cache_regenerate_needed = not self._mirrors_for_spec # First compare the mirror urls currently present in the cache to the # configured mirrors. If we have a cached index for a mirror which is # no longer configured, we should remove it from the cache. For any # cached indices corresponding to currently configured mirrors, we need # to check if the cache is still good, or needs to be updated. # Finally, if there are configured mirrors for which we don't have a # cache entry, we need to fetch and cache the indices from those # mirrors. # If, during this process, we find that any mirrors for which we # already have entries have either been removed, or their index # hash has changed, then our concrete spec cache (_mirrors_for_spec) # likely has entries that need to be removed, so we will clear it # and regenerate that data structure. # If, during this process, we find that there are new mirrors for # which do not yet have an entry in our index cache, then we simply # need to regenerate the concrete spec cache, but do not need to # clear it first. # Otherwise the concrete spec cache should not need to be updated at # all. fetch_errors: List[Exception] = [] all_methods_failed = True ttl = spack.config.get("config:binary_index_ttl", 600) now = time.time() for local_index_cache_key in self._local_index_cache: urlAndVersion = MirrorMetadata.from_string(local_index_cache_key) cached_mirror_url = urlAndVersion.url cache_entry = self._local_index_cache[local_index_cache_key] cached_index_path = cache_entry["index_path"] if urlAndVersion in configured_mirrors: # Only do a fetch if the last fetch was longer than TTL ago if ( with_cooldown and ttl > 0 and cached_mirror_url in self._last_fetch_times and now - self._last_fetch_times[urlAndVersion][0] < ttl ): # We're in the cooldown period, don't try to fetch again # If the fetch succeeded last time, consider this update a success, otherwise # re-report the error here if self._last_fetch_times[urlAndVersion][1]: all_methods_failed = False else: # May need to fetch the index and update the local caches needs_regen = False try: needs_regen = self._fetch_and_cache_index( urlAndVersion, cache_entry=cache_entry ) self._last_fetch_times[urlAndVersion] = (now, True) all_methods_failed = False except FetchIndexError as e: fetch_errors.append(e) self._last_fetch_times[urlAndVersion] = (now, False) except BuildcacheIndexNotExists as e: fetch_errors.append(e) self._last_fetch_times[urlAndVersion] = (now, False) # Binary caches are not required to have an index, don't raise # if it doesn't exist. all_methods_failed = False # The need to regenerate implies a need to clear as well. spec_cache_clear_needed |= needs_regen spec_cache_regenerate_needed |= needs_regen else: # No longer have this mirror, cached index should be removed items_to_remove.append( { "url": local_index_cache_key, "cache_key": os.path.join(self._index_cache_root, cached_index_path), } ) if urlAndVersion in self._last_fetch_times: del self._last_fetch_times[urlAndVersion] spec_cache_clear_needed = True spec_cache_regenerate_needed = True # Clean up items to be removed, identified above for item in items_to_remove: url = item["url"] cache_key = item["cache_key"] self._index_file_cache.remove(cache_key) del self._local_index_cache[url] # Iterate the configured mirrors now. Any mirror urls we do not # already have in our cache must be fetched, stored, and represented # locally. for urlAndVersion in configured_mirrors: if str(urlAndVersion) in self._local_index_cache: continue # Need to fetch the index and update the local caches needs_regen = False try: needs_regen = self._fetch_and_cache_index(urlAndVersion) self._last_fetch_times[urlAndVersion] = (now, True) all_methods_failed = False except FetchIndexError as e: fetch_errors.append(e) self._last_fetch_times[urlAndVersion] = (now, False) except BuildcacheIndexNotExists as e: fetch_errors.append(e) self._last_fetch_times[urlAndVersion] = (now, False) # Binary caches are not required to have an index, don't raise # if it doesn't exist. all_methods_failed = False # Generally speaking, a new mirror wouldn't imply the need to # clear the spec cache, so leave it as is. if needs_regen: spec_cache_regenerate_needed = True self._write_local_index_cache() if configured_mirrors and all_methods_failed: raise FetchCacheError(fetch_errors) if fetch_errors: tty.warn( "The following issues were ignored while updating the indices of binary caches", FetchCacheError(fetch_errors), ) if spec_cache_regenerate_needed: self.regenerate_spec_cache(clear_existing=spec_cache_clear_needed) def _fetch_and_cache_index(self, mirror_metadata: MirrorMetadata, cache_entry={}): """Fetch a buildcache index file from a remote mirror and cache it. If we already have a cached index from this mirror, then we first check if the hash has changed, and we avoid fetching it if not. Args: mirror_metadata: Contains mirror base url and target binary cache layout version cache_entry (dict): Old cache metadata with keys ``index_hash``, ``index_path``, ``etag`` Returns: True if the local index.json was updated. Throws: FetchIndexError BuildcacheIndexNotExists """ mirror_url = mirror_metadata.url mirror_view = mirror_metadata.view layout_version = mirror_metadata.version # TODO: get rid of this request, handle 404 better scheme = urllib.parse.urlparse(mirror_url).scheme if scheme != "oci": cache_class = get_url_buildcache_class(layout_version=layout_version) index_url = cache_class.get_index_url(mirror_url, mirror_view) if not web_util.url_exists(index_url): raise BuildcacheIndexNotExists(f"Index not found in cache {index_url}") fetcher: IndexFetcher = get_index_fetcher(scheme, mirror_metadata, cache_entry) result = fetcher.conditional_fetch() # Nothing to do if result.fresh: return False # Persist new index.json url_hash = compute_hash(str(mirror_metadata)) cache_key = "{}_{}.json".format(url_hash[:10], result.hash[:10]) with self._index_file_cache.write_transaction(cache_key) as (old, new): new.write(result.data) self._local_index_cache[str(mirror_metadata)] = { "index_hash": result.hash, "index_path": cache_key, "etag": result.etag, } # clean up the old cache_key if necessary old_cache_key = cache_entry.get("index_path", None) if old_cache_key: self._index_file_cache.remove(old_cache_key) # We fetched an index and updated the local index cache, we should # regenerate the spec cache as a result. return True def binary_index_location(): """Set up a BinaryCacheIndex for remote buildcache dbs in the user's homedir.""" cache_root = os.path.join(spack.caches.misc_cache_location(), "indices") return spack.util.path.canonicalize_path(cache_root) #: Default binary cache index instance BINARY_INDEX = cast(BinaryCacheIndex, spack.llnl.util.lang.Singleton(BinaryCacheIndex)) def compute_hash(data): if isinstance(data, str): data = data.encode("utf-8") return hashlib.sha256(data).hexdigest() def buildinfo_file_name(prefix): """Filename of the binary package meta-data file""" return os.path.join(prefix, ".spack", "binary_distribution") def read_buildinfo_file(prefix): """Read buildinfo file""" with open(buildinfo_file_name(prefix), "r", encoding="utf-8") as f: return syaml.load(f) def file_matches(f: IO[bytes], regex: spack.llnl.util.lang.PatternBytes) -> bool: try: return bool(regex.search(f.read())) finally: f.seek(0) def specs_to_relocate(spec: spack.spec.Spec) -> List[spack.spec.Spec]: """Return the set of specs that may be referenced in the install prefix of the provided spec. We currently include non-external transitive link and direct run dependencies.""" specs = [ s for s in itertools.chain( spec.traverse(root=True, deptype="link", order="breadth", key=traverse.by_dag_hash), spec.dependencies(deptype="run"), ) if not s.external ] return list(spack.llnl.util.lang.dedupe(specs, key=lambda s: s.dag_hash())) def get_buildinfo_dict(spec): """Create metadata for a tarball""" return { "sbang_install_path": spack.hooks.sbang.sbang_install_path(), "buildpath": spack.store.STORE.layout.root, "spackprefix": spack.paths.prefix, "relative_prefix": os.path.relpath(spec.prefix, spack.store.STORE.layout.root), # "relocate_textfiles": [], # "relocate_binaries": [], # "relocate_links": [], "hardlinks_deduped": True, "hash_to_prefix": {d.dag_hash(): str(d.prefix) for d in specs_to_relocate(spec)}, } def buildcache_relative_keys_path(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION): cache_class = get_url_buildcache_class(layout_version=layout_version) return os.path.join(*cache_class.get_relative_path_components(BuildcacheComponent.KEY)) def buildcache_relative_keys_url(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION): cache_class = get_url_buildcache_class(layout_version=layout_version) return url_util.join(*cache_class.get_relative_path_components(BuildcacheComponent.KEY)) def buildcache_relative_specs_path(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION): cache_class = get_url_buildcache_class(layout_version=layout_version) return os.path.join(*cache_class.get_relative_path_components(BuildcacheComponent.SPEC)) def buildcache_relative_specs_url(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION): cache_class = get_url_buildcache_class(layout_version=layout_version) return url_util.join(*cache_class.get_relative_path_components(BuildcacheComponent.SPEC)) def buildcache_relative_blobs_path(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION): cache_class = get_url_buildcache_class(layout_version=layout_version) return os.path.join(*cache_class.get_relative_path_components(BuildcacheComponent.BLOB)) def buildcache_relative_blobs_url(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION): cache_class = get_url_buildcache_class(layout_version=layout_version) return url_util.join(*cache_class.get_relative_path_components(BuildcacheComponent.BLOB)) def buildcache_relative_index_path(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION): cache_class = get_url_buildcache_class(layout_version=layout_version) return os.path.join(*cache_class.get_relative_path_components(BuildcacheComponent.INDEX)) def buildcache_relative_index_url(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION): cache_class = get_url_buildcache_class(layout_version=layout_version) return url_util.join(*cache_class.get_relative_path_components(BuildcacheComponent.INDEX)) @spack.llnl.util.lang.memoized def warn_v2_layout(mirror_url: str, action: str) -> bool: lines = textwrap.wrap( f"{action} from a v2 binary mirror layout, located at " f"{mirror_url} is deprecated. Support for this will be " "removed in a future version of spack. " "If you manage the buildcache please consider running:", width=72, subsequent_indent=" ", ) lines.extend( [ " 'spack buildcache migrate'", " or rebuilding the specs in this mirror. Otherwise, consider running:", " 'spack mirror list'", " 'spack mirror remove '", " with the for the mirror url shown in the list.", ] ) tty.warn("\n".join(lines)) return True def select_signing_key() -> str: keys = spack.util.gpg.signing_keys() num = len(keys) if num > 1: raise PickKeyException(str(keys)) elif num == 0: raise NoKeyException( "No default key available for signing.\n" "Use spack gpg init and spack gpg create" " to create a default key." ) return keys[0] def _push_index(db: BuildCacheDatabase, temp_dir: str, cache_prefix: str, name: str = ""): """Generate the index, compute its hash, and push the files to the mirror""" index_json_path = os.path.join(temp_dir, spack.database.INDEX_JSON_FILE) with open(index_json_path, "w", encoding="utf-8") as f: db._write_to_file(f) cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION) cache_class.push_local_file_as_blob( index_json_path, cache_prefix, url_util.join(name, "index") if name else "index", BuildcacheComponent.INDEX, compression="none", ) cache_class.maybe_push_layout_json(cache_prefix) def _read_specs_and_push_index( file_list: List[str], read_method: Callable[[str], URLBuildcacheEntry], name: str, filter_fn: Callable[[str], bool], cache_prefix: str, db: BuildCacheDatabase, temp_dir: str, *, timer=timer.NULL_TIMER, ): """Read listed specs, generate the index, and push it to the mirror. Args: file_list: List of urls or file paths pointing at spec files to read read_method: A function taking a single argument, either a url or a file path, and which reads the spec file at that location, and returns the spec. cache_prefix: prefix of the build cache on s3 where index should be pushed. db: A spack database used for adding specs and then writing the index. temp_dir: Location to write index.json and hash for pushing """ with timer.measure("read"): for file in file_list: # All supported versions of build caches put the hash as the last # parameter before the extension try: x = file.split("/")[-1].split("-")[-1].split(".")[0] except IndexError: raise GenerateIndexError(f"Malformed metadata file name detected {file}") if not filter_fn(x): continue cache_entry: Optional[URLBuildcacheEntry] = None try: cache_entry = read_method(file) spec_dict = cache_entry.fetch_metadata() fetched_spec = spack.spec.Spec.from_dict(spec_dict) except Exception as e: tty.warn(f"Unable to fetch spec for manifest {file} due to: {e}") continue finally: if cache_entry: cache_entry.destroy() db.add(fetched_spec) db.mark(fetched_spec, "in_buildcache", True) with timer.measure("push"): _push_index(db, temp_dir, cache_prefix, name) def _url_generate_package_index( url: str, tmpdir: str, db: Optional[BuildCacheDatabase] = None, name: str = "", filter_fn: Callable[[str], bool] = lambda x: True, *, timer=timer.NULL_TIMER, ): """Create or replace the build cache index on the given mirror. The buildcache index contains an entry for each binary package under the cache_prefix. Args: url: Base url of binary mirror. Return: None """ with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir: try: with timer.measure("list"): filename_to_mtime_mapping, read_fn = get_entries_from_cache( url, tmpspecsdir, component_type=BuildcacheComponent.SPEC ) file_list = list(filename_to_mtime_mapping.keys()) except ListMirrorSpecsError as e: raise GenerateIndexError(f"Unable to generate package index: {e}") from e tty.debug(f"Retrieving spec descriptor files from {url} to build index") if not db: db = BuildCacheDatabase(tmpdir) db._write() try: _read_specs_and_push_index( file_list, read_fn, name, filter_fn, url, db, str(db.database_directory), timer=timer, ) except Exception as e: raise GenerateIndexError( f"Encountered problem pushing package index to {url}: {e}" ) from e def generate_key_index(mirror_url: str, tmpdir: str) -> None: """Create the key index page. Creates (or replaces) the ``index.json`` page at the location given in mirror_url. This page contains an entry for each key under mirror_url. """ tty.debug(f"Retrieving key.pub files from {url_util.format(mirror_url)} to build key index") key_prefix = url_util.join(mirror_url, buildcache_relative_keys_url()) try: fingerprints = ( entry[:-18] for entry in web_util.list_url(key_prefix, recursive=False) if entry.endswith(".key.manifest.json") ) except Exception as e: raise CannotListKeys(f"Encountered problem listing keys at {key_prefix}: {e}") from e target = os.path.join(tmpdir, "index.json") index = {"keys": dict((fingerprint, {}) for fingerprint in sorted(set(fingerprints)))} with open(target, "w", encoding="utf-8") as f: sjson.dump(index, f) cache_class = get_url_buildcache_class() try: cache_class.push_local_file_as_blob( local_file_path=target, mirror_url=mirror_url, manifest_name="keys", component_type=BuildcacheComponent.KEY_INDEX, compression="none", ) cache_class.maybe_push_layout_json(mirror_url) except Exception as e: raise GenerateIndexError( f"Encountered problem pushing key index to {key_prefix}: {e}" ) from e class FileTypes: BINARY = 0 TEXT = 1 UNKNOWN = 2 NOT_ISO8859_1_TEXT = re.compile(b"[\x00\x7f-\x9f]") def file_type(f: IO[bytes]) -> int: try: # first check if this is an ELF or mach-o binary. magic = f.read(8) if len(magic) < 8: return FileTypes.UNKNOWN elif relocate.is_elf_magic(magic) or relocate.is_macho_magic(magic): return FileTypes.BINARY f.seek(0) # Then try utf-8, which has a fast exponential decay in false positive rate with file size. # Use chunked reads for fast early exit. f_txt = io.TextIOWrapper(f, encoding="utf-8", errors="strict") try: while f_txt.read(1024): pass return FileTypes.TEXT except UnicodeError: f_txt.seek(0) pass finally: f_txt.detach() # Finally try iso-8859-1 heuristically. In Python, all possible 256 byte values are valid. # We classify it as text if it does not contain any control characters / null bytes. data = f.read(1024) while data: if NOT_ISO8859_1_TEXT.search(data): break data = f.read(1024) else: return FileTypes.TEXT return FileTypes.UNKNOWN finally: f.seek(0) def tarfile_of_spec_prefix( tar: tarfile.TarFile, prefix: str, prefixes_to_relocate: List[str] ) -> dict: """Create a tarfile of an install prefix of a spec. Skips existing buildinfo file. Args: tar: tarfile object to add files to prefix: absolute install prefix of spec""" if not os.path.isabs(prefix) or not os.path.isdir(prefix): raise ValueError(f"prefix '{prefix}' must be an absolute path to a directory") stat_key = lambda stat: (stat.st_dev, stat.st_ino) try: # skip buildinfo file if it exists files_to_skip = [stat_key(os.lstat(buildinfo_file_name(prefix)))] skip = lambda entry: stat_key(entry.stat(follow_symlinks=False)) in files_to_skip except OSError: skip = lambda entry: False binary_regex = utf8_paths_to_single_binary_regex(prefixes_to_relocate) relocate_binaries = [] relocate_links = [] relocate_textfiles = [] # use callbacks to add files and symlinks, so we can register which files need relocation upon # extraction. def add_file(tar: tarfile.TarFile, info: tarfile.TarInfo, path: str): with open(path, "rb") as f: relpath = os.path.relpath(path, prefix) # no need to relocate anything in the .spack directory if relpath.split(os.sep, 1)[0] == ".spack": tar.addfile(info, f) return f_type = file_type(f) if f_type == FileTypes.BINARY: relocate_binaries.append(os.path.relpath(path, prefix)) elif f_type == FileTypes.TEXT and file_matches(f, binary_regex): relocate_textfiles.append(os.path.relpath(path, prefix)) tar.addfile(info, f) def add_symlink(tar: tarfile.TarFile, info: tarfile.TarInfo, path: str): if os.path.isabs(info.linkname) and binary_regex.match(info.linkname.encode("utf-8")): relocate_links.append(os.path.relpath(path, prefix)) tar.addfile(info) spack.util.archive.reproducible_tarfile_from_prefix( tar, prefix, # Spack <= 0.21 did not include parent directories, leading to issues when tarballs are # used in runtimes like AWS lambda. include_parent_directories=True, skip=skip, add_file=add_file, add_symlink=add_symlink, ) return { "relocate_binaries": relocate_binaries, "relocate_links": relocate_links, "relocate_textfiles": relocate_textfiles, } def create_tarball(spec: spack.spec.Spec, tarfile_path: str) -> Tuple[str, str]: """Create a tarball of a spec and return the checksums of the compressed tarfile and the uncompressed tarfile.""" return _do_create_tarball( tarfile_path, spec.prefix, buildinfo=get_buildinfo_dict(spec), prefixes_to_relocate=prefixes_to_relocate(spec), ) def _do_create_tarball( tarfile_path: str, prefix: str, buildinfo: dict, prefixes_to_relocate: List[str] ) -> Tuple[str, str]: with spack.util.archive.gzip_compressed_tarfile(tarfile_path) as ( tar, tar_gz_checksum, tar_checksum, ): # Tarball the install prefix files_to_relocate = tarfile_of_spec_prefix(tar, prefix, prefixes_to_relocate) buildinfo.update(files_to_relocate) # Serialize buildinfo for the tarball bstring = syaml.dump(buildinfo, default_flow_style=True).encode("utf-8") tarinfo = tarfile.TarInfo( name=spack.util.archive.default_path_to_name(buildinfo_file_name(prefix)) ) tarinfo.type = tarfile.REGTYPE tarinfo.size = len(bstring) tarinfo.mode = 0o644 tar.addfile(tarinfo, io.BytesIO(bstring)) return tar_gz_checksum.hexdigest(), tar_checksum.hexdigest() def _exists_in_buildcache( spec: spack.spec.Spec, out_url: str, allow_unsigned: bool = False ) -> URLBuildcacheEntry: """creates and returns (after checking existence) a URLBuildcacheEntry""" cache_type = get_url_buildcache_class(CURRENT_BUILD_CACHE_LAYOUT_VERSION) cache_entry = cache_type(out_url, spec, allow_unsigned=allow_unsigned) return cache_entry def prefixes_to_relocate(spec): prefixes = [s.prefix for s in specs_to_relocate(spec)] prefixes.append(spack.hooks.sbang.sbang_install_path()) prefixes.append(str(spack.store.STORE.layout.root)) return prefixes def _url_upload_tarball_and_specfile( spec: spack.spec.Spec, tmpdir: str, cache_entry: URLBuildcacheEntry, signing_key: Optional[str] ): tarball = os.path.join(tmpdir, f"{spec.dag_hash()}.tar.gz") checksum, _ = create_tarball(spec, tarball) cache_entry.push_binary_package(spec, tarball, "sha256", checksum, tmpdir, signing_key) class Uploader: def __init__(self, mirror: spack.mirrors.mirror.Mirror, force: bool, update_index: bool): self.mirror = mirror self.force = force self.update_index = update_index self.tmpdir: str self.executor: concurrent.futures.Executor # Verify if the mirror meets the requirements to push self.mirror.ensure_mirror_usable("push") def __enter__(self): self._tmpdir = tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) self._executor = spack.util.parallel.make_concurrent_executor() self.tmpdir = self._tmpdir.__enter__() self.executor = self.executor = self._executor.__enter__() return self def __exit__(self, *args): self._executor.__exit__(*args) self._tmpdir.__exit__(*args) def push_or_raise(self, specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]: skipped, errors = self.push(specs) if errors: raise PushToBuildCacheError( f"Failed to push {len(errors)} specs to {self.mirror.push_url}:\n" + "\n".join( f"Failed to push {_format_spec(spec)}: {error}" for spec, error in errors ) ) return skipped def push( self, specs: List[spack.spec.Spec] ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]: raise NotImplementedError def tag(self, tag: str, roots: List[spack.spec.Spec]): """Make a list of selected specs together available under the given tag""" pass class OCIUploader(Uploader): def __init__( self, mirror: spack.mirrors.mirror.Mirror, force: bool, update_index: bool, base_image: Optional[str], ) -> None: super().__init__(mirror, force, update_index) self.target_image = spack.oci.oci.image_from_mirror(mirror) self.base_image = ImageReference.from_string(base_image) if base_image else None def push( self, specs: List[spack.spec.Spec] ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]: skipped, base_images, checksums, upload_errors = _oci_push( target_image=self.target_image, base_image=self.base_image, installed_specs_with_deps=specs, force=self.force, tmpdir=self.tmpdir, executor=self.executor, ) self._base_images = base_images self._checksums = checksums # only update index if any binaries were uploaded if self.update_index and len(skipped) + len(upload_errors) < len(specs): _oci_update_index(self.target_image, self.tmpdir, self.executor) return skipped, upload_errors def tag(self, tag: str, roots: List[spack.spec.Spec]): tagged_image = self.target_image.with_tag(tag) # _push_oci may not populate self._base_images if binaries were already in the registry for spec in roots: _oci_update_base_images( base_image=self.base_image, target_image=self.target_image, spec=spec, base_image_cache=self._base_images, ) _oci_put_manifest( self._base_images, self._checksums, tagged_image, self.tmpdir, None, None, *roots ) tty.info(f"Tagged {tagged_image}") class URLUploader(Uploader): def __init__( self, mirror: spack.mirrors.mirror.Mirror, force: bool, update_index: bool, signing_key: Optional[str], ) -> None: super().__init__(mirror, force, update_index) self.url = mirror.push_url self.signing_key = signing_key def push( self, specs: List[spack.spec.Spec] ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]: return _url_push( specs, out_url=self.url, force=self.force, update_index=self.update_index, signing_key=self.signing_key, tmpdir=self.tmpdir, executor=self.executor, ) def make_uploader( mirror: spack.mirrors.mirror.Mirror, force: bool = False, update_index: bool = False, signing_key: Optional[str] = None, base_image: Optional[str] = None, ) -> Uploader: """Builder for the appropriate uploader based on the mirror type""" if spack.oci.image.is_oci_url(mirror.push_url): return OCIUploader( mirror=mirror, force=force, update_index=update_index, base_image=base_image ) else: return URLUploader( mirror=mirror, force=force, update_index=update_index, signing_key=signing_key ) def _format_spec(spec: spack.spec.Spec) -> str: return spec.cformat("{name}{@version}{/hash:7}") class FancyProgress: def __init__(self, total: int): self.n = 0 self.total = total self.running = False self.enable = sys.stdout.isatty() self.pretty_spec: str = "" self.pre = "" def _clear(self): if self.enable and self.running: sys.stdout.write("\033[F\033[K") def _progress(self): if self.total > 1: digits = len(str(self.total)) return f"[{self.n:{digits}}/{self.total}] " return "" def start(self, spec: spack.spec.Spec, running: bool) -> None: self.n += 1 self.running = running self.pre = self._progress() self.pretty_spec = _format_spec(spec) if self.enable and self.running: tty.info(f"{self.pre}Pushing {self.pretty_spec}...") def ok(self, msg: Optional[str] = None) -> None: self._clear() msg = msg or f"Pushed {self.pretty_spec}" tty.info(f"{self.pre}{msg}") def fail(self) -> None: self._clear() tty.info(f"{self.pre}Failed to push {self.pretty_spec}") def _url_push( specs: List[spack.spec.Spec], out_url: str, signing_key: Optional[str], force: bool, update_index: bool, tmpdir: str, executor: concurrent.futures.Executor, ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]: """Pushes to the provided build cache, and returns a list of skipped specs that were already present (when force=False), and a list of errors. Does not raise on error.""" skipped: List[spack.spec.Spec] = [] errors: List[Tuple[spack.spec.Spec, BaseException]] = [] exists_futures = [ executor.submit( _exists_in_buildcache, spec, out_url, allow_unsigned=False if signing_key else True ) for spec in specs ] cache_entries = { spec.dag_hash(): exists_future.result() for spec, exists_future in zip(specs, exists_futures) } if not force: specs_to_upload = [] for spec in specs: if cache_entries[spec.dag_hash()].exists( [BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL] ): skipped.append(spec) else: specs_to_upload.append(spec) else: specs_to_upload = specs if not specs_to_upload: return skipped, errors total = len(specs_to_upload) if total != len(specs): tty.info(f"{total} specs need to be pushed to {out_url}") upload_futures = [ executor.submit( _url_upload_tarball_and_specfile, spec, tmpdir, cache_entries[spec.dag_hash()], signing_key, ) for spec in specs_to_upload ] uploaded_any = False fancy_progress = FancyProgress(total) for spec, upload_future in zip(specs_to_upload, upload_futures): fancy_progress.start(spec, upload_future.running()) error = upload_future.exception() if error is None: uploaded_any = True fancy_progress.ok() else: fancy_progress.fail() errors.append((spec, error)) # don't bother pushing keys / index if all failed to upload if not uploaded_any: return skipped, errors # If the layout.json doesn't yet exist on this mirror, push it cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION) cache_class.maybe_push_layout_json(out_url) if signing_key: keys_tmpdir = os.path.join(tmpdir, "keys") os.mkdir(keys_tmpdir) _url_push_keys(out_url, keys=[signing_key], update_index=update_index, tmpdir=keys_tmpdir) if update_index: index_tmpdir = os.path.join(tmpdir, "index") os.mkdir(index_tmpdir) _url_generate_package_index(out_url, index_tmpdir) return skipped, errors def _oci_upload_success_msg(spec: spack.spec.Spec, digest: Digest, size: int, elapsed: float): elapsed = max(elapsed, 0.001) # guard against division by zero return ( f"Pushed {_format_spec(spec)}: {digest} ({elapsed:.2f}s, " f"{size / elapsed / 1024 / 1024:.2f} MB/s)" ) def _oci_get_blob_info(image_ref: ImageReference) -> Optional[spack.oci.oci.Blob]: """Get the spack tarball layer digests and size if it exists""" try: manifest, config = get_manifest_and_config_with_retry(image_ref) return spack.oci.oci.Blob( compressed_digest=Digest.from_string(manifest["layers"][-1]["digest"]), uncompressed_digest=Digest.from_string(config["rootfs"]["diff_ids"][-1]), size=manifest["layers"][-1]["size"], ) except Exception: return None def _oci_push_pkg_blob( image_ref: ImageReference, spec: spack.spec.Spec, tmpdir: str ) -> Tuple[spack.oci.oci.Blob, float]: """Push a package blob to the registry and return the blob info and the time taken""" filename = os.path.join(tmpdir, f"{spec.dag_hash()}.tar.gz") # Create an oci.image.layer aka tarball of the package tar_gz_checksum, tar_checksum = create_tarball(spec, filename) blob = spack.oci.oci.Blob( Digest.from_sha256(tar_gz_checksum), Digest.from_sha256(tar_checksum), os.path.getsize(filename), ) # Upload the blob start = time.time() upload_blob_with_retry(image_ref, file=filename, digest=blob.compressed_digest) elapsed = time.time() - start # delete the file os.unlink(filename) return blob, elapsed def _oci_retrieve_env_dict_from_config(config: dict) -> dict: """Retrieve the environment variables from the image config file. Sets a default value for PATH if it is not present. Args: config (dict): The image config file. Returns: dict: The environment variables. """ env = {"PATH": "/bin:/usr/bin"} if "Env" in config.get("config", {}): for entry in config["config"]["Env"]: key, value = entry.split("=", 1) env[key] = value return env def _oci_archspec_to_gooarch(spec: spack.spec.Spec) -> str: name = spec.target.family.name name_map = {"aarch64": "arm64", "x86_64": "amd64"} return name_map.get(name, name) def _oci_put_manifest( base_images: Dict[str, Tuple[dict, dict]], checksums: Dict[str, spack.oci.oci.Blob], image_ref: ImageReference, tmpdir: str, extra_config: Optional[dict], annotations: Optional[dict], *specs: spack.spec.Spec, ): architecture = _oci_archspec_to_gooarch(specs[0]) expected_blobs: List[spack.spec.Spec] = [ s for s in traverse.traverse_nodes(specs, order="topo", deptype=("link", "run"), root=True) if not s.external ] expected_blobs.reverse() base_manifest, base_config = base_images[architecture] env = _oci_retrieve_env_dict_from_config(base_config) # If the base image uses `vnd.docker.distribution.manifest.v2+json`, then we use that too. # This is because Singularity / Apptainer is very strict about not mixing them. base_manifest_mediaType = base_manifest.get( "mediaType", "application/vnd.oci.image.manifest.v1+json" ) use_docker_format = ( base_manifest_mediaType == "application/vnd.docker.distribution.manifest.v2+json" ) spack.user_environment.environment_modifications_for_specs(*specs).apply_modifications(env) # Create an oci.image.config file config = copy.deepcopy(base_config) # Add the diff ids of the blobs for s in expected_blobs: # If a layer for a dependency has gone missing (due to removed manifest in the registry, a # failed push, or a local forced uninstall), we cannot create a runnable container image. checksum = checksums.get(s.dag_hash()) if checksum: config["rootfs"]["diff_ids"].append(str(checksum.uncompressed_digest)) # Set the environment variables config["config"]["Env"] = [f"{k}={v}" for k, v in env.items()] if extra_config: # From the OCI v1.0 spec: # > Any extra fields in the Image JSON struct are considered implementation # > specific and MUST be ignored by any implementations which are unable to # > interpret them. config.update(extra_config) config_file = os.path.join(tmpdir, f"{specs[0].dag_hash()}.config.json") with open(config_file, "w", encoding="utf-8") as f: json.dump(config, f, separators=(",", ":")) config_file_checksum = Digest.from_sha256( spack.util.crypto.checksum(hashlib.sha256, config_file) ) # Upload the config file upload_blob_with_retry(image_ref, file=config_file, digest=config_file_checksum) manifest = { "mediaType": base_manifest_mediaType, "schemaVersion": 2, "config": { "mediaType": base_manifest["config"]["mediaType"], "digest": str(config_file_checksum), "size": os.path.getsize(config_file), }, "layers": [ *(layer for layer in base_manifest["layers"]), *( { "mediaType": ( "application/vnd.docker.image.rootfs.diff.tar.gzip" if use_docker_format else "application/vnd.oci.image.layer.v1.tar+gzip" ), "digest": str(checksums[s.dag_hash()].compressed_digest), "size": checksums[s.dag_hash()].size, } for s in expected_blobs if s.dag_hash() in checksums ), ], } if not use_docker_format and annotations: manifest["annotations"] = annotations # Finally upload the manifest upload_manifest_with_retry(image_ref, manifest=manifest) # delete the config file os.unlink(config_file) def _oci_update_base_images( *, base_image: Optional[ImageReference], target_image: ImageReference, spec: spack.spec.Spec, base_image_cache: Dict[str, Tuple[dict, dict]], ): """For a given spec and base image, copy the missing layers of the base image with matching arch to the registry of the target image. If no base image is specified, create a dummy manifest and config file.""" architecture = _oci_archspec_to_gooarch(spec) if architecture in base_image_cache: return if base_image is None: base_image_cache[architecture] = ( default_manifest(), default_config(architecture, "linux"), ) else: base_image_cache[architecture] = copy_missing_layers_with_retry( base_image, target_image, architecture ) def _oci_default_tag(spec: spack.spec.Spec) -> str: """Return a valid, default image tag for a spec.""" return ensure_valid_tag(f"{spec.name}-{spec.version}-{spec.dag_hash()}.spack") #: Default OCI index tag default_index_tag = "index.spack" def tag_is_spec(tag: str) -> bool: """Check if a tag is likely a Spec""" return tag.endswith(".spack") and tag != default_index_tag def _oci_push( *, target_image: ImageReference, base_image: Optional[ImageReference], installed_specs_with_deps: List[spack.spec.Spec], tmpdir: str, executor: concurrent.futures.Executor, force: bool = False, ) -> Tuple[ List[spack.spec.Spec], Dict[str, Tuple[dict, dict]], Dict[str, spack.oci.oci.Blob], List[Tuple[spack.spec.Spec, BaseException]], ]: # Spec dag hash -> blob checksums: Dict[str, spack.oci.oci.Blob] = {} # arch -> (manifest, config) base_images: Dict[str, Tuple[dict, dict]] = {} # Specs not uploaded because they already exist skipped: List[spack.spec.Spec] = [] if not force: tty.info("Checking for existing specs in the buildcache") blobs_to_upload = [] tags_to_check = ( target_image.with_tag(_oci_default_tag(s)) for s in installed_specs_with_deps ) available_blobs = executor.map(_oci_get_blob_info, tags_to_check) for spec, maybe_blob in zip(installed_specs_with_deps, available_blobs): if maybe_blob is not None: checksums[spec.dag_hash()] = maybe_blob skipped.append(spec) else: blobs_to_upload.append(spec) else: blobs_to_upload = installed_specs_with_deps if not blobs_to_upload: return skipped, base_images, checksums, [] if len(blobs_to_upload) != len(installed_specs_with_deps): tty.info( f"{len(blobs_to_upload)} specs need to be pushed to " f"{target_image.domain}/{target_image.name}" ) blob_progress = FancyProgress(len(blobs_to_upload)) # Upload blobs blob_futures = [ executor.submit(_oci_push_pkg_blob, target_image, spec, tmpdir) for spec in blobs_to_upload ] manifests_to_upload: List[spack.spec.Spec] = [] errors: List[Tuple[spack.spec.Spec, BaseException]] = [] # And update the spec to blob mapping for successful uploads for spec, blob_future in zip(blobs_to_upload, blob_futures): blob_progress.start(spec, blob_future.running()) error = blob_future.exception() if error is None: blob, elapsed = blob_future.result() blob_progress.ok( _oci_upload_success_msg(spec, blob.compressed_digest, blob.size, elapsed) ) manifests_to_upload.append(spec) checksums[spec.dag_hash()] = blob else: blob_progress.fail() errors.append((spec, error)) # Copy base images if necessary for spec in manifests_to_upload: _oci_update_base_images( base_image=base_image, target_image=target_image, spec=spec, base_image_cache=base_images, ) def extra_config(spec: spack.spec.Spec): spec_dict = spec.to_dict(hash=ht.dag_hash) spec_dict["buildcache_layout_version"] = CURRENT_BUILD_CACHE_LAYOUT_VERSION spec_dict["binary_cache_checksum"] = { "hash_algorithm": "sha256", "hash": checksums[spec.dag_hash()].compressed_digest.digest, } spec_dict["archive_size"] = checksums[spec.dag_hash()].size spec_dict["archive_timestamp"] = datetime.datetime.now().astimezone().isoformat() spec_dict["archive_compression"] = "gzip" return spec_dict # Upload manifests tty.info("Uploading manifests") manifest_futures = [ executor.submit( _oci_put_manifest, base_images, checksums, target_image.with_tag(_oci_default_tag(spec)), tmpdir, extra_config(spec), {"org.opencontainers.image.description": spec.format()}, spec, ) for spec in manifests_to_upload ] manifest_progress = FancyProgress(len(manifests_to_upload)) # Print the image names of the top-level specs for spec, manifest_future in zip(manifests_to_upload, manifest_futures): error = manifest_future.exception() manifest_progress.start(spec, manifest_future.running()) if error is None: manifest_progress.ok( f"Tagged {_format_spec(spec)} as {target_image.with_tag(_oci_default_tag(spec))}" ) else: manifest_progress.fail() errors.append((spec, error)) return skipped, base_images, checksums, errors def _oci_config_from_tag(image_ref_and_tag: Tuple[ImageReference, str]) -> Optional[dict]: image_ref, tag = image_ref_and_tag # Don't allow recursion here, since Spack itself always uploads # vnd.oci.image.manifest.v1+json, not vnd.oci.image.index.v1+json _, config = get_manifest_and_config_with_retry(image_ref.with_tag(tag), tag, recurse=0) # Do very basic validation: if "spec" is a key in the config, it # must be a Spec object too. return config if "spec" in config else None def _oci_update_index( image_ref: ImageReference, tmpdir: str, pool: concurrent.futures.Executor ) -> None: tags = list_tags(image_ref) # Fetch all image config files in parallel spec_dicts = pool.map( _oci_config_from_tag, ((image_ref, tag) for tag in tags if tag_is_spec(tag)) ) # Populate the database db_root_dir = os.path.join(tmpdir, "db_root") db = BuildCacheDatabase(db_root_dir) for spec_dict in spec_dicts: spec = spack.spec.Spec.from_dict(spec_dict) db.add(spec) db.mark(spec, "in_buildcache", True) # Create the index.json file index_json_path = os.path.join(tmpdir, spack.database.INDEX_JSON_FILE) with open(index_json_path, "w", encoding="utf-8") as f: db._write_to_file(f) # Create an empty config.json file empty_config_json_path = os.path.join(tmpdir, "config.json") with open(empty_config_json_path, "wb") as f: f.write(b"{}") # Upload the index.json file index_shasum = Digest.from_sha256(spack.util.crypto.checksum(hashlib.sha256, index_json_path)) upload_blob_with_retry(image_ref, file=index_json_path, digest=index_shasum) # Upload the config.json file empty_config_digest = Digest.from_sha256( spack.util.crypto.checksum(hashlib.sha256, empty_config_json_path) ) upload_blob_with_retry(image_ref, file=empty_config_json_path, digest=empty_config_digest) # Push a manifest file that references the index.json file as a layer # Notice that we push this as if it is an image, which it of course is not. # When the ORAS spec becomes official, we can use that instead of a fake image. # For now we just use the OCI image spec, so that we don't run into issues with # automatic garbage collection of blobs that are not referenced by any image manifest. oci_manifest = { "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, # Config is just an empty {} file for now, and irrelevant "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": str(empty_config_digest), "size": os.path.getsize(empty_config_json_path), }, # The buildcache index is the only layer, and is not a tarball, we lie here. "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": str(index_shasum), "size": os.path.getsize(index_json_path), } ], } upload_manifest_with_retry(image_ref.with_tag(default_index_tag), oci_manifest) def try_fetch(url_to_fetch): """Utility function to try and fetch a file from a url, stage it locally, and return the path to the staged file. Args: url_to_fetch (str): Url pointing to remote resource to fetch Returns: Path to locally staged resource or ``None`` if it could not be fetched. """ stage = Stage(url_to_fetch, keep=True) stage.create() try: stage.fetch() except spack.error.FetchError: stage.destroy() return None return stage def download_tarball( spec: spack.spec.Spec, unsigned: Optional[bool] = False, mirrors_for_spec: Optional[List[MirrorMetadata]] = None, ) -> Optional[spack.stage.Stage]: """Download binary tarball for given package Args: spec: a concrete spec unsigned: if ``True`` or ``False`` override the mirror signature verification defaults mirrors_for_spec: Optional list of mirrors known to have the spec. These will be checked in order first before looking in other configured mirrors. Returns: ``None`` if the tarball could not be downloaded, the signature verified (if required), and its checksum validated. Otherwise, return the stage containing the downloaded tarball. """ configured_mirrors: Iterable[spack.mirrors.mirror.Mirror] = ( spack.mirrors.mirror.MirrorCollection(binary=True).values() ) if not configured_mirrors: raise NoConfiguredBinaryMirrors() # Note on try_first and try_next: # mirrors_for_spec mostly likely came from spack caching remote # mirror indices locally and adding their specs to a local data # structure supporting quick lookup of concrete specs. Those # mirrors are likely a subset of all configured mirrors, and # we'll probably find what we need in one of them. But we'll # look in all configured mirrors if needed, as maybe the spec # we need was in an un-indexed mirror. No need to check any # mirror for the spec twice though. try_first = mirrors_for_spec or [] try_next = [ MirrorMetadata(mirror.fetch_url, layout, mirror.fetch_view) for mirror in configured_mirrors for layout in mirror.supported_layout_versions ] urls_and_versions = try_first + [uv for uv in try_next if uv not in try_first] # TODO: turn `mirrors_for_spec` into a list of Mirror instances, instead of doing that here. def fetch_url_to_mirror( mirror_metadata: MirrorMetadata, ) -> Tuple[spack.mirrors.mirror.Mirror, int]: url = mirror_metadata.url layout_version = mirror_metadata.version for mirror in configured_mirrors: if mirror.fetch_url == url: return mirror, layout_version return spack.mirrors.mirror.Mirror(url), layout_version mirrors = [fetch_url_to_mirror(mirror_metadata) for mirror_metadata in urls_and_versions] for mirror, layout_version in mirrors: # Override mirror's default if currently_unsigned = unsigned if unsigned is not None else not mirror.signed # If it's an OCI index, do things differently, since we cannot compose URLs. fetch_url = mirror.fetch_url # TODO: refactor this to some "nice" place. if spack.oci.image.is_oci_url(fetch_url): ref = ImageReference.from_url(fetch_url).with_tag(_oci_default_tag(spec)) # Fetch the manifest try: with spack.oci.opener.urlopen( urllib.request.Request( url=ref.manifest_url(), headers={"Accept": ", ".join(spack.oci.oci.manifest_content_type)}, ) ) as response: manifest = json.load(response) except Exception: continue # Download the config = spec.json and the relevant tarball try: spec_digest = spack.oci.image.Digest.from_string(manifest["config"]["digest"]) tarball_digest = spack.oci.image.Digest.from_string( manifest["layers"][-1]["digest"] ) except Exception: continue with spack.oci.oci.make_stage( ref.blob_url(spec_digest), spec_digest, keep=True ) as local_specfile_stage: try: local_specfile_stage.fetch() local_specfile_stage.check() try: get_valid_spec_file( local_specfile_stage.save_filename, CURRENT_BUILD_CACHE_LAYOUT_VERSION ) except InvalidMetadataFile as e: tty.warn( f"Ignoring binary package for {spec.name}/{spec.dag_hash()[:7]} " f"from {fetch_url} due to invalid metadata file: {e}" ) local_specfile_stage.destroy() continue except Exception: continue local_specfile_stage.cache_local() local_specfile_stage.destroy() with spack.oci.oci.make_stage( ref.blob_url(tarball_digest), tarball_digest, keep=True ) as tarball_stage: try: tarball_stage.fetch() tarball_stage.check() except Exception: continue tarball_stage.cache_local() return tarball_stage else: cache_type = get_url_buildcache_class(layout_version=layout_version) cache_entry = cache_type(fetch_url, spec, allow_unsigned=currently_unsigned) try: cache_entry.fetch_archive() except Exception as e: tty.debug( f"Encountered error attempting to fetch archive for " f"{spec.name}/{spec.dag_hash()[:7]} from {fetch_url} " f"(v{layout_version}) due to {e}" ) cache_entry.destroy() continue if layout_version == 2: warn_v2_layout(fetch_url, "Installing a spec") return cache_entry.get_archive_stage() # Falling through the nested loops means we exhaustively searched # for all known kinds of spec files on all mirrors and did not find # an acceptable one for which we could download a tarball and (if # needed) verify a signature. So at this point, we will proceed to # install from source. return None def dedupe_hardlinks_if_necessary(root, buildinfo): """Updates a buildinfo dict for old archives that did not dedupe hardlinks. De-duping hardlinks is necessary when relocating files in parallel and in-place. This means we must preserve inodes when relocating.""" # New archives don't need this. if buildinfo.get("hardlinks_deduped", False): return # Clearly we can assume that an inode is either in the # textfile or binary group, but let's just stick to # a single set of visited nodes. visited = set() # Note: we do *not* dedupe hardlinked symlinks, since # it seems difficult or even impossible to relink # symlinks while preserving inode. for key in ("relocate_textfiles", "relocate_binaries"): if key not in buildinfo: continue new_list = [] for rel_path in buildinfo[key]: stat_result = os.lstat(os.path.join(root, rel_path)) identifier = (stat_result.st_dev, stat_result.st_ino) if stat_result.st_nlink > 1: if identifier in visited: continue visited.add(identifier) new_list.append(rel_path) buildinfo[key] = new_list def relocate_package(spec: spack.spec.Spec) -> None: """Relocate binaries and text files in the given spec prefix, based on its buildinfo file.""" spec_prefix = str(spec.prefix) buildinfo = read_buildinfo_file(spec_prefix) old_layout_root = str(buildinfo["buildpath"]) # Warn about old style tarballs created with the --rel flag (removed in Spack v0.20) if buildinfo.get("relative_rpaths", False): tty.warn( f"Tarball for {spec} uses relative rpaths, which can cause library loading issues." ) # In Spack 0.19 and older prefix_to_hash was the default and externals were not dropped, so # prefixes were not unique. if "hash_to_prefix" in buildinfo: hash_to_old_prefix = buildinfo["hash_to_prefix"] elif "prefix_to_hash" in buildinfo: hash_to_old_prefix = {v: k for (k, v) in buildinfo["prefix_to_hash"].items()} else: raise NewLayoutException( "Package tarball was created from an install prefix with a different directory layout " "and an older buildcache create implementation. It cannot be relocated." ) prefix_to_prefix: Dict[str, str] = {} if "sbang_install_path" in buildinfo: old_sbang_install_path = str(buildinfo["sbang_install_path"]) prefix_to_prefix[old_sbang_install_path] = spack.hooks.sbang.sbang_install_path() # First match specific prefix paths. Possibly the *local* install prefix of some dependency is # in an upstream, so we cannot assume the original spack store root can be mapped uniformly to # the new spack store root. # If the spec is spliced, we need to handle the simultaneous mapping from the old install_tree # to the new install_tree and from the build_spec to the spliced spec. Because foo.build_spec # is foo for any non-spliced spec, we can simplify by checking for spliced-in nodes by checking # for nodes not in the build_spec without any explicit check for whether the spec is spliced. # An analog in this algorithm is any spec that shares a name or provides the same virtuals in # the context of the relevant root spec. This ensures that the analog for a spec s is the spec # that s replaced when we spliced. relocation_specs = specs_to_relocate(spec) build_spec_ids = set(id(s) for s in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD)) for s in relocation_specs: analog = s if id(s) not in build_spec_ids: analogs = [ d for d in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD) if s._splice_match(d, self_root=spec, other_root=spec.build_spec) ] if analogs: # Prefer same-name analogs and prefer higher versions # This matches the preferences in spack.spec.Spec.splice, so we # will find same node analog = max(analogs, key=lambda a: (a.name == s.name, a.version)) lookup_dag_hash = analog.dag_hash() if lookup_dag_hash in hash_to_old_prefix: old_dep_prefix = hash_to_old_prefix[lookup_dag_hash] prefix_to_prefix[old_dep_prefix] = str(s.prefix) # Only then add the generic fallback of install prefix -> install prefix. prefix_to_prefix[old_layout_root] = str(spack.store.STORE.layout.root) # Delete identity mappings from prefix_to_prefix prefix_to_prefix = {k: v for k, v in prefix_to_prefix.items() if k != v} # If there's nothing to relocate, we're done. if not prefix_to_prefix: return for old, new in prefix_to_prefix.items(): tty.debug(f"Relocating: {old} => {new}.") # Old archives may have hardlinks repeated. dedupe_hardlinks_if_necessary(spec_prefix, buildinfo) # Text files containing the prefix text textfiles = [os.path.join(spec_prefix, f) for f in buildinfo["relocate_textfiles"]] binaries = [os.path.join(spec_prefix, f) for f in buildinfo.get("relocate_binaries")] links = [os.path.join(spec_prefix, f) for f in buildinfo.get("relocate_links", [])] platform = spack.platforms.by_name(spec.platform) if "macho" in platform.binary_formats: relocate.relocate_macho_binaries(binaries, prefix_to_prefix) elif "elf" in platform.binary_formats: relocate.relocate_elf_binaries(binaries, prefix_to_prefix) relocate.relocate_links(links, prefix_to_prefix) relocate.relocate_text(textfiles, prefix_to_prefix) changed_files = relocate.relocate_text_bin(binaries, prefix_to_prefix) # Add ad-hoc signatures to patched macho files when on macOS. if "macho" in platform.binary_formats and sys.platform == "darwin": codesign = which("codesign") if not codesign: return for binary in changed_files: # preserve the original inode by running codesign on a copy with fsys.edit_in_place_through_temporary_file(binary) as tmp_binary: codesign("-fs-", tmp_binary) install_manifest = os.path.join( spec.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name, ) if not os.path.exists(install_manifest): spec_id = spec.format("{name}/{hash:7}") tty.warn("No manifest file in tarball for spec %s" % spec_id) # overwrite old metadata with new if spec.spliced: # rewrite spec on disk spack.store.STORE.layout.write_spec(spec, spack.store.STORE.layout.spec_file_path(spec)) # de-cache the install manifest with contextlib.suppress(FileNotFoundError): os.unlink(install_manifest) def _tar_strip_component(tar: tarfile.TarFile, prefix: str): """Yield all members of tarfile that start with given prefix, and strip that prefix (including symlinks)""" # Including trailing /, otherwise we end up with absolute paths. regex = re.compile(re.escape(prefix) + "/*") # Only yield members in the package prefix. # Note: when a tarfile is created, relative in-prefix symlinks are # expanded to matching member names of tarfile entries. So, we have # to ensure that those are updated too. # Absolute symlinks are copied verbatim -- relocation should take care of # them. for m in tar.getmembers(): result = regex.match(m.name) if not result: continue m.name = m.name[result.end() :] if m.linkname: result = regex.match(m.linkname) if result: m.linkname = m.linkname[result.end() :] yield m def extract_buildcache_tarball(tarfile_path: str, destination: str) -> None: with closing(tarfile.open(tarfile_path, "r")) as tar: # For consistent behavior across all supported Python versions tar.extraction_filter = lambda member, path: member # Remove common prefix from tarball entries and directly extract them to the install dir. tar.extractall( path=destination, members=_tar_strip_component(tar, prefix=_ensure_common_prefix(tar)) ) def extract_tarball(spec, tarball_stage: spack.stage.Stage, force=False, timer=timer.NULL_TIMER): """ extract binary tarball for given package into install area """ timer.start("extract") if os.path.exists(spec.prefix): if force: shutil.rmtree(spec.prefix) else: raise NoOverwriteException(str(spec.prefix)) # Create the install prefix fsys.mkdirp( spec.prefix, mode=get_package_dir_permissions(spec), group=get_package_group(spec), default_perms="parents", ) tarfile_path = tarball_stage.save_filename try: extract_buildcache_tarball(tarfile_path, destination=spec.prefix) except Exception: shutil.rmtree(spec.prefix, ignore_errors=True) tarball_stage.destroy() raise timer.stop("extract") timer.start("relocate") try: relocate_package(spec) except Exception as e: shutil.rmtree(spec.prefix, ignore_errors=True) raise e finally: tarball_stage.destroy() timer.stop("relocate") def _ensure_common_prefix(tar: tarfile.TarFile) -> str: # Find the lowest `binary_distribution` file (hard-coded forward slash is on purpose). binary_distribution = min( ( e.name for e in tar.getmembers() if e.isfile() and e.name.endswith(".spack/binary_distribution") ), key=len, default=None, ) if binary_distribution is None: raise ValueError("Tarball is not a Spack package, missing binary_distribution file") pkg_path = pathlib.PurePosixPath(binary_distribution).parent.parent # Even the most ancient Spack version has required to list the dir of the package itself, so # guard against broken tarballs where `path.parent.parent` is empty. if pkg_path == pathlib.PurePosixPath(): raise ValueError("Invalid tarball, missing package prefix dir") pkg_prefix = str(pkg_path) # Ensure all tar entries are in the pkg_prefix dir, and if they're not, they should be parent # dirs of it. has_prefix = False for member in tar.getmembers(): stripped = member.name.rstrip("/") if not ( stripped.startswith(pkg_prefix) or member.isdir() and pkg_prefix.startswith(stripped) ): raise ValueError(f"Tarball contains file {stripped} outside of prefix {pkg_prefix}") if member.isdir() and stripped == pkg_prefix: has_prefix = True # This is technically not required, but let's be defensive about the existence of the package # prefix dir. if not has_prefix: raise ValueError(f"Tarball does not contain a common prefix {pkg_prefix}") return pkg_prefix def install_root_node( spec: spack.spec.Spec, unsigned=False, force: bool = False, sha256: Optional[str] = None, allow_missing: bool = False, ) -> None: """Install the root node of a concrete spec from a buildcache. Checking the sha256 sum of a node before installation is usually needed only for software installed during Spack's bootstrapping (since we might not have a proper signature verification mechanism available). Args: spec: spec to be installed (note that only the root node will be installed) unsigned: if True allows installing unsigned binaries force: force installation if the spec is already present in the local store sha256: optional sha256 of the binary package, to be checked before installation allow_missing: when true, allows installing a node with missing dependencies """ # Early termination if spec.external or not spec.concrete: warnings.warn("Skipping external or abstract spec {0}".format(spec.format())) return elif spec.installed and not force: warnings.warn("Package for spec {0} already installed.".format(spec.format())) return tarball_stage = download_tarball(spec.build_spec, unsigned) if not tarball_stage: msg = 'download of binary cache file for spec "{0}" failed' raise RuntimeError(msg.format(spec.build_spec.format())) # don't print long padded paths while extracting/relocating binaries with spack.util.path.filter_padding(): tty.msg('Installing "{0}" from a buildcache'.format(spec.format())) extract_tarball(spec, tarball_stage, force) spec.package.windows_establish_runtime_linkage() spack.hooks.post_install(spec, False) spack.store.STORE.db.add(spec, allow_missing=allow_missing) def install_single_spec(spec, unsigned=False, force=False): """Install a single concrete spec from a buildcache. Args: spec (spack.spec.Spec): spec to be installed unsigned (bool): if True allows installing unsigned binaries force (bool): force installation if the spec is already present in the local store """ for node in spec.traverse(root=True, order="post", deptype=("link", "run")): install_root_node(node, unsigned=unsigned, force=force) def try_direct_fetch(spec: spack.spec.Spec) -> List[MirrorMetadata]: """Try to find the spec directly on the configured mirrors""" found_specs: List[MirrorMetadata] = [] binary_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True).values() for mirror in binary_mirrors: # TODO: OCI-support if spack.oci.image.is_oci_url(mirror.fetch_url): continue for layout_version in mirror.supported_layout_versions: # layout_version could eventually come from the mirror config cache_class = get_url_buildcache_class(layout_version=layout_version) cache_entry = cache_class(mirror.fetch_url, spec) try: spec_dict = cache_entry.fetch_metadata() except BuildcacheEntryError: continue finally: cache_entry.destroy() # All specs in build caches are concrete (as they are built) so we need # to mark this spec concrete on read-in. fetched_spec = spack.spec.Spec.from_dict(spec_dict) fetched_spec._mark_concrete() found_specs.append(MirrorMetadata(mirror.fetch_url, layout_version, mirror.fetch_view)) return found_specs def get_mirrors_for_spec(spec: spack.spec.Spec, index_only: bool = False) -> List[MirrorMetadata]: """ Check if concrete spec exists on mirrors and return a list indicating the mirrors on which it can be found Args: spec: The spec to look for in binary mirrors index_only: When ``index_only`` is set to ``True``, only the local cache is checked, no requests are made. """ if not spack.mirrors.mirror.MirrorCollection(binary=True): tty.debug("No Spack mirrors are currently configured") return [] results = BINARY_INDEX.find_built_spec(spec) # The index may be out-of-date. If we aren't only considering indices, try # to fetch directly since we know where the file should be. if not results and not index_only: results = try_direct_fetch(spec) # We found a spec by the direct fetch approach, we might as well # add it to our mapping. if results: BINARY_INDEX.update_spec(spec, results) return results def update_cache_and_get_specs(): """ Get all concrete specs for build caches available on configured mirrors. Initialization of internal cache data structures is done as lazily as possible, so this method will also attempt to initialize and update the local index cache (essentially a no-op if it has been done already and nothing has changed on the configured mirrors.) Raises: FetchCacheError """ BINARY_INDEX.update() return BINARY_INDEX.get_all_built_specs() def get_keys( install: bool = False, trust: bool = False, force: bool = False, mirrors: Optional[Mapping[str, spack.mirrors.mirror.Mirror]] = None, ): """Get pgp public keys available on mirror with suffix .pub""" mirror_collection = mirrors or spack.mirrors.mirror.MirrorCollection(binary=True) if not mirror_collection: tty.die("Please add a spack mirror to allow " + "download of build caches.") fingerprints = [] for mirror in mirror_collection.values(): if not mirror.signed: # Don't bother fetching keys for unsigned mirrors continue for layout_version in mirror.supported_layout_versions: fetch_url = mirror.fetch_url if layout_version == 2: mirror_layout_fingerprints = _get_keys_v2(fetch_url, install, trust, force) else: mirror_layout_fingerprints = _get_keys( fetch_url, layout_version, install, trust, force ) if mirror_layout_fingerprints: fingerprints.extend(mirror_layout_fingerprints) return fingerprints def _get_keys( mirror_url: str, layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION, install: bool = False, trust: bool = False, force: bool = False, ) -> Optional[List[str]]: cache_class = get_url_buildcache_class(layout_version=layout_version) tty.debug("Finding public keys in {0}".format(url_util.format(mirror_url))) keys_prefix = url_util.join( mirror_url, *cache_class.get_relative_path_components(BuildcacheComponent.KEY) ) key_index_manifest_url = url_util.join(keys_prefix, "keys.manifest.json") index_entry = cache_class(mirror_url, allow_unsigned=True) try: index_manifest = index_entry.read_manifest(manifest_url=key_index_manifest_url) index_blob_path = index_entry.fetch_blob(index_manifest.data[0]) except BuildcacheEntryError as e: tty.debug(f"Failed to fetch key index due to: {e}") index_entry.destroy() return None with open(index_blob_path, encoding="utf-8") as fd: json_index = json.load(fd) index_entry.destroy() saved_fingerprints = [] for fingerprint, _ in json_index["keys"].items(): key_manifest_url = url_util.join(keys_prefix, f"{fingerprint}.key.manifest.json") key_entry = cache_class(mirror_url, allow_unsigned=True) try: key_manifest = key_entry.read_manifest(manifest_url=key_manifest_url) key_blob_path = key_entry.fetch_blob(key_manifest.data[0]) except BuildcacheEntryError as e: tty.debug(f"Failed to fetch key {fingerprint} due to: {e}") key_entry.destroy() continue tty.debug("Found key {0}".format(fingerprint)) if install: if trust: spack.util.gpg.trust(key_blob_path) tty.debug(f"Added {fingerprint} to trusted keys.") saved_fingerprints.append(fingerprint) else: tty.debug( "Will not add this key to trusted keys.Use -t to install all downloaded keys" ) key_entry.destroy() return saved_fingerprints def _get_keys_v2(mirror_url, install=False, trust=False, force=False) -> Optional[List[str]]: cache_class = get_url_buildcache_class(layout_version=2) keys_url = url_util.join( mirror_url, *cache_class.get_relative_path_components(BuildcacheComponent.KEY) ) keys_index = url_util.join(keys_url, "index.json") tty.debug("Finding public keys in {0}".format(url_util.format(mirror_url))) try: json_index = web_util.read_json(keys_index) except (web_util.SpackWebError, OSError, ValueError) as url_err: # TODO: avoid repeated request if web_util.url_exists(keys_index): tty.error( f"Unable to find public keys in {url_util.format(mirror_url)}," f" caught exception attempting to read from {url_util.format(keys_index)}." ) tty.error(url_err) return None saved_fingerprints = [] for fingerprint, key_attributes in json_index["keys"].items(): link = os.path.join(keys_url, fingerprint + ".pub") with Stage(link, name="build_cache", keep=True) as stage: if os.path.exists(stage.save_filename) and force: os.remove(stage.save_filename) if not os.path.exists(stage.save_filename): try: stage.fetch() except spack.error.FetchError: continue tty.debug("Found key {0}".format(fingerprint)) if install: if trust: spack.util.gpg.trust(stage.save_filename) tty.debug("Added this key to trusted keys.") saved_fingerprints.append(fingerprint) else: tty.debug( "Will not add this key to trusted keys.Use -t to install all downloaded keys" ) return saved_fingerprints def _url_push_keys( *mirrors: Union[spack.mirrors.mirror.Mirror, str], keys: List[str], tmpdir: str, update_index: bool = False, ): """Upload pgp public keys to the given mirrors""" keys = spack.util.gpg.public_keys(*(keys or ())) files = [os.path.join(tmpdir, f"{key}.pub") for key in keys] for key, file in zip(keys, files): spack.util.gpg.export_keys(file, [key]) cache_class = get_url_buildcache_class() for mirror in mirrors: push_url = mirror if isinstance(mirror, str) else mirror.push_url tty.debug(f"Pushing public keys to {url_util.format(push_url)}") pushed_a_key = False for key, file in zip(keys, files): cache_class.push_local_file_as_blob( local_file_path=file, mirror_url=push_url, manifest_name=f"{key}.key", component_type=BuildcacheComponent.KEY, compression="none", ) pushed_a_key = True if update_index: generate_key_index(push_url, tmpdir=tmpdir) if pushed_a_key or update_index: cache_class.maybe_push_layout_json(push_url) def needs_rebuild(spec, mirror_url): if not spec.concrete: raise ValueError("spec must be concrete to check against mirror") pkg_name = spec.name pkg_version = spec.version pkg_hash = spec.dag_hash() tty.debug("Checking {0}-{1}, dag_hash = {2}".format(pkg_name, pkg_version, pkg_hash)) tty.debug(spec.tree()) # Try to retrieve the specfile directly, based on the known # format of the name, in order to determine if the package # needs to be rebuilt. cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION) cache_entry = cache_class(mirror_url, spec, allow_unsigned=True) exists = cache_entry.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL]) return not exists def check_specs_against_mirrors(mirrors, specs, output_file=None): """Check all the given specs against buildcaches on the given mirrors and determine if any of the specs need to be rebuilt. Specs need to be rebuilt when their hash doesn't exist in the mirror. Arguments: mirrors (dict): Mirrors to check against specs (typing.Iterable): Specs to check against mirrors output_file (str): Path to output file to be written. If provided, mirrors with missing or out-of-date specs will be formatted as a JSON object and written to this file. Returns: 1 if any spec was out-of-date on any mirror, 0 otherwise. """ rebuilds = {} for mirror in spack.mirrors.mirror.MirrorCollection(mirrors, binary=True).values(): tty.debug("Checking for built specs at {0}".format(mirror.fetch_url)) rebuild_list = [] for spec in specs: if needs_rebuild(spec, mirror.fetch_url): rebuild_list.append({"short_spec": spec.short_spec, "hash": spec.dag_hash()}) if rebuild_list: rebuilds[mirror.fetch_url] = { "mirrorName": mirror.name, "mirrorUrl": mirror.fetch_url, "rebuildSpecs": rebuild_list, } if output_file: with open(output_file, "w", encoding="utf-8") as outf: outf.write(json.dumps(rebuilds)) return 1 if rebuilds else 0 def download_single_spec( concrete_spec, destination, mirror_url=None, layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION, ): """Download the buildcache files for a single concrete spec. Args: concrete_spec: concrete spec to be downloaded destination (str): path where to put the downloaded buildcache mirror_url (str): url of the mirror from which to download """ if not mirror_url and not spack.mirrors.mirror.MirrorCollection(binary=True): tty.die( "Please provide or add a spack mirror to allow " + "download of buildcache entries." ) urls = ( [mirror_url] if mirror_url else [ mirror.fetch_url for mirror in spack.mirrors.mirror.MirrorCollection(binary=True).values() ] ) mkdirp(destination) for url in urls: cache_class = get_url_buildcache_class(layout_version=layout_version) cache_entry = cache_class(url, concrete_spec, allow_unsigned=True) try: cache_entry.fetch_metadata() cache_entry.fetch_archive() except BuildcacheEntryError as e: tty.warn(f"Error downloading {concrete_spec.name}/{concrete_spec.dag_hash()[:7]}: {e}") cache_entry.destroy() continue shutil.move(cache_entry.get_local_spec_path(), destination) shutil.move(cache_entry.get_local_archive_path(), destination) return True return False class BinaryCacheQuery: """Callable object to query if a spec is in a binary cache""" def __init__(self, all_architectures): """ Args: all_architectures (bool): if True consider all the spec for querying, otherwise restrict to the current default architecture """ self.all_architectures = all_architectures specs = update_cache_and_get_specs() if not self.all_architectures: arch = spack.spec.Spec.default_arch() specs = [s for s in specs if s.satisfies(arch)] self.possible_specs = specs def __call__(self, spec: spack.spec.Spec, **kwargs): """ Args: spec: The spec being searched for """ return [s for s in self.possible_specs if s.satisfies(spec)] class FetchIndexError(Exception): def __str__(self): if len(self.args) == 1: return str(self.args[0]) else: return "{}, due to: {}".format(self.args[0], self.args[1]) class BuildcacheIndexError(spack.error.SpackError): """Raised when a buildcache cannot be read for any reason""" class BuildcacheIndexNotExists(Exception): """Buildcache does not contain an index""" FetchIndexResult = collections.namedtuple("FetchIndexResult", "etag hash data fresh") class IndexFetcher: def conditional_fetch(self) -> FetchIndexResult: raise NotImplementedError(f"{self.__class__.__name__} is abstract") def get_index_manifest(self, manifest_response) -> BlobRecord: """Read the response of the manifest request and return a BlobRecord""" cache_class = get_url_buildcache_class(CURRENT_BUILD_CACHE_LAYOUT_VERSION) try: result = io.TextIOWrapper(manifest_response, encoding="utf-8").read() except (ValueError, OSError) as e: raise FetchIndexError(f"Remote index {manifest_response.url} is invalid", e) from e manifest = BuildcacheManifest.from_dict( # Currently we do not sign buildcache index, but we could cache_class.verify_and_extract_manifest(result, verify=False) ) blob_record = manifest.get_blob_records( cache_class.component_to_media_type(BuildcacheComponent.INDEX) )[0] return blob_record def fetch_index_blob( self, cache_entry: URLBuildcacheEntry, blob_record: BlobRecord ) -> Tuple[str, str]: """Fetch the index blob indicated by the BlobRecord, and return the (checksum, contents) of the blob""" try: staged_blob_path = cache_entry.fetch_blob(blob_record) except BuildcacheEntryError as e: cache_entry.destroy() raise FetchIndexError( f"Could not fetch index blob from {cache_entry.mirror_url}" ) from e with open(staged_blob_path, encoding="utf-8") as fd: blob_result = fd.read() computed_hash = compute_hash(blob_result) if computed_hash != blob_record.checksum: cache_entry.destroy() raise FetchIndexError(f"Remote index at {cache_entry.mirror_url} is invalid") return (computed_hash, blob_result) class DefaultIndexFetcherV2(IndexFetcher): """Fetcher for index.json, using separate index.json.hash as cache invalidation strategy""" def __init__(self, url, local_hash, urlopen=web_util.urlopen): self.url = url self.local_hash = local_hash self.urlopen = urlopen self.headers = {"User-Agent": web_util.SPACK_USER_AGENT} def get_remote_hash(self): # Failure to fetch index.json.hash is not fatal url_index_hash = url_util.join(self.url, "build_cache", "index.json.hash") try: with self.urlopen( urllib.request.Request(url_index_hash, headers=self.headers) ) as response: remote_hash = response.read(64) except OSError: return None # Validate the hash if not re.match(rb"[a-f\d]{64}$", remote_hash): return None return remote_hash.decode("utf-8") def conditional_fetch(self) -> FetchIndexResult: # Do an intermediate fetch for the hash # and a conditional fetch for the contents # Early exit if our cache is up to date. if self.local_hash and self.local_hash == self.get_remote_hash(): return FetchIndexResult(etag=None, hash=None, data=None, fresh=True) # Otherwise, download index.json url_index = url_util.join(self.url, "build_cache", spack.database.INDEX_JSON_FILE) try: response = self.urlopen(urllib.request.Request(url_index, headers=self.headers)) except OSError as e: raise FetchIndexError(f"Could not fetch index from {url_index}", e) from e with response: try: result = io.TextIOWrapper(response, encoding="utf-8").read() except (ValueError, OSError) as e: raise FetchIndexError(f"Remote index {url_index} is invalid") from e # For now we only handle etags on http(s), since 304 error handling # in s3:// is not there yet. if urllib.parse.urlparse(self.url).scheme not in ("http", "https"): etag = None else: etag = web_util.parse_etag( response.headers.get("Etag", None) or response.headers.get("etag", None) ) computed_hash = compute_hash(result) # We don't handle computed_hash != remote_hash here, which can happen # when remote index.json and index.json.hash are out of sync, or if # the hash algorithm changed. # The most likely scenario is that we got index.json got updated # while we fetched index.json.hash. Warning about an issue thus feels # wrong, as it's more of an issue with race conditions in the cache # invalidation strategy. warn_v2_layout(self.url, "Fetching an index") return FetchIndexResult(etag=etag, hash=computed_hash, data=result, fresh=False) class EtagIndexFetcherV2(IndexFetcher): """Fetcher for index.json, using ETags headers as cache invalidation strategy""" def __init__(self, url, etag, urlopen=web_util.urlopen): self.url = url self.etag = etag self.urlopen = urlopen def conditional_fetch(self) -> FetchIndexResult: # Just do a conditional fetch immediately url = url_util.join(self.url, "build_cache", spack.database.INDEX_JSON_FILE) headers = {"User-Agent": web_util.SPACK_USER_AGENT, "If-None-Match": f'"{self.etag}"'} try: response = self.urlopen(urllib.request.Request(url, headers=headers)) except urllib.error.HTTPError as e: if e.getcode() == 304: # Not modified; that means fresh. return FetchIndexResult(etag=None, hash=None, data=None, fresh=True) raise FetchIndexError(f"Could not fetch index {url}", e) from e except OSError as e: # URLError, socket.timeout, etc. raise FetchIndexError(f"Could not fetch index {url}", e) from e with response: try: result = io.TextIOWrapper(response, encoding="utf-8").read() except (ValueError, OSError) as e: raise FetchIndexError(f"Remote index {url} is invalid", e) from e warn_v2_layout(self.url, "Fetching an index") etag_header_value = response.headers.get("Etag", None) or response.headers.get( "etag", None ) return FetchIndexResult( etag=web_util.parse_etag(etag_header_value), hash=compute_hash(result), data=result, fresh=False, ) class OCIIndexFetcher(IndexFetcher): def __init__(self, mirror_metadata: MirrorMetadata, local_hash, urlopen=None) -> None: self.local_hash = local_hash self.ref = spack.oci.image.ImageReference.from_url(mirror_metadata.url) self.urlopen = urlopen or spack.oci.opener.urlopen def conditional_fetch(self) -> FetchIndexResult: """Download an index from an OCI registry type mirror.""" url_manifest = self.ref.with_tag(default_index_tag).manifest_url() try: response = self.urlopen( urllib.request.Request( url=url_manifest, headers={"Accept": "application/vnd.oci.image.manifest.v1+json"}, ) ) except OSError as e: raise FetchIndexError(f"Could not fetch manifest from {url_manifest}", e) from e with response: try: manifest = json.load(response) except Exception as e: raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e # Get first blob hash, which should be the index.json try: index_digest = spack.oci.image.Digest.from_string(manifest["layers"][0]["digest"]) except Exception as e: raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e # Fresh? if index_digest.digest == self.local_hash: return FetchIndexResult(etag=None, hash=None, data=None, fresh=True) # Otherwise fetch the blob / index.json try: with self.urlopen( urllib.request.Request( url=self.ref.blob_url(index_digest), headers={"Accept": "application/vnd.oci.image.layer.v1.tar+gzip"}, ) ) as response: result = io.TextIOWrapper(response, encoding="utf-8").read() except (OSError, ValueError) as e: raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e # Make sure the blob we download has the advertised hash if compute_hash(result) != index_digest.digest: raise FetchIndexError(f"Remote index {url_manifest} is invalid") return FetchIndexResult(etag=None, hash=index_digest.digest, data=result, fresh=False) class DefaultIndexFetcher(IndexFetcher): """Fetcher for buildcache index, cache invalidation via manifest contents""" def __init__(self, mirror_metadata: MirrorMetadata, local_hash, urlopen=web_util.urlopen): self.url = mirror_metadata.url self.view = mirror_metadata.view self.layout_version = mirror_metadata.version self.local_hash = local_hash self.urlopen = urlopen self.headers = {"User-Agent": web_util.SPACK_USER_AGENT} def conditional_fetch(self) -> FetchIndexResult: cache_class = get_url_buildcache_class(layout_version=self.layout_version) url_index_manifest = cache_class.get_index_url(self.url, self.view) try: response = self.urlopen( urllib.request.Request(url_index_manifest, headers=self.headers) ) except OSError as e: raise FetchIndexError( f"Could not read index manifest from {url_index_manifest}" ) from e with response: index_blob_record = self.get_index_manifest(response) # Early exit if our cache is up to date. if self.local_hash and self.local_hash == index_blob_record.checksum: return FetchIndexResult(etag=None, hash=None, data=None, fresh=True) # Otherwise, download the index blob cache_entry = cache_class(self.url, allow_unsigned=True) computed_hash, result = self.fetch_index_blob(cache_entry, index_blob_record) cache_entry.destroy() # For now we only handle etags on http(s), since 304 error handling # in s3:// is not there yet. if urllib.parse.urlparse(self.url).scheme not in ("http", "https"): etag = None else: etag = web_util.parse_etag( response.headers.get("Etag", None) or response.headers.get("etag", None) ) return FetchIndexResult(etag=etag, hash=computed_hash, data=result, fresh=False) class EtagIndexFetcher(IndexFetcher): """Fetcher for buildcache index, cache invalidation via ETags headers This class differs from the :class:`DefaultIndexFetcher` in the following ways: 1. It is provided with an etag value on creation, rather than an index checksum value. Note that since we never start out with an etag, the default fetcher must have been used initially and determined that the etag approach is valid. 2. It provides this etag value in the ``If-None-Match`` request header for the index manifest. 3. It checks for special exception type and response code indicating the index manifest is not modified, exiting early and returning ``Fresh``, if encountered. 4. If it needs to actually read the manifest, it does not need to do any checks of the url scheme to determine whether an etag should be included in the return value.""" def __init__(self, mirror_metadata: MirrorMetadata, etag, urlopen=web_util.urlopen): self.url = mirror_metadata.url self.view = mirror_metadata.view self.layout_version = mirror_metadata.version self.etag = etag self.urlopen = urlopen def conditional_fetch(self) -> FetchIndexResult: # Do a conditional fetch of the index manifest (i.e. using If-None-Match header) cache_class = get_url_buildcache_class(layout_version=self.layout_version) manifest_url = cache_class.get_index_url(self.url, self.view) headers = {"User-Agent": web_util.SPACK_USER_AGENT, "If-None-Match": f'"{self.etag}"'} try: response = self.urlopen(urllib.request.Request(manifest_url, headers=headers)) except urllib.error.HTTPError as e: if e.getcode() == 304: # The remote manifest has not been modified, i.e. the index we # already have is the freshest there is. return FetchIndexResult(etag=None, hash=None, data=None, fresh=True) raise FetchIndexError(f"Could not fetch index manifest {manifest_url}", e) from e except OSError as e: # URLError, socket.timeout, etc. raise FetchIndexError(f"Could not fetch index manifest {manifest_url}", e) from e # We need to read the index manifest and fetch the associated blob with response: index_blob_record = self.get_index_manifest(response) etag_header_value = response.headers.get("Etag", None) or response.headers.get( "etag", None ) cache_entry = cache_class(self.url, allow_unsigned=True) computed_hash, result = self.fetch_index_blob(cache_entry, index_blob_record) cache_entry.destroy() return FetchIndexResult( etag=web_util.parse_etag(etag_header_value), hash=computed_hash, data=result, fresh=False, ) def get_index_fetcher( scheme: str, mirror_metadata: MirrorMetadata, cache_entry: Dict[str, str] ) -> IndexFetcher: if scheme == "oci": # TODO: Actually etag and OCI are not mutually exclusive... return OCIIndexFetcher(mirror_metadata, cache_entry.get("index_hash", None)) elif cache_entry.get("etag"): if mirror_metadata.version < 3: return EtagIndexFetcherV2(mirror_metadata.url, cache_entry["etag"]) else: return EtagIndexFetcher(mirror_metadata, cache_entry["etag"]) else: if mirror_metadata.version < 3: return DefaultIndexFetcherV2( mirror_metadata.url, local_hash=cache_entry.get("index_hash", None) ) else: return DefaultIndexFetcher( mirror_metadata, local_hash=cache_entry.get("index_hash", None) ) class NoOverwriteException(spack.error.SpackError): """Raised when a file would be overwritten""" def __init__(self, file_path): super().__init__(f"Refusing to overwrite the following file: {file_path}") class NoGpgException(spack.error.SpackError): """ Raised when gpg2 is not in PATH """ def __init__(self, msg): super().__init__(msg) class NoKeyException(spack.error.SpackError): """ Raised when gpg has no default key added. """ def __init__(self, msg): super().__init__(msg) class PickKeyException(spack.error.SpackError): """ Raised when multiple keys can be used to sign. """ def __init__(self, keys): err_msg = "Multiple keys available for signing\n%s\n" % keys err_msg += "Use spack buildcache create -k to pick a key." super().__init__(err_msg) class NewLayoutException(spack.error.SpackError): """ Raised if directory layout is different from buildcache. """ def __init__(self, msg): super().__init__(msg) class UnsignedPackageException(spack.error.SpackError): """ Raised if installation of unsigned package is attempted without the use of ``--no-check-signature``. """ class GenerateIndexError(spack.error.SpackError): """Raised when unable to generate key or package index for mirror""" class CannotListKeys(GenerateIndexError): """Raised when unable to list keys when generating key index""" class PushToBuildCacheError(spack.error.SpackError): """Raised when unable to push objects to binary mirror""" class NoConfiguredBinaryMirrors(spack.error.SpackError): """Raised when no binary mirrors are configured but an operation requires them""" def __init__(self): super().__init__("Please add a spack mirror to allow download of pre-compiled packages.") ================================================ FILE: lib/spack/spack/bootstrap/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Function and classes needed to bootstrap Spack itself.""" from .config import ensure_bootstrap_configuration, is_bootstrapping, store_path from .core import ( all_core_root_specs, ensure_clingo_importable_or_raise, ensure_core_dependencies, ensure_gpg_in_path_or_raise, ensure_patchelf_in_path_or_raise, ensure_winsdk_external_or_raise, ) from .environment import BootstrapEnvironment, ensure_environment_dependencies from .status import status_message __all__ = [ "all_core_root_specs", "BootstrapEnvironment", "ensure_bootstrap_configuration", "ensure_clingo_importable_or_raise", "ensure_core_dependencies", "ensure_environment_dependencies", "ensure_gpg_in_path_or_raise", "ensure_patchelf_in_path_or_raise", "ensure_winsdk_external_or_raise", "is_bootstrapping", "status_message", "store_path", ] ================================================ FILE: lib/spack/spack/bootstrap/_common.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Common basic functions used through the spack.bootstrap package""" import fnmatch import glob import importlib import os import re import sys import sysconfig import warnings from typing import Optional, Sequence, Union import spack.vendor.archspec.cpu from spack.vendor.typing_extensions import TypedDict import spack.llnl.util.filesystem as fs import spack.platforms import spack.spec import spack.store import spack.util.environment import spack.util.executable from spack.llnl.util import tty from .config import spec_for_current_python class QueryInfo(TypedDict, total=False): spec: spack.spec.Spec command: spack.util.executable.Executable def _python_import(module: str) -> bool: try: importlib.import_module(module) except ImportError: return False return True def _try_import_from_store( module: str, query_spec: Union[str, "spack.spec.Spec"], query_info: Optional[QueryInfo] = None ) -> bool: """Return True if the module can be imported from an already installed spec, False otherwise. Args: module: Python module to be imported query_spec: spec that may provide the module query_info (dict or None): if a dict is passed it is populated with the command found and the concrete spec providing it """ # If it is a string assume it's one of the root specs by this module if isinstance(query_spec, str): # We have to run as part of this python interpreter query_spec += " ^" + spec_for_current_python() installed_specs = spack.store.STORE.db.query(query_spec, installed=True) for candidate_spec in installed_specs: # previously bootstrapped specs may not have a python-venv dependency. if candidate_spec.dependencies("python-venv"): python, *_ = candidate_spec.dependencies("python-venv") else: python, *_ = candidate_spec.dependencies("python") # if python is installed, ask it for the layout if python.installed: module_paths = [ os.path.join(candidate_spec.prefix, python.package.purelib), os.path.join(candidate_spec.prefix, python.package.platlib), ] # otherwise search for the site-packages directory # (clingo from binaries with truncated python-venv runtime) else: module_paths = glob.glob( os.path.join(candidate_spec.prefix, "lib", "python*", "site-packages") ) path_before = list(sys.path) # NOTE: try module_paths first and last, last allows an existing version in path # to be picked up and used, possibly depending on something in the store, first # allows the bootstrap version to work when an incompatible version is in # sys.path orders = [module_paths + sys.path, sys.path + module_paths] for path in orders: sys.path = path try: _fix_ext_suffix(candidate_spec) if _python_import(module): msg = ( f"[BOOTSTRAP MODULE {module}] The installed spec " f'"{query_spec}/{candidate_spec.dag_hash()}" ' f'provides the "{module}" Python module' ) tty.debug(msg) if query_info is not None: query_info["spec"] = candidate_spec return True except Exception as exc: # pylint: disable=broad-except msg = ( "unexpected error while trying to import module " f'"{module}" from spec "{candidate_spec}" [error="{str(exc)}"]' ) warnings.warn(msg) else: msg = "Spec {0} did not provide module {1}" warnings.warn(msg.format(candidate_spec, module)) sys.path = path_before return False def _fix_ext_suffix(candidate_spec: "spack.spec.Spec"): """Fix the external suffixes of Python extensions on the fly for platforms that may need it Args: candidate_spec (Spec): installed spec with a Python module to be checked. """ # Here we map target families to the patterns expected # by pristine CPython. Only architectures with known issues # are included. Known issues: # # [RHEL + ppc64le]: https://github.com/spack/spack/issues/25734 # _suffix_to_be_checked = { "ppc64le": { "glob": "*.cpython-*-powerpc64le-linux-gnu.so", "re": r".cpython-[\w]*-powerpc64le-linux-gnu.so", "fmt": r"{module}.cpython-{major}{minor}m-powerpc64le-linux-gnu.so", } } # If the current architecture is not problematic return generic_target = spack.vendor.archspec.cpu.host().family if str(generic_target) not in _suffix_to_be_checked: return # If there's no EXT_SUFFIX (Python < 3.5) or the suffix matches # the expectations, return since the package is surely good ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") if ext_suffix is None: return expected = _suffix_to_be_checked[str(generic_target)] if fnmatch.fnmatch(ext_suffix, expected["glob"]): return # If we are here it means the current interpreter expects different names # than pristine CPython. So: # 1. Find what we have installed # 2. Create symbolic links for the other names, it they're not there already # Check if standard names are installed and if we have to create # link for this interpreter standard_extensions = fs.find(candidate_spec.prefix, expected["glob"]) link_names = [re.sub(expected["re"], ext_suffix, s) for s in standard_extensions] for file_name, link_name in zip(standard_extensions, link_names): if os.path.exists(link_name): continue os.symlink(file_name, link_name) # Check if this interpreter installed something and we have to create # links for a standard CPython interpreter non_standard_extensions = fs.find(candidate_spec.prefix, "*" + ext_suffix) for abs_path in non_standard_extensions: directory, filename = os.path.split(abs_path) module = filename.split(".")[0] link_name = os.path.join( directory, expected["fmt"].format( module=module, major=sys.version_info[0], minor=sys.version_info[1] ), ) if os.path.exists(link_name): continue os.symlink(abs_path, link_name) def _executables_in_store( executables: Sequence[str], query_spec: Union["spack.spec.Spec", str], query_info: Optional[QueryInfo] = None, ) -> bool: """Return True if at least one of the executables can be retrieved from a spec in store, False otherwise. The different executables must provide the same functionality and are "alternate" to each other, i.e. the function will exit True on the first executable found. Args: executables: list of executables to be searched query_spec: spec that may provide the executable query_info (dict or None): if a dict is passed it is populated with the command found and the concrete spec providing it """ executables_str = ", ".join(executables) msg = "[BOOTSTRAP EXECUTABLES {0}] Try installed specs with query '{1}'" tty.debug(msg.format(executables_str, query_spec)) installed_specs = spack.store.STORE.db.query(query_spec, installed=True) if installed_specs: for concrete_spec in installed_specs: bin_dir = concrete_spec.prefix.bin # IF we have a "bin" directory and it contains # the executables we are looking for if ( os.path.exists(bin_dir) and os.path.isdir(bin_dir) and spack.util.executable.which_string(*executables, path=bin_dir) ): spack.util.environment.path_put_first("PATH", [bin_dir]) if query_info is not None: query_info["command"] = spack.util.executable.which( *executables, path=bin_dir, required=True ) query_info["spec"] = concrete_spec return True return False def _root_spec(spec_str: str) -> str: """Add a proper compiler and target to a spec used during bootstrapping. Args: spec_str: spec to be bootstrapped. Must be without compiler and target. """ # Add a compiler and platform requirement to the root spec. platform = str(spack.platforms.host()) spec_str += f" platform={platform}" target = spack.vendor.archspec.cpu.host().family spec_str += f" target={target}" tty.debug(f"[BOOTSTRAP ROOT SPEC] {spec_str}") return spec_str ================================================ FILE: lib/spack/spack/bootstrap/clingo.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Bootstrap concrete specs for clingo Spack uses clingo to concretize specs. When clingo itself needs to be bootstrapped from sources, we need to rely on another mechanism to get a concrete spec that fits the current host. This module contains the logic to get a concrete spec for clingo, starting from a prototype JSON file for a similar platform. """ import pathlib import sys from typing import Dict, Optional, Tuple, Type import spack.vendor.archspec.cpu import spack.compilers.config import spack.compilers.libraries import spack.config import spack.package_base import spack.platforms import spack.repo import spack.spec import spack.traverse import spack.version from .config import spec_for_current_python def _select_best_version( pkg_cls: Type["spack.package_base.PackageBase"], node: spack.spec.Spec, valid_versions: str ) -> None: """Try to attach the best known version to a node""" constraint = spack.version.from_string(valid_versions) allowed_versions = [v for v in pkg_cls.versions if v.satisfies(constraint)] try: best_version = spack.package_base.sort_by_pkg_preference(allowed_versions, pkg=pkg_cls)[0] except (KeyError, ValueError, IndexError): return node.versions.versions = [spack.version.from_string(f"={best_version}")] def _add_compilers_if_missing() -> None: arch = spack.spec.ArchSpec.default_arch() if not spack.compilers.config.compilers_for_arch(arch): spack.compilers.config.find_compilers() class ClingoBootstrapConcretizer: def __init__(self, configuration): _add_compilers_if_missing() self.host_platform = spack.platforms.host() self.host_os = self.host_platform.default_operating_system() self.host_target = spack.vendor.archspec.cpu.host().family self.host_architecture = spack.spec.ArchSpec.default_arch() self.host_architecture.target = str(self.host_target) self.host_compiler = self._valid_compiler_or_raise() self.host_python = self.python_external_spec() if str(self.host_platform) == "linux": self.host_libc = self.libc_external_spec() self.external_cmake, self.external_bison = self._externals_from_yaml(configuration) def _valid_compiler_or_raise(self): if str(self.host_platform) == "linux": compiler_name = "gcc" elif str(self.host_platform) == "darwin": compiler_name = "apple-clang" elif str(self.host_platform) == "windows": compiler_name = "msvc" elif str(self.host_platform) == "freebsd": compiler_name = "llvm" else: raise RuntimeError(f"Cannot bootstrap clingo from sources on {self.host_platform}") candidates = [ x for x in spack.compilers.config.CompilerFactory.from_packages_yaml(spack.config.CONFIG) if x.name == compiler_name ] if not candidates: raise RuntimeError( f"Cannot find any version of {compiler_name} to bootstrap clingo from sources" ) candidates.sort(key=lambda x: x.version, reverse=True) best = candidates[0] # Get compilers for bootstrapping from the 'builtin' repository best.namespace = "builtin" # If the compiler does not support C++ 14, fail with a legible error message try: _ = best.package.standard_flag(language="cxx", standard="14") except RuntimeError as e: raise RuntimeError( "cannot find a compiler supporting C++ 14 [needed to bootstrap clingo]" ) from e return candidates[0] def _externals_from_yaml( self, configuration: "spack.config.Configuration" ) -> Tuple[Optional["spack.spec.Spec"], Optional["spack.spec.Spec"]]: packages_yaml = configuration.get("packages") requirements = {"cmake": "@3.20:", "bison": "@2.5:"} selected: Dict[str, Optional["spack.spec.Spec"]] = {"cmake": None, "bison": None} for pkg_name in ["cmake", "bison"]: if pkg_name not in packages_yaml: continue candidates = packages_yaml[pkg_name].get("externals", []) for candidate in candidates: s = spack.spec.Spec(candidate["spec"], external_path=candidate["prefix"]) if not s.satisfies(requirements[pkg_name]): continue if not s.intersects(f"arch={self.host_architecture}"): continue selected[pkg_name] = self._external_spec(s) break return selected["cmake"], selected["bison"] def prototype_path(self) -> pathlib.Path: """Path to a prototype concrete specfile for clingo""" parent_dir = pathlib.Path(__file__).parent result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-{self.host_target}.json" if str(self.host_platform) == "linux": # Using aarch64 as a fallback, since it has gnuconfig (x86_64 doesn't have it) if not result.exists(): result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-aarch64.json" elif str(self.host_platform) == "freebsd": result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-amd64.json" elif not result.exists(): raise RuntimeError(f"Cannot bootstrap clingo from sources on {self.host_platform}") return result def concretize(self) -> "spack.spec.Spec": # Read the prototype and mark it NOT concrete s = spack.spec.Spec.from_specfile(str(self.prototype_path())) s._mark_concrete(False) # These are nodes in the cmake stack, whose versions are frequently deprecated for # security reasons. In case there is no external cmake on this machine, we'll update # their versions to the most preferred, within the valid range, according to the # repository we know. to_be_updated = { pkg_name: (spack.repo.PATH.get_pkg_class(pkg_name), valid_versions) for pkg_name, valid_versions in { "ca-certificates-mozilla": ":", "openssl": "3:3", "curl": "8:8", "cmake": "3.16:3", "libiconv": "1:1", "ncurses": "6:6", "m4": "1.4", }.items() } # Tweak it to conform to the host architecture + update the version of a few dependencies for node in s.traverse(): # Clear patches, we'll compute them correctly later node.patches.clear() if "patches" in node.variants: del node.variants["patches"] node.architecture.os = str(self.host_os) node.architecture = self.host_architecture if node.name == "gcc-runtime": node.versions = self.host_compiler.versions if node.name in to_be_updated: pkg_cls, valid_versions = to_be_updated[node.name] _select_best_version(pkg_cls=pkg_cls, node=node, valid_versions=valid_versions) # Can't use re2c@3.1 with Python 3.6 if self.host_python.satisfies("@3.6"): s["re2c"].versions.versions = [spack.version.from_string("=2.2")] for edge in spack.traverse.traverse_edges([s], cover="edges"): if edge.spec.name == "python": edge.spec = self.host_python if edge.spec.name == "bison" and self.external_bison: edge.spec = self.external_bison if edge.spec.name == "cmake" and self.external_cmake: edge.spec = self.external_cmake if edge.spec.name == self.host_compiler.name: edge.spec = self.host_compiler if "libc" in edge.virtuals: edge.spec = self.host_libc spack.spec._inject_patches_variant(s) s._finalize_concretization() # Work around the fact that the installer calls Spec.dependents() and # we modified edges inconsistently return s.copy() def python_external_spec(self) -> "spack.spec.Spec": """Python external spec corresponding to the current running interpreter""" result = spack.spec.Spec(spec_for_current_python(), external_path=sys.exec_prefix) return self._external_spec(result) def libc_external_spec(self) -> "spack.spec.Spec": detector = spack.compilers.libraries.CompilerPropertyDetector(self.host_compiler) result = detector.default_libc() return self._external_spec(result) def _external_spec(self, initial_spec) -> "spack.spec.Spec": initial_spec.namespace = "builtin" initial_spec.architecture = self.host_architecture for flag_type in spack.spec.FlagMap.valid_compiler_flags(): initial_spec.compiler_flags[flag_type] = [] return spack.spec.parse_with_version_concrete(initial_spec) ================================================ FILE: lib/spack/spack/bootstrap/config.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Manage configuration swapping for bootstrapping purposes""" import contextlib import os import sys from typing import Any, Dict, Generator, MutableSequence, Sequence import spack.config import spack.environment import spack.modules import spack.paths import spack.platforms import spack.repo import spack.store import spack.util.path from spack.llnl.util import tty #: Reference counter for the bootstrapping configuration context manager _REF_COUNT = 0 def is_bootstrapping() -> bool: """Return True if we are in a bootstrapping context, False otherwise.""" return _REF_COUNT > 0 def spec_for_current_python() -> str: """For bootstrapping purposes we are just interested in the Python minor version (all patches are ABI compatible with the same minor). See: * https://www.python.org/dev/peps/pep-0513/ * https://stackoverflow.com/a/35801395/771663 """ version_str = ".".join(str(x) for x in sys.version_info[:2]) return f"python@{version_str}" def root_path() -> str: """Root of all the bootstrap related folders""" return spack.util.path.canonicalize_path( spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path) ) def store_path() -> str: """Path to the store used for bootstrapped software""" enabled = spack.config.get("bootstrap:enable", True) if not enabled: msg = 'bootstrapping is currently disabled. Use "spack bootstrap enable" to enable it' raise RuntimeError(msg) return _store_path() @contextlib.contextmanager def spack_python_interpreter() -> Generator: """Override the current configuration to set the interpreter under which Spack is currently running as the only Python external spec available. """ python_prefix = sys.exec_prefix external_python = spec_for_current_python() entry = { "buildable": False, "externals": [{"prefix": python_prefix, "spec": str(external_python)}], } with spack.config.override("packages:python::", entry): yield def _store_path() -> str: bootstrap_root_path = root_path() return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "store")) def _config_path() -> str: bootstrap_root_path = root_path() return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "config")) @contextlib.contextmanager def ensure_bootstrap_configuration() -> Generator: """Swap the current configuration for the one used to bootstrap Spack. The context manager is reference counted to ensure we don't swap multiple times if there's nested use of it in the stack. One compelling use case is bootstrapping patchelf during the bootstrap of clingo. """ global _REF_COUNT # pylint: disable=global-statement already_swapped = bool(_REF_COUNT) _REF_COUNT += 1 try: if already_swapped: yield else: with _ensure_bootstrap_configuration(): yield finally: _REF_COUNT -= 1 def _read_and_sanitize_configuration() -> Dict[str, Any]: """Read the user configuration that needs to be reused for bootstrapping and remove the entries that should not be copied over. """ # Read the "config" section but pop the install tree (the entry will not be # considered due to the use_store context manager, so it will be confusing # to have it in the configuration). config_yaml = spack.config.get("config") config_yaml.pop("install_tree", None) return { "bootstrap": spack.config.get("bootstrap"), "config": config_yaml, "repos": spack.config.get("repos"), } def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]: tty.debug("[BOOTSTRAP CONFIG SCOPE] name=_builtin") config_scopes: MutableSequence["spack.config.ConfigScope"] = [ spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS) ] configuration_paths = (spack.config.CONFIGURATION_DEFAULTS_PATH, ("bootstrap", _config_path())) for name, path in configuration_paths: generic_scope = spack.config.DirectoryConfigScope(name, path) config_scopes.append(generic_scope) msg = "[BOOTSTRAP CONFIG SCOPE] name={0}, path={1}" tty.debug(msg.format(generic_scope.name, generic_scope.path)) return config_scopes @contextlib.contextmanager def _ensure_bootstrap_configuration() -> Generator: spack.repo.PATH.repos # ensure this is instantiated from current config. spack.store.ensure_singleton_created() bootstrap_store_path = store_path() user_configuration = _read_and_sanitize_configuration() with spack.environment.no_active_environment(), spack.platforms.use_platform( spack.platforms.real_host() ), spack.config.use_configuration( # Default configuration scopes excluding command line and builtin *_bootstrap_config_scopes() ), spack.store.use_store(bootstrap_store_path, extra_data={"padded_length": 0}): spack.config.set("bootstrap", user_configuration["bootstrap"]) spack.config.set("config", user_configuration["config"]) spack.config.set("repos", user_configuration["repos"]) with spack.modules.disable_modules(), spack_python_interpreter(): yield ================================================ FILE: lib/spack/spack/bootstrap/core.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Bootstrap Spack core dependencies from binaries. This module contains logic to bootstrap software required by Spack from binaries served in the bootstrapping mirrors. The logic is quite different from an installation done from a Spack user, because of the following reasons: 1. The binaries are all compiled on the same OS for a given platform (e.g. they are compiled on ``centos7`` on ``linux``), but they will be installed and used on the host OS. They are also targeted at the most generic architecture possible. That makes the binaries difficult to reuse with other specs in an environment without ad-hoc logic. 2. Bootstrapping has a fallback procedure where we try to install software by default from the most recent binaries, and proceed to older versions of the mirror, until we try building from sources as a last resort. This allows us not to be blocked on architectures where we don't have binaries readily available, but is also not compatible with the working of environments (they don't have fallback procedures). 3. Among the binaries we have clingo, so we can't concretize that with clingo :-) 4. clingo, GnuPG and patchelf binaries need to be verified by sha256 sum (all the other binaries we might add on top of that in principle can be verified with GPG signatures). """ import copy import functools import json import os import sys import uuid from typing import Any, Callable, Dict, List, Optional, Tuple import spack.binary_distribution import spack.concretize import spack.config import spack.detection import spack.error import spack.installer_dispatch import spack.mirrors.mirror import spack.platforms import spack.spec import spack.store import spack.user_environment import spack.util.executable import spack.util.path import spack.util.spack_yaml import spack.util.url import spack.version from spack.llnl.util import tty from spack.llnl.util.lang import GroupedExceptionHandler from ._common import ( QueryInfo, _executables_in_store, _python_import, _root_spec, _try_import_from_store, ) from .clingo import ClingoBootstrapConcretizer from .config import spack_python_interpreter, spec_for_current_python #: Name of the file containing metadata about the bootstrapping source METADATA_YAML_FILENAME = "metadata.yaml" #: Whether the current platform is Windows IS_WINDOWS = sys.platform == "win32" #: Map a bootstrapper type to the corresponding class _bootstrap_methods = {} ConfigDictionary = Dict[str, Any] def bootstrapper(bootstrapper_type: str): """Decorator to register classes implementing bootstrapping methods. Args: bootstrapper_type: string identifying the class """ def _register(cls): _bootstrap_methods[bootstrapper_type] = cls return cls return _register class Bootstrapper: """Interface for "core" software bootstrappers""" config_scope_name = "" def __init__(self, conf: ConfigDictionary) -> None: self.conf = conf self.name = conf["name"] self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"]) # Check for relative paths, and turn them into absolute paths # root is the metadata_dir maybe_url = conf["info"]["url"] if spack.util.url.is_path_instead_of_url(maybe_url) and not os.path.isabs(maybe_url): maybe_url = os.path.join(self.metadata_dir, maybe_url) self.url = spack.mirrors.mirror.Mirror(maybe_url).fetch_url @property def mirror_scope(self) -> spack.config.InternalConfigScope: """Mirror scope to be pushed onto the bootstrapping configuration when using this bootstrapper. """ return spack.config.InternalConfigScope( self.config_scope_name, {"mirrors:": {self.name: self.url}} ) def try_import(self, module: str, abstract_spec_str: str) -> bool: """Try to import a Python module from a spec satisfying the abstract spec passed as argument. Args: module: Python module name to try importing abstract_spec_str: abstract spec that can provide the Python module Return: True if the Python module could be imported, False otherwise """ return False def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool: """Try to search some executables in the prefix of specs satisfying the abstract spec passed as argument. Args: executables: executables to be found abstract_spec_str: abstract spec that can provide the Python module Return: True if the executables are found, False otherwise """ return False @bootstrapper(bootstrapper_type="buildcache") class BuildcacheBootstrapper(Bootstrapper): """Install the software needed during bootstrapping from a buildcache.""" def __init__(self, conf) -> None: super().__init__(conf) self.last_search: Optional[QueryInfo] = None self.config_scope_name = f"bootstrap_buildcache-{uuid.uuid4()}" @staticmethod def _spec_and_platform( abstract_spec_str: str, ) -> Tuple[spack.spec.Spec, spack.platforms.Platform]: """Return the spec object and platform we need to use when querying the buildcache. Args: abstract_spec_str: abstract spec string we are looking for """ # Try to install from an unsigned binary cache abstract_spec = spack.spec.Spec(abstract_spec_str) # On Cray we want to use Linux binaries if available from mirrors bincache_platform = spack.platforms.real_host() return abstract_spec, bincache_platform def _read_metadata(self, package_name: str) -> Any: """Return metadata about the given package.""" json_filename = f"{package_name}.json" json_dir = self.metadata_dir json_path = os.path.join(json_dir, json_filename) with open(json_path, encoding="utf-8") as stream: data = json.load(stream) return data def _install_by_hash( self, pkg_hash: str, pkg_sha256: str, bincache_platform: spack.platforms.Platform ) -> None: with spack.platforms.use_platform(bincache_platform): query = spack.binary_distribution.BinaryCacheQuery(all_architectures=True) for match in spack.store.find([f"/{pkg_hash}"], multiple=False, query_fn=query): spack.binary_distribution.install_root_node( # allow_missing is true since when bootstrapping clingo we truncate runtime # deps such as gcc-runtime, since we link libstdc++ statically, and the other # further runtime deps are loaded by the Python interpreter. This just silences # warnings about missing dependencies. match, unsigned=True, force=True, sha256=pkg_sha256, allow_missing=True, ) def _install_and_test( self, abstract_spec: spack.spec.Spec, bincache_platform: spack.platforms.Platform, bincache_data, test_fn, ) -> bool: # Ensure we see only the buildcache being used to bootstrap with spack.config.override(self.mirror_scope): # This index is currently needed to get the compiler used to build some # specs that we know by dag hash. spack.binary_distribution.BINARY_INDEX.regenerate_spec_cache() index = spack.binary_distribution.update_cache_and_get_specs() if not index: raise RuntimeError("The binary index is empty") for item in bincache_data["verified"]: candidate_spec = item["spec"] # This will be None for things that don't depend on python python_spec = item.get("python", None) # Skip specs which are not compatible if not abstract_spec.intersects(candidate_spec): continue if python_spec is not None and not abstract_spec.intersects(f"^{python_spec}"): continue for _, pkg_hash, pkg_sha256 in item["binaries"]: self._install_by_hash(pkg_hash, pkg_sha256, bincache_platform) info: QueryInfo = {} if test_fn(query_spec=abstract_spec, query_info=info): self.last_search = info return True return False def try_import(self, module: str, abstract_spec_str: str) -> bool: info: QueryInfo test_fn, info = functools.partial(_try_import_from_store, module), {} if test_fn(query_spec=abstract_spec_str, query_info=info): return True tty.debug(f"Bootstrapping {module} from pre-built binaries") abstract_spec, bincache_platform = self._spec_and_platform( abstract_spec_str + " ^" + spec_for_current_python() ) data = self._read_metadata(module) return self._install_and_test(abstract_spec, bincache_platform, data, test_fn) def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool: info: QueryInfo test_fn, info = functools.partial(_executables_in_store, executables), {} if test_fn(query_spec=abstract_spec_str, query_info=info): self.last_search = info return True abstract_spec, bincache_platform = self._spec_and_platform(abstract_spec_str) tty.debug(f"Bootstrapping {abstract_spec.name} from pre-built binaries") data = self._read_metadata(abstract_spec.name) return self._install_and_test(abstract_spec, bincache_platform, data, test_fn) @bootstrapper(bootstrapper_type="install") class SourceBootstrapper(Bootstrapper): """Install the software needed during bootstrapping from sources.""" def __init__(self, conf) -> None: super().__init__(conf) self.last_search: Optional[QueryInfo] = None self.config_scope_name = f"bootstrap_source-{uuid.uuid4()}" def try_import(self, module: str, abstract_spec_str: str) -> bool: info: QueryInfo = {} if _try_import_from_store(module, abstract_spec_str, query_info=info): self.last_search = info return True tty.debug(f"Bootstrapping {module} from sources") # If we compile code from sources detecting a few build tools # might reduce compilation time by a fair amount _add_externals_if_missing() # Try to build and install from sources with spack_python_interpreter(): if module == "clingo": bootstrapper = ClingoBootstrapConcretizer(configuration=spack.config.CONFIG) concrete_spec = bootstrapper.concretize() else: abstract_spec = spack.spec.Spec( abstract_spec_str + " ^" + spec_for_current_python() ) concrete_spec = spack.concretize.concretize_one(abstract_spec) msg = "[BOOTSTRAP MODULE {0}] Try installing '{1}' from sources" tty.debug(msg.format(module, abstract_spec_str)) # Install the spec that should make the module importable with spack.config.override(self.mirror_scope): spack.installer_dispatch.create_installer( [concrete_spec.package], fail_fast=True, root_policy="source_only", dependencies_policy="source_only", ).install() if _try_import_from_store(module, query_spec=concrete_spec, query_info=info): self.last_search = info return True return False def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool: info: QueryInfo = {} if _executables_in_store(executables, abstract_spec_str, query_info=info): self.last_search = info return True tty.debug(f"Bootstrapping {abstract_spec_str} from sources") # If we compile code from sources detecting a few build tools # might reduce compilation time by a fair amount _add_externals_if_missing() concrete_spec = spack.concretize.concretize_one(abstract_spec_str) msg = "[BOOTSTRAP] Try installing '{0}' from sources" tty.debug(msg.format(abstract_spec_str)) with spack.config.override(self.mirror_scope): spack.installer_dispatch.create_installer([concrete_spec.package]).install() if _executables_in_store(executables, concrete_spec, query_info=info): self.last_search = info return True return False def create_bootstrapper(conf: ConfigDictionary): """Return a bootstrap object built according to the configuration argument""" btype = conf["type"] return _bootstrap_methods[btype](conf) def source_is_enabled(conf: ConfigDictionary) -> bool: """Returns true if the source is not enabled for bootstrapping""" return spack.config.get("bootstrap:trusted").get(conf["name"], False) def ensure_module_importable_or_raise(module: str, abstract_spec: Optional[str] = None): """Make the requested module available for import, or raise. This function tries to import a Python module in the current interpreter using, in order, the methods configured in bootstrap.yaml. If none of the methods succeed, an exception is raised. The function exits on first success. Args: module: module to be imported in the current interpreter abstract_spec: abstract spec that might provide the module. If not given it defaults to "module" Raises: ImportError: if the module couldn't be imported """ # If we can import it already, that's great tty.debug(f"[BOOTSTRAP MODULE {module}] Try importing from Python") if _python_import(module): return abstract_spec = abstract_spec or module exception_handler = GroupedExceptionHandler() for current_config in bootstrapping_sources(): if not source_is_enabled(current_config): continue with exception_handler.forward(current_config["name"], Exception): if create_bootstrapper(current_config).try_import(module, abstract_spec): return msg = f'cannot bootstrap the "{module}" Python module ' if abstract_spec: msg += f'from spec "{abstract_spec}" ' if not exception_handler: msg += ": no bootstrapping sources are enabled" elif spack.error.debug or spack.error.SHOW_BACKTRACE: msg += exception_handler.grouped_message(with_tracebacks=True) else: msg += exception_handler.grouped_message(with_tracebacks=False) msg += "\nRun `spack --backtrace ...` for more detailed errors" raise ImportError(msg) def ensure_executables_in_path_or_raise( executables: list, abstract_spec: str, cmd_check: Optional[Callable[[spack.util.executable.Executable], bool]] = None, ): """Ensure that some executables are in path or raise. Args: executables (list): list of executables to be searched in the PATH, in order. The function exits on the first one found. abstract_spec (str): abstract spec that provides the executables cmd_check (object): callable predicate that takes a ``spack.util.executable.Executable`` command and validate it. Should return ``True`` if the executable is acceptable, ``False`` otherwise. Can be used to, e.g., ensure a suitable version of the command before accepting for bootstrapping. Raises: RuntimeError: if the executables cannot be ensured to be in PATH Return: Executable object """ cmd = spack.util.executable.which(*executables) if cmd: if not cmd_check or cmd_check(cmd): return cmd executables_str = ", ".join(executables) exception_handler = GroupedExceptionHandler() for current_config in bootstrapping_sources(): if not source_is_enabled(current_config): continue with exception_handler.forward(current_config["name"], Exception): current_bootstrapper = create_bootstrapper(current_config) if current_bootstrapper.try_search_path(executables, abstract_spec): # Additional environment variables needed concrete_spec, cmd = ( current_bootstrapper.last_search["spec"], current_bootstrapper.last_search["command"], ) assert cmd is not None, "expected an Executable" cmd.add_default_envmod( spack.user_environment.environment_modifications_for_specs( concrete_spec, set_package_py_globals=False ) ) return cmd msg = f"cannot bootstrap any of the {executables_str} executables " if abstract_spec: msg += f'from spec "{abstract_spec}" ' if not exception_handler: msg += ": no bootstrapping sources are enabled" elif spack.error.debug or spack.error.SHOW_BACKTRACE: msg += exception_handler.grouped_message(with_tracebacks=True) else: msg += exception_handler.grouped_message(with_tracebacks=False) msg += "\nRun `spack --backtrace ...` for more detailed errors" raise RuntimeError(msg) def _add_externals_if_missing() -> None: search_list = [ # clingo "cmake", "bison", # GnuPG "gawk", # develop deps "git", ] if IS_WINDOWS: search_list.append("winbison") externals = spack.detection.by_path(search_list) # System git is typically deprecated, so mark as non-buildable to force it as external non_buildable_externals = {k: externals.pop(k) for k in ("git",) if k in externals} spack.detection.update_configuration(externals, scope="bootstrap", buildable=True) spack.detection.update_configuration( non_buildable_externals, scope="bootstrap", buildable=False ) def clingo_root_spec() -> str: """Return the root spec used to bootstrap clingo""" return _root_spec("clingo-bootstrap@spack+python") def ensure_clingo_importable_or_raise() -> None: """Ensure that the clingo module is available for import.""" ensure_module_importable_or_raise(module="clingo", abstract_spec=clingo_root_spec()) def gnupg_root_spec() -> str: """Return the root spec used to bootstrap GnuPG""" root_spec_name = "win-gpg" if IS_WINDOWS else "gnupg" return _root_spec(f"{root_spec_name}@2.3:") def ensure_gpg_in_path_or_raise() -> None: """Ensure gpg or gpg2 are in the PATH or raise.""" return ensure_executables_in_path_or_raise( executables=["gpg2", "gpg"], abstract_spec=gnupg_root_spec() ) def patchelf_root_spec() -> str: """Return the root spec used to bootstrap patchelf""" # 0.13.1 is the last version not to require C++17. return _root_spec("patchelf@0.13.1:") def verify_patchelf(patchelf: "spack.util.executable.Executable") -> bool: """Older patchelf versions can produce broken binaries, so we verify the version here. Arguments: patchelf: patchelf executable """ out = patchelf("--version", output=str, error=os.devnull, fail_on_error=False).strip() if patchelf.returncode != 0: return False parts = out.split(" ") if len(parts) < 2: return False try: version = spack.version.Version(parts[1]) except ValueError: return False return version >= spack.version.Version("0.13.1") def ensure_patchelf_in_path_or_raise() -> spack.util.executable.Executable: """Ensure patchelf is in the PATH or raise.""" # The old concretizer is not smart and we're doing its job: if the latest patchelf # does not concretize because the compiler doesn't support C++17, we try to # concretize again with an upperbound @:13. try: return ensure_executables_in_path_or_raise( executables=["patchelf"], abstract_spec=patchelf_root_spec(), cmd_check=verify_patchelf ) except RuntimeError: return ensure_executables_in_path_or_raise( executables=["patchelf"], abstract_spec=_root_spec("patchelf@0.13.1:0.13"), cmd_check=verify_patchelf, ) def ensure_winsdk_external_or_raise() -> None: """Ensure the Windows SDK + WGL are available on system If both of these package are found, the Spack user or bootstrap configuration (depending on where Spack is running) will be updated to include all versions and variants detected. If either the WDK or WSDK are not found, this method will raise a RuntimeError. **NOTE:** This modifies the Spack config in the current scope, either user or environment depending on the calling context. This is different from all other current bootstrap dependency checks. """ if set(["win-sdk", "wgl"]).issubset(spack.config.get("packages").keys()): return tty.debug("Detecting Windows SDK and WGL installations") # find the externals sequentially to avoid subprocesses being spawned externals = spack.detection.by_path(["win-sdk", "wgl"], max_workers=1) if not set(["win-sdk", "wgl"]) == externals.keys(): missing_packages_lst = [] if "wgl" not in externals: missing_packages_lst.append("wgl") if "win-sdk" not in externals: missing_packages_lst.append("win-sdk") missing_packages = " & ".join(missing_packages_lst) raise RuntimeError( f"Unable to find the {missing_packages}, please install these packages via the Visual " "Studio installer before proceeding with Spack or provide the path to a non standard " "install with 'spack external find --path'" ) # wgl/sdk are not required for bootstrapping Spack, but # are required for building anything non trivial # add to user config so they can be used by subsequent Spack ops spack.detection.update_configuration(externals, buildable=False) def ensure_core_dependencies() -> None: """Ensure the presence of all the core dependencies.""" if sys.platform.lower() == "linux": ensure_patchelf_in_path_or_raise() ensure_gpg_in_path_or_raise() ensure_clingo_importable_or_raise() def all_core_root_specs() -> List[str]: """Return a list of all the core root specs that may be used to bootstrap Spack""" return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec()] def bootstrapping_sources(scope: Optional[str] = None): """Return the list of configured sources of software for bootstrapping Spack Args: scope: if a valid configuration scope is given, return the list only from that scope """ source_configs = spack.config.get("bootstrap:sources", default=None, scope=scope) source_configs = source_configs or [] list_of_sources = [] for entry in source_configs: current = copy.copy(entry) metadata_dir = spack.util.path.canonicalize_path(entry["metadata"]) metadata_yaml = os.path.join(metadata_dir, METADATA_YAML_FILENAME) try: with open(metadata_yaml, encoding="utf-8") as stream: current.update(spack.util.spack_yaml.load(stream)) list_of_sources.append(current) except OSError: pass return list_of_sources ================================================ FILE: lib/spack/spack/bootstrap/environment.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Bootstrap non-core Spack dependencies from an environment.""" import contextlib import hashlib import os import pathlib import sys from typing import Iterable, List import spack.vendor.archspec.cpu import spack.binary_distribution import spack.config import spack.environment import spack.spec import spack.tengine import spack.util.gpg import spack.util.path from spack.llnl.util import tty from .config import root_path, spec_for_current_python, store_path from .core import _add_externals_if_missing class BootstrapEnvironment(spack.environment.Environment): """Environment to install dependencies of Spack for a given interpreter and architecture""" def __init__(self) -> None: if not self.spack_yaml().exists(): self._write_spack_yaml_file() super().__init__(self.environment_root()) # Remove python package roots created before python-venv was introduced for s in self.concrete_roots(): if "python" in s.package.extendees and not s.dependencies("python-venv"): self.deconcretize_by_hash(s.dag_hash()) @classmethod def spack_dev_requirements(cls) -> List[str]: """Spack development requirements""" return [pytest_root_spec(), ruff_root_spec(), mypy_root_spec()] @classmethod def environment_root(cls) -> pathlib.Path: """Environment root directory""" bootstrap_root_path = root_path() python_part = spec_for_current_python().replace("@", "") arch_part = spack.vendor.archspec.cpu.host().family interpreter_part = hashlib.md5(sys.exec_prefix.encode()).hexdigest()[:5] environment_dir = f"{python_part}-{arch_part}-{interpreter_part}" return pathlib.Path( spack.util.path.canonicalize_path( os.path.join(bootstrap_root_path, "environments", environment_dir) ) ) @classmethod def view_root(cls) -> pathlib.Path: """Location of the view""" return cls.environment_root().joinpath("view") @classmethod def bin_dir(cls) -> pathlib.Path: """Paths to be added to PATH""" return cls.view_root().joinpath("bin") def python_dirs(self) -> Iterable[pathlib.Path]: python = next(s for s in self.all_specs_generator() if s.name == "python-venv").package return {self.view_root().joinpath(p) for p in (python.platlib, python.purelib)} @classmethod def spack_yaml(cls) -> pathlib.Path: """Environment spack.yaml file""" return cls.environment_root().joinpath("spack.yaml") @contextlib.contextmanager def trust_bootstrap_mirror_keys(self): with spack.util.gpg.gnupghome_override(os.path.join(root_path(), ".bootstrap-gpg")): spack.binary_distribution.get_keys(install=True, trust=True) yield def update_installations(self) -> None: """Update the installations of this environment.""" log_enabled = tty.is_debug() or tty.is_verbose() with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled): specs = self.concretize() if specs: colorized_specs = [ spack.spec.Spec(x).cformat("{name}{@version}") for x in self.spack_dev_requirements() ] tty.msg(f"[BOOTSTRAPPING] Installing dependencies ({', '.join(colorized_specs)})") self.write(regenerate=False) with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled): with self.trust_bootstrap_mirror_keys(): fetch_policy = ( "cache_only" if not spack.config.get("bootstrap:dev:enable_source", False) else "auto" ) self.install_all( fail_fast=True, root_policy=fetch_policy, dependencies_policy=fetch_policy ) self.write(regenerate=True) def load(self) -> None: """Update PATH and sys.path.""" # Make executables available (shouldn't need PYTHONPATH) os.environ["PATH"] = f"{self.bin_dir()}{os.pathsep}{os.environ.get('PATH', '')}" # Spack itself imports pytest sys.path.extend(str(p) for p in self.python_dirs()) def _write_spack_yaml_file(self) -> None: tty.msg( "[BOOTSTRAPPING] Spack has missing dependencies, creating a bootstrapping environment" ) env = spack.tengine.make_environment() template = env.get_template("bootstrap/spack.yaml") context = { "python_spec": f"{spec_for_current_python()}+ctypes", "python_prefix": sys.exec_prefix, "architecture": spack.vendor.archspec.cpu.host().family, "environment_path": self.environment_root(), "environment_specs": self.spack_dev_requirements(), "store_path": store_path(), "bootstrap_mirrors": dev_bootstrap_mirror_names(), } self.environment_root().mkdir(parents=True, exist_ok=True) self.spack_yaml().write_text(template.render(context), encoding="utf-8") def mypy_root_spec() -> str: """Return the root spec used to bootstrap mypy""" return "py-mypy@0.900: ^py-mypy-extensions@:1.0" def pytest_root_spec() -> str: """Return the root spec used to bootstrap pytest""" return "py-pytest@6.2.4:" def ruff_root_spec() -> str: """Return the root spec used to bootstrap ruff""" return "py-ruff@0.15.0" def dev_bootstrap_mirror_names() -> List[str]: """Return the mirror names used for bootstrapping dev requirements""" return [ "developer-tools-darwin", "developer-tools-x86_64_v3-linux-gnu", "developer-tools-aarch64-linux-gnu", ] def ensure_environment_dependencies() -> None: """Ensure Spack dependencies from the bootstrap environment are installed and ready to use""" _add_externals_if_missing() with BootstrapEnvironment() as env: env.update_installations() env.load() ================================================ FILE: lib/spack/spack/bootstrap/prototypes/clingo-darwin-aarch64.json ================================================ {"spec":{"_meta":{"version":5},"nodes":[{"name":"clingo-bootstrap","version":"spack","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","docs":false,"generator":"make","ipo":false,"optimized":false,"python":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"bison","hash":"mtmvlzy7mfjfrcecjxk6wgjwxlqtmzpj","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"cmake","hash":"zlqbht6siyfbw65vi7eg3g2kkjbgeb3s","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"ubenpqhxb6v4lsnefcot2naoyufhtvlq","parameters":{"deptypes":["build","link","run"],"virtuals":[]}},{"name":"python-venv","hash":"4duigy4ujnstkq5c542w4okhszygw72h","parameters":{"deptypes":["build","run"],"virtuals":[]}},{"name":"re2c","hash":"uhkg474hzahckbth32ydtp24etgavq76","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"lssvl522otfuhyr7zw6rr65fpyd5eicp"},{"name":"apple-clang","version":"16.0.0","arch":{"platform":"darwin","platform_os":"sonoma","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin","parameters":{"build_system":"bundle","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{"compilers":{"c":"/usr/bin/clang","cxx":"/usr/bin/clang++"}}},"package_hash":"7iabceub7ckyfs2h5g75jxtolk253q6nm3r5hyqbztckky25gnpa====","annotations":{"original_specfile_version":5},"hash":"nea2oy52arwgstum7vyornhbnk3poj32"},{"name":"bison","version":"3.8.2","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","color":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"4p2v6lqdlosuxav6qtrmemodqnp7p7ql","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"m4","hash":"4u7ml457pqh6mzvundcjcv4xzvcjwhw3","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"mtmvlzy7mfjfrcecjxk6wgjwxlqtmzpj"},{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"darwin","platform_os":"sonoma","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====","annotations":{"original_specfile_version":5},"hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4"},{"name":"diffutils","version":"3.10","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"hvarrkkr7z5tujmt45xhns2kljvnunof","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}}],"annotations":{"original_specfile_version":5},"hash":"4p2v6lqdlosuxav6qtrmemodqnp7p7ql"},{"name":"gmake","version":"4.4.1","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","guile":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk"},{"name":"gnuconfig","version":"2024-07-27","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"aar2tabf35425kgzryprq775xycug7xlbt4rkwvm4aj76dhlychq====","annotations":{"original_specfile_version":5},"hash":"rzeea2himrnudsunposb2rlyw6mjhmr7"},{"name":"libiconv","version":"1.17","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"hvarrkkr7z5tujmt45xhns2kljvnunof"},{"name":"m4","version":"1.4.19","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573","bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89"],"sigsegv":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89","9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573"],"package_hash":"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"4p2v6lqdlosuxav6qtrmemodqnp7p7ql","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libsigsegv","hash":"j36atbspivp2gr7gnthqzakoitzzkstp","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"4u7ml457pqh6mzvundcjcv4xzvcjwhw3"},{"name":"libsigsegv","version":"2.14","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"j36atbspivp2gr7gnthqzakoitzzkstp"},{"name":"cmake","version":"3.31.2","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":true,"ownlibs":true,"qtgui":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"curl","hash":"jkilt3drjtni4pwxwvujwpasnef4xzqx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"7ikuk745stdvecdqbcrmrczgozpfwopt","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"zlqbht6siyfbw65vi7eg3g2kkjbgeb3s"},{"name":"curl","version":"8.10.1","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","gssapi":false,"ldap":false,"libidn2":false,"librtmp":false,"libs":["shared","static"],"libssh":false,"libssh2":false,"nghttp2":true,"tls":["secure_transport"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"nghttp2","hash":"7ypebulcls2zoubgisasufif4lbsvfyv","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"ub6rqfbiqdmmttgtgywljfealg3xnjmh","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"zlib-ng","hash":"7ikuk745stdvecdqbcrmrczgozpfwopt","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"jkilt3drjtni4pwxwvujwpasnef4xzqx"},{"name":"nghttp2","version":"1.64.0","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"4p2v6lqdlosuxav6qtrmemodqnp7p7ql","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"ub6rqfbiqdmmttgtgywljfealg3xnjmh","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"7ypebulcls2zoubgisasufif4lbsvfyv"},{"name":"pkgconf","version":"2.2.0","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"ub6rqfbiqdmmttgtgywljfealg3xnjmh"},{"name":"zlib-ng","version":"2.2.1","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","compat":true,"new_strategies":true,"opt":true,"pic":true,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"7ikuk745stdvecdqbcrmrczgozpfwopt"},{"name":"ncurses","version":"6.5","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"abi":"none","build_system":"autotools","patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"symlinks":false,"termlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"package_hash":"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"ub6rqfbiqdmmttgtgywljfealg3xnjmh","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t"},{"name":"python","version":"3.13.0","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","bz2":true,"ctypes":true,"dbm":true,"debug":false,"libxml2":true,"lzma":true,"optimizations":false,"pic":true,"pyexpat":true,"pythoncmd":true,"readline":true,"shared":true,"sqlite3":true,"ssl":true,"tkinter":false,"uuid":true,"zlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"apple-libuuid","hash":"c6v7lfkcgosqqnc2zkzqulybqqg4e22s","parameters":{"deptypes":["build","link"],"virtuals":["uuid"]}},{"name":"bzip2","hash":"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"expat","hash":"22rzbasmo5pkkhqri5vplsxujq3fkyzd","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gdbm","hash":"pa6dzar6nbm3muyi5wm7wdqcyjom3rrt","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gettext","hash":"bah7ymgn6hajzicktrj3d26sltksrdyf","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libffi","hash":"z533r2ia3xmkjsr2raqwtgnnwqdjhrxf","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"7etpau7yq3ctzzymjvlqo64yxykumskl","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"ub6rqfbiqdmmttgtgywljfealg3xnjmh","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"readline","hash":"vuu3chywd42dhwbavg2bqogsaqb5u2vw","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"sqlite","hash":"4g3wvdt4hvdmqqrckn5aqzhebx6cf7t3","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"xz","hash":"4aws4yus4ottcqw63gx2ppktuev2z2qw","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"7ikuk745stdvecdqbcrmrczgozpfwopt","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"ubenpqhxb6v4lsnefcot2naoyufhtvlq"},{"name":"apple-libuuid","version":"1353.100.2","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"bundle","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk","module":null,"extra_attributes":{}},"package_hash":"rv7eeukm7m2umg6ulafeco2qz2kvaqpx2bjoita6g27hrs6vfmiq====","annotations":{"original_specfile_version":5},"hash":"c6v7lfkcgosqqnc2zkzqulybqqg4e22s"},{"name":"bzip2","version":"1.0.8","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","debug":false,"pic":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"4p2v6lqdlosuxav6qtrmemodqnp7p7ql","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr"},{"name":"expat","version":"2.6.4","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","libbsd":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"22rzbasmo5pkkhqri5vplsxujq3fkyzd"},{"name":"gdbm","version":"1.23","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"vuu3chywd42dhwbavg2bqogsaqb5u2vw","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"pa6dzar6nbm3muyi5wm7wdqcyjom3rrt"},{"name":"readline","version":"8.2","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"package_hash":"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"ncurses","hash":"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"vuu3chywd42dhwbavg2bqogsaqb5u2vw"},{"name":"gettext","version":"0.22.5","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","bzip2":true,"curses":true,"git":true,"libunistring":false,"libxml2":true,"pic":true,"shared":true,"tar":true,"xz":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"bzip2","hash":"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"hvarrkkr7z5tujmt45xhns2kljvnunof","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"libxml2","hash":"zh4o2buvhkcq2ayeoww6bw6p7lpc53lr","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"tar","hash":"4hde5fjjlzaxmazsaeen6cdlejigf4ts","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"4aws4yus4ottcqw63gx2ppktuev2z2qw","parameters":{"deptypes":["build","link","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"bah7ymgn6hajzicktrj3d26sltksrdyf"},{"name":"libxml2","version":"2.13.4","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","pic":true,"python":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"hvarrkkr7z5tujmt45xhns2kljvnunof","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pkgconf","hash":"ub6rqfbiqdmmttgtgywljfealg3xnjmh","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"xz","hash":"4aws4yus4ottcqw63gx2ppktuev2z2qw","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"7ikuk745stdvecdqbcrmrczgozpfwopt","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"zh4o2buvhkcq2ayeoww6bw6p7lpc53lr"},{"name":"xz","version":"5.4.6","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"pic":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"4aws4yus4ottcqw63gx2ppktuev2z2qw"},{"name":"tar","version":"1.35","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","zip":"pigz","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"bzip2","hash":"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"hvarrkkr7z5tujmt45xhns2kljvnunof","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pigz","hash":"l3lu3os3j4yjdpdvtooaxc74ece64qy6","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"4aws4yus4ottcqw63gx2ppktuev2z2qw","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"zstd","hash":"eantpzna3rm5ccxgz3z6p4kdaqcm22lr","parameters":{"deptypes":["run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"4hde5fjjlzaxmazsaeen6cdlejigf4ts"},{"name":"pigz","version":"2.8","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"7ikuk745stdvecdqbcrmrczgozpfwopt","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"l3lu3os3j4yjdpdvtooaxc74ece64qy6"},{"name":"zstd","version":"1.5.6","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"makefile","compression":["none"],"libs":["shared","static"],"programs":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"eantpzna3rm5ccxgz3z6p4kdaqcm22lr"},{"name":"libffi","version":"3.4.6","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"z533r2ia3xmkjsr2raqwtgnnwqdjhrxf"},{"name":"openssl","version":"3.4.0","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","certs":"mozilla","docs":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"ca-certificates-mozilla","hash":"v5ocihzb43rasxqelwzx4o3htu4xavgu","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"perl","hash":"wc57ocdnd6w5f5apre2ywlaca3mcvgks","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"7ikuk745stdvecdqbcrmrczgozpfwopt","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"7etpau7yq3ctzzymjvlqo64yxykumskl"},{"name":"ca-certificates-mozilla","version":"2023-05-30","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====","annotations":{"original_specfile_version":5},"hash":"v5ocihzb43rasxqelwzx4o3htu4xavgu"},{"name":"perl","version":"5.40.0","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cpanm":true,"opcode":true,"open":true,"shared":true,"threads":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"berkeley-db","hash":"cwabnmpcy7hllcma4zyjhwi2mhxcrfti","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"bzip2","hash":"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gdbm","hash":"pa6dzar6nbm3muyi5wm7wdqcyjom3rrt","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"7ikuk745stdvecdqbcrmrczgozpfwopt","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"wc57ocdnd6w5f5apre2ywlaca3mcvgks"},{"name":"berkeley-db","version":"18.1.40","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cxx":true,"docs":false,"patches":["26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3","b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522"],"stl":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522","26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3"],"package_hash":"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"cwabnmpcy7hllcma4zyjhwi2mhxcrfti"},{"name":"sqlite","version":"3.46.0","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","column_metadata":true,"dynamic_extensions":true,"fts":true,"functions":false,"rtree":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"vuu3chywd42dhwbavg2bqogsaqb5u2vw","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"7ikuk745stdvecdqbcrmrczgozpfwopt","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"4g3wvdt4hvdmqqrckn5aqzhebx6cf7t3"},{"name":"python-venv","version":"1.0","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====","dependencies":[{"name":"python","hash":"ubenpqhxb6v4lsnefcot2naoyufhtvlq","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"4duigy4ujnstkq5c542w4okhszygw72h"},{"name":"re2c","version":"3.1","arch":{"platform":"darwin","platform_os":"sonoma","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====","dependencies":[{"name":"apple-clang","hash":"nea2oy52arwgstum7vyornhbnk3poj32","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"b3urggrazcghz2ngfudq7ndzyhkjstj4","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"3b47stocf6w7bbkc3yqakyrjv72ywszk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"rzeea2himrnudsunposb2rlyw6mjhmr7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"ubenpqhxb6v4lsnefcot2naoyufhtvlq","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"uhkg474hzahckbth32ydtp24etgavq76"}]}} ================================================ FILE: lib/spack/spack/bootstrap/prototypes/clingo-darwin-x86_64.json ================================================ {"spec":{"_meta":{"version":5},"nodes":[{"name":"clingo-bootstrap","version":"spack","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","docs":false,"generator":"make","ipo":false,"optimized":false,"python":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"bison","hash":"e575uqnqgn6zxpyrfphfw35vihuc3af3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"cmake","hash":"ev3zcv2blhxx2checfszy6736ya2ve45","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"3tktfceps6thsraftda3svkdlypt47vx","parameters":{"deptypes":["build","link","run"],"virtuals":[]}},{"name":"python-venv","hash":"mg6k4cfdhg6dore5avimwxdc7jn6onzs","parameters":{"deptypes":["build","run"],"virtuals":[]}},{"name":"re2c","hash":"4ym4yrdx4hfbj5rcevsdidy6zdc77om4","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"qrttp2eu44r35mtyem2njmtdo2tr5xvf"},{"name":"apple-clang","version":"16.0.0","arch":{"platform":"darwin","platform_os":"sequoia","target":{"name":"cannonlake","vendor":"GenuineIntel","features":["adx","aes","avx","avx2","avx512bw","avx512cd","avx512dq","avx512f","avx512ifma","avx512vbmi","avx512vl","bmi1","bmi2","clflushopt","f16c","fma","mmx","movbe","pclmulqdq","popcnt","rdrand","rdseed","sha","sse","sse2","sse4_1","sse4_2","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["skylake"],"cpupart":""}},"namespace":"builtin","parameters":{"build_system":"bundle","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{"compilers":{"c":"/usr/bin/clang","cxx":"/usr/bin/clang++"}}},"package_hash":"7iabceub7ckyfs2h5g75jxtolk253q6nm3r5hyqbztckky25gnpa====","annotations":{"original_specfile_version":5},"hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv"},{"name":"bison","version":"3.8.2","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","color":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"acwjalgeefeymuhyv4umstcnz465ar6e","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"m4","hash":"d36fz4p3yx77w6b272r5yr74owsvwvfm","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"e575uqnqgn6zxpyrfphfw35vihuc3af3"},{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"darwin","platform_os":"sequoia","target":{"name":"cannonlake","vendor":"GenuineIntel","features":["adx","aes","avx","avx2","avx512bw","avx512cd","avx512dq","avx512f","avx512ifma","avx512vbmi","avx512vl","bmi1","bmi2","clflushopt","f16c","fma","mmx","movbe","pclmulqdq","popcnt","rdrand","rdseed","sha","sse","sse2","sse4_1","sse4_2","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["skylake"],"cpupart":""}},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====","annotations":{"original_specfile_version":5},"hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge"},{"name":"diffutils","version":"3.10","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}}],"annotations":{"original_specfile_version":5},"hash":"acwjalgeefeymuhyv4umstcnz465ar6e"},{"name":"gmake","version":"4.4.1","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","guile":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"fszz4zptmmipakokiufglsphlmdgb6x3"},{"name":"libiconv","version":"1.17","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct"},{"name":"m4","version":"1.4.19","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573","bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89"],"sigsegv":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89","9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573"],"package_hash":"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"acwjalgeefeymuhyv4umstcnz465ar6e","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libsigsegv","hash":"mtowncjrriz2jjl6onql3wyacciix4ne","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"d36fz4p3yx77w6b272r5yr74owsvwvfm"},{"name":"libsigsegv","version":"2.14","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"mtowncjrriz2jjl6onql3wyacciix4ne"},{"name":"cmake","version":"3.31.2","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":true,"ownlibs":true,"qtgui":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"curl","hash":"uugvk6k3zupw4xzto2hwjfe647pqsyyf","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"nykye5s3jvzc2zwtpx4xljlos6xnorsw","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"l2fyfx2t7sesnitglbumuds2wqflfir6","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"ev3zcv2blhxx2checfszy6736ya2ve45"},{"name":"curl","version":"8.10.1","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","gssapi":false,"ldap":false,"libidn2":false,"librtmp":false,"libs":["shared","static"],"libssh":false,"libssh2":false,"nghttp2":true,"tls":["secure_transport"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"nghttp2","hash":"bm2f7poacyin2wyvgq2axmbynhaslhgb","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"e65fgchge7g22kbiqdpyxu4fmvqehlqb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"zlib-ng","hash":"l2fyfx2t7sesnitglbumuds2wqflfir6","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"uugvk6k3zupw4xzto2hwjfe647pqsyyf"},{"name":"nghttp2","version":"1.64.0","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"acwjalgeefeymuhyv4umstcnz465ar6e","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"e65fgchge7g22kbiqdpyxu4fmvqehlqb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"bm2f7poacyin2wyvgq2axmbynhaslhgb"},{"name":"pkgconf","version":"2.2.0","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"e65fgchge7g22kbiqdpyxu4fmvqehlqb"},{"name":"zlib-ng","version":"2.2.1","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","compat":true,"new_strategies":true,"opt":true,"pic":true,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"l2fyfx2t7sesnitglbumuds2wqflfir6"},{"name":"ncurses","version":"6.5","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"abi":"none","build_system":"autotools","patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"symlinks":false,"termlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"package_hash":"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"e65fgchge7g22kbiqdpyxu4fmvqehlqb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"nykye5s3jvzc2zwtpx4xljlos6xnorsw"},{"name":"python","version":"3.13.0","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","bz2":true,"ctypes":true,"dbm":true,"debug":false,"libxml2":true,"lzma":true,"optimizations":false,"pic":true,"pyexpat":true,"pythoncmd":true,"readline":true,"shared":true,"sqlite3":true,"ssl":true,"tkinter":false,"uuid":true,"zlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"apple-libuuid","hash":"m5z7kt64hlhnwisipfs5nqqorpi6u6vm","parameters":{"deptypes":["build","link"],"virtuals":["uuid"]}},{"name":"bzip2","hash":"h4bmmb7myvboscsbvlgq46twwacgahk3","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"expat","hash":"bqc34odirjhz2jaue7n3dk7sux6hmojn","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gdbm","hash":"h4qbc6g2v5yotoalpyvddbcmqyric4v7","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gettext","hash":"m5uhx7get2eeuftgoadtv7r2vfh7u5ds","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libffi","hash":"ofcuigaaxlvvv6tgzetfjwfhabzlkbo7","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"nykye5s3jvzc2zwtpx4xljlos6xnorsw","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"bwnlb7yiy67eabhgaei64susdxgas3to","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"e65fgchge7g22kbiqdpyxu4fmvqehlqb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"readline","hash":"5wymz4fw6majnuwaoopp3m7dmjqbbvrx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"sqlite","hash":"qtxu5k6vkkyr2oii62lz7r4ubs7sz3xq","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"xz","hash":"oxzkcdzjdywney64q6tnmmjib33u6ms7","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"l2fyfx2t7sesnitglbumuds2wqflfir6","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"3tktfceps6thsraftda3svkdlypt47vx"},{"name":"apple-libuuid","version":"1353.100.2","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"bundle","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk","module":null,"extra_attributes":{}},"package_hash":"rv7eeukm7m2umg6ulafeco2qz2kvaqpx2bjoita6g27hrs6vfmiq====","annotations":{"original_specfile_version":5},"hash":"m5z7kt64hlhnwisipfs5nqqorpi6u6vm"},{"name":"bzip2","version":"1.0.8","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","debug":false,"pic":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"acwjalgeefeymuhyv4umstcnz465ar6e","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"h4bmmb7myvboscsbvlgq46twwacgahk3"},{"name":"expat","version":"2.6.4","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","libbsd":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"bqc34odirjhz2jaue7n3dk7sux6hmojn"},{"name":"gdbm","version":"1.23","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"5wymz4fw6majnuwaoopp3m7dmjqbbvrx","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"h4qbc6g2v5yotoalpyvddbcmqyric4v7"},{"name":"readline","version":"8.2","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"package_hash":"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"ncurses","hash":"nykye5s3jvzc2zwtpx4xljlos6xnorsw","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"5wymz4fw6majnuwaoopp3m7dmjqbbvrx"},{"name":"gettext","version":"0.22.5","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","bzip2":true,"curses":true,"git":true,"libunistring":false,"libxml2":true,"pic":true,"shared":true,"tar":true,"xz":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"bzip2","hash":"h4bmmb7myvboscsbvlgq46twwacgahk3","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"libxml2","hash":"patsnnun4o2w3vupeontcjecxeoyh2js","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"nykye5s3jvzc2zwtpx4xljlos6xnorsw","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"tar","hash":"tegwze36okijyiui4nbbnkn2ngkqmxlm","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"oxzkcdzjdywney64q6tnmmjib33u6ms7","parameters":{"deptypes":["build","link","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"m5uhx7get2eeuftgoadtv7r2vfh7u5ds"},{"name":"libxml2","version":"2.13.4","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","pic":true,"python":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pkgconf","hash":"e65fgchge7g22kbiqdpyxu4fmvqehlqb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"xz","hash":"oxzkcdzjdywney64q6tnmmjib33u6ms7","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"l2fyfx2t7sesnitglbumuds2wqflfir6","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"patsnnun4o2w3vupeontcjecxeoyh2js"},{"name":"xz","version":"5.4.6","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"pic":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"oxzkcdzjdywney64q6tnmmjib33u6ms7"},{"name":"tar","version":"1.35","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","zip":"pigz","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"bzip2","hash":"h4bmmb7myvboscsbvlgq46twwacgahk3","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pigz","hash":"rv3drbcskvc7snlhqex2byavaddd6xfy","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"oxzkcdzjdywney64q6tnmmjib33u6ms7","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"zstd","hash":"isadyc5phrez7pmz4spx4zly5wu5pslt","parameters":{"deptypes":["run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"tegwze36okijyiui4nbbnkn2ngkqmxlm"},{"name":"pigz","version":"2.8","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"l2fyfx2t7sesnitglbumuds2wqflfir6","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"rv3drbcskvc7snlhqex2byavaddd6xfy"},{"name":"zstd","version":"1.5.6","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"makefile","compression":["none"],"libs":["shared","static"],"programs":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"isadyc5phrez7pmz4spx4zly5wu5pslt"},{"name":"libffi","version":"3.4.6","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"ofcuigaaxlvvv6tgzetfjwfhabzlkbo7"},{"name":"openssl","version":"3.4.0","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","certs":"mozilla","docs":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"ca-certificates-mozilla","hash":"xinl4agw3xhagk74cw2pclmlbqoq223j","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"perl","hash":"wxl5qpzezncbick5ygjx3fnqwpd3ousb","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"l2fyfx2t7sesnitglbumuds2wqflfir6","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"bwnlb7yiy67eabhgaei64susdxgas3to"},{"name":"ca-certificates-mozilla","version":"2023-05-30","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====","annotations":{"original_specfile_version":5},"hash":"xinl4agw3xhagk74cw2pclmlbqoq223j"},{"name":"perl","version":"5.40.0","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","cpanm":true,"opcode":true,"open":true,"shared":true,"threads":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"berkeley-db","hash":"wamcmlsv3jtpzy7qvmfful4fabex5q7y","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"bzip2","hash":"h4bmmb7myvboscsbvlgq46twwacgahk3","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gdbm","hash":"h4qbc6g2v5yotoalpyvddbcmqyric4v7","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"l2fyfx2t7sesnitglbumuds2wqflfir6","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"wxl5qpzezncbick5ygjx3fnqwpd3ousb"},{"name":"berkeley-db","version":"18.1.40","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cxx":true,"docs":false,"patches":["26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3","b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522"],"stl":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522","26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3"],"package_hash":"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"wamcmlsv3jtpzy7qvmfful4fabex5q7y"},{"name":"sqlite","version":"3.46.0","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","column_metadata":true,"dynamic_extensions":true,"fts":true,"functions":false,"rtree":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"5wymz4fw6majnuwaoopp3m7dmjqbbvrx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"l2fyfx2t7sesnitglbumuds2wqflfir6","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"qtxu5k6vkkyr2oii62lz7r4ubs7sz3xq"},{"name":"python-venv","version":"1.0","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====","dependencies":[{"name":"python","hash":"3tktfceps6thsraftda3svkdlypt47vx","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"mg6k4cfdhg6dore5avimwxdc7jn6onzs"},{"name":"re2c","version":"3.1","arch":{"platform":"darwin","platform_os":"sequoia","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====","dependencies":[{"name":"apple-clang","hash":"qj3zadkktznahfizazbfvmqvkhzd4bqv","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"compiler-wrapper","hash":"2uitb26t2s6nfpj244fbsh7gntsiwvge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"fszz4zptmmipakokiufglsphlmdgb6x3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"3tktfceps6thsraftda3svkdlypt47vx","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"4ym4yrdx4hfbj5rcevsdidy6zdc77om4"}]}} ================================================ FILE: lib/spack/spack/bootstrap/prototypes/clingo-freebsd-amd64.json ================================================ {"spec":{"_meta":{"version":5},"nodes":[{"name":"clingo-bootstrap","version":"spack","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","docs":false,"generator":"make","ipo":false,"optimized":false,"python":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====","dependencies":[{"name":"bison","hash":"l4llzdyliqbeor66ht54qkezfdofmwj6","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"cmake","hash":"okz75726c4grndc4kadvpivfbr6546ud","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"python","hash":"syeuozebaclogvjl7izswkitiduyniob","parameters":{"deptypes":["build","link","run"],"virtuals":[]}},{"name":"python-venv","hash":"nw53taerhuinrvwfc6gcg4hztg77dkq5","parameters":{"deptypes":["build","run"],"virtuals":[]}},{"name":"re2c","hash":"prd7dmeald2bitrpbt6cqdcslfap5aay","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"je2szed32t2zoajsczveb4bokeitrcan"},{"name":"bison","version":"3.8.2","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","color":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"nk5z5kralivpxqazpvgmxvqdm73mimpx","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"m4","hash":"hoq7tejwrsetrepd4kjww3yvxfraycsa","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"l4llzdyliqbeor66ht54qkezfdofmwj6"},{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====","annotations":{"original_specfile_version":5},"hash":"6o4jkave5ri3ooytknfil4p55ifcwxju"},{"name":"diffutils","version":"3.10","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"nnr7brz74vmypk3gfhyykql5rvshhxiu","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}}],"annotations":{"original_specfile_version":5},"hash":"nk5z5kralivpxqazpvgmxvqdm73mimpx"},{"name":"gmake","version":"4.4.1","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","guile":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}}],"annotations":{"original_specfile_version":5},"hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3"},{"name":"llvm","version":"18.1.5","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","clang":true,"compiler-rt":"runtime","cuda":false,"flang":false,"generator":"ninja","gold":true,"libcxx":"runtime","libomptarget":true,"libomptarget_debug":false,"libunwind":"runtime","link_llvm_dylib":false,"lld":false,"lldb":true,"llvm_dylib":true,"lua":true,"mlir":false,"openmp":"runtime","polly":true,"python":false,"shlib_symbol_version":"none","split_dwarf":false,"targets":["all"],"version_suffix":"none","z3":false,"zstd":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{"compilers":{"c":"/usr/bin/clang","cxx":"/usr/bin/clang++"}}},"package_hash":"7iourbijxpsp23e2wj3fel2fmmk23jzyzidcpqdgeux7g7ff2wxq====","annotations":{"original_specfile_version":5},"hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg"},{"name":"libiconv","version":"1.17","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}}],"annotations":{"original_specfile_version":5},"hash":"nnr7brz74vmypk3gfhyykql5rvshhxiu"},{"name":"m4","version":"1.4.19","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573","bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89"],"sigsegv":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89","9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573"],"package_hash":"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"nk5z5kralivpxqazpvgmxvqdm73mimpx","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libsigsegv","hash":"guzz5zr4juvhrq4pqxnibvoma5z3djfi","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}}],"annotations":{"original_specfile_version":5},"hash":"hoq7tejwrsetrepd4kjww3yvxfraycsa"},{"name":"libsigsegv","version":"2.14","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}}],"annotations":{"original_specfile_version":5},"hash":"guzz5zr4juvhrq4pqxnibvoma5z3djfi"},{"name":"cmake","version":"3.31.2","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":true,"ownlibs":true,"qtgui":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"curl","hash":"rkyymoo7xqnswutyvauf3iv5dddmaygt","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"ncurses","hash":"p2m3nzytg5lh6474vclnqtklvk6jpqos","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"okz75726c4grndc4kadvpivfbr6546ud"},{"name":"curl","version":"8.10.1","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","gssapi":false,"ldap":false,"libidn2":false,"librtmp":false,"libs":["shared","static"],"libssh":false,"libssh2":false,"nghttp2":true,"tls":["openssl"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"nghttp2","hash":"uuslnsztro7in3mxykjmrolg2wfdoyat","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"c6ojqefenrbxkupgaqznti6q2x3g22qf","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"yc2rz24ll3ulloccgxroltp5243csskb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"zlib-ng","hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"rkyymoo7xqnswutyvauf3iv5dddmaygt"},{"name":"nghttp2","version":"1.64.0","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"nk5z5kralivpxqazpvgmxvqdm73mimpx","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"pkgconf","hash":"yc2rz24ll3ulloccgxroltp5243csskb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"uuslnsztro7in3mxykjmrolg2wfdoyat"},{"name":"pkgconf","version":"2.2.0","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}}],"annotations":{"original_specfile_version":5},"hash":"yc2rz24ll3ulloccgxroltp5243csskb"},{"name":"openssl","version":"3.4.0","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","certs":"mozilla","docs":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====","dependencies":[{"name":"ca-certificates-mozilla","hash":"hm3nrr2yydcptn7fvphwvg6bwyo75bwf","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"perl","hash":"sadirf62yvikut4yghjhph6o5tztfwao","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"c6ojqefenrbxkupgaqznti6q2x3g22qf"},{"name":"ca-certificates-mozilla","version":"2023-05-30","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====","annotations":{"original_specfile_version":5},"hash":"hm3nrr2yydcptn7fvphwvg6bwyo75bwf"},{"name":"perl","version":"5.40.0","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","cpanm":true,"opcode":true,"open":true,"shared":true,"threads":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====","dependencies":[{"name":"berkeley-db","hash":"vncqfho5tjvizrhfpr4vft5nfyawkhw2","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"bzip2","hash":"utn5hm325756qkbf3ve5na2qtac7zxc5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gdbm","hash":"ktpz7bar56pafbw2ab5rerdejfwnngjd","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"zlib-ng","hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"sadirf62yvikut4yghjhph6o5tztfwao"},{"name":"berkeley-db","version":"18.1.40","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cxx":true,"docs":false,"patches":["26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3","b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522"],"stl":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522","26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3"],"package_hash":"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}}],"annotations":{"original_specfile_version":5},"hash":"vncqfho5tjvizrhfpr4vft5nfyawkhw2"},{"name":"bzip2","version":"1.0.8","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","debug":false,"pic":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"nk5z5kralivpxqazpvgmxvqdm73mimpx","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}}],"annotations":{"original_specfile_version":5},"hash":"utn5hm325756qkbf3ve5na2qtac7zxc5"},{"name":"gdbm","version":"1.23","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"readline","hash":"nixpi6ugx6vmxbxln5ceyqxnu2sypnlx","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"ktpz7bar56pafbw2ab5rerdejfwnngjd"},{"name":"readline","version":"8.2","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"package_hash":"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"ncurses","hash":"p2m3nzytg5lh6474vclnqtklvk6jpqos","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"nixpi6ugx6vmxbxln5ceyqxnu2sypnlx"},{"name":"ncurses","version":"6.5","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"abi":"none","build_system":"autotools","patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"symlinks":false,"termlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"package_hash":"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"pkgconf","hash":"yc2rz24ll3ulloccgxroltp5243csskb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"p2m3nzytg5lh6474vclnqtklvk6jpqos"},{"name":"zlib-ng","version":"2.2.1","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","compat":true,"new_strategies":true,"opt":true,"pic":true,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}}],"annotations":{"original_specfile_version":5},"hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix"},{"name":"python","version":"3.13.0","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","bz2":true,"ctypes":true,"dbm":true,"debug":false,"libxml2":true,"lzma":true,"optimizations":false,"pic":true,"pyexpat":true,"pythoncmd":true,"readline":true,"shared":true,"sqlite3":true,"ssl":true,"tkinter":false,"uuid":true,"zlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====","dependencies":[{"name":"bzip2","hash":"utn5hm325756qkbf3ve5na2qtac7zxc5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"expat","hash":"djhfx5nxzsatwcklt743hizybmgvq75l","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gdbm","hash":"ktpz7bar56pafbw2ab5rerdejfwnngjd","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gettext","hash":"nfyjnvifb6n3v55esjgk7rinnq6e7av2","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libffi","hash":"657brzxsad4zh6ajeiriuatlxaco5beg","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"ncurses","hash":"p2m3nzytg5lh6474vclnqtklvk6jpqos","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"c6ojqefenrbxkupgaqznti6q2x3g22qf","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"yc2rz24ll3ulloccgxroltp5243csskb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"readline","hash":"nixpi6ugx6vmxbxln5ceyqxnu2sypnlx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"sqlite","hash":"dcqokkasxhtuu7g7htoi2v5btc2b63qf","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"util-linux-uuid","hash":"5u5klk6jrayvbilllhrlbszendi5liip","parameters":{"deptypes":["build","link"],"virtuals":["uuid"]}},{"name":"xz","hash":"6cqtdj22u47rdbvycoylphh7d6jbrvq4","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"syeuozebaclogvjl7izswkitiduyniob"},{"name":"expat","version":"2.6.4","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","libbsd":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}}],"annotations":{"original_specfile_version":5},"hash":"djhfx5nxzsatwcklt743hizybmgvq75l"},{"name":"gettext","version":"0.22.5","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","bzip2":true,"curses":true,"git":true,"libunistring":false,"libxml2":true,"pic":true,"shared":true,"tar":true,"xz":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====","dependencies":[{"name":"bzip2","hash":"utn5hm325756qkbf3ve5na2qtac7zxc5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"nnr7brz74vmypk3gfhyykql5rvshhxiu","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"libxml2","hash":"gkoikjiianqwi3r7ynsrj5kczj36mufp","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"ncurses","hash":"p2m3nzytg5lh6474vclnqtklvk6jpqos","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"tar","hash":"pygs7gph2cxutw2jktsvex3vxb2nl7hl","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"6cqtdj22u47rdbvycoylphh7d6jbrvq4","parameters":{"deptypes":["build","link","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"nfyjnvifb6n3v55esjgk7rinnq6e7av2"},{"name":"libxml2","version":"2.13.4","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","pic":true,"python":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"nnr7brz74vmypk3gfhyykql5rvshhxiu","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"pkgconf","hash":"yc2rz24ll3ulloccgxroltp5243csskb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"xz","hash":"6cqtdj22u47rdbvycoylphh7d6jbrvq4","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"gkoikjiianqwi3r7ynsrj5kczj36mufp"},{"name":"xz","version":"5.4.6","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"pic":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}}],"annotations":{"original_specfile_version":5},"hash":"6cqtdj22u47rdbvycoylphh7d6jbrvq4"},{"name":"tar","version":"1.35","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","zip":"pigz","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====","dependencies":[{"name":"bzip2","hash":"utn5hm325756qkbf3ve5na2qtac7zxc5","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"nnr7brz74vmypk3gfhyykql5rvshhxiu","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"pigz","hash":"f5jym2egytrgpubdtunmqolh7ioaaudm","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"6cqtdj22u47rdbvycoylphh7d6jbrvq4","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"zstd","hash":"7niz2hlqarxclxncsbbzl7zx5uo3btrq","parameters":{"deptypes":["run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"pygs7gph2cxutw2jktsvex3vxb2nl7hl"},{"name":"pigz","version":"2.8","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"zlib-ng","hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"f5jym2egytrgpubdtunmqolh7ioaaudm"},{"name":"zstd","version":"1.5.6","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"makefile","compression":["none"],"libs":["shared","static"],"programs":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}}],"annotations":{"original_specfile_version":5},"hash":"7niz2hlqarxclxncsbbzl7zx5uo3btrq"},{"name":"libffi","version":"3.4.6","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}}],"annotations":{"original_specfile_version":5},"hash":"657brzxsad4zh6ajeiriuatlxaco5beg"},{"name":"sqlite","version":"3.46.0","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","column_metadata":true,"dynamic_extensions":true,"fts":true,"functions":false,"rtree":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"readline","hash":"nixpi6ugx6vmxbxln5ceyqxnu2sypnlx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"c2jgry3yzjofxxjuqjckluoqbcm5exix","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"dcqokkasxhtuu7g7htoi2v5btc2b63qf"},{"name":"util-linux-uuid","version":"2.40.2","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"eo6au7zhsz344imzoomhuskbl3cmrqq6ja6mcmrc3li3fnppqs6q====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"pkgconf","hash":"yc2rz24ll3ulloccgxroltp5243csskb","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"5u5klk6jrayvbilllhrlbszendi5liip"},{"name":"python-venv","version":"1.0","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====","dependencies":[{"name":"python","hash":"syeuozebaclogvjl7izswkitiduyniob","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"nw53taerhuinrvwfc6gcg4hztg77dkq5"},{"name":"re2c","version":"3.1","arch":{"platform":"freebsd","platform_os":"freebsd14.1","target":"amd64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====","dependencies":[{"name":"compiler-wrapper","hash":"6o4jkave5ri3ooytknfil4p55ifcwxju","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"clafylgtxlepfvfrhjfqgfg2fc52vho3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"llvm","hash":"ujjiokwbw55sm7o6zoajb3xtcs65utxg","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"python","hash":"syeuozebaclogvjl7izswkitiduyniob","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"prd7dmeald2bitrpbt6cqdcslfap5aay"}]}} ================================================ FILE: lib/spack/spack/bootstrap/prototypes/clingo-linux-aarch64.json ================================================ {"spec":{"_meta":{"version":5},"nodes":[{"name":"clingo-bootstrap","version":"spack","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","docs":false,"generator":"make","ipo":false,"optimized":false,"python":true,"static_libstdcpp":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====","dependencies":[{"name":"bison","hash":"smnn2cumnp72tnrnnr6igudxyvtriqdk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"cmake","hash":"ltrb7aes3hwdnz27nzndzsmbv2vnw6wy","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"gygxuzpdf33jg2ya6imlbn4bd5zghbcd","parameters":{"deptypes":["build","link","run"],"virtuals":[]}},{"name":"python-venv","hash":"hf5bgjk6fsdycb4zovjap4t4g6tjfcvx","parameters":{"deptypes":["build","run"],"virtuals":[]}},{"name":"re2c","hash":"eavspn7qgilrfiby4v6in34pmjg5le6b","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"fxkanrgnzq7yhegi7z5de6ax7i5dablo"},{"name":"bison","version":"3.8.2","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","color":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"kntg5epaheq5s2cpiqskcfu3do6nikge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"m4","hash":"jjtr2n3inumkcqn26fnznvt3ek5ddknd","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"smnn2cumnp72tnrnnr6igudxyvtriqdk"},{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"linux","platform_os":"rhel9","target":{"name":"neoverse_v2","vendor":"ARM","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","bf16","cpuid","crc32","dcpodp","dcpop","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","i8mm","ilrcpc","jscvt","lrcpc","pmull","sb","sha1","sha2","sha3","sha512","sve","sve2","svebf16","svei8mm","uscat"],"generation":0,"parents":["neoverse_n1","armv9.0a"],"cpupart":"0xd4f"}},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====","annotations":{"original_specfile_version":5},"hash":"rximc5jq3c544fhhnloem4mbccot26tv"},{"name":"diffutils","version":"3.10","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"bqcb2qmnv3vsz5u7b3whbrortoieu6bx","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}}],"annotations":{"original_specfile_version":5},"hash":"kntg5epaheq5s2cpiqskcfu3do6nikge"},{"name":"gcc","version":"8.5.0","arch":{"platform":"linux","platform_os":"rhel9","target":{"name":"neoverse_v2","vendor":"ARM","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","bf16","cpuid","crc32","dcpodp","dcpop","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","i8mm","ilrcpc","jscvt","lrcpc","pmull","sb","sha1","sha2","sha3","sha512","sve","sve2","svebf16","svei8mm","uscat"],"generation":0,"parents":["neoverse_n1","armv9.0a"],"cpupart":"0xd4f"}},"namespace":"builtin","parameters":{"binutils":false,"bootstrap":true,"build_system":"autotools","build_type":"RelWithDebInfo","graphite":false,"languages":["c","c++","fortran"],"nvptx":false,"patches":["98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76","d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56"],"piclibs":false,"strip":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{"compilers":{"c":"/usr/bin/gcc","cxx":"/usr/bin/g++"}}},"patches":["98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76","d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56"],"package_hash":"hnbtowhwympdfoqukgir3chmkqzzasrgcwxbot7im4bncvqtxvxq====","annotations":{"original_specfile_version":5},"hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol"},{"name":"gcc-runtime","version":"8.5.0","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"aud4d72goxupc5p3p6mdkwgtshpygn7uuj2ewx3zm6wudcgw4fzq====","dependencies":[{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}}],"annotations":{"original_specfile_version":5},"hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj"},{"name":"glibc","version":"2.34","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{}},"package_hash":"4z35ntbdhytzlhaviffrorrqxvspd6k6jf3pqj7gbday4c2hld5q====","annotations":{"original_specfile_version":5},"hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh"},{"name":"gmake","version":"4.4.1","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","guile":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}}],"annotations":{"original_specfile_version":5},"hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld"},{"name":"gnuconfig","version":"2024-07-27","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"aar2tabf35425kgzryprq775xycug7xlbt4rkwvm4aj76dhlychq====","annotations":{"original_specfile_version":5},"hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh"},{"name":"libiconv","version":"1.17","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"bqcb2qmnv3vsz5u7b3whbrortoieu6bx"},{"name":"m4","version":"1.4.19","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573","bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89"],"sigsegv":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89","9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573"],"package_hash":"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"kntg5epaheq5s2cpiqskcfu3do6nikge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libsigsegv","hash":"fkqxgj3yfnk4vl3iczancsoq5yc2bgye","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"jjtr2n3inumkcqn26fnznvt3ek5ddknd"},{"name":"libsigsegv","version":"2.14","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"fkqxgj3yfnk4vl3iczancsoq5yc2bgye"},{"name":"cmake","version":"3.31.2","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":true,"ownlibs":true,"qtgui":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"curl","hash":"ntpxjnhrnsjzadlmrkier3pqoxqpng3t","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"d7rkispaw64fota6iabiom2hbawedpgj","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"ltrb7aes3hwdnz27nzndzsmbv2vnw6wy"},{"name":"curl","version":"8.10.1","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","gssapi":false,"ldap":false,"libidn2":false,"librtmp":false,"libs":["shared","static"],"libssh":false,"libssh2":false,"nghttp2":true,"tls":["openssl"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"nghttp2","hash":"cznkg4nmmy62b3zdogggospnuuy3g5pc","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"h4j2u76c7rhqompivzi4whe4hjw3cze7","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"mxoabqjj7kluh3md2xo4qyof524orfwl","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"zlib-ng","hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"ntpxjnhrnsjzadlmrkier3pqoxqpng3t"},{"name":"nghttp2","version":"1.64.0","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"kntg5epaheq5s2cpiqskcfu3do6nikge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"mxoabqjj7kluh3md2xo4qyof524orfwl","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"cznkg4nmmy62b3zdogggospnuuy3g5pc"},{"name":"pkgconf","version":"2.2.0","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"mxoabqjj7kluh3md2xo4qyof524orfwl"},{"name":"openssl","version":"3.4.0","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","certs":"mozilla","docs":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====","dependencies":[{"name":"ca-certificates-mozilla","hash":"qeszxs4rv5nw7zezjc3524ztgkoz33ig","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"perl","hash":"4tn6es2ac3gd2dsnvskwle4etlpk6qv3","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"h4j2u76c7rhqompivzi4whe4hjw3cze7"},{"name":"ca-certificates-mozilla","version":"2023-05-30","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====","annotations":{"original_specfile_version":5},"hash":"qeszxs4rv5nw7zezjc3524ztgkoz33ig"},{"name":"perl","version":"5.40.0","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cpanm":true,"opcode":true,"open":true,"shared":true,"threads":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====","dependencies":[{"name":"berkeley-db","hash":"tmpewsx4vcxbciz63y3sjwqld577hzom","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"bzip2","hash":"gefii4i45qgge6oeyibc4a6neierycc5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"gdbm","hash":"3tpau5775md4363pqnphjbr2ufir6rno","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"4tn6es2ac3gd2dsnvskwle4etlpk6qv3"},{"name":"berkeley-db","version":"18.1.40","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cxx":true,"docs":false,"patches":["26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3","b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522"],"stl":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522","26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3"],"package_hash":"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"tmpewsx4vcxbciz63y3sjwqld577hzom"},{"name":"bzip2","version":"1.0.8","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","debug":false,"pic":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"kntg5epaheq5s2cpiqskcfu3do6nikge","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"gefii4i45qgge6oeyibc4a6neierycc5"},{"name":"gdbm","version":"1.23","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"6z2sfif7stzpfvb54eoqiiki5edutguc","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"3tpau5775md4363pqnphjbr2ufir6rno"},{"name":"readline","version":"8.2","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"package_hash":"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"ncurses","hash":"d7rkispaw64fota6iabiom2hbawedpgj","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"6z2sfif7stzpfvb54eoqiiki5edutguc"},{"name":"ncurses","version":"6.5","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"abi":"none","build_system":"autotools","patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"symlinks":false,"termlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"package_hash":"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"mxoabqjj7kluh3md2xo4qyof524orfwl","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"d7rkispaw64fota6iabiom2hbawedpgj"},{"name":"zlib-ng","version":"2.2.1","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","compat":true,"new_strategies":true,"opt":true,"pic":true,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53"},{"name":"python","version":"3.13.0","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","bz2":true,"ctypes":true,"dbm":true,"debug":false,"libxml2":true,"lzma":true,"optimizations":false,"pic":true,"pyexpat":true,"pythoncmd":true,"readline":true,"shared":true,"sqlite3":true,"ssl":true,"tkinter":false,"uuid":true,"zlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====","dependencies":[{"name":"bzip2","hash":"gefii4i45qgge6oeyibc4a6neierycc5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"expat","hash":"hjwfi4iuk7mecmsuh75z74wycjlw7lzi","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"gdbm","hash":"3tpau5775md4363pqnphjbr2ufir6rno","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gettext","hash":"hheyf7ak3sjcfohvfgegvdded4wppbvr","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libffi","hash":"qegr7o5zly6cqypzzsm7s6hxcwqsgtqj","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"d7rkispaw64fota6iabiom2hbawedpgj","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"h4j2u76c7rhqompivzi4whe4hjw3cze7","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"mxoabqjj7kluh3md2xo4qyof524orfwl","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"readline","hash":"6z2sfif7stzpfvb54eoqiiki5edutguc","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"sqlite","hash":"kz4n2vtbxcj72s2teh2g6k6eefy6zxpe","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"util-linux-uuid","hash":"rmy7tbekh4lfetlh55swl74gqwlvrm3y","parameters":{"deptypes":["build","link"],"virtuals":["uuid"]}},{"name":"xz","hash":"vqzih6qodsu52uopsr42t7h5esj4jd2v","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"gygxuzpdf33jg2ya6imlbn4bd5zghbcd"},{"name":"expat","version":"2.6.4","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","libbsd":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libbsd","hash":"ukmaajw26pw7xfaklkrklqha4rrrsgra","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"hjwfi4iuk7mecmsuh75z74wycjlw7lzi"},{"name":"libbsd","version":"0.12.2","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"debyg3en7sgggswkdhcyd6lbp7arawzmyujthyyuaiad5jqd5msa====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libmd","hash":"il5ykbrdnhlzimhloyrwyymx3aicprt3","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"ukmaajw26pw7xfaklkrklqha4rrrsgra"},{"name":"libmd","version":"1.0.4","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zs2e7fqr4dzthpj5fascqvfn7xcahf7dtc5bzdwfv6vqkzi7oncq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"il5ykbrdnhlzimhloyrwyymx3aicprt3"},{"name":"gettext","version":"0.22.5","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","bzip2":true,"curses":true,"git":true,"libunistring":false,"libxml2":true,"pic":true,"shared":true,"tar":true,"xz":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====","dependencies":[{"name":"bzip2","hash":"gefii4i45qgge6oeyibc4a6neierycc5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"bqcb2qmnv3vsz5u7b3whbrortoieu6bx","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"libxml2","hash":"6r76q5qnwa6ydovyzag7dghcfxsm6rlk","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"d7rkispaw64fota6iabiom2hbawedpgj","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"tar","hash":"peh5t7ttvsvzqas4gor63twpxwj7ei6i","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"vqzih6qodsu52uopsr42t7h5esj4jd2v","parameters":{"deptypes":["build","link","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"hheyf7ak3sjcfohvfgegvdded4wppbvr"},{"name":"libxml2","version":"2.13.4","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","pic":true,"python":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"bqcb2qmnv3vsz5u7b3whbrortoieu6bx","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pkgconf","hash":"mxoabqjj7kluh3md2xo4qyof524orfwl","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"xz","hash":"vqzih6qodsu52uopsr42t7h5esj4jd2v","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"6r76q5qnwa6ydovyzag7dghcfxsm6rlk"},{"name":"xz","version":"5.4.6","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"pic":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"vqzih6qodsu52uopsr42t7h5esj4jd2v"},{"name":"tar","version":"1.35","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","zip":"pigz","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====","dependencies":[{"name":"bzip2","hash":"gefii4i45qgge6oeyibc4a6neierycc5","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"bqcb2qmnv3vsz5u7b3whbrortoieu6bx","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pigz","hash":"h5quuhwtol6qrxznml2ffjex2nfndg3e","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"vqzih6qodsu52uopsr42t7h5esj4jd2v","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"zstd","hash":"u5inbr2rrtinstce7l5krqqpnsal4vxo","parameters":{"deptypes":["run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"peh5t7ttvsvzqas4gor63twpxwj7ei6i"},{"name":"pigz","version":"2.8","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"h5quuhwtol6qrxznml2ffjex2nfndg3e"},{"name":"zstd","version":"1.5.6","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"makefile","compression":["none"],"libs":["shared","static"],"programs":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"u5inbr2rrtinstce7l5krqqpnsal4vxo"},{"name":"libffi","version":"3.4.6","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"qegr7o5zly6cqypzzsm7s6hxcwqsgtqj"},{"name":"sqlite","version":"3.46.0","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","column_metadata":true,"dynamic_extensions":true,"fts":true,"functions":false,"rtree":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"6z2sfif7stzpfvb54eoqiiki5edutguc","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"hvwclyptyu46ird3xmb6gx4ii33rqd53","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"kz4n2vtbxcj72s2teh2g6k6eefy6zxpe"},{"name":"util-linux-uuid","version":"2.40.2","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"eo6au7zhsz344imzoomhuskbl3cmrqq6ja6mcmrc3li3fnppqs6q====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"mxoabqjj7kluh3md2xo4qyof524orfwl","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"rmy7tbekh4lfetlh55swl74gqwlvrm3y"},{"name":"python-venv","version":"1.0","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====","dependencies":[{"name":"python","hash":"gygxuzpdf33jg2ya6imlbn4bd5zghbcd","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"hf5bgjk6fsdycb4zovjap4t4g6tjfcvx"},{"name":"re2c","version":"3.1","arch":{"platform":"linux","platform_os":"rhel9","target":"aarch64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====","dependencies":[{"name":"compiler-wrapper","hash":"rximc5jq3c544fhhnloem4mbccot26tv","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vojoispd6oa5kvdlyebgdgddrmhfpkol","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"23zkc4xomaptugrl5ueoh3tv3oyaqjnj","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"2mq2filwjgkrv6j6cxvispjqvtirsssh","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"gjaj5hopp3pqqbupult3vmvokhzzfhld","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"z2hnkln52bnc5tbjkhtjv7n2av52a5eh","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"gygxuzpdf33jg2ya6imlbn4bd5zghbcd","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"eavspn7qgilrfiby4v6in34pmjg5le6b"}]}} ================================================ FILE: lib/spack/spack/bootstrap/prototypes/clingo-linux-ppc64le.json ================================================ {"spec":{"_meta":{"version":5},"nodes":[{"name":"clingo-bootstrap","version":"spack","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","docs":false,"generator":"make","ipo":false,"optimized":false,"python":true,"static_libstdcpp":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====","dependencies":[{"name":"bison","hash":"lgghcjqpoodrawadw7vibeiul7wrnqog","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"cmake","hash":"p4cntzqqcfg5a6ymiyjpk6ykqcwwirym","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"lk4r47znptvkiszmntnetz6kgen7tgm3","parameters":{"deptypes":["build","link","run"],"virtuals":[]}},{"name":"python-venv","hash":"dlduozijjwp5o7vnrdghszehqh5j4rim","parameters":{"deptypes":["build","run"],"virtuals":[]}},{"name":"re2c","hash":"xfl6wrih72mane3eeobwpkyjwtfn2y76","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"7bxhwjxs6euecw5nkz4pi2hoi6lqz6ee"},{"name":"bison","version":"3.8.2","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","color":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"m4","hash":"twjk5wpmcqes4w4biqdwwrillznv5qaq","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"lgghcjqpoodrawadw7vibeiul7wrnqog"},{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"linux","platform_os":"rhel8","target":{"name":"power9le","vendor":"IBM","features":[],"generation":9,"parents":["power8le"],"cpupart":""}},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====","annotations":{"original_specfile_version":5},"hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai"},{"name":"diffutils","version":"3.10","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"jjemqnzesqbyw5tdlrk3nnuuajptekss","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}}],"annotations":{"original_specfile_version":5},"hash":"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt"},{"name":"gcc","version":"8.5.0","arch":{"platform":"linux","platform_os":"rhel8","target":{"name":"power9le","vendor":"IBM","features":[],"generation":9,"parents":["power8le"],"cpupart":""}},"namespace":"builtin","parameters":{"binutils":false,"bootstrap":true,"build_system":"autotools","build_type":"RelWithDebInfo","graphite":false,"languages":["c","c++","fortran"],"nvptx":false,"patches":["98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76","d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56"],"piclibs":false,"strip":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{"compilers":{"c":"/usr/bin/gcc","cxx":"/usr/bin/g++"}}},"patches":["98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76","d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56"],"package_hash":"fnrebjvblgu5vg2gnwreotucmf67pkyu6dzgo5afxngtphp66biq====","annotations":{"original_specfile_version":5},"hash":"ezexv4wrroazd3i26siktomcoagxii3l"},{"name":"gcc-runtime","version":"8.5.0","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"aud4d72goxupc5p3p6mdkwgtshpygn7uuj2ewx3zm6wudcgw4fzq====","dependencies":[{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}}],"annotations":{"original_specfile_version":5},"hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp"},{"name":"glibc","version":"2.28","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{}},"package_hash":"riktbfk2yybad7tgbvdkntk5c5msjcm5pk3x7naszgbvfm57h4rq====","annotations":{"original_specfile_version":5},"hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac"},{"name":"gmake","version":"4.4.1","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","guile":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}}],"annotations":{"original_specfile_version":5},"hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk"},{"name":"gnuconfig","version":"2024-07-27","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"aar2tabf35425kgzryprq775xycug7xlbt4rkwvm4aj76dhlychq====","annotations":{"original_specfile_version":5},"hash":"klycihpzvu77okocxw42le5dbhwduu2z"},{"name":"libiconv","version":"1.17","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"jjemqnzesqbyw5tdlrk3nnuuajptekss"},{"name":"m4","version":"1.4.19","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573","bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89"],"sigsegv":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89","9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573"],"package_hash":"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libsigsegv","hash":"mgnur44dzhyu7j6gqkqqfaa6odgp4ox2","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"twjk5wpmcqes4w4biqdwwrillznv5qaq"},{"name":"libsigsegv","version":"2.14","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"mgnur44dzhyu7j6gqkqqfaa6odgp4ox2"},{"name":"cmake","version":"3.31.2","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":true,"ownlibs":true,"qtgui":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"curl","hash":"7cw4rfec7mv444ok2avp3qpq62upmims","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"s3oz4dsdxhvwkoekfjly6x3q4netali4","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"p4cntzqqcfg5a6ymiyjpk6ykqcwwirym"},{"name":"curl","version":"8.10.1","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","gssapi":false,"ldap":false,"libidn2":false,"librtmp":false,"libs":["shared","static"],"libssh":false,"libssh2":false,"nghttp2":true,"tls":["openssl"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"nghttp2","hash":"bkgwuueh4jnhdcu6gvtyxldelsp3nrf2","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"j7xymvpa4nhwhjxb2hhahjcyjvvezyho","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"dylr2p25oj5nqbtq3zhtfkktbocbe4jm","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"zlib-ng","hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"7cw4rfec7mv444ok2avp3qpq62upmims"},{"name":"nghttp2","version":"1.64.0","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"dylr2p25oj5nqbtq3zhtfkktbocbe4jm","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"bkgwuueh4jnhdcu6gvtyxldelsp3nrf2"},{"name":"pkgconf","version":"2.2.0","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"dylr2p25oj5nqbtq3zhtfkktbocbe4jm"},{"name":"openssl","version":"3.4.0","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","certs":"mozilla","docs":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====","dependencies":[{"name":"ca-certificates-mozilla","hash":"6aunhqikyb5jmxkapuhzc43lapta4gaa","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"perl","hash":"ly3b5hxhkeavnar35daa3xolmbb7guv2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"j7xymvpa4nhwhjxb2hhahjcyjvvezyho"},{"name":"ca-certificates-mozilla","version":"2023-05-30","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====","annotations":{"original_specfile_version":5},"hash":"6aunhqikyb5jmxkapuhzc43lapta4gaa"},{"name":"perl","version":"5.40.0","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","cpanm":true,"opcode":true,"open":true,"shared":true,"threads":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====","dependencies":[{"name":"berkeley-db","hash":"n3eeghdelxrza3mezn7guy6qsqhjcon4","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"bzip2","hash":"5myohalomy2tb2s3oxd5zninc6u7v4pr","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"gdbm","hash":"bhsabxvim2eymbj3w3chcjwv4boripys","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"ly3b5hxhkeavnar35daa3xolmbb7guv2"},{"name":"berkeley-db","version":"18.1.40","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cxx":true,"docs":false,"patches":["26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3","b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522"],"stl":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522","26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3"],"package_hash":"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"n3eeghdelxrza3mezn7guy6qsqhjcon4"},{"name":"bzip2","version":"1.0.8","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","debug":false,"pic":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"5myohalomy2tb2s3oxd5zninc6u7v4pr"},{"name":"gdbm","version":"1.23","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"jxqfcixv66kiwdfxcnbxadbrxmnhpiqf","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"bhsabxvim2eymbj3w3chcjwv4boripys"},{"name":"readline","version":"8.2","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"package_hash":"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"ncurses","hash":"s3oz4dsdxhvwkoekfjly6x3q4netali4","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"jxqfcixv66kiwdfxcnbxadbrxmnhpiqf"},{"name":"ncurses","version":"6.5","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"abi":"none","build_system":"autotools","patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"symlinks":false,"termlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"package_hash":"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"dylr2p25oj5nqbtq3zhtfkktbocbe4jm","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"s3oz4dsdxhvwkoekfjly6x3q4netali4"},{"name":"zlib-ng","version":"2.2.1","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","compat":true,"new_strategies":true,"opt":true,"pic":true,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs"},{"name":"python","version":"3.13.0","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","bz2":true,"ctypes":true,"dbm":true,"debug":false,"libxml2":true,"lzma":true,"optimizations":false,"pic":true,"pyexpat":true,"pythoncmd":true,"readline":true,"shared":true,"sqlite3":true,"ssl":true,"tkinter":false,"uuid":true,"zlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====","dependencies":[{"name":"bzip2","hash":"5myohalomy2tb2s3oxd5zninc6u7v4pr","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"expat","hash":"vvi4w4c2ibhcbc653rqnvf2cgkp6lhxm","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"gdbm","hash":"bhsabxvim2eymbj3w3chcjwv4boripys","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gettext","hash":"7o3dt5k4qbnr632i3gyiaaexuc3utv4w","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libffi","hash":"ibk3s5narlwxsakc4bsawr3npleftvjs","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"s3oz4dsdxhvwkoekfjly6x3q4netali4","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"j7xymvpa4nhwhjxb2hhahjcyjvvezyho","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"dylr2p25oj5nqbtq3zhtfkktbocbe4jm","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"readline","hash":"jxqfcixv66kiwdfxcnbxadbrxmnhpiqf","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"sqlite","hash":"4q6cdje3u6oxg3eww63oxmoy2dlks3ml","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"util-linux-uuid","hash":"dzc4fsrtt5bt5rn3hrq6mguskici66i7","parameters":{"deptypes":["build","link"],"virtuals":["uuid"]}},{"name":"xz","hash":"4bw3ito7ggkxzqxl6jryedkokkjdbgjv","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"lk4r47znptvkiszmntnetz6kgen7tgm3"},{"name":"expat","version":"2.6.4","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","libbsd":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libbsd","hash":"swmg4rlaebhq37ufiskqf3hz5vq76ybj","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"vvi4w4c2ibhcbc653rqnvf2cgkp6lhxm"},{"name":"libbsd","version":"0.12.2","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"debyg3en7sgggswkdhcyd6lbp7arawzmyujthyyuaiad5jqd5msa====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libmd","hash":"lhb5nmg7qo67plifgcchtgqnjuxa633a","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"swmg4rlaebhq37ufiskqf3hz5vq76ybj"},{"name":"libmd","version":"1.0.4","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zs2e7fqr4dzthpj5fascqvfn7xcahf7dtc5bzdwfv6vqkzi7oncq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"lhb5nmg7qo67plifgcchtgqnjuxa633a"},{"name":"gettext","version":"0.22.5","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","bzip2":true,"curses":true,"git":true,"libunistring":false,"libxml2":true,"pic":true,"shared":true,"tar":true,"xz":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====","dependencies":[{"name":"bzip2","hash":"5myohalomy2tb2s3oxd5zninc6u7v4pr","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"jjemqnzesqbyw5tdlrk3nnuuajptekss","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"libxml2","hash":"2hjspbs3neipsef47zhcjtswkg4x6wzo","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"s3oz4dsdxhvwkoekfjly6x3q4netali4","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"tar","hash":"p52zhlsdvvqcwgswuev2qkv4lhfk3zpr","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"4bw3ito7ggkxzqxl6jryedkokkjdbgjv","parameters":{"deptypes":["build","link","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"7o3dt5k4qbnr632i3gyiaaexuc3utv4w"},{"name":"libxml2","version":"2.13.4","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","pic":true,"python":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"jjemqnzesqbyw5tdlrk3nnuuajptekss","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pkgconf","hash":"dylr2p25oj5nqbtq3zhtfkktbocbe4jm","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"xz","hash":"4bw3ito7ggkxzqxl6jryedkokkjdbgjv","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"2hjspbs3neipsef47zhcjtswkg4x6wzo"},{"name":"xz","version":"5.4.6","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"pic":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"4bw3ito7ggkxzqxl6jryedkokkjdbgjv"},{"name":"tar","version":"1.35","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","zip":"pigz","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====","dependencies":[{"name":"bzip2","hash":"5myohalomy2tb2s3oxd5zninc6u7v4pr","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"jjemqnzesqbyw5tdlrk3nnuuajptekss","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pigz","hash":"7otxklss5g77i5xarpyasp4thet2fqis","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"4bw3ito7ggkxzqxl6jryedkokkjdbgjv","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"zstd","hash":"dkk3fhqzamznskkzgijs3dn5p4yqosv3","parameters":{"deptypes":["run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"p52zhlsdvvqcwgswuev2qkv4lhfk3zpr"},{"name":"pigz","version":"2.8","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"7otxklss5g77i5xarpyasp4thet2fqis"},{"name":"zstd","version":"1.5.6","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"makefile","compression":["none"],"libs":["shared","static"],"programs":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"dkk3fhqzamznskkzgijs3dn5p4yqosv3"},{"name":"libffi","version":"3.4.6","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"ibk3s5narlwxsakc4bsawr3npleftvjs"},{"name":"sqlite","version":"3.46.0","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","column_metadata":true,"dynamic_extensions":true,"fts":true,"functions":false,"rtree":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"jxqfcixv66kiwdfxcnbxadbrxmnhpiqf","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"2q57rihnetmc5erpl6vw3nusqw7ycjqs","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"4q6cdje3u6oxg3eww63oxmoy2dlks3ml"},{"name":"util-linux-uuid","version":"2.40.2","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"eo6au7zhsz344imzoomhuskbl3cmrqq6ja6mcmrc3li3fnppqs6q====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"dylr2p25oj5nqbtq3zhtfkktbocbe4jm","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"dzc4fsrtt5bt5rn3hrq6mguskici66i7"},{"name":"python-venv","version":"1.0","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====","dependencies":[{"name":"python","hash":"lk4r47znptvkiszmntnetz6kgen7tgm3","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"dlduozijjwp5o7vnrdghszehqh5j4rim"},{"name":"re2c","version":"3.1","arch":{"platform":"linux","platform_os":"rhel8","target":"ppc64le"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====","dependencies":[{"name":"compiler-wrapper","hash":"ktcmkdaifi35awtk4wu3logfsi4nvtai","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"ezexv4wrroazd3i26siktomcoagxii3l","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"gfkhbpchfu2fk7m5yz4dax52d7yt5etp","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"mhe3tozpzp7hwolo3dxeh3zzqh45rlac","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"vjbibbd23up7c3c4cxpgawbz63krxjpk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gnuconfig","hash":"klycihpzvu77okocxw42le5dbhwduu2z","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"lk4r47znptvkiszmntnetz6kgen7tgm3","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"xfl6wrih72mane3eeobwpkyjwtfn2y76"}]}} ================================================ FILE: lib/spack/spack/bootstrap/prototypes/clingo-linux-x86_64.json ================================================ {"spec":{"_meta":{"version":5},"nodes":[{"name":"clingo-bootstrap","version":"spack","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","docs":false,"generator":"make","ipo":false,"optimized":false,"python":true,"static_libstdcpp":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====","dependencies":[{"name":"bison","hash":"ipu4y2n34za3lzhgwsqxha3pag2v2dn7","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"cmake","hash":"2i4zyafripteq6cssiyrmo67n6tmypfs","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"7f76ydmj6f4epvepmak2y5qfllqow5db","parameters":{"deptypes":["build","link","run"],"virtuals":[]}},{"name":"python-venv","hash":"mvybzkpm37r4xrt3eip5nn2padrhnlrm","parameters":{"deptypes":["build","run"],"virtuals":[]}},{"name":"re2c","hash":"ketxaszk5wezamuffgkdpie66tkd7rbl","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"5mxtdduhp3wsqlifimjzb53eswxqgd5b"},{"name":"bison","version":"3.8.2","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","color":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"eso2orwqs33nyzewrf6ccckvkfoxdzn2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"m4","hash":"gb5idois57zldhovt7rx44bd2ou4yiwr","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"ipu4y2n34za3lzhgwsqxha3pag2v2dn7"},{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"linux","platform_os":"rhel8","target":{"name":"skylake_avx512","vendor":"GenuineIntel","features":["adx","aes","avx","avx2","avx512bw","avx512cd","avx512dq","avx512f","avx512vl","bmi1","bmi2","clflushopt","clwb","f16c","fma","mmx","movbe","pclmulqdq","popcnt","rdrand","rdseed","sse","sse2","sse4_1","sse4_2","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["skylake","x86_64_v4"],"cpupart":""}},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====","annotations":{"original_specfile_version":5},"hash":"3wjlvksj4tr3qckfozocbeziogwilggn"},{"name":"diffutils","version":"3.10","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"3mozqilguvrkepcixf5v5czrvz64sn7a","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}}],"annotations":{"original_specfile_version":5},"hash":"eso2orwqs33nyzewrf6ccckvkfoxdzn2"},{"name":"gcc","version":"8.5.0","arch":{"platform":"linux","platform_os":"rhel8","target":{"name":"skylake_avx512","vendor":"GenuineIntel","features":["adx","aes","avx","avx2","avx512bw","avx512cd","avx512dq","avx512f","avx512vl","bmi1","bmi2","clflushopt","clwb","f16c","fma","mmx","movbe","pclmulqdq","popcnt","rdrand","rdseed","sse","sse2","sse4_1","sse4_2","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["skylake","x86_64_v4"],"cpupart":""}},"namespace":"builtin","parameters":{"binutils":false,"bootstrap":true,"build_system":"autotools","build_type":"RelWithDebInfo","graphite":false,"languages":["c","c++","fortran"],"nvptx":false,"patches":["98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76","d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56"],"piclibs":false,"strip":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{"compilers":{"c":"/usr/bin/gcc","cxx":"/usr/bin/g++"}}},"patches":["98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76","d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56"],"package_hash":"fnrebjvblgu5vg2gnwreotucmf67pkyu6dzgo5afxngtphp66biq====","annotations":{"original_specfile_version":5},"hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr"},{"name":"gcc-runtime","version":"8.5.0","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"aud4d72goxupc5p3p6mdkwgtshpygn7uuj2ewx3zm6wudcgw4fzq====","dependencies":[{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}}],"annotations":{"original_specfile_version":5},"hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu"},{"name":"glibc","version":"2.28","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{}},"package_hash":"riktbfk2yybad7tgbvdkntk5c5msjcm5pk3x7naszgbvfm57h4rq====","annotations":{"original_specfile_version":5},"hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s"},{"name":"gmake","version":"4.4.1","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","guile":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}}],"annotations":{"original_specfile_version":5},"hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe"},{"name":"libiconv","version":"1.17","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"3mozqilguvrkepcixf5v5czrvz64sn7a"},{"name":"m4","version":"1.4.19","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573","bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89"],"sigsegv":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89","9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573"],"package_hash":"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"eso2orwqs33nyzewrf6ccckvkfoxdzn2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libsigsegv","hash":"4joh2v5wzpcg5cd5m4fnpwebagp47lai","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"gb5idois57zldhovt7rx44bd2ou4yiwr"},{"name":"libsigsegv","version":"2.14","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"4joh2v5wzpcg5cd5m4fnpwebagp47lai"},{"name":"cmake","version":"3.31.2","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":true,"ownlibs":true,"qtgui":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"curl","hash":"dp2opcfk3d74hz2nokrdthwa4xc7ghmb","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"dm74bntb4otcekmwea6jmevqvhnono72","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"2i4zyafripteq6cssiyrmo67n6tmypfs"},{"name":"curl","version":"8.10.1","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","gssapi":false,"ldap":false,"libidn2":false,"librtmp":false,"libs":["shared","static"],"libssh":false,"libssh2":false,"nghttp2":true,"tls":["openssl"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"nghttp2","hash":"ztbzbssc6u4bylezsl6fc4hou2p3syju","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"tdkectn77qw2zzxkgwduylz57p7zgi66","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"zlib-ng","hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"dp2opcfk3d74hz2nokrdthwa4xc7ghmb"},{"name":"nghttp2","version":"1.64.0","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"eso2orwqs33nyzewrf6ccckvkfoxdzn2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"ztbzbssc6u4bylezsl6fc4hou2p3syju"},{"name":"pkgconf","version":"2.2.0","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2"},{"name":"openssl","version":"3.4.0","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","certs":"mozilla","docs":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====","dependencies":[{"name":"ca-certificates-mozilla","hash":"oe3ftgfbeukmc6dzcmqjfgda7cccgx77","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"perl","hash":"zjvu7ocv2zwrg4krarhjh3vvi2u3ha2h","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"tdkectn77qw2zzxkgwduylz57p7zgi66"},{"name":"ca-certificates-mozilla","version":"2023-05-30","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====","annotations":{"original_specfile_version":5},"hash":"oe3ftgfbeukmc6dzcmqjfgda7cccgx77"},{"name":"perl","version":"5.40.0","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","cpanm":true,"opcode":true,"open":true,"shared":true,"threads":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====","dependencies":[{"name":"berkeley-db","hash":"lfuzet6lgtupoudoympe7rzjb4yndv2d","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"bzip2","hash":"mhpnc4vabp2r5fxmq6aakyvofvnnmldt","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"gdbm","hash":"5dd4vl5on3dfg6dd6yxy5t5vrpfwaii5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"zjvu7ocv2zwrg4krarhjh3vvi2u3ha2h"},{"name":"berkeley-db","version":"18.1.40","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cxx":true,"docs":false,"patches":["26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3","b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522"],"stl":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522","26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3"],"package_hash":"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"lfuzet6lgtupoudoympe7rzjb4yndv2d"},{"name":"bzip2","version":"1.0.8","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","debug":false,"pic":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"diffutils","hash":"eso2orwqs33nyzewrf6ccckvkfoxdzn2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"mhpnc4vabp2r5fxmq6aakyvofvnnmldt"},{"name":"gdbm","version":"1.23","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"23lcaxfxq4fy5hchfratqxywajwjgspx","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"5dd4vl5on3dfg6dd6yxy5t5vrpfwaii5"},{"name":"readline","version":"8.2","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7"],"package_hash":"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"ncurses","hash":"dm74bntb4otcekmwea6jmevqvhnono72","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"23lcaxfxq4fy5hchfratqxywajwjgspx"},{"name":"ncurses","version":"6.5","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"abi":"none","build_system":"autotools","patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"symlinks":false,"termlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535"],"package_hash":"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"dm74bntb4otcekmwea6jmevqvhnono72"},{"name":"zlib-ng","version":"2.2.1","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","compat":true,"new_strategies":true,"opt":true,"pic":true,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7"},{"name":"python","version":"3.13.0","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","bz2":true,"ctypes":true,"dbm":true,"debug":false,"libxml2":true,"lzma":true,"optimizations":false,"pic":true,"pyexpat":true,"pythoncmd":true,"readline":true,"shared":true,"sqlite3":true,"ssl":true,"tkinter":false,"uuid":true,"zlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====","dependencies":[{"name":"bzip2","hash":"mhpnc4vabp2r5fxmq6aakyvofvnnmldt","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"expat","hash":"cnsgli2fxpinhfywuenoz2t4dhc47hqw","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"gdbm","hash":"5dd4vl5on3dfg6dd6yxy5t5vrpfwaii5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"gettext","hash":"fdsw6uskzn4ddgrmdqcseatiziy2pdtx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libffi","hash":"yht6xjipvotkpf3t56t4qhzzng4gbluj","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"dm74bntb4otcekmwea6jmevqvhnono72","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"openssl","hash":"tdkectn77qw2zzxkgwduylz57p7zgi66","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"pkgconf","hash":"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"readline","hash":"23lcaxfxq4fy5hchfratqxywajwjgspx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"sqlite","hash":"ypawguzvgqolvimqyrun5r3rfbdphfsg","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"util-linux-uuid","hash":"l7pvs6vnv6exgs4uci6ulfrcqb7codqp","parameters":{"deptypes":["build","link"],"virtuals":["uuid"]}},{"name":"xz","hash":"ojolxif3gv5pmuc3zveqie7zcbtpgjfd","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"7f76ydmj6f4epvepmak2y5qfllqow5db"},{"name":"expat","version":"2.6.4","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","libbsd":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libbsd","hash":"xlv3vxthk3ra5fsoe7e55pcroy6njci2","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"cnsgli2fxpinhfywuenoz2t4dhc47hqw"},{"name":"libbsd","version":"0.12.2","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"debyg3en7sgggswkdhcyd6lbp7arawzmyujthyyuaiad5jqd5msa====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libmd","hash":"w5jj3yfzzxvwjoptrwnna3rbooo44i3b","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"xlv3vxthk3ra5fsoe7e55pcroy6njci2"},{"name":"libmd","version":"1.0.4","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zs2e7fqr4dzthpj5fascqvfn7xcahf7dtc5bzdwfv6vqkzi7oncq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"w5jj3yfzzxvwjoptrwnna3rbooo44i3b"},{"name":"gettext","version":"0.22.5","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","bzip2":true,"curses":true,"git":true,"libunistring":false,"libxml2":true,"pic":true,"shared":true,"tar":true,"xz":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====","dependencies":[{"name":"bzip2","hash":"mhpnc4vabp2r5fxmq6aakyvofvnnmldt","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"3mozqilguvrkepcixf5v5czrvz64sn7a","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"libxml2","hash":"xv3omnzedrjqkpn4sda6suxsfeauzkvz","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ncurses","hash":"dm74bntb4otcekmwea6jmevqvhnono72","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"tar","hash":"y5e5unbos2j4egc75khytcwtvfmznsxx","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"ojolxif3gv5pmuc3zveqie7zcbtpgjfd","parameters":{"deptypes":["build","link","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"fdsw6uskzn4ddgrmdqcseatiziy2pdtx"},{"name":"libxml2","version":"2.13.4","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","pic":true,"python":false,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"3mozqilguvrkepcixf5v5czrvz64sn7a","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pkgconf","hash":"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}},{"name":"xz","hash":"ojolxif3gv5pmuc3zveqie7zcbtpgjfd","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"xv3omnzedrjqkpn4sda6suxsfeauzkvz"},{"name":"xz","version":"5.4.6","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","libs":["shared","static"],"pic":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"ojolxif3gv5pmuc3zveqie7zcbtpgjfd"},{"name":"tar","version":"1.35","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","zip":"pigz","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====","dependencies":[{"name":"bzip2","hash":"mhpnc4vabp2r5fxmq6aakyvofvnnmldt","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"libiconv","hash":"3mozqilguvrkepcixf5v5czrvz64sn7a","parameters":{"deptypes":["build","link"],"virtuals":["iconv"]}},{"name":"pigz","hash":"6gltt7sf6leoizgacsyxcvkfjhfajubf","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"xz","hash":"ojolxif3gv5pmuc3zveqie7zcbtpgjfd","parameters":{"deptypes":["run"],"virtuals":[]}},{"name":"zstd","hash":"7hd6zzagnpahpiu46rg2i4ht32mdndmj","parameters":{"deptypes":["run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"y5e5unbos2j4egc75khytcwtvfmznsxx"},{"name":"pigz","version":"2.8","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"makefile","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib-ng","hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"6gltt7sf6leoizgacsyxcvkfjhfajubf"},{"name":"zstd","version":"1.5.6","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"makefile","compression":["none"],"libs":["shared","static"],"programs":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"7hd6zzagnpahpiu46rg2i4ht32mdndmj"},{"name":"libffi","version":"3.4.6","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"yht6xjipvotkpf3t56t4qhzzng4gbluj"},{"name":"sqlite","version":"3.46.0","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","column_metadata":true,"dynamic_extensions":true,"fts":true,"functions":false,"rtree":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"readline","hash":"23lcaxfxq4fy5hchfratqxywajwjgspx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib-ng","hash":"wrvzbh5ldwur22ypf3aa3srtdj77ufe7","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"ypawguzvgqolvimqyrun5r3rfbdphfsg"},{"name":"util-linux-uuid","version":"2.40.2","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"eo6au7zhsz344imzoomhuskbl3cmrqq6ja6mcmrc3li3fnppqs6q====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"pkgconf","hash":"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2","parameters":{"deptypes":["build"],"virtuals":["pkgconfig"]}}],"annotations":{"original_specfile_version":5},"hash":"l7pvs6vnv6exgs4uci6ulfrcqb7codqp"},{"name":"python-venv","version":"1.0","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====","dependencies":[{"name":"python","hash":"7f76ydmj6f4epvepmak2y5qfllqow5db","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"mvybzkpm37r4xrt3eip5nn2padrhnlrm"},{"name":"re2c","version":"3.1","arch":{"platform":"linux","platform_os":"rhel8","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"autotools","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====","dependencies":[{"name":"compiler-wrapper","hash":"3wjlvksj4tr3qckfozocbeziogwilggn","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"xfrx6wio34o7fhpwtv6kjypvxlurblwr","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"hfi7ird7tq2ektlpntoru7znszd7lkbu","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"glibc","hash":"z3v4q7z2ksjom7krlru22p27j4mdyw2s","parameters":{"deptypes":["link"],"virtuals":["libc"]}},{"name":"gmake","hash":"l65jstphe3wyvixgkd3lv4dp5boxxjhe","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"7f76ydmj6f4epvepmak2y5qfllqow5db","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"ketxaszk5wezamuffgkdpie66tkd7rbl"}]}} ================================================ FILE: lib/spack/spack/bootstrap/prototypes/clingo-windows-x86_64.json ================================================ {"spec":{"_meta":{"version":5},"nodes":[{"name":"clingo-bootstrap","version":"spack","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","docs":false,"generator":"ninja","ipo":false,"optimized":false,"patches":["311bd2ae3f2f5274d1d36a2d65f887dfdf4c309a3c6bb29a53bbafb82b42ba7a","4ccfd173d439ed1e23eff42d5a01a8fbb21341c632d86b5691242dc270dbf065","c5c4db292a920ded6eecfbb6749d88ce9c4f179500aee6aee3a417b93c7c5c7a"],"python":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["4ccfd173d439ed1e23eff42d5a01a8fbb21341c632d86b5691242dc270dbf065","311bd2ae3f2f5274d1d36a2d65f887dfdf4c309a3c6bb29a53bbafb82b42ba7a","c5c4db292a920ded6eecfbb6749d88ce9c4f179500aee6aee3a417b93c7c5c7a"],"package_hash":"4c42opkd2w53rbrvk73mrxvy2ynkvq5wj2lang7ov2ptpimldsxa====","dependencies":[{"name":"cmake","hash":"zumu22rfkjg3krutmigxxkx2me42efes","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"msvc","hash":"skajkv74f2oyno7p5xp25no66w2mrtrk","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"ninja","hash":"bqypodje25rvy7ozbsyhzve42m6mcpsx","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"j4qa7xsbagk4dex5qo3777lv4jdgbpwn","parameters":{"deptypes":["build","link","run"],"virtuals":[]}},{"name":"python-venv","hash":"po2f6c4cf4nfwd57jshovkkp6zhsxpuc","parameters":{"deptypes":["build","run"],"virtuals":[]}},{"name":"re2c","hash":"mf2atm3mtzukanuqcuk6vxmtcnvrjfm6","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"winbison","hash":"sjrbf3m2ypcbf2quglw26qfn3kksigyu","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"wzcmsgouevrl3jpzwoh2gh7upehzxta3"},{"name":"cmake","version":"3.31.2","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":false,"ownlibs":true,"qtgui":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====","dependencies":[{"name":"compiler-wrapper","hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"curl","hash":"etpxh45rduqsnd6fap5uj5qzhijabs4g","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"msvc","hash":"skajkv74f2oyno7p5xp25no66w2mrtrk","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"ninja","hash":"bqypodje25rvy7ozbsyhzve42m6mcpsx","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib","hash":"sweajh5242hgibn2nsvapphwztahxzpo","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"zumu22rfkjg3krutmigxxkx2me42efes"},{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":{"name":"broadwell","vendor":"GenuineIntel","features":["adx","aes","avx","avx2","bmi1","bmi2","f16c","fma","mmx","movbe","pclmulqdq","popcnt","rdrand","rdseed","sse","sse2","sse4_1","sse4_2","ssse3"],"generation":0,"parents":["haswell"],"cpupart":""}},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"f2cvl7ifstxe4onighf2lrijbckr3wwlzjaqt3yaxtxmepeldkwq====","annotations":{"original_specfile_version":5},"hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2"},{"name":"curl","version":"8.10.1","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"nmake","gssapi":false,"ldap":false,"libidn2":false,"librtmp":false,"libs":"shared","libssh":false,"libssh2":false,"nghttp2":false,"tls":["sspi"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====","dependencies":[{"name":"compiler-wrapper","hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"msvc","hash":"skajkv74f2oyno7p5xp25no66w2mrtrk","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"zlib","hash":"sweajh5242hgibn2nsvapphwztahxzpo","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"annotations":{"original_specfile_version":5},"hash":"etpxh45rduqsnd6fap5uj5qzhijabs4g"},{"name":"msvc","version":"19.41.34120","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":{"name":"broadwell","vendor":"GenuineIntel","features":["adx","aes","avx","avx2","bmi1","bmi2","f16c","fma","mmx","movbe","pclmulqdq","popcnt","rdrand","rdseed","sse","sse2","sse4_1","sse4_2","ssse3"],"generation":0,"parents":["haswell"],"cpupart":""}},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC\\Tools\\MSVC\\14.41.34120","module":null,"extra_attributes":{"compilers":{"c":"C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC\\Tools\\MSVC\\14.41.34120\\bin\\Hostx64\\x64\\cl.exe","cxx":"C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC\\Tools\\MSVC\\14.41.34120\\bin\\Hostx64\\x64\\cl.exe"}}},"package_hash":"xywxjwuwneitqkaxzvyewhvhhr4zzuxhewmj6vmvf3cq7nf24k2a====","annotations":{"original_specfile_version":5},"hash":"skajkv74f2oyno7p5xp25no66w2mrtrk"},{"name":"zlib","version":"1.3.1","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","optimize":true,"pic":true,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"7m5x6iihfcayy4fhcdurbffk4krn7ykq2vo6wxbr2ue2pgtetf4a====","dependencies":[{"name":"compiler-wrapper","hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"msvc","hash":"skajkv74f2oyno7p5xp25no66w2mrtrk","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}}],"annotations":{"original_specfile_version":5},"hash":"sweajh5242hgibn2nsvapphwztahxzpo"},{"name":"ninja","version":"1.12.1","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","re2c":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"eanqnmavyldorxcgxf6z3j76hehc37sw55hhjbnnjy4gsvrtji3a====","dependencies":[{"name":"compiler-wrapper","hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"msvc","hash":"skajkv74f2oyno7p5xp25no66w2mrtrk","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"python","hash":"j4qa7xsbagk4dex5qo3777lv4jdgbpwn","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"bqypodje25rvy7ozbsyhzve42m6mcpsx"},{"name":"python","version":"3.13.0","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","bz2":true,"ctypes":true,"dbm":true,"debug":false,"libxml2":true,"lzma":true,"optimizations":false,"pic":true,"pyexpat":true,"pythoncmd":false,"readline":false,"shared":true,"sqlite3":true,"ssl":true,"tkinter":false,"uuid":true,"zlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====","dependencies":[{"name":"compiler-wrapper","hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"msvc","hash":"skajkv74f2oyno7p5xp25no66w2mrtrk","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}}],"annotations":{"original_specfile_version":5},"hash":"j4qa7xsbagk4dex5qo3777lv4jdgbpwn"},{"name":"python-venv","version":"1.0","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====","dependencies":[{"name":"python","hash":"j4qa7xsbagk4dex5qo3777lv4jdgbpwn","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"po2f6c4cf4nfwd57jshovkkp6zhsxpuc"},{"name":"re2c","version":"3.1","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","generator":"ninja","ipo":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====","dependencies":[{"name":"cmake","hash":"zumu22rfkjg3krutmigxxkx2me42efes","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"msvc","hash":"skajkv74f2oyno7p5xp25no66w2mrtrk","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"ninja","hash":"bqypodje25rvy7ozbsyhzve42m6mcpsx","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"j4qa7xsbagk4dex5qo3777lv4jdgbpwn","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"mf2atm3mtzukanuqcuk6vxmtcnvrjfm6"},{"name":"winbison","version":"2.5.25","arch":{"platform":"windows","platform_os":"windows10.0.19045","target":"x86_64"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","generator":"ninja","ipo":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"t3g2slcnnleieqtz66oly6vsfe5ibje6b2wmamxv5chuewwds5la====","dependencies":[{"name":"cmake","hash":"zumu22rfkjg3krutmigxxkx2me42efes","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"compiler-wrapper","hash":"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"msvc","hash":"skajkv74f2oyno7p5xp25no66w2mrtrk","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"ninja","hash":"bqypodje25rvy7ozbsyhzve42m6mcpsx","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"sjrbf3m2ypcbf2quglw26qfn3kksigyu"}]}} ================================================ FILE: lib/spack/spack/bootstrap/status.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Query the status of bootstrapping on this machine""" import sys from typing import List, Optional, Sequence, Tuple, Union import spack.util.executable from ._common import _executables_in_store, _python_import, _try_import_from_store from .config import ensure_bootstrap_configuration from .core import clingo_root_spec, gnupg_root_spec, patchelf_root_spec from .environment import BootstrapEnvironment, mypy_root_spec, pytest_root_spec, ruff_root_spec ExecutablesType = Union[str, Sequence[str]] RequiredResponseType = Tuple[bool, Optional[str]] SpecLike = Union["spack.spec.Spec", str] def _required_system_executable(exes: ExecutablesType, msg: str) -> RequiredResponseType: """Search for an executable is the system path only.""" if isinstance(exes, str): exes = (exes,) if spack.util.executable.which_string(*exes): return True, None return False, msg def _required_executable( exes: ExecutablesType, query_spec: SpecLike, msg: str ) -> RequiredResponseType: """Search for an executable in the system path or in the bootstrap store.""" if isinstance(exes, str): exes = (exes,) if spack.util.executable.which_string(*exes) or _executables_in_store(exes, query_spec): return True, None return False, msg def _required_python_module(module: str, query_spec: SpecLike, msg: str) -> RequiredResponseType: """Check if a Python module is available in the current interpreter or if it can be loaded from the bootstrap store """ if _python_import(module) or _try_import_from_store(module, query_spec): return True, None return False, msg def _missing(name: str, purpose: str, system_only: bool = True) -> str: """Message to be printed if an executable is not found""" msg = '[{2}] MISSING "{0}": {1}' if not system_only: return msg.format(name, purpose, "@*y{{B}}") return msg.format(name, purpose, "@*y{{-}}") def _core_requirements() -> List[RequiredResponseType]: _core_system_exes = { "patch": _missing("patch", "required to patch source code before building"), "tar": _missing("tar", "required to manage code archives"), "gzip": _missing("gzip", "required to compress/decompress code archives"), "unzip": _missing("unzip", "required to compress/decompress code archives"), "bzip2": _missing("bzip2", "required to compress/decompress code archives"), "git": _missing("git", "required to fetch/manage git repositories"), } if sys.platform == "linux": _core_system_exes["xz"] = _missing("xz", "required to compress/decompress code archives") # Executables that are not bootstrapped yet result = [_required_system_executable(exe, msg) for exe, msg in _core_system_exes.items()] # Python modules result.append( _required_python_module( "clingo", clingo_root_spec(), _missing("clingo", "required to concretize specs", False) ) ) return result def _buildcache_requirements() -> List[RequiredResponseType]: # Add bootstrappable executables (these can be in PATH or bootstrapped) # GPG/GPG2 - used for signing and verifying buildcaches result = [ _required_executable( ("gpg2", "gpg"), gnupg_root_spec(), _missing("gpg2", "required to sign/verify buildcaches", False), ) ] # Patchelf - only needed on Linux, used for binary relocation if sys.platform == "linux": result.append( _required_executable( "patchelf", patchelf_root_spec(), _missing("patchelf", "required to relocate binaries", False), ) ) return result def _optional_requirements() -> List[RequiredResponseType]: _optional_exes = { "zstd": _missing("zstd", "required to compress/decompress code archives"), "svn": _missing("svn", "required to manage subversion repositories"), "hg": _missing("hg", "required to manage mercurial repositories"), } # Executables that are not bootstrapped yet result = [_required_system_executable(exe, msg) for exe, msg in _optional_exes.items()] return result def _development_requirements() -> List[RequiredResponseType]: # Ensure we trigger environment modifications if we have an environment if BootstrapEnvironment.spack_yaml().exists(): with BootstrapEnvironment() as env: env.load() return [ _required_python_module( "pytest", pytest_root_spec(), _missing("pytest", "required to run unit-test", False) ), _required_executable( "ruff", ruff_root_spec(), _missing("ruff", "required for code checking/formatting", False), ), _required_executable( "mypy", mypy_root_spec(), _missing("mypy", "required for type checks", False) ), ] def status_message(section) -> Tuple[str, bool]: """Return a status message to be printed to screen that refers to the section passed as argument and a bool which is True if there are missing dependencies. Args: section (str): either 'core' or 'buildcache' or 'optional' or 'develop' """ pass_token, fail_token = "@*g{[PASS]}", "@*r{[FAIL]}" # Contain the header of the section and a list of requirements spack_sections = { "core": ("{0} @*{{Core Functionalities}}", _core_requirements), "buildcache": ("{0} @*{{Binary packages}}", _buildcache_requirements), "optional": ("{0} @*{{Optional Features}}", _optional_requirements), "develop": ("{0} @*{{Development Dependencies}}", _development_requirements), } msg, required_software = spack_sections[section] with ensure_bootstrap_configuration(): missing_software = False for found, err_msg in required_software(): if not found and err_msg: missing_software = True msg += "\n " + err_msg msg += "\n" msg = msg.format(pass_token if not missing_software else fail_token) return msg, missing_software ================================================ FILE: lib/spack/spack/build_environment.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This module contains all routines related to setting up the package build environment. All of this is set up by package.py just before install() is called. There are two parts to the build environment: 1. Python build environment (i.e. install() method) This is how things are set up when install() is called. Spack takes advantage of each package being in its own module by adding a bunch of command-like functions (like configure(), make(), etc.) in the package's module scope. This allows package writers to call them all directly in Package.install() without writing 'self.' everywhere. No, this isn't Pythonic. Yes, it makes the code more readable and more like the shell script from which someone is likely porting. 2. Build execution environment This is the set of environment variables, like PATH, CC, CXX, etc. that control the build. There are also a number of environment variables used to pass information (like RPATHs and other information about dependencies) to Spack's compiler wrappers. All of these env vars are also set up here. Skimming this module is a nice way to get acquainted with the types of calls you can make from within the install() function. """ import inspect import io import multiprocessing import os import re import signal import sys import traceback import types import warnings from collections import defaultdict from enum import Flag, auto from itertools import chain from multiprocessing.connection import Connection from typing import ( Any, BinaryIO, Callable, Dict, List, Optional, Sequence, Set, Tuple, Type, Union, overload, ) import spack.vendor.archspec.cpu import spack.builder import spack.compilers.libraries import spack.config import spack.deptypes as dt import spack.error import spack.llnl.util.tty as tty import spack.multimethod import spack.package_base import spack.paths import spack.platforms import spack.schema.environment import spack.spec import spack.stage import spack.store import spack.subprocess_context import spack.util.executable import spack.util.module_cmd from spack import traverse from spack.context import Context from spack.error import InstallError, NoHeadersError, NoLibrariesError from spack.install_test import spack_install_test_log from spack.llnl.string import plural from spack.llnl.util.filesystem import join_path, symlink from spack.llnl.util.lang import dedupe, stable_partition from spack.llnl.util.tty.color import cescape, colorize from spack.util.environment import ( SYSTEM_DIR_CASE_ENTRY, EnvironmentModifications, ModificationList, PrependPath, env_flag, filter_system_paths, get_path, is_system_path, validate, ) from spack.util.executable import Executable from spack.util.log_parse import make_log_context, parse_log_events # # This can be set by the user to globally disable parallel builds. # SPACK_NO_PARALLEL_MAKE = "SPACK_NO_PARALLEL_MAKE" # # These environment variables are set by # set_wrapper_variables and used to pass parameters to # Spack's compiler wrappers. # SPACK_COMPILER_WRAPPER_PATH = "SPACK_COMPILER_WRAPPER_PATH" SPACK_MANAGED_DIRS = "SPACK_MANAGED_DIRS" SPACK_INCLUDE_DIRS = "SPACK_INCLUDE_DIRS" SPACK_LINK_DIRS = "SPACK_LINK_DIRS" SPACK_RPATH_DIRS = "SPACK_RPATH_DIRS" SPACK_STORE_INCLUDE_DIRS = "SPACK_STORE_INCLUDE_DIRS" SPACK_STORE_LINK_DIRS = "SPACK_STORE_LINK_DIRS" SPACK_STORE_RPATH_DIRS = "SPACK_STORE_RPATH_DIRS" SPACK_RPATH_DEPS = "SPACK_RPATH_DEPS" SPACK_LINK_DEPS = "SPACK_LINK_DEPS" SPACK_PREFIX = "SPACK_PREFIX" SPACK_INSTALL = "SPACK_INSTALL" SPACK_DEBUG = "SPACK_DEBUG" SPACK_SHORT_SPEC = "SPACK_SHORT_SPEC" SPACK_DEBUG_LOG_ID = "SPACK_DEBUG_LOG_ID" SPACK_DEBUG_LOG_DIR = "SPACK_DEBUG_LOG_DIR" SPACK_CCACHE_BINARY = "SPACK_CCACHE_BINARY" SPACK_SYSTEM_DIRS = "SPACK_SYSTEM_DIRS" # Platform-specific library suffix (deprecated) if sys.platform == "darwin": dso_suffix = "dylib" elif sys.platform == "win32": dso_suffix = "dll" else: dso_suffix = "so" stat_suffix = "lib" if sys.platform == "win32" else "a" def shared_library_suffix(spec: spack.spec.Spec) -> str: """Return the shared library suffix for the given spec.""" if spec.platform == "darwin": return "dylib" elif spec.platform == "windows": return "dll" else: return "so" def static_library_suffix(spec: spack.spec.Spec) -> str: """Return the static library suffix for the given spec.""" if spec.platform == "windows": return "lib" else: return "a" def jobserver_enabled(): """Returns true if a posix jobserver (make) is detected.""" return "MAKEFLAGS" in os.environ and "--jobserver" in os.environ["MAKEFLAGS"] def get_effective_jobs( jobs, parallel: bool = True, supports_jobserver: bool = False ) -> Optional[int]: """Return the number of jobs, or None if supports_jobserver and a jobserver is detected.""" if not parallel or jobs <= 1 or env_flag(SPACK_NO_PARALLEL_MAKE): return 1 if supports_jobserver and jobserver_enabled(): return None return jobs class MakeExecutable(Executable): """Special callable executable object for make so the user can specify parallelism options on a per-invocation basis. """ def __init__(self, name: str, *, jobs: int, supports_jobserver: bool = True) -> None: super().__init__(name) self.supports_jobserver = supports_jobserver self.jobs = jobs @overload def __call__( self, *args: str, parallel: bool = ..., jobs_env: Optional[str] = ..., jobs_env_supports_jobserver: bool = ..., fail_on_error: bool = ..., ignore_errors: Union[int, Sequence[int]] = ..., ignore_quotes: Optional[bool] = ..., timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., input: Optional[BinaryIO] = ..., output: Union[Optional[BinaryIO], str] = ..., error: Union[Optional[BinaryIO], str] = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> None: ... @overload def __call__( self, *args: str, parallel: bool = ..., jobs_env: Optional[str] = ..., jobs_env_supports_jobserver: bool = ..., fail_on_error: bool = ..., ignore_errors: Union[int, Sequence[int]] = ..., ignore_quotes: Optional[bool] = ..., timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., input: Optional[BinaryIO] = ..., output: Union[Type[str], Callable] = ..., error: spack.util.executable.OutType = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> str: ... @overload def __call__( self, *args: str, parallel: bool = ..., jobs_env: Optional[str] = ..., jobs_env_supports_jobserver: bool = ..., fail_on_error: bool = ..., ignore_errors: Union[int, Sequence[int]] = ..., ignore_quotes: Optional[bool] = ..., timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., input: Optional[BinaryIO] = ..., output: spack.util.executable.OutType = ..., error: Union[Type[str], Callable] = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> str: ... def __call__( self, *args: str, parallel: bool = True, jobs_env: Optional[str] = None, jobs_env_supports_jobserver: bool = False, **kwargs, ) -> Optional[str]: """Runs this ``make`` executable in a subprocess. Args: parallel: if False, parallelism is disabled jobs_env: environment variable that will be set to the current level of parallelism jobs_env_supports_jobserver: whether the jobs env supports a job server For all the other ``**kwargs``, refer to :func:`spack.util.executable.Executable.__call__`. """ jobs = get_effective_jobs( self.jobs, parallel=parallel, supports_jobserver=self.supports_jobserver ) if jobs is not None: args = (f"-j{jobs}",) + args if jobs_env: # Caller wants us to set an environment variable to control the parallelism jobs_env_jobs = get_effective_jobs( self.jobs, parallel=parallel, supports_jobserver=jobs_env_supports_jobserver ) if jobs_env_jobs is not None: extra_env = kwargs.setdefault("extra_env", {}) extra_env.update({jobs_env: str(jobs_env_jobs)}) return super().__call__(*args, **kwargs) class UndeclaredDependencyError(spack.error.SpackError): """Raised if a dependency is invoking an executable through a module global, without declaring a dependency on it. """ class DeprecatedExecutable: def __init__(self, pkg: str, exe: str, exe_pkg: str) -> None: self.pkg = pkg self.exe = exe self.exe_pkg = exe_pkg def __call__(self, *args, **kwargs): raise UndeclaredDependencyError( f"{self.pkg} is using {self.exe} without declaring a dependency on {self.exe_pkg}" ) def add_default_env(self, key: str, value: str): self.__call__() def clean_environment(): # Stuff in here sanitizes the build environment to eliminate # anything the user has set that may interfere. We apply it immediately # unlike the other functions so it doesn't overwrite what the modules load. env = EnvironmentModifications() # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset("LD_LIBRARY_PATH") env.unset("LD_RUN_PATH") env.unset("DYLD_LIBRARY_PATH") env.unset("DYLD_FALLBACK_LIBRARY_PATH") # These vars affect how the compiler finds libraries and include dirs. env.unset("LIBRARY_PATH") env.unset("CPATH") env.unset("C_INCLUDE_PATH") env.unset("CPLUS_INCLUDE_PATH") env.unset("OBJC_INCLUDE_PATH") # prevent configure scripts from sourcing variables from config site file (AC_SITE_LOAD). env.set("CONFIG_SITE", os.devnull) env.unset("CMAKE_PREFIX_PATH") env.unset("PYTHONPATH") env.unset("R_HOME") env.unset("R_ENVIRON") env.unset("LUA_PATH") env.unset("LUA_CPATH") # Affects GNU make, can e.g. indirectly inhibit enabling parallel build # env.unset('MAKEFLAGS') # Avoid that libraries of build dependencies get hijacked. env.unset("LD_PRELOAD") env.unset("DYLD_INSERT_LIBRARIES") # Avoid _ROOT user variables overriding spack dependencies # https://cmake.org/cmake/help/latest/variable/PackageName_ROOT.html # Spack needs SPACK_ROOT though, so we need to exclude that for varname in os.environ.keys(): if varname.endswith("_ROOT") and varname != "SPACK_ROOT": env.unset(varname) # Unset the following variables because they can affect installation of # Autotools and CMake packages. build_system_vars = [ "CC", "CFLAGS", "CPP", "CPPFLAGS", # C variables "CXX", "CCC", "CXXFLAGS", "CXXCPP", # C++ variables "F77", "FFLAGS", "FLIBS", # Fortran77 variables "FC", "FCFLAGS", "FCLIBS", # Fortran variables "LDFLAGS", "LIBS", # linker variables ] for v in build_system_vars: env.unset(v) # Unset mpi environment vars. These flags should only be set by # mpi providers for packages with mpi dependencies mpi_vars = ["MPICC", "MPICXX", "MPIFC", "MPIF77", "MPIF90"] for v in mpi_vars: env.unset(v) build_lang = spack.config.get("config:build_language") if build_lang: # Override language-related variables. This can be used to force # English compiler messages etc., which allows parse_log_events to # show useful matches. env.set("LC_ALL", build_lang) # Remove any macports installs from the PATH. The macports ld can # cause conflicts with the built-in linker on el capitan. Solves # assembler issues, e.g.: # suffix or operands invalid for `movq'" path = get_path("PATH") for p in path: if "/macports/" in p: env.remove_path("PATH", p) return env def _add_werror_handling(keep_werror, env): keep_flags = set() # set of pairs replace_flags: List[Tuple[str, str]] = [] if keep_werror == "all": keep_flags.add("-Werror*") else: if keep_werror == "specific": keep_flags.add("-Werror-*") keep_flags.add("-Werror=*") # This extra case is to handle -Werror-implicit-function-declaration replace_flags.append(("-Werror-", "-Wno-error=")) replace_flags.append(("-Werror", "-Wno-error")) env.set("SPACK_COMPILER_FLAGS_KEEP", "|".join(keep_flags)) env.set("SPACK_COMPILER_FLAGS_REPLACE", " ".join(["|".join(item) for item in replace_flags])) def set_wrapper_environment_variables_for_flags(pkg, env): assert pkg.spec.concrete spec = pkg.spec if pkg.keep_werror is not None: keep_werror = pkg.keep_werror else: keep_werror = spack.config.get("config:flags:keep_werror") _add_werror_handling(keep_werror, env) # Trap spack-tracked compiler flags as appropriate. # env_flags are easy to accidentally override. inject_flags = {} env_flags = {} build_system_flags = {} for flag in spack.spec.FlagMap.valid_compiler_flags(): # Always convert flag_handler to function type. # This avoids discrepancies in calling conventions between functions # and methods, or between bound and unbound methods in python 2. # We cannot effectively convert everything to a bound method, which # would be the simpler solution. if isinstance(pkg.flag_handler, types.FunctionType): handler = pkg.flag_handler else: handler = pkg.flag_handler.__func__ injf, envf, bsf = handler(pkg, flag, spec.compiler_flags[flag][:]) inject_flags[flag] = injf or [] env_flags[flag] = envf or [] build_system_flags[flag] = bsf or [] # Place compiler flags as specified by flag_handler for flag in spack.spec.FlagMap.valid_compiler_flags(): # Concreteness guarantees key safety here if inject_flags[flag]: # variables SPACK_ inject flags through wrapper var_name = "SPACK_{0}".format(flag.upper()) env.set(var_name, " ".join(f for f in inject_flags[flag])) if env_flags[flag]: # implicit variables env.set(flag.upper(), " ".join(f for f in env_flags[flag])) pkg.flags_to_build_system_args(build_system_flags) env.set("SPACK_SYSTEM_DIRS", SYSTEM_DIR_CASE_ENTRY) return env def optimization_flags(compiler, target): # Try to check if the current compiler comes with a version number or # has an unexpected suffix. If so, treat it as a compiler with a # custom spec. version_number, _ = spack.vendor.archspec.cpu.version_components( compiler.version.dotted_numeric_string ) try: result = target.optimization_flags(compiler.name, version_number) except (ValueError, spack.vendor.archspec.cpu.UnsupportedMicroarchitecture): result = "" return result def set_wrapper_variables(pkg, env): """Set environment variables used by the Spack compiler wrapper (which have the prefix ``SPACK_``) and also add the compiler wrappers to PATH. This determines the injected -L/-I/-rpath options; each of these specifies a search order and this function computes these options in a manner that is intended to match the DAG traversal order in ``SetupContext``. TODO: this is not the case yet, we're using post order, ``SetupContext`` is using topo order.""" # Set compiler flags injected from the spec set_wrapper_environment_variables_for_flags(pkg, env) # Working directory for the spack command itself, for debug logs. if spack.config.get("config:debug"): env.set(SPACK_DEBUG, "TRUE") env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format("{name}-{hash:7}")) env.set(SPACK_DEBUG_LOG_DIR, spack.paths.spack_working_dir) if spack.config.get("config:ccache"): # Enable ccache in the compiler wrapper env.set(SPACK_CCACHE_BINARY, spack.util.executable.which_string("ccache", required=True)) else: # Avoid cache pollution if a build system forces `ccache `. env.set("CCACHE_DISABLE", "1") # Gather information about various types of dependencies rpath_hashes = set(s.dag_hash() for s in get_rpath_deps(pkg)) link_deps = pkg.spec.traverse(root=False, order="topo", deptype=dt.LINK) external_link_deps, nonexternal_link_deps = stable_partition(link_deps, lambda d: d.external) link_dirs = [] include_dirs = [] rpath_dirs = [] for dep in chain(external_link_deps, nonexternal_link_deps): # TODO: is_system_path is wrong, but even if we knew default -L, -I flags from the compiler # and default search dirs from the dynamic linker, it's not obvious how to avoid a possibly # expensive search in `query.libs.directories` and `query.headers.directories`, which is # what this branch is trying to avoid. if is_system_path(dep.prefix): continue # TODO: as of Spack 0.22, multiple instances of the same package may occur among the link # deps, so keying by name is wrong. In practice it is not problematic: we obtain the same # gcc-runtime / glibc here, and repeatedly add the same dirs that are later deduped. query = pkg.spec[dep.name] dep_link_dirs = [] try: # Locating libraries can be time consuming, so log start and finish. tty.debug(f"Collecting libraries for {dep.name}") dep_link_dirs.extend(query.libs.directories) tty.debug(f"Libraries for {dep.name} have been collected.") except NoLibrariesError: tty.debug(f"No libraries found for {dep.name}") for default_lib_dir in ("lib", "lib64"): default_lib_prefix = os.path.join(dep.prefix, default_lib_dir) if os.path.isdir(default_lib_prefix): dep_link_dirs.append(default_lib_prefix) link_dirs[:0] = dep_link_dirs if dep.dag_hash() in rpath_hashes: rpath_dirs[:0] = dep_link_dirs try: tty.debug(f"Collecting headers for {dep.name}") include_dirs[:0] = query.headers.directories tty.debug(f"Headers for {dep.name} have been collected.") except NoHeadersError: tty.debug(f"No headers found for {dep.name}") # The top-level package is heuristically rpath'ed. for libdir in ("lib64", "lib"): lib_path = os.path.join(pkg.prefix, libdir) rpath_dirs.insert(0, lib_path) # TODO: filter_system_paths is again wrong (and probably unnecessary due to the is_system_path # branch above). link_dirs should be filtered with entries from _parse_link_paths. link_dirs = list(dedupe(filter_system_paths(link_dirs))) include_dirs = list(dedupe(filter_system_paths(include_dirs))) rpath_dirs = list(dedupe(filter_system_paths(rpath_dirs))) default_dynamic_linker_filter = spack.compilers.libraries.dynamic_linker_filter_for(pkg.spec) if default_dynamic_linker_filter: rpath_dirs = default_dynamic_linker_filter(rpath_dirs) # Spack managed directories include the stage, store and upstream stores. We extend this with # their real paths to make it more robust (e.g. /tmp vs /private/tmp on macOS). spack_managed_dirs: Set[str] = { spack.stage.get_stage_root(), spack.store.STORE.db.root, *(db.root for db in spack.store.STORE.db.upstream_dbs), } spack_managed_dirs.update([os.path.realpath(p) for p in spack_managed_dirs]) env.set(SPACK_MANAGED_DIRS, "|".join(f'"{p}/"*' for p in sorted(spack_managed_dirs))) is_spack_managed = lambda p: any(p.startswith(store) for store in spack_managed_dirs) link_dirs_spack, link_dirs_system = stable_partition(link_dirs, is_spack_managed) include_dirs_spack, include_dirs_system = stable_partition(include_dirs, is_spack_managed) rpath_dirs_spack, rpath_dirs_system = stable_partition(rpath_dirs, is_spack_managed) env.set(SPACK_LINK_DIRS, ":".join(link_dirs_system)) env.set(SPACK_INCLUDE_DIRS, ":".join(include_dirs_system)) env.set(SPACK_RPATH_DIRS, ":".join(rpath_dirs_system)) env.set(SPACK_STORE_LINK_DIRS, ":".join(link_dirs_spack)) env.set(SPACK_STORE_INCLUDE_DIRS, ":".join(include_dirs_spack)) env.set(SPACK_STORE_RPATH_DIRS, ":".join(rpath_dirs_spack)) def set_package_py_globals(pkg, context: Context = Context.BUILD): """Populate the Python module of a package with some useful global names. This makes things easier for package writers. """ module = ModuleChangePropagator(pkg) jobs = spack.config.determine_number_of_jobs(parallel=pkg.parallel) module.make_jobs = jobs module.make = DeprecatedExecutable(pkg.name, "make", "gmake") module.gmake = DeprecatedExecutable(pkg.name, "gmake", "gmake") module.ninja = DeprecatedExecutable(pkg.name, "ninja", "ninja") if sys.platform == "win32": module.nmake = DeprecatedExecutable(pkg.name, "nmake", "msvc") module.msbuild = DeprecatedExecutable(pkg.name, "msbuild", "msvc") # analog to configure for win32 module.cscript = Executable("cscript") # Find the configure script in the archive path # Don't use which for this; we want to find it in the current dir. module.configure = Executable("./configure") # Useful directories within the prefix are encapsulated in # a Prefix object. module.prefix = pkg.prefix # Platform-specific library suffix. module.dso_suffix = dso_suffix def static_to_shared_library(static_lib, shared_lib=None, **kwargs): compiler_path = kwargs.get("compiler", module.spack_cc) compiler = Executable(compiler_path) return _static_to_shared_library( pkg.spec.architecture, compiler, static_lib, shared_lib, **kwargs ) module.static_to_shared_library = static_to_shared_library module.propagate_changes_to_mro() def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, **kwargs): """ Converts a static library to a shared library. The static library has to be built with PIC for the conversion to work. Parameters: static_lib (str): Path to the static library. shared_lib (str): Path to the shared library. Default is to derive from the static library's path. Keyword arguments: compiler (str): Path to the compiler. Default is spack_cc. compiler_output: Where to print compiler output to. arguments (str list): Additional arguments for the compiler. version (str): Library version. Default is unspecified. compat_version (str): Library compatibility version. Default is version. """ compiler_output = kwargs.get("compiler_output", None) arguments = kwargs.get("arguments", []) version = kwargs.get("version", None) compat_version = kwargs.get("compat_version", version) if not shared_lib: shared_lib = "{0}.{1}".format(os.path.splitext(static_lib)[0], dso_suffix) compiler_args = [] # TODO: Compiler arguments should not be hardcoded but provided by # the different compiler classes. if "linux" in arch or "cray" in arch: soname = os.path.basename(shared_lib) if compat_version: soname += ".{0}".format(compat_version) compiler_args = [ "-shared", "-Wl,-soname,{0}".format(soname), "-Wl,--whole-archive", static_lib, "-Wl,--no-whole-archive", ] elif "darwin" in arch: install_name = shared_lib if compat_version: install_name += ".{0}".format(compat_version) compiler_args = [ "-dynamiclib", "-install_name", "{0}".format(install_name), "-Wl,-force_load,{0}".format(static_lib), ] if compat_version: compiler_args.extend(["-compatibility_version", "{0}".format(compat_version)]) if version: compiler_args.extend(["-current_version", "{0}".format(version)]) if len(arguments) > 0: compiler_args.extend(arguments) shared_lib_base = shared_lib if version: shared_lib += ".{0}".format(version) elif compat_version: shared_lib += ".{0}".format(compat_version) compiler_args.extend(["-o", shared_lib]) # Create symlinks for version and compat_version shared_lib_link = os.path.basename(shared_lib) if version or compat_version: symlink(shared_lib_link, shared_lib_base) if compat_version and compat_version != version: symlink(shared_lib_link, "{0}.{1}".format(shared_lib_base, compat_version)) return compiler(*compiler_args, output=compiler_output) def _get_rpath_deps_from_spec( spec: spack.spec.Spec, transitive_rpaths: bool ) -> List[spack.spec.Spec]: if not transitive_rpaths: return spec.dependencies(deptype=dt.LINK) by_name: Dict[str, spack.spec.Spec] = {} for dep in spec.traverse(root=False, deptype=dt.LINK): lookup = by_name.get(dep.name) if lookup is None: by_name[dep.name] = dep elif lookup.version < dep.version: by_name[dep.name] = dep return list(by_name.values()) def get_rpath_deps(pkg: spack.package_base.PackageBase) -> List[spack.spec.Spec]: """Return immediate or transitive dependencies (depending on the package) that need to be rpath'ed. If a package occurs multiple times, the newest version is kept.""" return _get_rpath_deps_from_spec(pkg.spec, pkg.transitive_rpaths) def get_cmake_prefix_path(pkg: spack.package_base.PackageBase) -> List[str]: """Obtain the ``CMAKE_PREFIX_PATH`` entries for a package, based on the :attr:`~spack.package_base.PackageBase.cmake_prefix_paths` package attribute of direct build/test and transitive link dependencies.""" edges = traverse.traverse_topo_edges_generator( traverse.with_artificial_edges([pkg.spec]), visitor=traverse.MixedDepthVisitor( direct=dt.BUILD | dt.TEST, transitive=dt.LINK, key=traverse.by_dag_hash ), key=traverse.by_dag_hash, root=False, all_edges=False, # cover all nodes, not all edges ) ordered_specs = [edge.spec for edge in edges] # Separate out externals so they do not shadow Spack prefixes externals, spack_built = stable_partition((s for s in ordered_specs), lambda x: x.external) return filter_system_paths( path for spec in chain(spack_built, externals) for path in spec.package.cmake_prefix_paths ) def setup_package(pkg, dirty, context: Context = Context.BUILD): """Execute all environment setup routines.""" if context not in (Context.BUILD, Context.TEST): raise ValueError(f"'context' must be Context.BUILD or Context.TEST - got {context}") # First populate the package.py's module with the relevant globals that could be used in any # of the setup_* functions. setup_context = SetupContext(pkg.spec, context=context) setup_context.set_all_package_py_globals() # Keep track of env changes from packages separately, since we want to # issue warnings when packages make "suspicious" modifications. env_base = EnvironmentModifications() if dirty else clean_environment() env_mods = EnvironmentModifications() # setup compilers for build contexts need_compiler = context == Context.BUILD or ( context == Context.TEST and pkg.test_requires_compiler ) if need_compiler: set_wrapper_variables(pkg, env_mods) # Platform specific setup goes before package specific setup. This is for setting # defaults like MACOSX_DEPLOYMENT_TARGET on macOS. platform = spack.platforms.by_name(pkg.spec.architecture.platform) platform.setup_platform_environment(pkg, env_mods) tty.debug("setup_package: grabbing modifications from dependencies") env_mods.extend(setup_context.get_env_modifications()) tty.debug("setup_package: collected all modifications from dependencies") tty.debug("setup_package: adding compiler wrappers paths") env_by_name = env_mods.group_by_name() for x in env_by_name["SPACK_COMPILER_WRAPPER_PATH"]: assert isinstance(x, PrependPath), ( "unexpected setting used for SPACK_COMPILER_WRAPPER_PATH" ) env_mods.prepend_path("PATH", x.value) # Check whether we want to force RPATH or RUNPATH enable_var_name, disable_var_name = "SPACK_ENABLE_NEW_DTAGS", "SPACK_DISABLE_NEW_DTAGS" if enable_var_name in env_by_name and disable_var_name in env_by_name: enable_new_dtags = _extract_dtags_arg(env_by_name, var_name=enable_var_name) disable_new_dtags = _extract_dtags_arg(env_by_name, var_name=disable_var_name) if spack.config.CONFIG.get("config:shared_linking:type") == "rpath": env_mods.set("SPACK_DTAGS_TO_STRIP", enable_new_dtags) env_mods.set("SPACK_DTAGS_TO_ADD", disable_new_dtags) else: env_mods.set("SPACK_DTAGS_TO_STRIP", disable_new_dtags) env_mods.set("SPACK_DTAGS_TO_ADD", enable_new_dtags) if context == Context.TEST: env_mods.prepend_path("PATH", ".") elif context == Context.BUILD and not dirty and not env_mods.is_unset("CPATH"): tty.debug( "A dependency has updated CPATH, this may lead pkg-config to assume that the package " "is part of the system includes and omit it when invoked with '--cflags'." ) # First apply the clean environment changes env_base.apply_modifications() # Load modules on an already clean environment, just before applying Spack's # own environment modifications. This ensures Spack controls CC/CXX/... variables. load_external_modules(setup_context) # Make sure nothing's strange about the Spack environment. validate(env_mods, tty.warn) env_mods.apply_modifications() # Return all env modifications we controlled (excluding module related ones) env_base.extend(env_mods) return env_base def _extract_dtags_arg(env_by_name: Dict[str, ModificationList], *, var_name: str) -> str: try: enable_new_dtags = env_by_name[var_name][0].value # type: ignore[union-attr] except (KeyError, IndexError, AttributeError): enable_new_dtags = "" return enable_new_dtags class EnvironmentVisitor: def __init__(self, *roots: spack.spec.Spec, context: Context): # For the roots (well, marked specs) we follow different edges # than for their deps, depending on the context. self.root_hashes = set(s.dag_hash() for s in roots) if context == Context.BUILD: # Drop direct run deps in build context # We don't really distinguish between install and build time test deps, # so we include them here as build-time test deps. self.root_depflag = dt.BUILD | dt.TEST | dt.LINK elif context == Context.TEST: # This is more of an extended run environment self.root_depflag = dt.TEST | dt.RUN | dt.LINK elif context == Context.RUN: self.root_depflag = dt.RUN | dt.LINK def accept(self, item): return True def neighbors(self, item): spec = item.edge.spec if spec.dag_hash() in self.root_hashes: depflag = self.root_depflag else: depflag = dt.LINK | dt.RUN return traverse.sort_edges(spec.edges_to_dependencies(depflag=depflag)) class UseMode(Flag): #: Entrypoint spec (a spec to be built; an env root, etc) ROOT = auto() #: A spec used at runtime, but no executables in PATH RUNTIME = auto() #: A spec used at runtime, with executables in PATH RUNTIME_EXECUTABLE = auto() #: A spec that's a direct build or test dep BUILDTIME_DIRECT = auto() #: A spec that should be visible in search paths in a build env. BUILDTIME = auto() #: Flag is set when the (node, mode) is finalized ADDED = auto() def effective_deptypes( *specs: spack.spec.Spec, context: Context = Context.BUILD ) -> List[Tuple[spack.spec.Spec, UseMode]]: """Given a list of input specs and a context, return a list of tuples of all specs that contribute to (environment) modifications, together with a flag specifying in what way they do so. The list is ordered topologically from root to leaf, meaning that environment modifications should be applied in reverse so that dependents override dependencies, not the other way around.""" topo_sorted_edges = traverse.traverse_topo_edges_generator( traverse.with_artificial_edges(specs), visitor=traverse.CoverEdgesVisitor( EnvironmentVisitor(*specs, context=context), key=traverse.by_dag_hash ), key=traverse.by_dag_hash, root=True, all_edges=True, ) # Dictionary with "no mode" as default value, so it's easy to write modes[x] |= flag. use_modes = defaultdict(lambda: UseMode(0)) nodes_with_type = [] for edge in topo_sorted_edges: parent, child, depflag = edge.parent, edge.spec, edge.depflag # Mark the starting point if parent is None: use_modes[child] = UseMode.ROOT continue parent_mode = use_modes[parent] # Nothing to propagate. if not parent_mode: continue # Depending on the context, include particular deps from the root. if UseMode.ROOT & parent_mode: if context == Context.BUILD: if (dt.BUILD | dt.TEST) & depflag: use_modes[child] |= UseMode.BUILDTIME_DIRECT if dt.LINK & depflag: use_modes[child] |= UseMode.BUILDTIME elif context == Context.TEST: if (dt.RUN | dt.TEST) & depflag: use_modes[child] |= UseMode.RUNTIME_EXECUTABLE elif dt.LINK & depflag: use_modes[child] |= UseMode.RUNTIME elif context == Context.RUN: if dt.RUN & depflag: use_modes[child] |= UseMode.RUNTIME_EXECUTABLE elif dt.LINK & depflag: use_modes[child] |= UseMode.RUNTIME # Propagate RUNTIME and RUNTIME_EXECUTABLE through link and run deps. if (UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE | UseMode.BUILDTIME_DIRECT) & parent_mode: if dt.LINK & depflag: use_modes[child] |= UseMode.RUNTIME if dt.RUN & depflag: use_modes[child] |= UseMode.RUNTIME_EXECUTABLE # Propagate BUILDTIME through link deps. if UseMode.BUILDTIME & parent_mode: if dt.LINK & depflag: use_modes[child] |= UseMode.BUILDTIME # Finalize the spec; the invariant is that all in-edges are processed # before out-edges, meaning that parent is done. if not (UseMode.ADDED & parent_mode): use_modes[parent] |= UseMode.ADDED nodes_with_type.append((parent, parent_mode)) # Attach the leaf nodes, since we only added nodes with out-edges. for spec, parent_mode in use_modes.items(): if parent_mode and not (UseMode.ADDED & parent_mode): nodes_with_type.append((spec, parent_mode)) return nodes_with_type class SetupContext: """This class encapsulates the logic to determine environment modifications, and is used as well to set globals in modules of package.py.""" def __init__(self, *specs: spack.spec.Spec, context: Context) -> None: """Construct a ModificationsFromDag object. Args: specs: single root spec for build/test context, possibly more for run context context: build, run, or test""" if (context == Context.BUILD or context == Context.TEST) and not len(specs) == 1: raise ValueError("Cannot setup build environment for multiple specs") specs_with_type = effective_deptypes(*specs, context=context) self.specs = specs self.context = context self.external: List[Tuple[spack.spec.Spec, UseMode]] self.nonexternal: List[Tuple[spack.spec.Spec, UseMode]] # Reverse so we go from leaf to root self.nodes_in_subdag = set(id(s) for s, _ in specs_with_type) # Split into non-external and external, maintaining topo order per group. self.external, self.nonexternal = stable_partition( reversed(specs_with_type), lambda t: t[0].external ) self.should_be_runnable = UseMode.BUILDTIME_DIRECT | UseMode.RUNTIME_EXECUTABLE self.should_setup_run_env = ( UseMode.BUILDTIME_DIRECT | UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE ) self.should_setup_dependent_build_env = UseMode.BUILDTIME | UseMode.BUILDTIME_DIRECT self.should_setup_build_env = UseMode.ROOT if context == Context.BUILD else UseMode(0) if context == Context.RUN or context == Context.TEST: self.should_be_runnable |= UseMode.ROOT self.should_setup_run_env |= UseMode.ROOT # Everything that calls setup_run_environment and setup_dependent_* needs globals set. self.should_set_package_py_globals = ( self.should_setup_dependent_build_env | self.should_setup_run_env | UseMode.ROOT ) # In a build context, the root needs build-specific globals set. self.needs_build_context = UseMode.ROOT def set_all_package_py_globals(self): """Set the globals in modules of package.py files.""" for dspec, flag in chain(self.external, self.nonexternal): pkg = dspec.package if self.should_set_package_py_globals & flag: if self.context == Context.BUILD and self.needs_build_context & flag: set_package_py_globals(pkg, context=Context.BUILD) else: # This includes runtime dependencies, also runtime deps of direct build deps. set_package_py_globals(pkg, context=Context.RUN) # Looping over the set of packages a second time # ensures all globals are loaded into the module space prior to # any package setup. This guarantees package setup methods have # access to expected module level definitions such as "spack_cc" for dspec, flag in chain(self.external, self.nonexternal): pkg = dspec.package for spec in dspec.dependents(): # Note: some specs have dependents that are unreachable from the root, so avoid # setting globals for those. if id(spec) not in self.nodes_in_subdag: continue dependent_module = ModuleChangePropagator(spec.package) pkg.setup_dependent_package(dependent_module, spec) dependent_module.propagate_changes_to_mro() def get_env_modifications(self) -> EnvironmentModifications: """Returns the environment variable modifications for the given input specs and context. Environment modifications include: - Updating PATH for packages that are required at runtime - Updating CMAKE_PREFIX_PATH and PKG_CONFIG_PATH so that their respective tools can find Spack-built dependencies (when context=build) - Running custom package environment modifications: setup_run_environment, setup_dependent_run_environment, setup_build_environment, setup_dependent_build_environment. The (partial) order imposed on the specs is externals first, then topological from leaf to root. That way externals cannot contribute search paths that would shadow Spack's prefixes, and dependents override variables set by dependencies.""" env = EnvironmentModifications() for dspec, flag in chain(self.external, self.nonexternal): tty.debug(f"Adding env modifications for {dspec.name}") pkg = dspec.package if self.should_setup_dependent_build_env & flag: self._make_buildtime_detectable(dspec, env) for root in self.specs: # there is only one root in build context spack.builder.create(pkg).setup_dependent_build_environment(env, root) if self.should_setup_build_env & flag: spack.builder.create(pkg).setup_build_environment(env) if self.should_be_runnable & flag: self._make_runnable(dspec, env) if self.should_setup_run_env & flag: run_env_mods = EnvironmentModifications() for spec in dspec.dependents(deptype=dt.LINK | dt.RUN): if id(spec) in self.nodes_in_subdag: pkg.setup_dependent_run_environment(run_env_mods, spec) pkg.setup_run_environment(run_env_mods) external_env = (dspec.extra_attributes or {}).get("environment", {}) if external_env: run_env_mods.extend(spack.schema.environment.parse(external_env)) if self.context == Context.BUILD: # Don't let the runtime environment of compiler like dependencies leak into the # build env run_env_mods.drop("CC", "CXX", "F77", "FC") env.extend(run_env_mods) return env def _make_buildtime_detectable(self, dep: spack.spec.Spec, env: EnvironmentModifications): if is_system_path(dep.prefix): return env.prepend_path("CMAKE_PREFIX_PATH", dep.prefix) for d in ("lib", "lib64", "share"): pcdir = os.path.join(dep.prefix, d, "pkgconfig") if os.path.isdir(pcdir): env.prepend_path("PKG_CONFIG_PATH", pcdir) def _make_runnable(self, dep: spack.spec.Spec, env: EnvironmentModifications): if is_system_path(dep.prefix): return for d in ("bin", "bin64"): bin_dir = os.path.join(dep.prefix, d) if os.path.isdir(bin_dir): env.prepend_path("PATH", bin_dir) def load_external_modules(context: SetupContext) -> None: """Traverse a package's spec DAG and load any external modules. Traverse a package's dependencies and load any external modules associated with them. Args: context: A populated SetupContext object """ for spec, _ in context.external: external_modules = spec.external_modules or [] for external_module in external_modules: spack.util.module_cmd.load_module(external_module) def _setup_pkg_and_run( serialized_pkg: "spack.subprocess_context.PackageInstallContext", function: Callable, kwargs: Dict, write_pipe: Connection, input_pipe: Optional[Connection], jsfd1: Optional[Connection], jsfd2: Optional[Connection], stdout_pipe: Optional[Connection] = None, stderr_pipe: Optional[Connection] = None, ): """Main entry point in the child process for Spack builds. ``_setup_pkg_and_run`` is called by the child process created in ``start_build_process()``, and its main job is to run ``function()`` on behalf of some Spack installation (see :ref:`spack.installer.PackageInstaller._complete_task`). The child process is passed a ``write_pipe``, on which it's expected to send one of the following: * ``StopPhase``: error raised by a build process indicating it's stopping at a particular build phase. * ``BaseException``: any exception raised by a child build process, which will be wrapped in ``ChildError`` (which adds a bunch of debug info and log context) and raised in the parent. * The return value of ``function()``, which can be anything (except an exception). This is returned to the caller. Note: ``jsfd1`` and ``jsfd2`` are passed solely to ensure that the child process does not close these file descriptors. Some ``multiprocessing`` backends will close them automatically in the child if they are not passed at process creation time. Arguments: serialized_pkg: Spack package install context object (serialized form of the package that we'll build in the child process). function: function to call in the child process; serialized_pkg is passed to this as the first argument. kwargs: additional keyword arguments to pass to ``function()``. write_pipe: multiprocessing ``Connection`` to the parent process, to which the child *must* send a result (or an error) back to parent on. input_multiprocess_fd: stdin from the parent (not passed currently on Windows) jsfd1: gmake Jobserver file descriptor 1. jsfd2: gmake Jobserver file descriptor 2. stdout_pipe: pipe to redirect stdout to stderr_pipe: pipe to redirect stderr to """ context: str = kwargs.get("context", "build") try: # We are in the child process. Python sets sys.stdin to open(os.devnull) to prevent our # process and its parent from simultaneously reading from the original stdin. But, we # assume that the parent process is not going to read from it till we are done with the # child, so we undo Python's precaution. closefd=False since Connection has ownership. if input_pipe is not None: sys.stdin = os.fdopen(input_pipe.fileno(), closefd=False) if stdout_pipe is not None: os.dup2(stdout_pipe.fileno(), sys.stdout.fileno()) stdout_pipe.close() if stderr_pipe is not None: os.dup2(stderr_pipe.fileno(), sys.stderr.fileno()) stderr_pipe.close() pkg = serialized_pkg.restore() if not kwargs.get("fake", False): kwargs["unmodified_env"] = os.environ.copy() kwargs["env_modifications"] = setup_package( pkg, dirty=kwargs.get("dirty", False), context=Context.from_string(context) ) return_value = function(pkg, kwargs) write_pipe.send(return_value) except spack.error.StopPhase as e: # Do not create a full ChildError from this, it's not an error # it's a control statement. write_pipe.send(e) except BaseException as e: # catch ANYTHING that goes wrong in the child process # Need to unwind the traceback in the child because traceback # objects can't be sent to the parent. exc_type = type(e) tb = e.__traceback__ tb_string = "".join(traceback.format_exception(exc_type, e, tb)) # build up some context from the offending package so we can # show that, too. package_context = get_package_context(tb) logfile = None if context == "build": try: if hasattr(pkg, "log_path"): logfile = pkg.log_path except NameError: # 'pkg' is not defined yet pass elif context == "test": logfile = os.path.join( pkg.test_suite.stage, # type: ignore[union-attr] pkg.test_suite.test_log_name(pkg.spec), # type: ignore[union-attr] ) error_msg = str(e) if isinstance(e, (spack.multimethod.NoSuchMethodError, AttributeError)): process = "test the installation" if context == "test" else "build from sources" error_msg = ( "The '{}' package cannot find an attribute while trying to {}. You can fix this " "by updating the {} recipe, and you can also report the issue as a build-error or " "a bug at https://github.com/spack/spack/issues" ).format(pkg.name, process, context) error_msg = colorize("@*R{{{}}}".format(error_msg)) error_msg = "{}\n\n{}".format(str(e), error_msg) # make a pickleable exception to send to parent. msg = "%s: %s" % (exc_type.__name__, error_msg) ce = ChildError( msg, exc_type.__module__, exc_type.__name__, tb_string, logfile, context, package_context, ) write_pipe.send(ce) finally: write_pipe.close() if input_pipe is not None: input_pipe.close() class BuildProcess: """Class used to manage builds launched by Spack. Each build is launched in its own child process, and the main Spack process tracks each child with a ``BuildProcess`` object. ``BuildProcess`` is used to: - Start and monitor an active child process. - Clean up its processes and resources when the child process completes. - Kill the child process if needed. See also ``start_build_process()`` and ``complete_build_process()``. """ def __init__( self, *, target: Callable, args: Tuple[Any, ...], pkg: "spack.package_base.PackageBase", read_pipe: Connection, timeout: Optional[int], ) -> None: self.p = multiprocessing.Process(target=target, args=args) self.pkg = pkg self.read_pipe = read_pipe self.timeout = timeout def start(self) -> None: self.p.start() def poll(self) -> bool: """Check if there is data available to receive from the read pipe.""" return self.read_pipe.poll() def complete(self): """Wait (if needed) for child process to complete and return its exit status. See ``complete_build_process()``. """ return complete_build_process(self) def is_alive(self) -> bool: return self.p.is_alive() def join(self, *, timeout: Optional[int] = None): self.p.join(timeout=timeout) def terminate(self): # Opportunity for graceful termination self.p.terminate() self.p.join(timeout=1) # If the process didn't gracefully terminate, forcefully kill if self.p.is_alive(): # TODO (python 3.6 removal): use self.p.kill() instead, consider removing this class assert isinstance(self.p.pid, int), f"unexpected value for PID: {self.p.pid}" os.kill(self.p.pid, signal.SIGKILL) self.p.join() @property def pid(self): return self.p.pid @property def exitcode(self): return self.p.exitcode def start_build_process( pkg: "spack.package_base.PackageBase", function: Callable, kwargs: Dict[str, Any], *, timeout: Optional[int] = None, ) -> BuildProcess: """Create a child process to do part of a spack build. Args: pkg: package whose environment we should set up the child process for. function: argless function to run in the child process. kwargs: additional keyword arguments to pass to ``function()`` timeout: maximum time allowed to finish the execution of function Usage:: def child_fun(): # do stuff process = build_env.start_build_process(pkg, child_fun) complete_build_process(process) The child process is run with the build environment set up by spack.build_environment. This allows package authors to have full control over the environment, etc. without affecting other builds that might be executed in the same spack call. """ read_pipe, write_pipe = multiprocessing.Pipe(duplex=False) input_fd = None stdout_fd = None stderr_fd = None jobserver_fd1 = None jobserver_fd2 = None serialized_pkg = spack.subprocess_context.PackageInstallContext(pkg) try: # Forward sys.stdin when appropriate, to allow toggling verbosity if sys.platform != "win32" and sys.stdin.isatty() and hasattr(sys.stdin, "fileno"): input_fd = Connection(os.dup(sys.stdin.fileno())) # If our process has redirected stdout/stderr after the forkserver was started, we need to # make the forked processes use the new file descriptors. if multiprocessing.get_start_method() == "forkserver": try: stdout_fd = Connection(os.dup(sys.stdout.fileno())) stderr_fd = Connection(os.dup(sys.stderr.fileno())) except Exception: pass mflags = os.environ.get("MAKEFLAGS") if mflags is not None: m = re.search(r"--jobserver-[^=]*=(\d),(\d)", mflags) if m: jobserver_fd1 = Connection(int(m.group(1))) jobserver_fd2 = Connection(int(m.group(2))) p = BuildProcess( target=_setup_pkg_and_run, args=( serialized_pkg, function, kwargs, write_pipe, input_fd, jobserver_fd1, jobserver_fd2, stdout_fd, stderr_fd, ), read_pipe=read_pipe, timeout=timeout, pkg=pkg, ) p.start() # We close the writable end of the pipe now to be sure that p is the # only process which owns a handle for it. This ensures that when p # closes its handle for the writable end, read_pipe.recv() will # promptly report the readable end as being ready. write_pipe.close() except InstallError as e: e.pkg = pkg raise finally: # Close the input stream in the parent process if input_fd is not None: input_fd.close() if stdout_fd is not None: stdout_fd.close() if stderr_fd is not None: stderr_fd.close() return p def complete_build_process(process: BuildProcess): """ Wait for the child process to complete and handles its exit status. If something goes wrong, the child process catches the error and passes it to the parent wrapped in a ChildError. The parent is expected to handle (or re-raise) the ChildError. """ def exitcode_msg(process): typ = "exit" if process.exitcode >= 0 else "signal" return f"{typ} {abs(process.exitcode)}" try: # Check if information from the read pipe has been received. child_result = process.read_pipe.recv() except EOFError: raise InstallError(f"The process has stopped unexpectedly ({exitcode_msg(process)})") finally: timeout = process.timeout process.join(timeout=timeout) if process.is_alive(): warnings.warn(f"Terminating process, since the timeout of {timeout}s was exceeded") process.terminate() # If returns a StopPhase, raise it if isinstance(child_result, spack.error.StopPhase): raise child_result # let the caller know which package went wrong. if isinstance(child_result, InstallError): child_result.pkg = process.pkg if isinstance(child_result, ChildError): # If the child process raised an error, print its output here rather # than waiting until the call to SpackError.die() in main(). This # allows exception handling output to be logged from within Spack. # see spack.main.SpackCommand. child_result.print_context() raise child_result # Fallback. Usually caught beforehand in EOFError above. if process.exitcode != 0: raise InstallError(f"The process failed unexpectedly ({exitcode_msg(process)})") return child_result CONTEXT_BASES = (spack.package_base.PackageBase, spack.builder.BaseBuilder) def get_package_context(traceback, context=3): """Return some context for an error message when the build fails. Args: traceback: A traceback from some exception raised during install context (int): Lines of context to show before and after the line where the error happened This function inspects the stack to find where we failed in the package file, and it adds detailed context to the long_message from there. """ def make_stack(tb, stack=None): """Tracebacks come out of the system in caller -> callee order. Return an array in callee -> caller order so we can traverse it.""" if stack is None: stack = [] if tb is not None: make_stack(tb.tb_next, stack) stack.append(tb) return stack stack = make_stack(traceback) basenames = tuple(base.__name__ for base in CONTEXT_BASES) for tb in stack: frame = tb.tb_frame if "self" in frame.f_locals: # Find the first proper subclass of the PackageBase or BaseBuilder, but # don't provide context if the code is actually in the base classes. obj = frame.f_locals["self"] func = getattr(obj, tb.tb_frame.f_code.co_name, "") if func and hasattr(func, "__qualname__"): typename, *_ = func.__qualname__.partition(".") if isinstance(obj, CONTEXT_BASES) and typename not in basenames: break else: return None # We found obj, the Package implementation we care about. # Point out the location in the install method where we failed. filename = inspect.getfile(frame.f_code) lines = [f"{filename}:{frame.f_lineno}, in {frame.f_code.co_name}:"] # Build a message showing context in the install method. sourcelines, start = inspect.getsourcelines(frame) # Calculate lineno of the error relative to the start of the function. fun_lineno = frame.f_lineno - start start_ctx = max(0, fun_lineno - context) sourcelines = sourcelines[start_ctx : fun_lineno + context + 1] for i, line in enumerate(sourcelines): is_error = start_ctx + i == fun_lineno # Add start to get lineno relative to start of file, not function. marked = f" {'>> ' if is_error else ' '}{start + start_ctx + i:-6d}{line.rstrip()}" if is_error: marked = colorize("@R{%s}" % cescape(marked)) lines.append(marked) return lines class ChildError(InstallError): """Special exception class for wrapping exceptions from child processes in Spack's build environment. The main features of a ChildError are: 1. They're serializable, so when a child build fails, we can send one of these to the parent and let the parent report what happened. 2. They have a ``traceback`` field containing a traceback generated on the child immediately after failure. Spack will print this on failure in lieu of trying to run sys.excepthook on the parent process, so users will see the correct stack trace from a child. 3. They also contain context, which shows context in the Package implementation where the error happened. This helps people debug Python code in their packages. To get it, Spack searches the stack trace for the deepest frame where ``self`` is in scope and is an instance of PackageBase. This will generally find a useful spot in the ``package.py`` file. The long_message of a ChildError displays one of two things: 1. If the original error was a ProcessError, indicating a command died during the build, we'll show context from the build log. 2. If the original error was any other type of error, we'll show context from the Python code. SpackError handles displaying the special traceback if we're in debug mode with spack -d. """ # List of errors considered "build errors", for which we'll show log # context instead of Python context. build_errors = [("spack.util.executable", "ProcessError")] def __init__(self, msg, module, classname, traceback_string, log_name, log_type, context): super().__init__(msg) self.module = module self.name = classname self.traceback = traceback_string self.log_name = log_name self.log_type = log_type self.context = context @property def long_message(self): out = io.StringIO() out.write(self._long_message if self._long_message else "") have_log = self.log_name and os.path.exists(self.log_name) if (self.module, self.name) in ChildError.build_errors: # The error happened in some external executed process. Show # the log with errors or warnings highlighted. if have_log: write_log_summary(out, self.log_type, self.log_name) else: # The error happened in the Python code, so try to show # some context from the Package itself. if self.context: out.write("\n") out.write("\n".join(self.context)) out.write("\n") if out.getvalue(): out.write("\n") if have_log: out.write("See {0} log for details:\n".format(self.log_type)) out.write(" {0}\n".format(self.log_name)) # Also output the test log path IF it exists if self.context != "test" and have_log: test_log = join_path(os.path.dirname(self.log_name), spack_install_test_log) if os.path.isfile(test_log): out.write("\nSee test log for details:\n") out.write(" {0}\n".format(test_log)) return out.getvalue() def __str__(self): return self.message def __reduce__(self): """__reduce__ is used to serialize (pickle) ChildErrors. Return a function to reconstruct a ChildError, along with the salient properties we'll need. """ return _make_child_error, ( self.message, self.module, self.name, self.traceback, self.log_name, self.log_type, self.context, ) def _make_child_error(msg, module, name, traceback, log, log_type, context): """Used by __reduce__ in ChildError to reconstruct pickled errors.""" return ChildError(msg, module, name, traceback, log, log_type, context) def write_log_summary(out, log_type, log, last=None): errors, warnings = parse_log_events(log) nerr = len(errors) nwar = len(warnings) if nerr > 0: if last and nerr > last: errors = errors[-last:] nerr = last # If errors are found, only display errors out.write("\n%s found in %s log:\n" % (plural(nerr, "error"), log_type)) out.write(make_log_context(errors)) elif nwar > 0: if last and nwar > last: warnings = warnings[-last:] nwar = last # If no errors are found but warnings are, display warnings out.write("\n%s found in %s log:\n" % (plural(nwar, "warning"), log_type)) out.write(make_log_context(warnings)) class ModuleChangePropagator: """The function :meth:`spack.package_base.PackageBase.setup_dependent_package` receives an instance of this class for the ``module`` argument. It's used to set global variables in the module of a package, and propagate those globals to the modules of all classes in the inheritance hierarchy of the package. It's reminiscent of :class:`spack.util.environment.EnvironmentModifications`, but sets Python variables instead of environment variables. This class should typically not be instantiated in packages directly. """ _PROTECTED_NAMES = ("package", "current_module", "modules_in_mro", "_set_attributes") def __init__(self, package: spack.package_base.PackageBase) -> None: self._set_self_attributes("package", package) self._set_self_attributes("current_module", package.module) #: Modules for the classes in the MRO up to PackageBase modules_in_mro = [] for cls in package.__class__.__mro__: module = getattr(cls, "module", None) if module is None or module is spack.package_base: break if module is self.current_module: continue modules_in_mro.append(module) self._set_self_attributes("modules_in_mro", modules_in_mro) self._set_self_attributes("_set_attributes", {}) def _set_self_attributes(self, key, value): super().__setattr__(key, value) def __getattr__(self, item): return getattr(self.current_module, item) def __setattr__(self, key, value): if key in ModuleChangePropagator._PROTECTED_NAMES: msg = f'Cannot set attribute "{key}" in ModuleMonkeyPatcher' return AttributeError(msg) setattr(self.current_module, key, value) self._set_attributes[key] = value def propagate_changes_to_mro(self): for module_in_mro in self.modules_in_mro: module_in_mro.__dict__.update(self._set_attributes) ================================================ FILE: lib/spack/spack/buildcache_migrate.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import json import os import pathlib import tempfile from typing import NamedTuple import spack.binary_distribution import spack.database as spack_db import spack.error import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.spec import spack.stage import spack.util.crypto import spack.util.parallel import spack.util.url as url_util import spack.util.web as web_util from .enums import InstallRecordStatus from .url_buildcache import ( BlobRecord, BuildcacheComponent, compressed_json_from_dict, get_url_buildcache_class, sign_file, try_verify, ) def v2_tarball_directory_name(spec): """ Return name of the tarball directory according to the convention -//-/ """ return spec.format_path("{architecture}/{compiler.name}-{compiler.version}/{name}-{version}") def v2_tarball_name(spec, ext): """ Return the name of the tarfile according to the convention --- """ spec_formatted = spec.format_path( "{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}" ) return f"{spec_formatted}{ext}" def v2_tarball_path_name(spec, ext): """ Return the full path+name for a given spec according to the convention / """ return os.path.join(v2_tarball_directory_name(spec), v2_tarball_name(spec, ext)) class MigrateSpecResult(NamedTuple): success: bool message: str class MigrationException(spack.error.SpackError): """ Raised when migration fails irrevocably """ def __init__(self, msg): super().__init__(msg) def _migrate_spec( s: spack.spec.Spec, mirror_url: str, tmpdir: str, unsigned: bool = False, signing_key: str = "" ) -> MigrateSpecResult: """Parallelizable function to migrate a single spec""" print_spec = f"{s.name}/{s.dag_hash()[:7]}" # Check if the spec file exists in the new location and exit early if so v3_cache_class = get_url_buildcache_class(layout_version=3) v3_cache_entry = v3_cache_class(mirror_url, s, allow_unsigned=unsigned) exists = v3_cache_entry.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL]) v3_cache_entry.destroy() if exists: msg = f"No need to migrate {print_spec}" return MigrateSpecResult(True, msg) # Try to fetch the spec metadata v2_metadata_urls = [ url_util.join(mirror_url, "build_cache", v2_tarball_name(s, ".spec.json.sig")) ] if unsigned: v2_metadata_urls.append( url_util.join(mirror_url, "build_cache", v2_tarball_name(s, ".spec.json")) ) spec_contents = None for meta_url in v2_metadata_urls: try: spec_contents = web_util.read_text(meta_url) v2_spec_url = meta_url break except (web_util.SpackWebError, OSError): pass else: msg = f"Unable to read metadata for {print_spec}" return MigrateSpecResult(False, msg) spec_dict = {} if unsigned: # User asked for unsigned, if we found a signed specfile, just ignore # the signature if v2_spec_url.endswith(".sig"): spec_dict = spack.spec.Spec.extract_json_from_clearsig(spec_contents) else: spec_dict = json.loads(spec_contents) else: # User asked for signed, we must successfully verify the signature local_signed_pre_verify = os.path.join( tmpdir, f"{s.name}_{s.dag_hash()}_verify.spec.json.sig" ) with open(local_signed_pre_verify, "w", encoding="utf-8") as fd: fd.write(spec_contents) if not try_verify(local_signed_pre_verify): return MigrateSpecResult(False, f"Failed to verify signature of {print_spec}") with open(local_signed_pre_verify, encoding="utf-8") as fd: spec_dict = spack.spec.Spec.extract_json_from_clearsig(fd.read()) # Read out and remove the bits needed to rename and position the archive bcc = spec_dict.pop("binary_cache_checksum", None) if not bcc: msg = "Cannot migrate a spec that does not have 'binary_cache_checksum'" return MigrateSpecResult(False, msg) algorithm = bcc["hash_algorithm"] checksum = bcc["hash"] # TODO: Remove this key once oci buildcache no longer uses it spec_dict["buildcache_layout_version"] = 2 v2_archive_url = url_util.join(mirror_url, "build_cache", v2_tarball_path_name(s, ".spack")) # spacks web utilities do not include direct copying of s3 objects, so we # need to download the archive locally, and then push it back to the target # location archive_stage_path = os.path.join(tmpdir, f"archive_stage_{s.name}_{s.dag_hash()}") archive_stage = spack.stage.Stage(v2_archive_url, path=archive_stage_path) try: archive_stage.create() archive_stage.fetch() except spack.error.FetchError: return MigrateSpecResult(False, f"Unable to fetch archive for {print_spec}") local_tarfile_path = archive_stage.save_filename # As long as we have to download the tarball anyway, we might as well compute the # checksum locally and check it against the expected value local_checksum = spack.util.crypto.checksum( spack.util.crypto.hash_fun_for_algo(algorithm), local_tarfile_path ) if local_checksum != checksum: return MigrateSpecResult( False, f"Checksum mismatch for {print_spec}: expected {checksum}, got {local_checksum}" ) spec_dict["archive_size"] = os.stat(local_tarfile_path).st_size # Compress the spec dict and compute its checksum metadata_checksum_algo = "sha256" spec_json_path = os.path.join(tmpdir, f"{s.name}_{s.dag_hash()}.spec.json") metadata_checksum, metadata_size = compressed_json_from_dict( spec_json_path, spec_dict, metadata_checksum_algo ) tarball_blob_record = BlobRecord( spec_dict["archive_size"], v3_cache_class.TARBALL_MEDIATYPE, "gzip", algorithm, checksum ) metadata_blob_record = BlobRecord( metadata_size, v3_cache_class.SPEC_MEDIATYPE, "gzip", metadata_checksum_algo, metadata_checksum, ) # Compute the urls to the new blobs v3_archive_url = v3_cache_class.get_blob_url(mirror_url, tarball_blob_record) v3_spec_url = v3_cache_class.get_blob_url(mirror_url, metadata_blob_record) # First push the tarball tty.debug(f"Pushing {local_tarfile_path} to {v3_archive_url}") try: web_util.push_to_url(local_tarfile_path, v3_archive_url, keep_original=True) except Exception: return MigrateSpecResult(False, f"Failed to push archive for {print_spec}") # Then push the spec file tty.debug(f"Pushing {spec_json_path} to {v3_spec_url}") try: web_util.push_to_url(spec_json_path, v3_spec_url, keep_original=True) except Exception: return MigrateSpecResult(False, f"Failed to push spec metadata for {print_spec}") # Generate the manifest and write it to a temporary location manifest = { "version": v3_cache_class.get_layout_version(), "data": [tarball_blob_record.to_dict(), metadata_blob_record.to_dict()], } manifest_path = os.path.join(tmpdir, f"{s.dag_hash()}.manifest.json") with open(manifest_path, "w", encoding="utf-8") as f: json.dump(manifest, f, indent=0, separators=(",", ":")) # Note: when using gpg clear sign, we need to avoid long lines (19995 # chars). If lines are longer, they are truncated without error. So, # here we still add newlines, but no indent, so save on file size and # line length. # Possibly sign the manifest if not unsigned: manifest_path = sign_file(signing_key, manifest_path) v3_manifest_url = v3_cache_class.get_manifest_url(s, mirror_url) # Push the manifest try: web_util.push_to_url(manifest_path, v3_manifest_url, keep_original=True) except Exception: return MigrateSpecResult(False, f"Failed to push manifest for {print_spec}") return MigrateSpecResult(True, f"Successfully migrated {print_spec}") def migrate( mirror: spack.mirrors.mirror.Mirror, unsigned: bool = False, delete_existing: bool = False ) -> None: """Perform migration of the given mirror If unsigned is True, signatures on signed specs will be ignored, and specs will not be re-signed before pushing to the new location. Otherwise, spack will attempt to verify signatures and re-sign specs, and will fail if not able to do so. If delete_existing is True, spack will delete the original contents of the mirror once the migration is complete.""" signing_key = "" if not unsigned: try: signing_key = spack.binary_distribution.select_signing_key() except ( spack.binary_distribution.NoKeyException, spack.binary_distribution.PickKeyException, ): raise MigrationException( "Signed migration requires exactly one secret key in keychain" ) delete_action = "deleting" if delete_existing else "keeping" sign_action = "an unsigned" if unsigned else "a signed" mirror_url = mirror.fetch_url tty.msg( f"Performing {sign_action} migration of {mirror.push_url} " f"and {delete_action} existing contents" ) index_url = url_util.join(mirror_url, "build_cache", spack_db.INDEX_JSON_FILE) contents = None try: contents = web_util.read_text(index_url) except (web_util.SpackWebError, OSError): raise MigrationException("Buildcache migration requires a buildcache index") with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: index_path = os.path.join(tmpdir, "_tmp_index.json") with open(index_path, "w", encoding="utf-8") as fd: fd.write(contents) db = spack.binary_distribution.BuildCacheDatabase(tmpdir) db._read_from_file(pathlib.Path(index_path)) specs_to_migrate = [ s for s in db.query_local(installed=InstallRecordStatus.ANY) # todo, make it easier to get install records associated with specs if not s.external and db._data[s.dag_hash()].in_buildcache ] # Run the tasks in parallel if possible executor = spack.util.parallel.make_concurrent_executor() migrate_futures = [ executor.submit(_migrate_spec, spec, mirror_url, tmpdir, unsigned, signing_key) for spec in specs_to_migrate ] success_count = 0 tty.msg("Migration summary:") for spec, migrate_future in zip(specs_to_migrate, migrate_futures): result = migrate_future.result() msg = f" {spec.name}/{spec.dag_hash()[:7]}: {result.message}" if result.success: success_count += 1 tty.msg(msg) else: tty.error(msg) # The migrated index should have the same specs as the original index, # modulo any specs that we failed to migrate for whatever reason. So # to avoid having to re-fetch all the spec files now, just mark them # appropriately in the existing database and push that. db.mark(spec, "in_buildcache", result.success) if success_count > 0: tty.msg("Updating index and pushing keys") # If the layout.json doesn't yet exist on this mirror, push it v3_cache_class = get_url_buildcache_class(layout_version=3) v3_cache_class.maybe_push_layout_json(mirror_url) # Push the migrated mirror index index_tmpdir = os.path.join(tmpdir, "rebuild_index") os.mkdir(index_tmpdir) spack.binary_distribution._push_index(db, index_tmpdir, mirror_url) # Push the public part of the signing key if not unsigned: keys_tmpdir = os.path.join(tmpdir, "keys") os.mkdir(keys_tmpdir) spack.binary_distribution._url_push_keys( mirror_url, keys=[signing_key], update_index=True, tmpdir=keys_tmpdir ) else: tty.warn("No specs migrated, did you mean to perform an unsigned migration instead?") # Delete the old layout if the user requested it if delete_existing: delete_prefix = url_util.join(mirror_url, "build_cache") tty.msg(f"Recursively deleting {delete_prefix}") web_util.remove_url(delete_prefix, recursive=True) tty.msg("Migration complete") ================================================ FILE: lib/spack/spack/buildcache_prune.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import re import tempfile import uuid from concurrent.futures import Future, as_completed from fnmatch import fnmatch from pathlib import Path from typing import Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, cast import spack.binary_distribution import spack.error import spack.llnl.util.tty as tty import spack.stage import spack.util.parallel import spack.util.url as url_util import spack.util.web as web_util from spack.util.executable import which from .mirrors.mirror import Mirror from .url_buildcache import ( CURRENT_BUILD_CACHE_LAYOUT_VERSION, BuildcacheComponent, URLBuildcacheEntry, get_entries_from_cache, get_url_buildcache_class, ) def _fetch_manifests( mirror: Mirror, tmpspecsdir: str ) -> Tuple[Dict[str, float], Callable[[str], URLBuildcacheEntry], List[str]]: """ Fetch all manifests from the buildcache for a given mirror. This function retrieves all the manifest files from the buildcache of the specified mirror and returns a list of tuples containing the file names and a callable to read each manifest. :param mirror: The mirror from which to fetch the manifests. :return: A tuple with three elements - a list of manifest files in the mirror, a callable to read each manifest, and a list of blobs in the mirror. """ manifest_file_to_mtime_mapping, read_fn = get_entries_from_cache( mirror.fetch_url, tmpspecsdir, BuildcacheComponent.MANIFEST ) url_to_list = url_util.join( mirror.fetch_url, spack.binary_distribution.buildcache_relative_blobs_path() ) tty.debug(f"Listing blobs in {url_to_list}") blobs = web_util.list_url(url_to_list, recursive=True) or [] if not blobs: tty.warn(f"Unable to list blobs in {url_to_list}") blobs = [ url_util.join( mirror.fetch_url, spack.binary_distribution.buildcache_relative_blobs_path(), blob_name ) for blob_name in blobs ] return manifest_file_to_mtime_mapping, read_fn, blobs def _delete_manifests_from_cache_aws( url: str, tmpspecsdir: str, urls_to_delete: Set[str] ) -> Optional[int]: aws = which("aws") if not aws: tty.warn("AWS CLI not found, skipping deletion of cache entries.") return None cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION) include_pattern = cache_class.get_buildcache_component_include_pattern( BuildcacheComponent.MANIFEST ) file_count_before_deletion = len(list(pathlib.Path(tmpspecsdir).rglob(include_pattern))) tty.debug(f"Deleting {len(urls_to_delete)} entries from cache at {url}") deleted = _delete_entries_from_cache_manual(tmpspecsdir, urls_to_delete) tty.debug(f"Deleted {deleted} entries from cache at {url}") sync_command_args = [ "s3", "sync", "--delete", "--exclude", "*", "--include", include_pattern, tmpspecsdir, url, ] try: aws(*sync_command_args, output=os.devnull, error=os.devnull) # `aws s3 sync` doesn't return the number of deleted files, # but we can calculate it based on the local file count from # before and after the deletion. return file_count_before_deletion - len( list(pathlib.Path(tmpspecsdir).rglob(include_pattern)) ) except Exception: tty.warn( "Failed to use aws s3 sync to delete manifests, falling back to parallel deletion." ) return None def _delete_entries_from_cache_manual(url: str, urls_to_delete: Set[str]) -> int: pruned_objects = 0 futures: List[Future] = [] with spack.util.parallel.make_concurrent_executor() as executor: for url in urls_to_delete: futures.append(executor.submit(_delete_object, url)) for manifest_or_blob_future in as_completed(futures): pruned_objects += manifest_or_blob_future.result() return pruned_objects def _delete_entries_from_cache( mirror: Mirror, tmpspecsdir: str, manifests_to_delete: Set[str], blobs_to_delete: Set[str] ) -> int: pruned_manifests: Optional[int] = None if mirror.fetch_url.startswith("s3://"): pruned_manifests = _delete_manifests_from_cache_aws( url=mirror.fetch_url, tmpspecsdir=tmpspecsdir, urls_to_delete=manifests_to_delete ) if pruned_manifests is None: # If the AWS CLI deletion failed, we fall back to deleting both manifests # and blobs with the fallback method. objects_to_delete = blobs_to_delete.union(manifests_to_delete) pruned_objects = 0 else: # If the AWS CLI deletion succeeded, we only need to worry about # deleting the blobs, since the manifests have already been deleted. objects_to_delete = blobs_to_delete pruned_objects = pruned_manifests return pruned_objects + _delete_entries_from_cache_manual( url=mirror.fetch_url, urls_to_delete=objects_to_delete ) def _delete_object(url: str) -> int: try: web_util.remove_url(url=url) tty.info(f"Removed object {url}") return 1 except Exception as e: tty.warn(f"Unable to remove object {url} due to: {e}") return 0 def _object_has_prunable_mtime(url: str, pruning_started_at: float) -> Tuple[str, bool]: """Check if an object's modification time makes it eligible for pruning. Objects modified after pruning started should not be pruned to avoid race conditions with concurrent uploads. """ stat_result = web_util.stat_url(url) assert stat_result is not None if stat_result[1] > pruning_started_at: tty.verbose(f"Skipping deletion of {url} because it was modified after pruning started") return url, False return url, True def _filter_new_specs(urls: Iterable[str], pruning_started_at: float) -> Iterator[str]: """Filter out URLs that were modified after pruning started. Runs parallel modification time checks on all URLs and yields only those that are old enough to be safely pruned. """ with spack.util.parallel.make_concurrent_executor() as executor: futures = [] for url in urls: futures.append(executor.submit(_object_has_prunable_mtime, url, pruning_started_at)) for manifest_or_blob_future in as_completed(futures): url, has_prunable_mtime = manifest_or_blob_future.result() if has_prunable_mtime: yield url def _prune_orphans( mirror: Mirror, manifests: List[str], read_fn: Callable[[str], URLBuildcacheEntry], blobs: List[str], pruning_started_at: float, tmpspecsdir: str, dry_run: bool, ) -> int: """ Prune orphaned manifests and blobs from the buildcache. This function crawls the buildcache for a given mirror and identifies orphaned manifests and blobs. An "orphaned manifest" is one that references blobs that are not present in the cache, while an "orphaned blob" is one that is present in the cache but not referenced in any manifest. It uses the following steps to identify and prune orphaned objects: 1. Fetch all the manifests in the cache and build up a list of all the blobs that they reference. 2. List all the blobs in the buildcache, resulting in a list of all the blobs that *actually* exist in the cache. 3. Compare the two lists and use the difference to determine which objects are orphaned. - If a blob is listed in the cache but not in any manifest, that blob is orphaned. - If a blob is listed in a manifest but not in the cache, that manifest is orphaned. """ # As part of the pruning process, we need to keep track of the mapping between # blob URLs and their corresponding manifest URLs. Once we start computing # which blobs are referenced by a manifest but not present in the cache, # we will need to know which manifest to prune. blob_to_manifest_mapping: Dict[str, str] = {} for manifest in manifests: cache_entry: Optional[URLBuildcacheEntry] = None try: cache_entry = cast(URLBuildcacheEntry, read_fn(manifest)) assert cache_entry.manifest is not None # to satisfy type checker blob_to_manifest_mapping.update( { cache_entry.get_blob_url(mirror_url=mirror.fetch_url, record=data): manifest for data in cache_entry.manifest.data } ) except Exception as e: tty.warn(f"Unable to fetch manifest {manifest} due to: {e}") continue finally: if cache_entry: cache_entry.destroy() # Blobs that are referenced in a manifest file (but not necessarily present in the cache) blob_urls_referenced_by_manifest = set(blob_to_manifest_mapping.keys()) # Blobs that are actually present in the cache (but not necessarily referenced in any manifest) blob_urls_present_in_cache: Set[str] = set(blobs) # Compute set of blobs that are present in the cache but not referenced in any manifest orphaned_blobs = blob_urls_present_in_cache - blob_urls_referenced_by_manifest # Compute set of blobs that are referenced in a manifest but not present in the cache nonexisting_referenced_blobs = blob_urls_referenced_by_manifest - blob_urls_present_in_cache # Compute set of manifests that are orphaned (i.e., they reference blobs that are not # present in the cache) orphaned_manifests = { blob_to_manifest_mapping[blob_url] for blob_url in nonexisting_referenced_blobs } if not orphaned_blobs and not orphaned_manifests: return 0 # Filter out any new specs that have been uploaded since the pruning started orphaned_blobs = set(_filter_new_specs(orphaned_blobs, pruning_started_at)) orphaned_manifests = set(_filter_new_specs(orphaned_manifests, pruning_started_at)) if orphaned_blobs: tty.info(f"Found {len(orphaned_blobs)} blob(s) with no manifest") if orphaned_manifests: tty.info(f"Found {len(orphaned_manifests)} manifest(s) that are missing blobs") # If dry run, just print the manifests and blobs that would be deleted # and exit early. if dry_run: pruned_object_count = len(orphaned_blobs) + len(orphaned_manifests) for manifest in orphaned_manifests: manifests.remove(manifest) tty.info(f" Would prune manifest: {manifest}") for blob in orphaned_blobs: blobs.remove(blob) tty.info(f" Would prune blob: {blob}") return pruned_object_count # Otherwise, perform the deletions. pruned_object_count = _delete_entries_from_cache( mirror=mirror, tmpspecsdir=tmpspecsdir, manifests_to_delete=orphaned_manifests, blobs_to_delete=orphaned_blobs, ) for manifest in orphaned_manifests: manifests.remove(manifest) for blob in orphaned_blobs: blobs.remove(blob) return pruned_object_count def prune_direct( mirror: Mirror, keeplist_file: pathlib.Path, pruning_started_at: float, dry_run: bool ) -> None: """ Execute direct pruning for a given mirror using a keeplist file. This function reads a file containing spec hashes to keep, then deletes all other spec manifests from the buildcache. Note that this function does *not* prune the blobs associated with the manifests; to do that, `prune_orphan` must be invoked to clean up the now-orphaned blobs. Args: mirror: Mirror to prune keeplist_file: Path to file containing newline-delimited hashes to keep pruning_started_at: Timestamp of when the pruning started dry_run: Whether to perform a dry run without actually deleting """ tty.info("Running Direct Pruning") tty.debug(f"Direct pruning mirror: {mirror.fetch_url}" + (" (dry run)" if dry_run else "")) keep_hashes: Set[str] = set() for line in keeplist_file.read_text().splitlines(): keep_hash = line.strip().lstrip("/") if len(keep_hash) != 32: raise MalformedKeepListException(f"Found malformed hash in keeplist: {line}") keep_hashes.add(keep_hash) if not keep_hashes: raise BuildcachePruningException(f"No hashes found in keeplist file: {keeplist_file}") tty.info(f"Loaded {len(keep_hashes)} hashes to keep from {keeplist_file}") total_pruned: Optional[int] = None with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir: try: manifest_to_mtime_mapping, read_fn, blob_list = _fetch_manifests(mirror, tmpspecsdir) except Exception as e: raise BuildcachePruningException("Error getting entries from buildcache") from e # Determine which manifests correspond to specs we want to prune manifests_to_prune: List[str] = [] specs_to_prune: List[str] = [] for manifest in manifest_to_mtime_mapping.keys(): if not fnmatch( manifest, URLBuildcacheEntry.get_buildcache_component_include_pattern( BuildcacheComponent.SPEC ), ): tty.info(f"Found a non-spec manifest at {manifest}, skipping...") continue # Attempt to regex match the manifest name in order to extract the name, version, # and hash for the spec. manifest_name = manifest.split("/")[-1] # strip off parent directories regex_match = re.match(r"([^ ]+)-([^- ]+)[-_]([^-_\. ]+)", manifest_name) if regex_match is None: # This should never happen, unless the buildcache is somehow corrupted # and/or there is a bug. raise BuildcachePruningException( "Unable to extract spec name, version, and hash from " f'the manifest named "{manifest_name}"' ) spec_name, spec_version, spec_hash = regex_match.groups() # Chop off any prefix/parent file path to get just the name spec_name = pathlib.Path(spec_name).name if spec_hash not in keep_hashes: manifests_to_prune.append(manifest) specs_to_prune.append(f"{spec_name}/{spec_hash[:7]}") if not manifests_to_prune: tty.info("No specs to prune - all specs are in the keeplist") return tty.info(f"Found {len(manifests_to_prune)} spec(s) to prune") if dry_run: for spec_name in specs_to_prune: tty.info(f" Would prune: {spec_name}") total_pruned = len(manifests_to_prune) else: manifests_to_delete = set(_filter_new_specs(manifests_to_prune, pruning_started_at)) total_pruned = _delete_entries_from_cache( mirror=mirror, tmpspecsdir=tmpspecsdir, manifests_to_delete=manifests_to_delete, blobs_to_delete=set(), ) if dry_run: tty.info(f"Would have pruned {total_pruned} objects from mirror: {mirror.fetch_url}") else: tty.info(f"Pruned {total_pruned} objects from mirror: {mirror.fetch_url}") if total_pruned > 0: tty.info( "As a consequence of pruning, the buildcache index is now likely out of date." ) tty.info("Run `spack buildcache update-index` to update the index for this mirror.") def prune_orphan(mirror: Mirror, pruning_started_at: float, dry_run: bool) -> None: """ Execute the pruning process for a given mirror. Currently, this function only performs the pruning of orphaned manifests and blobs. """ tty.info("=== Orphan Pruning Phase ===") tty.debug(f"Pruning mirror: {mirror.fetch_url}" + (" (dry run)" if dry_run else "")) total_pruned = 0 with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir: try: manifest_to_mtime_mapping, read_fn, blob_list = _fetch_manifests(mirror, tmpspecsdir) manifests = list(manifest_to_mtime_mapping.keys()) except Exception as e: raise BuildcachePruningException("Error getting entries from buildcache") from e while True: # Continue pruning until no more orphaned objects are found pruned = _prune_orphans( mirror=mirror, manifests=manifests, read_fn=read_fn, blobs=blob_list, pruning_started_at=pruning_started_at, tmpspecsdir=tmpspecsdir, dry_run=dry_run, ) if pruned == 0: break total_pruned += pruned if dry_run: tty.info( f"Would have pruned {total_pruned} orphaned objects from mirror: " + mirror.fetch_url ) else: tty.info(f"Pruned {total_pruned} orphaned objects from mirror: {mirror.fetch_url}") if total_pruned > 0: # If we pruned any objects, the buildcache index is likely out of date. # Inform the user about this. tty.info( "As a consequence of pruning, the buildcache index is now likely out of date." ) tty.info( "Run `spack buildcache update-index` to update the index for this mirror." ) def get_buildcache_normalized_time(mirror: Mirror) -> float: """ Get the current time as reported by the buildcache. This is necessary because different buildcache implementations may use different time formats/time zones. This function creates a temporary file, calls `stat_url` on it, and then deletes it. This guarantees that the time used for the beginning of the pruning is consistent across all buildcache implementations. """ with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as f: tmpdir = Path(f) touch_file = tmpdir / f".spack-prune-marker-{uuid.uuid4()}" touch_file.touch() remote_path = url_util.join(mirror.push_url, touch_file.name) web_util.push_to_url( local_file_path=str(touch_file), remote_path=remote_path, keep_original=True ) stat_info = web_util.stat_url(remote_path) assert stat_info is not None start_time = stat_info[1] web_util.remove_url(remote_path) return start_time def prune_buildcache(mirror: Mirror, keeplist: Optional[str] = None, dry_run: bool = False): """ Runs buildcache pruning for a given mirror. Args: mirror: Mirror to prune keeplist_file: Path to file containing newline-delimited hashes to keep dry_run: Whether to perform a dry run without actually deleting """ # Determine the time to use as the "started at" time for pruning. # If a cache index exists, use that time. Otherwise, use the current time (normalized # to the buildcache's time zone). cache_index_url = URLBuildcacheEntry.get_index_url(mirror_url=mirror.fetch_url) stat_result = web_util.stat_url(cache_index_url) if stat_result is not None: started_at = stat_result[1] else: started_at = get_buildcache_normalized_time(mirror) if keeplist: prune_direct(mirror, pathlib.Path(keeplist), started_at, dry_run) prune_orphan(mirror, started_at, dry_run) class BuildcachePruningException(spack.error.SpackError): """ Raised when pruning fails irrevocably """ pass class MalformedKeepListException(BuildcachePruningException): """ Raised when the keeplist passed to the direct pruner is invalid or malformed in some way """ pass ================================================ FILE: lib/spack/spack/builder.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import collections.abc import copy import functools import os from typing import Callable, Dict, List, Optional, Tuple, Type import spack.directives import spack.error import spack.multimethod import spack.package_base import spack.phase_callbacks import spack.relocate import spack.repo import spack.spec import spack.util.environment from spack.error import SpackError from spack.util.prefix import Prefix #: Builder classes, as registered by the ``builder`` decorator BUILDER_CLS: Dict[str, Type["Builder"]] = {} #: Map id(pkg) to a builder, to avoid creating multiple #: builders for the same package object. _BUILDERS: Dict[int, "Builder"] = {} def register_builder(build_system_name: str): """Class decorator used to register the default builder for a given build system. The name corresponds to the ``build_system`` variant value of the package. Example:: @register_builder("cmake") class CMakeBuilder(BuilderWithDefaults): pass Args: build_system_name: name of the build system """ def _decorator(cls): cls.build_system = build_system_name BUILDER_CLS[build_system_name] = cls return cls return _decorator def create(pkg: spack.package_base.PackageBase) -> "Builder": """Given a package object with an associated concrete spec, return the builder object that can install it.""" if id(pkg) not in _BUILDERS: _BUILDERS[id(pkg)] = _create(pkg) return _BUILDERS[id(pkg)] class _PhaseAdapter: def __init__(self, builder, phase_fn): self.builder = builder self.phase_fn = phase_fn def __call__(self, spec, prefix): return self.phase_fn(self.builder.pkg, spec, prefix) def get_builder_class(pkg, name: str) -> Optional[Type["Builder"]]: """Return the builder class if a package module defines it.""" for current_cls in type(pkg).__mro__: if not hasattr(current_cls, "module"): continue maybe_builder = getattr(current_cls.module, name, None) if maybe_builder and spack.repo.is_package_module(maybe_builder.__module__): return maybe_builder return None def _create(pkg: spack.package_base.PackageBase) -> "Builder": """Return a new builder object for the package object being passed as argument. The function inspects the build-system used by the package object and try to: 1. Return a custom builder, if any is defined in the same ``package.py`` file. 2. Return a customization of more generic builders, if any is defined in the class hierarchy (look at AspellDictPackage for an example of that) 3. Return a run-time generated adapter builder otherwise The run-time generated adapter builder is capable of adapting an old-style package to the new architecture, where the installation procedure has been extracted from the ``*Package`` hierarchy into a ``*Builder`` hierarchy. This means that the adapter looks for attribute or method overrides preferably in the ``*Package`` before using the default builder implementation. Note that in case a builder is explicitly coded in ``package.py``, no attempt is made to look for build-related methods in the ``*Package``. Args: pkg: package object for which we need a builder """ package_buildsystem = buildsystem_name(pkg) default_builder_cls = BUILDER_CLS[package_buildsystem] builder_cls_name = default_builder_cls.__name__ builder_class = get_builder_class(pkg, builder_cls_name) if builder_class: return builder_class(pkg) # Specialized version of a given buildsystem can subclass some # base classes and specialize certain phases or methods or attributes. # In that case they can store their builder class as a class level attribute. # See e.g. AspellDictPackage as an example. base_cls = getattr(pkg, builder_cls_name, default_builder_cls) # From here on we define classes to construct a special builder that adapts to the # old, single class, package format. The adapter forwards any call or access to an # attribute related to the installation procedure to a package object wrapped in # a class that falls-back on calling the base builder if no override is found on the # package. The semantic should be the same as the method in the base builder were still # present in the base class of the package. class _ForwardToBaseBuilder: def __init__(self, wrapped_pkg_object, root_builder): self.wrapped_package_object = wrapped_pkg_object self.root_builder = root_builder package_cls = type(wrapped_pkg_object) wrapper_cls = type(self) bases = (package_cls, wrapper_cls) new_cls_name = package_cls.__name__ + "Wrapper" # Forward attributes that might be monkey patched later new_cls = type( new_cls_name, bases, { "__module__": package_cls.__module__, "run_tests": property(lambda x: x.wrapped_package_object.run_tests), "test_requires_compiler": property( lambda x: x.wrapped_package_object.test_requires_compiler ), "test_suite": property(lambda x: x.wrapped_package_object.test_suite), "tester": property(lambda x: x.wrapped_package_object.tester), }, ) self.__class__ = new_cls self.__dict__.update(wrapped_pkg_object.__dict__) def __getattr__(self, item): result = getattr(super(type(self.root_builder), self.root_builder), item) if item in super(type(self.root_builder), self.root_builder).phases: result = _PhaseAdapter(self.root_builder, result) return result def forward_method_to_getattr(fn_name): def __forward(self, *args, **kwargs): return self.__getattr__(fn_name)(*args, **kwargs) return __forward # Add fallback methods for the Package object to refer to the builder. If a method # with the same name is defined in the Package, it will override this definition # (when _ForwardToBaseBuilder is initialized) for method_name in ( base_cls.phases # type: ignore + package_methods(base_cls) # type: ignore + package_long_methods(base_cls) # type: ignore + ("setup_build_environment", "setup_dependent_build_environment") ): setattr(_ForwardToBaseBuilder, method_name, forward_method_to_getattr(method_name)) def forward_property_to_getattr(property_name): def __forward(self): return self.__getattr__(property_name) return __forward for attribute_name in package_attributes(base_cls): # type: ignore setattr( _ForwardToBaseBuilder, attribute_name, property(forward_property_to_getattr(attribute_name)), ) class Adapter(base_cls, metaclass=_PackageAdapterMeta): # type: ignore def __init__(self, pkg): # Deal with custom phases in packages here if hasattr(pkg, "phases"): self.phases = pkg.phases for phase in self.phases: setattr(Adapter, phase, _PackageAdapterMeta.phase_method_adapter(phase)) # Attribute containing the package wrapped in dispatcher with a `__getattr__` # method that will forward certain calls to the default builder. self.pkg_with_dispatcher = _ForwardToBaseBuilder(pkg, root_builder=self) super().__init__(pkg) # These two methods don't follow the (self, spec, prefix) signature of phases nor # the (self) signature of methods, so they are added explicitly to avoid using a # catch-all (*args, **kwargs) def setup_build_environment( self, env: spack.util.environment.EnvironmentModifications ) -> None: return self.pkg_with_dispatcher.setup_build_environment(env) def setup_dependent_build_environment( self, env: spack.util.environment.EnvironmentModifications, dependent_spec: spack.spec.Spec, ) -> None: return self.pkg_with_dispatcher.setup_dependent_build_environment(env, dependent_spec) return Adapter(pkg) def buildsystem_name(pkg: spack.package_base.PackageBase) -> str: """Given a package object with an associated concrete spec, return the name of its build system.""" try: return pkg.spec.variants["build_system"].value except KeyError as e: # We are reading an old spec without the build_system variant if hasattr(pkg, "default_buildsystem"): # Package API v2.2 return pkg.default_buildsystem elif hasattr(pkg, "legacy_buildsystem"): return pkg.legacy_buildsystem raise SpackError(f"Package {pkg.name} does not define a build system.") from e class BuilderMeta( spack.phase_callbacks.PhaseCallbacksMeta, spack.multimethod.MultiMethodMeta, type(collections.abc.Sequence), # type: ignore ): pass class _PackageAdapterMeta(BuilderMeta): """Metaclass to adapt old-style packages to the new architecture based on builders for the installation phase. This class does the necessary mangling to function argument so that a call to a builder object can delegate to a package object. """ @staticmethod def phase_method_adapter(phase_name): def _adapter(self, pkg, spec, prefix): phase_fn = getattr(self.pkg_with_dispatcher, phase_name) return phase_fn(spec, prefix) return _adapter @staticmethod def legacy_long_method_adapter(method_name): def _adapter(self, spec, prefix): bind_method = getattr(self.pkg_with_dispatcher, method_name) return bind_method(spec, prefix) return _adapter @staticmethod def legacy_method_adapter(method_name): def _adapter(self): bind_method = getattr(self.pkg_with_dispatcher, method_name) return bind_method() return _adapter @staticmethod def legacy_attribute_adapter(attribute_name): def _adapter(self): return getattr(self.pkg_with_dispatcher, attribute_name) return property(_adapter) @staticmethod def combine_callbacks(pipeline_attribute_name): """This function combines callbacks from old-style packages with callbacks that might be registered for the default builder. It works by: 1. Extracting the callbacks from the old-style package 2. Transforming those callbacks by adding an adapter that receives a builder as argument and calls the wrapped function with ``builder.pkg`` 3. Combining the list of transformed callbacks with those that might be present in the default builder """ def _adapter(self): def unwrap_pkg(fn): @functools.wraps(fn) def _wrapped(builder): return fn(builder.pkg_with_dispatcher) return _wrapped # Concatenate the current list with the one from package callbacks_from_package = getattr(self.pkg, pipeline_attribute_name, []) callbacks_from_package = [(key, unwrap_pkg(x)) for key, x in callbacks_from_package] callbacks_from_builder = getattr(super(type(self), self), pipeline_attribute_name, []) return callbacks_from_package + callbacks_from_builder return property(_adapter) def __new__(mcs, name, bases, attr_dict): # Add ways to intercept methods and attribute calls and dispatch # them first to a package object default_builder_cls = bases[0] for phase_name in default_builder_cls.phases: attr_dict[phase_name] = _PackageAdapterMeta.phase_method_adapter(phase_name) for method_name in package_methods(default_builder_cls): attr_dict[method_name] = _PackageAdapterMeta.legacy_method_adapter(method_name) # These exist e.g. for Python, see discussion in https://github.com/spack/spack/pull/32068 for method_name in package_long_methods(default_builder_cls): attr_dict[method_name] = _PackageAdapterMeta.legacy_long_method_adapter(method_name) for attribute_name in package_attributes(default_builder_cls): attr_dict[attribute_name] = _PackageAdapterMeta.legacy_attribute_adapter( attribute_name ) combine_callbacks = _PackageAdapterMeta.combine_callbacks attr_dict[spack.phase_callbacks._RUN_BEFORE.attribute_name] = combine_callbacks( spack.phase_callbacks._RUN_BEFORE.attribute_name ) attr_dict[spack.phase_callbacks._RUN_AFTER.attribute_name] = combine_callbacks( spack.phase_callbacks._RUN_AFTER.attribute_name ) return super(_PackageAdapterMeta, mcs).__new__(mcs, name, bases, attr_dict) class InstallationPhase: """Manages a single phase of the installation. This descriptor stores at creation time the name of the method it should search for execution. The method is retrieved at __get__ time, so that it can be overridden by subclasses of whatever class declared the phases. It also provides hooks to execute arbitrary callbacks before and after the phase. """ def __init__(self, name, builder): self.name = name self.builder = builder self.phase_fn = self._select_phase_fn() self.run_before = self._make_callbacks(spack.phase_callbacks._RUN_BEFORE.attribute_name) self.run_after = self._make_callbacks(spack.phase_callbacks._RUN_AFTER.attribute_name) def _make_callbacks(self, callbacks_attribute): result = [] callbacks = getattr(self.builder, callbacks_attribute, []) for (phase, condition), fn in callbacks: # Same if it is for another phase if phase != self.name: continue # If we have no condition or the callback satisfies a condition, register it if condition is None or self.builder.pkg.spec.satisfies(condition): result.append(fn) return result def __str__(self): msg = '{0}: executing "{1}" phase' return msg.format(self.builder, self.name) def execute(self): pkg = self.builder.pkg self._on_phase_start(pkg) for callback in self.run_before: callback(self.builder) self.phase_fn(pkg, pkg.spec, pkg.prefix) for callback in self.run_after: callback(self.builder) self._on_phase_exit(pkg) def _select_phase_fn(self): phase_fn = getattr(self.builder, self.name, None) if not phase_fn: msg = ( 'unexpected error: package "{0.fullname}" must implement an ' '"{1}" phase for the "{2}" build system' ) raise RuntimeError(msg.format(self.builder.pkg, self.name, self.builder.build_system)) return phase_fn def _on_phase_start(self, instance): # If a phase has a matching stop_before_phase attribute, # stop the installation process raising a StopPhase if getattr(instance, "stop_before_phase", None) == self.name: raise spack.error.StopPhase("Stopping before '{0}' phase".format(self.name)) def _on_phase_exit(self, instance): # If a phase has a matching last_phase attribute, # stop the installation process raising a StopPhase if getattr(instance, "last_phase", None) == self.name: raise spack.error.StopPhase("Stopping at '{0}' phase".format(self.name)) def copy(self): return copy.deepcopy(self) class BaseBuilder(metaclass=BuilderMeta): """An interface for builders, without any phases defined. This class is exposed in the package API, so that packagers can create a single class to define :meth:`setup_build_environment` and :func:`spack.phase_callbacks.run_before` and :func:`spack.phase_callbacks.run_after` callbacks that can be shared among different builders. Example: .. code-block:: python class AnyBuilder(BaseBuilder): @run_after("install") def fixup_install(self): # do something after the package is installed pass def setup_build_environment(self, env: EnvironmentModifications) -> None: env.set("MY_ENV_VAR", "my_value") class CMakeBuilder(cmake.CMakeBuilder, AnyBuilder): pass class AutotoolsBuilder(autotools.AutotoolsBuilder, AnyBuilder): pass """ def __init__(self, pkg: spack.package_base.PackageBase) -> None: self.pkg = pkg @property def spec(self) -> spack.spec.Spec: return self.pkg.spec @property def stage(self): return self.pkg.stage @property def prefix(self): return self.pkg.prefix def setup_build_environment( self, env: spack.util.environment.EnvironmentModifications ) -> None: """Sets up the build environment for a package. This method will be called before the current package prefix exists in Spack's store. Args: env: environment modifications to be applied when the package is built. Package authors can call methods on it to alter the build environment. """ if not hasattr(super(), "setup_build_environment"): return super().setup_build_environment(env) # type: ignore def setup_dependent_build_environment( self, env: spack.util.environment.EnvironmentModifications, dependent_spec: spack.spec.Spec ) -> None: """Sets up the build environment of a package that depends on this one. This is similar to ``setup_build_environment``, but it is used to modify the build environment of a package that *depends* on this one. This gives packages the ability to set environment variables for the build of the dependent, which can be useful to provide search hints for headers or libraries if they are not in standard locations. This method will be called before the dependent package prefix exists in Spack's store. Args: env: environment modifications to be applied when the dependent package is built. Package authors can call methods on it to alter the build environment. dependent_spec: the spec of the dependent package about to be built. This allows the extendee (self) to query the dependent's state. Note that *this* package's spec is available as ``self.spec`` """ if not hasattr(super(), "setup_dependent_build_environment"): return super().setup_dependent_build_environment(env, dependent_spec) # type: ignore def __repr__(self): fmt = "{name}{/hash:7}" return f"{self.__class__.__name__}({self.spec.format(fmt)})" def __str__(self): fmt = "{name}{/hash:7}" return f'"{self.__class__.__name__}" builder for "{self.spec.format(fmt)}"' class Builder(BaseBuilder, collections.abc.Sequence): """A builder is a class that, given a package object (i.e. associated with concrete spec), knows how to install it. The builder behaves like a sequence, and when iterated over return the ``phases`` of the installation in the correct order. """ #: Sequence of phases. Must be defined in derived classes phases: Tuple[str, ...] = () #: Build system name. Must also be defined in derived classes. build_system: Optional[str] = None #: Methods, with no arguments, that the adapter can find in Package classes, #: if a builder is not defined. package_methods: Tuple[str, ...] # Use :attr:`package_methods` instead of this attribute, which is deprecated legacy_methods: Tuple[str, ...] = () #: Methods with the same signature as phases, that the adapter can find in Package classes, #: if a builder is not defined. package_long_methods: Tuple[str, ...] # Use :attr:`package_long_methods` instead of this attribute, which is deprecated legacy_long_methods: Tuple[str, ...] #: Attributes that the adapter can find in Package classes, if a builder is not defined package_attributes: Tuple[str, ...] # Use :attr:`package_attributes` instead of this attribute, which is deprecated legacy_attributes: Tuple[str, ...] = () # type hints for some of the legacy methods build_time_test_callbacks: List[str] install_time_test_callbacks: List[str] #: List of glob expressions. Each expression must either be absolute or relative to the package #: source path. Matching artifacts found at the end of the build process will be copied in the #: same directory tree as _spack_build_logfile and _spack_build_envfile. @property def archive_files(self) -> List[str]: return [] def __init__(self, pkg: spack.package_base.PackageBase) -> None: super().__init__(pkg) self.callbacks = {} for phase in self.phases: self.callbacks[phase] = InstallationPhase(phase, self) def __getitem__(self, idx): key = self.phases[idx] return self.callbacks[key] def __len__(self): return len(self.phases) def package_methods(builder: Type[Builder]) -> Tuple[str, ...]: """Returns the list of methods, taking no arguments, that are defined in the package class and are associated with the builder. """ if hasattr(builder, "package_methods"): # Package API v2.2 return builder.package_methods return builder.legacy_methods def package_attributes(builder: Type[Builder]) -> Tuple[str, ...]: """Returns the list of attributes that are defined in the package class and are associated with the builder. """ if hasattr(builder, "package_attributes"): # Package API v2.2 return builder.package_attributes return builder.legacy_attributes def package_long_methods(builder: Type[Builder]) -> Tuple[str, ...]: """Returns the list of methods, with the same signature as phases, that are defined in the package class and are associated with the builder. """ if hasattr(builder, "package_long_methods"): # Package API v2.2 return builder.package_long_methods return getattr(builder, "legacy_long_methods", tuple()) def sanity_check_prefix(builder: Builder): """Check that specific directories and files are created after installation. The files to be checked are in the ``sanity_check_is_file`` attribute of the package object, while the directories are in the ``sanity_check_is_dir``. Args: builder: builder that installed the package """ pkg = builder.pkg def check_paths(path_list: List[str], filetype: str, predicate: Callable[[str], bool]) -> None: if isinstance(path_list, str): path_list = [path_list] for path in path_list: if not predicate(os.path.join(pkg.prefix, path)): raise spack.error.InstallError( f"Install failed for {pkg.name}. No such {filetype} in prefix: {path}" ) check_paths(pkg.sanity_check_is_file, "file", os.path.isfile) check_paths(pkg.sanity_check_is_dir, "directory", os.path.isdir) # Check that the prefix is not empty apart from the .spack/ directory with os.scandir(pkg.prefix) as entries: f = next( (f for f in entries if not (f.name == ".spack" and f.is_dir(follow_symlinks=False))), None, ) if f is None: raise spack.error.InstallError(f"Install failed for {pkg.name}. Nothing was installed!") class BuilderWithDefaults(Builder): """Base class for all specific builders with common callbacks registered.""" # Check that self.prefix is there after installation spack.phase_callbacks.run_after("install")(sanity_check_prefix) def apply_macos_rpath_fixups(builder: Builder): """On Darwin, make installed libraries more easily relocatable. Some build systems (handrolled, autotools, makefiles) can set their own rpaths that are duplicated by spack's compiler wrapper. This fixup interrogates, and postprocesses if necessary, all libraries installed by the code. It should be added as a :func:`~spack.phase_callbacks.run_after` to packaging systems (or individual packages) that do not install relocatable libraries by default. Example:: run_after("install", when="platform=darwin")(apply_macos_rpath_fixups) Args: builder: builder that installed the package """ spack.relocate.fixup_macos_rpaths(builder.spec) def execute_install_time_tests(builder: Builder): """Execute the install-time tests prescribed by builder. Args: builder: builder prescribing the test callbacks. The name of the callbacks is stored as a list of strings in the ``install_time_test_callbacks`` attribute. """ if not builder.pkg.run_tests or not builder.install_time_test_callbacks: return builder.pkg.tester.phase_tests(builder, "install", builder.install_time_test_callbacks) class Package(spack.package_base.PackageBase): """Build system base class for packages that do not use a specific build system. It adds the ``build_system=generic`` variant to the package. This is the only build system base class defined in Spack core. All other build systems are defined in the builtin package repository :mod:`spack_repo.builtin.build_systems`. The associated builder is :class:`GenericBuilder`, which is only necessary when the package has multiple build systems. Example:: from spack.package import * class MyPackage(Package): \"\"\"A package that does not use a specific build system.\"\"\" homepage = "https://example.com/mypackage" url = "https://example.com/mypackage-1.0.tar.gz" version("1.0", sha256="...") def install(self, spec: Spec, prefix: Prefix) -> None: # Custom installation logic here pass .. note:: The difference between :class:`Package` and :class:`~spack.package_base.PackageBase` is that :class:`~spack.package_base.PackageBase` is the universal base class for all package classes, no matter their build system. The :class:`Package` class is a *build system base class*, similar to ``CMakePackage``, and ``AutotoolsPackage``. It is called ``Package`` and not ``GenericPackage`` for legacy reasons. """ #: This attribute is used in UI queries that require to know which #: build-system class we are using build_system_class = "Package" #: Legacy buildsystem attribute used to deserialize and install old specs default_buildsystem = "generic" spack.directives.build_system("generic") @register_builder("generic") class GenericBuilder(BuilderWithDefaults): """The associated builder for the :class:`Package` base class. This class is typically only used in ``package.py`` files when a package has multiple build systems. Packagers need to implement the :meth:`install` phase to define how the package is installed. This is the only builder that is defined in the Spack core, all other builders are defined in the builtin package repository :mod:`spack_repo.builtin.build_systems`. Example:: from spack.package import * class MyPackage(Package): \"\"\"A package that does not use a specific build system.\"\"\" homepage = "https://example.com/mypackage" url = "https://example.com/mypackage-1.0.tar.gz" version("1.0", sha256="...") class GenericBuilder(GenericBuilder): def install(self, pkg: Package, spec: Spec, prefix: Prefix) -> None: pass """ #: A generic package has only the ``install`` phase phases = ("install",) #: Names associated with package methods in the old build-system format package_methods: Tuple[str, ...] = () #: Names associated with package attributes in the old build-system format package_attributes: Tuple[str, ...] = ("archive_files", "install_time_test_callbacks") #: Callback names for post-install phase tests install_time_test_callbacks = [] # On macOS, force rpaths for shared library IDs and remove duplicate rpaths spack.phase_callbacks.run_after("install", when="platform=darwin")(apply_macos_rpath_fixups) # unconditionally perform any post-install phase tests spack.phase_callbacks.run_after("install")(execute_install_time_tests) def install(self, pkg: Package, spec: spack.spec.Spec, prefix: Prefix) -> None: """Install phase for the generic builder, to be implemented by packagers.""" raise NotImplementedError ================================================ FILE: lib/spack/spack/caches.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Caches used by Spack to store data""" import os from typing import cast import spack.config import spack.fetch_strategy import spack.llnl.util.lang import spack.paths import spack.util.file_cache import spack.util.path from spack.llnl.util.filesystem import mkdirp def misc_cache_location(): """The ``MISC_CACHE`` is Spack's cache for small data. Currently the ``MISC_CACHE`` stores indexes for virtual dependency providers and for which packages provide which tags. """ path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path) return spack.util.path.canonicalize_path(path) def _misc_cache(): path = misc_cache_location() return spack.util.file_cache.FileCache(path) #: Spack's cache for small data MISC_CACHE = cast(spack.util.file_cache.FileCache, spack.llnl.util.lang.Singleton(_misc_cache)) def fetch_cache_location(): """Filesystem cache of downloaded archives. This prevents Spack from repeatedly fetch the same files when building the same package different ways or multiple times. """ path = spack.config.get("config:source_cache") if not path: path = spack.paths.default_fetch_cache_path path = spack.util.path.canonicalize_path(path) return path def _fetch_cache(): path = fetch_cache_location() return spack.fetch_strategy.FsCache(path) class MirrorCache: def __init__(self, root, skip_unstable_versions): self.root = os.path.abspath(root) self.skip_unstable_versions = skip_unstable_versions def store(self, fetcher, relative_dest): """Fetch and relocate the fetcher's target into our mirror cache.""" # Note this will archive package sources even if they would not # normally be cached (e.g. the current tip of an hg/git branch) dst = os.path.join(self.root, relative_dest) mkdirp(os.path.dirname(dst)) fetcher.archive(dst) #: Spack's local cache for downloaded source archives FETCH_CACHE = cast(spack.fetch_strategy.FsCache, spack.llnl.util.lang.Singleton(_fetch_cache)) ================================================ FILE: lib/spack/spack/ci/README.md ================================================ # Spack CI generators This document describes how the ci module can be extended to provide novel ci generators. The module currently has only a single generator for gitlab. The unit-tests for the ci module define a small custom generator for testing purposes as well. The process of generating a pipeline involves creating a ci-enabled spack environment, activating it, and running `spack ci generate`, possibly with arguments describing things like where the output should be written. Internally pipeline generation is broken into two components: general and ci platform specific. ## General pipeline functionality General pipeline functionality includes building a pipeline graph (really, a forest), pruning it in a variety of ways, and gathering attributes for all the generated spec build jobs from the spack configuration. All of the above functionality is defined in the `__init__.py` of the top-level ci module, and should be roughly the same for pipelines generated for any platform. ## CI platform specific functionality Functionality specific to CI platforms (e.g. gitlab, gha, etc.) should be defined in a dedicated module. In order to define a generator for a new platform, there are only a few requirements: 1. add a file under `ci` in which you define a generator method decorated with the `@generator` attribute. . 1. import it from `lib/spack/spack/ci/__init__.py`, so that your new generator is registered. 1. the generator method must take as arguments PipelineDag, SpackCIConfig, and PipelineOptions objects, in that order. 1. the generator method must produce an output file containing the generated pipeline. ================================================ FILE: lib/spack/spack/ci/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import base64 import json import os import pathlib import re import shutil import stat import subprocess import tempfile import zipfile from collections import namedtuple from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple from urllib.request import Request import spack import spack.binary_distribution import spack.builder import spack.config as cfg import spack.environment as ev import spack.llnl.path import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.main import spack.mirrors.mirror import spack.paths import spack.repo import spack.spec import spack.stage import spack.store import spack.util.git import spack.util.gpg as gpg_util import spack.util.spack_yaml as syaml import spack.util.url as url_util import spack.util.web as web_util from spack import traverse from spack.error import SpackError from spack.llnl.util.tty.color import cescape, colorize from spack.reporters.cdash import SPACK_CDASH_TIMEOUT from .common import ( IS_WINDOWS, CDashHandler, PipelineDag, PipelineOptions, PipelineType, SpackCIConfig, SpackCIError, copy_files_to_artifacts, ) from .generator_registry import UnknownGeneratorException, get_generator # Import any modules with generator functions from here, so they get # registered without introducing any import cycles. from .gitlab import generate_gitlab_yaml # noqa: F401 spack_gpg = spack.main.SpackCommand("gpg") spack_compiler = spack.main.SpackCommand("compiler") PushResult = namedtuple("PushResult", "success url") urlopen = web_util.urlopen # alias for mocking in tests def get_git_root(path: str) -> Optional[str]: git = spack.util.git.git(required=True) try: with fs.working_dir(path): # Raises SpackError on command failure git_dir = git("rev-parse", "--show-toplevel", fail_on_error=True, output=str).strip() tty.debug(f"{path} git toplevel at {git_dir}") return git_dir except SpackError: return None def get_change_revisions(path: str) -> Tuple[Optional[str], Optional[str]]: """If this is a git repo get the revisions to use when checking for changed packages and spack core modules.""" if get_git_root(path): # TODO: This will only find changed packages from the last # TODO: commit. While this may work for single merge commits # TODO: when merging the topic branch into the base, it will # TODO: require more thought outside of that narrow case. return "HEAD^", "HEAD" else: return None, None def filter_added_checksums( checksums: Iterable[str], path: str, from_ref: str = "HEAD~1", to_ref: str = "HEAD" ) -> List[str]: """Get a list of the version checksums added between ``from_ref`` and ``to_ref``. Args: checksums: an iterable of checksums to look for in the diff path: path to the package.py from_ref: oldest git ref, defaults to ``HEAD~1`` to_ref: newer git ref, defaults to ``HEAD`` Returns: list of version checksums added between refs """ git_exe = spack.util.git.git(required=True) # Gather git diff diff_lines = git_exe("diff", from_ref, to_ref, "--", path, output=str).split("\n") # Store added and removed versions # Removed versions are tracked here to determine when versions are moved in a file # and show up as both added and removed in a git diff. added_checksums: Set[str] = set() removed_checksums: Set[str] = set() # Scrape diff for modified versions and prune added versions if they show up # as also removed (which means they've actually just moved in the file and # we shouldn't need to rechecksum them) for checksum in checksums: for line in diff_lines: if checksum in line: if line.startswith("+"): added_checksums.add(checksum) if line.startswith("-"): removed_checksums.add(checksum) return list(added_checksums - removed_checksums) def stack_changed(env_path: str) -> bool: """Given an environment manifest path, return whether or not the stack was changed. Returns True iff the environment manifest changed between the provided revisions (or additionally if the ``.gitlab-ci.yml`` file itself changed).""" # git returns posix paths always, normalize input to be compatible with that env_path = spack.llnl.path.convert_to_posix_path(os.path.dirname(env_path)) git = spack.util.git.git(required=True) git_dir = get_git_root(env_path) if git_dir is None: return False with fs.working_dir(git_dir): diff = git( "diff", "--name-only", "HEAD^", "HEAD", output=str, error=os.devnull, fail_on_error=False, ).strip() if not diff: return False for path in diff.split(): if ".gitlab-ci.yml" in path or path in env_path: tty.debug(f"env represented by {env_path} changed") tty.debug(f"touched file: {path}") return True return False def compute_affected_packages( repo: spack.repo.Repo, rev1: str = "HEAD^", rev2: str = "HEAD" ) -> Set[str]: """Determine which packages were added, removed or changed between rev1 and rev2, and return the names as a set""" return spack.repo.get_all_package_diffs("ARC", repo, rev1=rev1, rev2=rev2) def get_spec_filter_list( env: ev.Environment, affected_pkgs: Set[str], dependent_traverse_depth: Optional[int] = None ) -> Set[spack.spec.Spec]: """Given a list of package names and an active/concretized environment, return the set of all concrete specs from the environment that could have been affected by changing the list of packages. If a ``dependent_traverse_depth`` is given, it is used to limit upward (in the parent direction) traversal of specs of touched packages. E.g. if 1 is provided, then only direct dependents of touched package specs are traversed to produce specs that could have been affected by changing the package, while if 0 is provided, only the changed specs themselves are traversed. If ``None`` is given, upward traversal of touched package specs is done all the way to the environment roots. Providing a negative number results in no traversals at all, yielding an empty set. Arguments: env: Active concrete environment affected_pkgs: Affected package names dependent_traverse_depth: Integer to limit dependent traversal, None means no limit Returns: A set of concrete specs from the active environment including those associated with affected packages, their dependencies and dependents, as well as their dependents dependencies. """ affected_specs: Set[spack.spec.Spec] = set() all_concrete_specs = env.all_specs() env_matches = [s for s in all_concrete_specs if s.name in affected_pkgs] visited: Set[str] = set() for depth, parent in traverse.traverse_nodes( env_matches, direction="parents", key=traverse.by_dag_hash, depth=True, order="breadth" ): if dependent_traverse_depth is not None and depth > dependent_traverse_depth: break affected_specs.update( parent.traverse(direction="children", visited=visited, key=traverse.by_dag_hash) ) return affected_specs # Pruning functions should take a spack.spec.Spec object and # return a RebuildDecision containing the pruners opinion on # whether or not to keep (rebuild) the spec and a message # containing the reason for the decision. class RebuildDecision: def __init__(self, rebuild: bool = True, reason: str = ""): self.rebuild = rebuild self.reason = reason PrunerCallback = Callable[[spack.spec.Spec], RebuildDecision] def create_unaffected_pruner(affected_specs: Set[spack.spec.Spec]) -> PrunerCallback: """Given a set of "affected" specs, return a filter that prunes specs not in the set.""" def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision: if s in affected_specs: return RebuildDecision(True, "affected by change") return RebuildDecision(False, "unaffected by change") return rebuild_filter def create_already_built_pruner(check_index_only: bool = True) -> PrunerCallback: """Return a filter that prunes specs already present on any configured mirrors""" try: spack.binary_distribution.BINARY_INDEX.update() except spack.binary_distribution.FetchCacheError as e: tty.warn(e) def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision: spec_locations = spack.binary_distribution.get_mirrors_for_spec( spec=s, index_only=check_index_only ) if not spec_locations: return RebuildDecision(True, "not found anywhere") urls = ",".join(f"{loc.url}@v{loc.version}" for loc in spec_locations) message = f"up-to-date [{urls}]" return RebuildDecision(False, message) return rebuild_filter def create_external_pruner() -> PrunerCallback: """Return a filter that prunes external specs""" def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision: if not s.external: return RebuildDecision(True, "not external") return RebuildDecision(False, "external spec") return rebuild_filter def _format_pruning_message(spec: spack.spec.Spec, prune: bool, reasons: List[str]) -> str: reason_msg = ", ".join(reasons) spec_fmt = "{name}{@version}{/hash:7}{compilers}" if not prune: status = colorize("@*g{[x]} ") return f" {status}{spec.cformat(spec_fmt)} ({reason_msg})" msg = f"{spec.format(spec_fmt)} ({reason_msg})" return colorize(f" @K - {cescape(msg)}@.") def prune_pipeline( pipeline: PipelineDag, pruning_filters: List[PrunerCallback], print_summary: bool = False ) -> None: """Given a PipelineDag and a list of pruning filters, return a modified PipelineDag containing only the nodes that survive pruning by all of the filters.""" keys_to_prune = set() keys_to_rebuild = set() specs: Dict[str, spack.spec.Spec] = {} reasons: Dict[str, List[str]] = {} for _, node in pipeline.traverse_nodes(direction="children"): filter_results = [keepSpec(node.spec) for keepSpec in pruning_filters] reasons[node.key] = [r.reason for r in filter_results] specs[node.key] = node.spec if not all(r.rebuild for r in filter_results): keys_to_prune.add(node.key) else: keys_to_rebuild.add(node.key) for key in keys_to_prune: pipeline.prune(key) if print_summary: sort_key = lambda k: f"{specs[k].name}/{specs[k].dag_hash(7)}" tty.msg("Pipeline pruning summary:") if keys_to_rebuild: tty.msg(" Rebuild list:") for key in sorted(keys_to_rebuild, key=sort_key): tty.msg(_format_pruning_message(specs[key], False, reasons[key])) if keys_to_prune: tty.msg(" Prune list:") for key in sorted(keys_to_prune, key=sort_key): tty.msg(_format_pruning_message(specs[key], True, reasons[key])) def check_for_broken_specs(pipeline_specs: List[spack.spec.Spec], broken_specs_url: str) -> bool: """Check the pipeline specs against the list of known broken specs and return True if there were any matches, False otherwise.""" if broken_specs_url.startswith("http"): # To make checking each spec against the list faster, we require # a url protocol that allows us to iterate the url in advance. tty.msg("Cannot use an http(s) url for broken specs, ignoring") return False broken_spec_urls = web_util.list_url(broken_specs_url) if broken_spec_urls is None: return False known_broken_specs_encountered = [] for release_spec in pipeline_specs: release_spec_dag_hash = release_spec.dag_hash() if release_spec_dag_hash in broken_spec_urls: known_broken_specs_encountered.append(release_spec_dag_hash) if known_broken_specs_encountered: tty.error("This pipeline generated hashes known to be broken on develop:") display_broken_spec_messages(broken_specs_url, known_broken_specs_encountered) return True return False def collect_pipeline_options(env: ev.Environment, args) -> PipelineOptions: """Gather pipeline options from cli args, spack environment, and os environment variables""" pipeline_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True) if "buildcache-destination" not in pipeline_mirrors: raise SpackCIError("spack ci generate requires a mirror named 'buildcache-destination'") buildcache_destination = pipeline_mirrors["buildcache-destination"] options = PipelineOptions(env, buildcache_destination) options.env = env options.artifacts_root = args.artifacts_root options.output_file = args.output_file options.prune_up_to_date = args.prune_dag options.prune_unaffected = args.prune_unaffected options.prune_external = args.prune_externals options.check_index_only = args.index_only options.forward_variables = args.forward_variable or [] ci_config = cfg.get("ci") cdash_config = cfg.get("cdash") if "build-group" in cdash_config: options.cdash_handler = CDashHandler(cdash_config) dependent_depth = os.environ.get("SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH", None) if dependent_depth is not None: try: options.untouched_pruning_dependent_depth = int(dependent_depth) except (TypeError, ValueError): tty.warn( f"Unrecognized value ({dependent_depth}) " "provided for SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH, " "ignoring it." ) spack_prune_untouched = str(os.environ.get("SPACK_PRUNE_UNTOUCHED", options.prune_unaffected)) options.prune_untouched = ( spack_prune_untouched is not None and spack_prune_untouched.lower() == "true" ) # Allow overriding --prune-dag cli opt with environment variable prune_dag_override = os.environ.get("SPACK_PRUNE_UP_TO_DATE", None) if prune_dag_override is not None: options.prune_up_to_date = True if prune_dag_override.lower() == "true" else False options.stack_name = os.environ.get("SPACK_CI_STACK_NAME", None) require_signing = os.environ.get("SPACK_REQUIRE_SIGNING", None) options.require_signing = ( True if require_signing and require_signing.lower() == "true" else False ) # Get the type of pipeline, which is optional spack_pipeline_type = os.environ.get("SPACK_PIPELINE_TYPE", None) if spack_pipeline_type: try: options.pipeline_type = PipelineType[spack_pipeline_type] except KeyError: options.pipeline_type = None if "broken-specs-url" in ci_config: options.broken_specs_url = ci_config["broken-specs-url"] if "rebuild-index" in ci_config and ci_config["rebuild-index"] is False: options.rebuild_index = False return options def get_unaffected_pruners( env: ev.Environment, untouched_pruning_dependent_depth: Optional[int] ) -> Optional[PrunerCallback]: # If the stack env has changed, do not apply unaffected pruning if stack_changed(env.manifest_path): tty.info("Skipping unaffected pruning: stack environment changed") return None # TODO: This should be configurable to only check for changed packages # in specific configured repos that are being tested with CI. For now # it assumes all configured repos are merge commits that contain relevant # changes to run CI on. affected_pkgs: Set[str] = set() for repo in spack.repo.PATH.repos: rev1, rev2 = get_change_revisions(repo.root) if not (rev1 and rev2): continue tty.debug(f"repo {repo.namespace}: revisions rev1={rev1}, rev2={rev2}") repo_affected_pkgs = compute_affected_packages(repo, rev1=rev1, rev2=rev2) tty.debug(f"repo {repo.namespace}: affected pkgs") for p in repo_affected_pkgs: tty.debug(f" {p}") affected_pkgs.update(repo_affected_pkgs) if not affected_pkgs: tty.info("Skipping unaffected pruning: no package changes were detected") return None affected_specs = get_spec_filter_list( env, affected_pkgs, dependent_traverse_depth=untouched_pruning_dependent_depth ) tty.debug(f"dependent_traverse_depth={untouched_pruning_dependent_depth}, affected specs:") for s in affected_specs: tty.debug(f" {PipelineDag.key(s)}") return create_unaffected_pruner(affected_specs) def generate_pipeline(env: ev.Environment, args) -> None: """Given an environment and the command-line args, generate a pipeline. Arguments: env (spack.environment.Environment): Activated environment object which must contain a ci section describing attributes for all jobs and a target which should specify an existing pipeline generator. args: (spack.main.SpackArgumentParser): Parsed arguments from the command line. """ with env.write_transaction(): env.concretize() env.write() options = collect_pipeline_options(env, args) # Get the joined "ci" config with all of the current scopes resolved ci_config = cfg.get("ci") if not ci_config: raise SpackCIError("Environment does not have a `ci` configuration") # Get the target platform we should generate a pipeline for ci_target = ci_config.get("target", "gitlab") try: generate_method = get_generator(ci_target) except UnknownGeneratorException: raise SpackCIError(f"Spack CI module cannot generate a pipeline for format {ci_target}") # If we are not doing any kind of pruning, we are rebuilding everything rebuild_everything = not options.prune_up_to_date and not options.prune_untouched # Build a pipeline from the specs in the concrete environment pipeline = PipelineDag([env.specs_by_hash[x.hash] for x in env.concretized_roots]) # Optionally add various pruning filters pruning_filters = [] # Possibly prune specs that were unaffected by the change if options.prune_untouched: # If we don't have two revisions to compare, or if either the spack.yaml # associated with the active env or the .gitlab-ci.yml files changed # between the provided revisions, then don't do any "untouched spec" # pruning. Otherwise, list the names of all packages touched between # rev1 and rev2, and prune from the pipeline any node whose spec has a # packagen name not in that list. unaffected_pruner = get_unaffected_pruners(env, options.untouched_pruning_dependent_depth) if unaffected_pruner: tty.info("Enabling Unaffected Pruner") pruning_filters.append(unaffected_pruner) # Possibly prune specs that are already built on some configured mirror if options.prune_up_to_date: tty.info("Enabling Up-to-date Pruner") pruning_filters.append( create_already_built_pruner(check_index_only=options.check_index_only) ) # Possibly prune specs that are external if options.prune_external: tty.info("Enabling Externals Pruner") pruning_filters.append(create_external_pruner()) # Do all the pruning prune_pipeline(pipeline, pruning_filters, options.print_summary) # List all specs remaining after any pruning pipeline_specs = [n.spec for _, n in pipeline.traverse_nodes(direction="children")] # If this is configured, spack will fail "spack ci generate" if it # generates any hash which exists under the broken specs url. if options.broken_specs_url and not options.pipeline_type == PipelineType.COPY_ONLY: broken = check_for_broken_specs(pipeline_specs, options.broken_specs_url) if broken and not rebuild_everything: raise SpackCIError("spack ci generate failed broken specs check") spack_ci_config = SpackCIConfig(ci_config) spack_ci_config.init_pipeline_jobs(pipeline) # Format the pipeline using the formatter specified in the configs generate_method(pipeline, spack_ci_config, options) # Use all unpruned specs to populate the build group for this set cdash_config = cfg.get("cdash") if options.cdash_handler and options.cdash_handler.auth_token: options.cdash_handler.create_buildgroup() elif cdash_config: # warn only if there was actually a CDash configuration. tty.warn("Unable to populate buildgroup without CDash credentials") def import_signing_key(base64_signing_key: str) -> None: """Given Base64-encoded gpg key, decode and import it to use for signing packages. Arguments: base64_signing_key: A gpg key including the secret key, armor-exported and base64 encoded, so it can be stored in a gitlab CI variable. For an example of how to generate such a key, see https://github.com/spack/spack-infrastructure/blob/main/gitlab-docker/files/gen-key. """ if not base64_signing_key: tty.warn("No key found for signing/verifying packages") return tty.debug("ci.import_signing_key() will attempt to import a key") # This command has the side-effect of creating the directory referred # to as GNUPGHOME in setup_environment() list_output = spack_gpg("list") tty.debug("spack gpg list:") tty.debug(list_output) decoded_key = base64.b64decode(base64_signing_key).decode("utf-8") with tempfile.TemporaryDirectory() as tmpdir: sign_key_path = os.path.join(tmpdir, "signing_key") with open(sign_key_path, "w", encoding="utf-8") as fd: fd.write(decoded_key) key_import_output = spack_gpg("trust", sign_key_path) tty.debug(f"spack gpg trust {sign_key_path}") tty.debug(key_import_output) # Now print the keys we have for verifying and signing trusted_keys_output = spack_gpg("list", "--trusted") signing_keys_output = spack_gpg("list", "--signing") tty.debug("spack gpg list --trusted") tty.debug(trusted_keys_output) tty.debug("spack gpg list --signing") tty.debug(signing_keys_output) def can_sign_binaries(): """Utility method to determine if this spack instance is capable of signing binary packages. This is currently only possible if the spack gpg keystore contains exactly one secret key.""" return len(gpg_util.signing_keys()) == 1 def can_verify_binaries(): """Utility method to determine if this spack instance is capable (at least in theory) of verifying signed binaries.""" return len(gpg_util.public_keys()) >= 1 def push_to_build_cache(spec: spack.spec.Spec, mirror_url: str, sign_binaries: bool) -> bool: """Push one or more binary packages to the mirror. Arguments: spec: Installed spec to push mirror_url: URL of target mirror sign_binaries: If True, spack will attempt to sign binary package before pushing. """ tty.debug(f"Pushing to build cache ({'signed' if sign_binaries else 'unsigned'})") signing_key = spack.binary_distribution.select_signing_key() if sign_binaries else None mirror = spack.mirrors.mirror.Mirror.from_url(mirror_url) try: with spack.binary_distribution.make_uploader(mirror, signing_key=signing_key) as uploader: uploader.push_or_raise([spec]) return True except spack.binary_distribution.PushToBuildCacheError as e: tty.error(f"Problem writing to {mirror_url}: {e}") return False def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) -> None: """Copy selected build stage file(s) to the given artifacts directory Looks for build logs in the stage directory of the given job_spec, and attempts to copy the files into the directory given by job_log_dir. Parameters: job_spec: spec associated with spack install log job_log_dir: path into which build log should be copied """ tty.debug(f"job spec: {job_spec}") if not job_spec.concrete: tty.warn("Cannot copy artifacts for non-concrete specs") return package_metadata_root = pathlib.Path(spack.store.STORE.layout.metadata_path(job_spec)) if not os.path.isdir(package_metadata_root): # Fallback to using the stage directory job_pkg = job_spec.package package_metadata_root = pathlib.Path(job_pkg.stage.path) archive_files = spack.builder.create(job_pkg).archive_files tty.warn("Package not installed, falling back to use stage dir") tty.debug(f"stage dir: {package_metadata_root}") else: # Get the package's archived files archive_files = [] archive_root = package_metadata_root / "archived-files" if os.path.isdir(archive_root): archive_files = [str(f) for f in archive_root.rglob("*") if os.path.isfile(f)] else: tty.debug(f"No archived files detected at {archive_root}") # Try zipped and unzipped versions of the build log build_log_zipped = package_metadata_root / "spack-build-out.txt.gz" build_log = package_metadata_root / "spack-build-out.txt" build_env_mods = package_metadata_root / "spack-build-env.txt" for f in [build_log_zipped, build_log, build_env_mods, *archive_files]: copy_files_to_artifacts(str(f), job_log_dir, compress_artifacts=True) def copy_test_logs_to_artifacts(test_stage, job_test_dir): """ Copy test log file(s) to the given artifacts directory Parameters: test_stage (str): test stage path job_test_dir (str): the destination artifacts test directory """ tty.debug(f"test stage: {test_stage}") if not os.path.exists(test_stage): tty.error(f"Cannot copy test logs: job test stage ({test_stage}) does not exist") return copy_files_to_artifacts( os.path.join(test_stage, "*", "*.txt"), job_test_dir, compress_artifacts=True ) def download_and_extract_artifacts(url: str, work_dir: str) -> str: """Look for gitlab artifacts.zip at the given url, and attempt to download and extract the contents into the given work_dir Arguments: url: Complete url to artifacts.zip file work_dir: Path to destination where artifacts should be extracted Returns: Artifacts root path relative to the archive root """ tty.msg(f"Fetching artifacts from: {url}") headers = {"Content-Type": "application/zip"} token = os.environ.get("GITLAB_PRIVATE_TOKEN", None) if token: headers["PRIVATE-TOKEN"] = token request = Request(url, headers=headers, method="GET") artifacts_zip_path = os.path.join(work_dir, "artifacts.zip") os.makedirs(work_dir, exist_ok=True) try: with urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response: with open(artifacts_zip_path, "wb") as out_file: shutil.copyfileobj(response, out_file) with zipfile.ZipFile(artifacts_zip_path) as zip_file: zip_file.extractall(work_dir) # Get the artifact root artifact_root = "" for f in zip_file.filelist: if "spack.lock" in f.filename: artifact_root = os.path.dirname(os.path.dirname(f.filename)) break except OSError as e: raise SpackError(f"Error fetching artifacts: {e}") finally: try: os.remove(artifacts_zip_path) except FileNotFoundError: # If the file doesn't exist we are already raising pass return artifact_root def get_spack_info(): """If spack is running from a git repo, return the most recent git log entry, otherwise, return a string containing the spack version.""" git_path = os.path.join(spack.paths.prefix, ".git") if os.path.exists(git_path): git = spack.util.git.git() if git: with fs.working_dir(spack.paths.prefix): git_log = git("log", "-1", output=str, error=os.devnull, fail_on_error=False) return git_log return f"no git repo, use spack {spack.spack_version}" def setup_spack_repro_version( repro_dir: str, checkout_commit: str, merge_commit: Optional[str] = None ) -> bool: """Look in the local spack clone to find the checkout_commit, and if provided, the merge_commit given as arguments. If those commits can be found locally, then clone spack and attempt to recreate a merge commit with the same parent commits as tested in gitlab. This looks something like 1. ``git clone repo && cd repo`` 2. ``git checkout `` 3. ``git merge `` If there is no merge_commit provided, then skip step (3). Arguments: repro_dir: Location where spack should be cloned checkout_commit: SHA of PR branch commit merge_commit: SHA of target branch parent Returns: True iff the git repo state was successfully recreated """ # figure out the path to the spack git version being used for the # reproduction tty.info(f"checkout_commit: {checkout_commit}") tty.info(f"merge_commit: {merge_commit}") dot_git_path = os.path.join(spack.paths.prefix, ".git") if not os.path.exists(dot_git_path): tty.error("Unable to find the path to your local spack clone") return False spack_git_path = spack.paths.prefix git = spack.util.git.git() if not git: tty.error("reproduction of pipeline job requires git") return False # Check if we can find the tested commits in your local spack repo with fs.working_dir(spack_git_path): git("log", "-1", checkout_commit, output=str, error=os.devnull, fail_on_error=False) if git.returncode != 0: tty.error(f"Missing commit: {checkout_commit}") return False if merge_commit: git("log", "-1", merge_commit, output=str, error=os.devnull, fail_on_error=False) if git.returncode != 0: tty.error(f"Missing commit: {merge_commit}") return False # Next attempt to clone your local spack repo into the repro dir with fs.working_dir(repro_dir): clone_out = git( "clone", spack_git_path, "spack", output=str, error=os.devnull, fail_on_error=False ) if git.returncode != 0: tty.error("Unable to clone your local spack repo:") tty.msg(clone_out) return False # Finally, attempt to put the cloned repo into the same state used during # the pipeline build job repro_spack_path = os.path.join(repro_dir, "spack") with fs.working_dir(repro_spack_path): co_out = git( "checkout", checkout_commit, output=str, error=os.devnull, fail_on_error=False ) if git.returncode != 0: tty.error(f"Unable to checkout {checkout_commit}") tty.msg(co_out) return False if merge_commit: merge_out = git( "-c", "user.name=cirepro", "-c", "user.email=user@email.org", "merge", "--no-edit", merge_commit, output=str, error=os.devnull, fail_on_error=False, ) if git.returncode != 0: tty.error(f"Unable to merge {merge_commit}") tty.msg(merge_out) return False return True def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime, use_local_head): """Given a url to gitlab artifacts.zip from a failed ``spack ci rebuild`` job, attempt to setup an environment in which the failure can be reproduced locally. This entails the following: First download and extract artifacts. Then look through those artifacts to glean some information needed for the reproduer (e.g. one of the artifacts contains information about the version of spack tested by gitlab, another is the generated pipeline yaml containing details of the job like the docker image used to run it). The output of this function is a set of printed instructions for running docker and then commands to run to reproduce the build once inside the container. """ work_dir = os.path.realpath(work_dir) if os.path.exists(work_dir) and os.listdir(work_dir): raise SpackError(f"Cannot run reproducer in non-empty working dir:\n {work_dir}") platform_script_ext = "ps1" if IS_WINDOWS else "sh" artifact_root = download_and_extract_artifacts(url, work_dir) gpg_path = None if gpg_url: gpg_path = web_util.fetch_url_text(gpg_url, dest_dir=os.path.join(work_dir, "_pgp")) rel_gpg_path = gpg_path.replace(work_dir, "").lstrip(os.path.sep) lock_file = fs.find(work_dir, "spack.lock")[0] repro_lock_dir = os.path.dirname(lock_file) tty.debug(f"Found lock file in: {repro_lock_dir}") yaml_files = fs.find(work_dir, ["*.yaml", "*.yml"]) tty.debug("yaml files:") for yaml_file in yaml_files: tty.debug(f" {yaml_file}") pipeline_yaml = None # Try to find the dynamically generated pipeline yaml file in the # reproducer. If the user did not put it in the artifacts root, # but rather somewhere else and exported it as an artifact from # that location, we won't be able to find it. for yf in yaml_files: with open(yf, encoding="utf-8") as y_fd: yaml_obj = syaml.load(y_fd) if "variables" in yaml_obj and "stages" in yaml_obj: pipeline_yaml = yaml_obj if pipeline_yaml: tty.debug(f"\n{yf} is likely your pipeline file") relative_concrete_env_dir = pipeline_yaml["variables"]["SPACK_CONCRETE_ENV_DIR"] tty.debug(f"Relative environment path used by cloud job: {relative_concrete_env_dir}") # Using the relative concrete environment path found in the generated # pipeline variable above, copy the spack environment files so they'll # be found in the same location as when the job ran in the cloud. concrete_env_dir = os.path.join(work_dir, relative_concrete_env_dir) os.makedirs(concrete_env_dir, exist_ok=True) copy_lock_path = os.path.join(concrete_env_dir, "spack.lock") orig_yaml_path = os.path.join(repro_lock_dir, "spack.yaml") copy_yaml_path = os.path.join(concrete_env_dir, "spack.yaml") shutil.copyfile(lock_file, copy_lock_path) shutil.copyfile(orig_yaml_path, copy_yaml_path) # Find the install script in the unzipped artifacts and make it executable install_script = fs.find(work_dir, f"install.{platform_script_ext}")[0] if not IS_WINDOWS: # pointless on Windows st = os.stat(install_script) os.chmod(install_script, st.st_mode | stat.S_IEXEC) # Find the repro details file. This just includes some values we wrote # during `spack ci rebuild` to make reproduction easier. E.g. the job # name is written here so we can easily find the configuration of the # job from the generated pipeline file. repro_file = fs.find(work_dir, "repro.json")[0] repro_details = None with open(repro_file, encoding="utf-8") as fd: repro_details = json.load(fd) spec_file = fs.find(work_dir, repro_details["job_spec_json"])[0] reproducer_spec = spack.spec.Spec.from_specfile(spec_file) repro_dir = os.path.dirname(repro_file) rel_repro_dir = repro_dir.replace(work_dir, "").lstrip(os.path.sep) # Find the spack info text file that should contain the git log # of the HEAD commit used during the CI build spack_info_file = fs.find(work_dir, "spack_info.txt")[0] with open(spack_info_file, encoding="utf-8") as fd: spack_info = fd.read() # Access the specific job configuration job_name = repro_details["job_name"] job_yaml = None if job_name in pipeline_yaml: job_yaml = pipeline_yaml[job_name] if job_yaml: tty.debug("Found job:") tty.debug(job_yaml) job_image = None setup_result = False if "image" in job_yaml: job_image_elt = job_yaml["image"] if "name" in job_image_elt: job_image = job_image_elt["name"] else: job_image = job_image_elt tty.msg(f"Job ran with the following image: {job_image}") # Because we found this job was run with a docker image, so we will try # to print a "docker run" command that bind-mounts the directory where # we extracted the artifacts. # Destination of bind-mounted reproduction directory. It makes for a # more faithful reproducer if everything appears to run in the same # absolute path used during the CI build. mount_as_dir = "/work" mounted_workdir = "/reproducer" if repro_details: mount_as_dir = repro_details["ci_project_dir"] mounted_repro_dir = os.path.join(mount_as_dir, rel_repro_dir) mounted_env_dir = os.path.join(mount_as_dir, relative_concrete_env_dir) if gpg_path: mounted_gpg_path = os.path.join(mounted_workdir, rel_gpg_path) # We will also try to clone spack from your local checkout and # reproduce the state present during the CI build, and put that into # the bind-mounted reproducer directory. # Regular expressions for parsing that HEAD commit. If the pipeline # was on the gitlab spack mirror, it will have been a merge commit made by # github and pushed by the sync script. If the pipeline was run on some # environment repo, then the tested spack commit will likely have been # a regular commit. commit_1 = None commit_2 = None commit_regex = re.compile(r"commit\s+([^\s]+)") merge_commit_regex = re.compile(r"Merge\s+([^\s]+)\s+into\s+([^\s]+)") if use_local_head: commit_1 = "HEAD" else: # Try the more specific merge commit regex first m = merge_commit_regex.search(spack_info) if m: # This was a merge commit and we captured the parents commit_1 = m.group(1) commit_2 = m.group(2) else: # Not a merge commit, just get the commit sha m = commit_regex.search(spack_info) if m: commit_1 = m.group(1) setup_result = False if commit_1: if commit_2: setup_result = setup_spack_repro_version(work_dir, commit_2, merge_commit=commit_1) else: setup_result = setup_spack_repro_version(work_dir, commit_1) if not setup_result: setup_msg = """ This can happen if the spack you are using to run this command is not a git repo, or if it is a git repo, but it does not have the commits needed to recreate the tested merge commit. If you are trying to reproduce a spack PR pipeline job failure, try fetching the latest develop commits from mainline spack and make sure you have the most recent commit of the PR branch in your local spack repo. Then run this command again. Alternatively, you can also manually clone spack if you know the version you want to test. """ tty.error( "Failed to automatically setup the tested version of spack " "in your local reproduction directory." ) tty.info(setup_msg) # In cases where CI build was run on a shell runner, it might be useful # to see what tags were applied to the job so the user knows what shell # runner was used. But in that case in general, we cannot do nearly as # much to set up the reproducer. job_tags = None if "tags" in job_yaml: job_tags = job_yaml["tags"] tty.msg(f"Job ran with the following tags: {job_tags}") entrypoint_script = [ ["git", "config", "--global", "--add", "safe.directory", mount_as_dir], [ ".", os.path.join( mount_as_dir if job_image else work_dir, f"share/spack/setup-env.{platform_script_ext}", ), ], ["spack", "gpg", "trust", mounted_gpg_path if job_image else gpg_path] if gpg_path else [], ["spack", "env", "activate", mounted_env_dir if job_image else repro_dir], [ ( os.path.join(mounted_repro_dir, f"install.{platform_script_ext}") if job_image else install_script ) ], ] entry_script = os.path.join(mounted_workdir, f"entrypoint.{platform_script_ext}") inst_list = [] # Finally, print out some instructions to reproduce the build if job_image: # Allow interactive install_mechanism = ( os.path.join(mounted_repro_dir, f"install.{platform_script_ext}") if job_image else install_script ) entrypoint_script.append(["echo", f"Re-run install script using:\n\t{install_mechanism}"]) # Allow interactive if IS_WINDOWS: entrypoint_script.append(["&", "($args -Join ' ')", "-NoExit"]) else: entrypoint_script.append(["exec", "$@"]) process_command( "entrypoint", entrypoint_script, work_dir, run=False, exit_on_failure=False ) # Attempt to create a unique name for the reproducer container container_suffix = "_" + reproducer_spec.dag_hash() if reproducer_spec else "" docker_command = [ runtime, "run", "-i", "-t", "--rm", "--name", f"spack_reproducer{container_suffix}", "-v", ":".join([work_dir, mounted_workdir, "Z"]), "-v", ":".join( [ os.path.join(work_dir, artifact_root), os.path.join(mount_as_dir, artifact_root), "Z", ] ), "-v", ":".join([os.path.join(work_dir, "spack"), mount_as_dir, "Z"]), "--entrypoint", ] if IS_WINDOWS: docker_command.extend(["powershell.exe", job_image, entry_script, "powershell.exe"]) else: docker_command.extend([entry_script, job_image, "bash"]) docker_command = [docker_command] autostart = autostart and setup_result process_command("start", docker_command, work_dir, run=autostart) if not autostart: inst_list.append("\nTo run the docker reproducer:\n\n") inst_list.extend( [ " - Start the docker container install", f" $ {work_dir}/start.{platform_script_ext}", ] ) else: autostart = autostart and setup_result process_command("reproducer", entrypoint_script, work_dir, run=autostart) inst_list.append("\nOnce on the tagged runner:\n\n") inst_list.extend( [ " - Run the reproducer script", f" $ {work_dir}/reproducer.{platform_script_ext}", ] ) if not setup_result: inst_list.append("\n - Clone spack and acquire tested commit") inst_list.append(f"\n {spack_info}\n") inst_list.append("\n") inst_list.append(f"\n Path to clone spack: {work_dir}/spack\n\n") tty.msg("".join(inst_list)) def process_command(name, commands, repro_dir, run=True, exit_on_failure=True): """ Create a script for and run the command. Copy the script to the reproducibility directory. Arguments: name (str): name of the command being processed commands (list): list of arguments for single command or list of lists of arguments for multiple commands. No shell escape is performed. repro_dir (str): Job reproducibility directory run (bool): Run the script and return the exit code if True Returns: the exit code from processing the command """ tty.debug(f"spack {name} arguments: {commands}") if len(commands) == 0 or isinstance(commands[0], str): commands = [commands] def compose_command_err_handling(args): if not IS_WINDOWS: args = [f'"{arg}"' for arg in args] arg_str = " ".join(args) result = arg_str + "\n" # ErrorActionPreference will handle PWSH commandlets (Spack calls), # but we need to handle EXEs (git, etc) ourselves catch_exe_failure = ( """ if ($LASTEXITCODE -ne 0){{ throw 'Command {} has failed' }} """ if IS_WINDOWS else "" ) if exit_on_failure and catch_exe_failure: result += catch_exe_failure.format(arg_str) return result # Create a string [command 1] \n [command 2] \n ... \n [command n] with # commands composed into a platform dependent shell script, pwsh on Windows, full_command = "\n".join(map(compose_command_err_handling, commands)) # Write the command to a python script if IS_WINDOWS: script = f"{name}.ps1" script_content = [f"\n# spack {name} command\n"] if exit_on_failure: script_content.append('$ErrorActionPreference = "Stop"\n') if os.environ.get("SPACK_VERBOSE_SCRIPT"): script_content.append("Set-PSDebug -Trace 2\n") else: script = f"{name}.sh" script_content = ["#!/bin/sh\n\n", f"\n# spack {name} command\n"] if exit_on_failure: script_content.append("set -e\n") if os.environ.get("SPACK_VERBOSE_SCRIPT"): script_content.append("set -x\n") script_content.append(full_command) script_content.append("\n") with open(script, "w", encoding="utf-8") as fd: for line in script_content: fd.write(line) copy_path = os.path.join(repro_dir, script) shutil.copyfile(script, copy_path) if not IS_WINDOWS: st = os.stat(copy_path) os.chmod(copy_path, st.st_mode | stat.S_IEXEC) # Run the generated shell script as if it were being run in # a login shell. exit_code = None if run: try: # We use sh as executor on Linux like platforms, pwsh on Windows interpreter = "powershell.exe" if IS_WINDOWS else "/bin/sh" cmd_process = subprocess.Popen([interpreter, f"./{script}"]) cmd_process.wait() exit_code = cmd_process.returncode except (ValueError, subprocess.CalledProcessError, OSError) as err: tty.error(f"Encountered error running {name} script") tty.error(err) exit_code = 1 tty.debug(f"spack {name} exited {exit_code}") else: # Delete the script, it is copied to the destination dir os.remove(script) return exit_code def create_buildcache( input_spec: spack.spec.Spec, *, destination_mirror_urls: List[str], sign_binaries: bool = False ) -> List[PushResult]: """Create the buildcache at the provided mirror(s). Arguments: input_spec: Installed spec to package and push destination_mirror_urls: List of urls to push to sign_binaries: Whether or not to sign buildcache entry Returns: A list of PushResults, indicating success or failure. """ results = [] for mirror_url in destination_mirror_urls: results.append( PushResult( success=push_to_build_cache(input_spec, mirror_url, sign_binaries), url=mirror_url ) ) return results def write_broken_spec(url, pkg_name, stack_name, job_url, pipeline_url, spec_dict): """Given a url to write to and the details of the failed job, write an entry in the broken specs list. """ with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: file_path = os.path.join(tmpdir, "broken.txt") broken_spec_details = { "broken-spec": { "job-name": pkg_name, "job-stack": stack_name, "job-url": job_url, "pipeline-url": pipeline_url, "concrete-spec-dict": spec_dict, } } try: with open(file_path, "w", encoding="utf-8") as fd: syaml.dump(broken_spec_details, fd) web_util.push_to_url( file_path, url, keep_original=False, extra_args={"ContentType": "text/plain"} ) except Exception as err: # If there is an S3 error (e.g., access denied or connection # error), the first non boto-specific class in the exception # hierarchy is Exception. Just print a warning and return msg = f"Error writing to broken specs list {url}: {err}" tty.warn(msg) def read_broken_spec(broken_spec_url): """Read data from broken specs file located at the url, return as a yaml object. """ try: broken_spec_contents = web_util.read_text(broken_spec_url) except web_util.SpackWebError: tty.warn(f"Unable to read broken spec from {broken_spec_url}") return None return syaml.load(broken_spec_contents) def display_broken_spec_messages(base_url, hashes): """Fetch the broken spec file for each of the hashes under the base_url and print a message with some details about each one. """ broken_specs = [(h, read_broken_spec(url_util.join(base_url, h))) for h in hashes] for spec_hash, broken_spec in [tup for tup in broken_specs if tup[1]]: details = broken_spec["broken-spec"] if "job-name" in details: item_name = f"{details['job-name']}/{spec_hash[:7]}" else: item_name = spec_hash if "job-stack" in details: item_name = f"{item_name} (in stack {details['job-stack']})" msg = f" {item_name} was reported broken here: {details['job-url']}" tty.msg(msg) def run_standalone_tests( *, cdash: Optional[CDashHandler] = None, fail_fast: bool = False, log_file: Optional[str] = None, job_spec: Optional[spack.spec.Spec] = None, repro_dir: Optional[str] = None, timeout: Optional[int] = None, ): """Run stand-alone tests on the current spec. Args: cdash: cdash handler instance fail_fast: terminate tests after the first failure log_file: test log file name if NOT CDash reporting job_spec: spec that was built repro_dir: reproduction directory timeout: maximum time (in seconds) that tests are allowed to run """ if cdash and log_file: tty.msg(f"The test log file {log_file} option is ignored with CDash reporting") log_file = None # Error out but do NOT terminate if there are missing required arguments. if not job_spec: tty.error("Job spec is required to run stand-alone tests") return if not repro_dir: tty.error("Reproduction directory is required for stand-alone tests") return test_args = ["spack", "--color=always", "--backtrace", "--verbose", "test", "run"] if fail_fast: test_args.append("--fail-fast") if timeout is not None: test_args.extend(["--timeout", str(timeout)]) if cdash: test_args.extend(cdash.args()) else: test_args.extend(["--log-format", "junit"]) if log_file: test_args.extend(["--log-file", log_file]) test_args.append(job_spec.name) tty.debug(f"Running {job_spec.name} stand-alone tests") exit_code = process_command("test", test_args, repro_dir) tty.debug(f"spack test exited {exit_code}") ================================================ FILE: lib/spack/spack/ci/common.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import copy import errno import glob import gzip import json import os import re import shutil import sys import time from collections import deque from enum import Enum from typing import Dict, Generator, List, Optional, Set, Tuple from urllib.parse import quote, urlencode, urlparse from urllib.request import Request import spack.binary_distribution import spack.config as cfg import spack.deptypes as dt import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.schema import spack.spec import spack.util.compression as compression import spack.util.web as web_util from spack import traverse from spack.llnl.util.lang import memoized from spack.reporters import CDash, CDashConfiguration from spack.reporters.cdash import SPACK_CDASH_TIMEOUT from spack.reporters.cdash import build_stamp as cdash_build_stamp from spack.url_buildcache import get_url_buildcache_class IS_WINDOWS = sys.platform == "win32" SPACK_RESERVED_TAGS = ["public", "protected", "notary"] # this exists purely for testing purposes _urlopen = web_util.urlopen def copy_gzipped(glob_or_path: str, dest: str) -> None: """Copy all of the files in the source glob/path to the destination. Args: glob_or_path: path to file to test dest: destination path to copy to """ files = glob.glob(glob_or_path) if not files: raise OSError("No such file or directory: '{0}'".format(glob_or_path), errno.ENOENT) if len(files) > 1 and not os.path.isdir(dest): raise ValueError( "'{0}' matches multiple files but '{1}' is not a directory".format(glob_or_path, dest) ) def is_gzipped(path): with open(path, "rb") as fd: return compression.GZipFileType().matches_magic(fd) for src in files: if is_gzipped(src): fs.copy(src, dest) else: # Compress and copy in one step src_name = os.path.basename(src) if os.path.isdir(dest): zipped = os.path.join(dest, f"{src_name}.gz") elif not dest.endswith(".gz"): zipped = f"{dest}.gz" else: zipped = dest with open(src, "rb") as fin, gzip.open(zipped, "wb") as fout: shutil.copyfileobj(fin, fout) def copy_files_to_artifacts( src: str, artifacts_dir: str, *, compress_artifacts: bool = False ) -> None: """ Copy file(s) to the given artifacts directory Args: src (str): the glob-friendly path expression for the file(s) to copy artifacts_dir (str): the destination directory compress_artifacts (bool): option to compress copied artifacts using Gzip """ try: if compress_artifacts: copy_gzipped(src, artifacts_dir) else: fs.copy(src, artifacts_dir) except Exception as err: tty.warn( ( f"Unable to copy files ({src}) to artifacts {artifacts_dir} due to " f"exception: {str(err)}" ) ) def win_quote(quote_str: str) -> str: if IS_WINDOWS: quote_str = f'"{quote_str}"' return quote_str def _spec_matches(spec, match_string): return spec.intersects(match_string) def _noop(x): return x def unpack_script(script_section, op=_noop): script = [] for cmd in script_section: if isinstance(cmd, list): for subcmd in cmd: script.append(op(subcmd)) else: script.append(op(cmd)) return script def ensure_expected_target_path(path: str) -> str: """Returns passed paths with all Windows path separators exchanged for posix separators TODO (johnwparent): Refactor config + cli read/write to deal only in posix style paths """ if path: return path.replace("\\", "/") return path def write_pipeline_manifest(specs, src_prefix, dest_prefix, output_file): """Write out the file describing specs that should be copied""" buildcache_copies = {} for release_spec in specs: release_spec_dag_hash = release_spec.dag_hash() cache_class = get_url_buildcache_class( layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ) buildcache_copies[release_spec_dag_hash] = { "src": cache_class.get_manifest_url(release_spec, src_prefix), "dest": cache_class.get_manifest_url(release_spec, dest_prefix), } target_dir = os.path.dirname(output_file) if not os.path.exists(target_dir): os.makedirs(target_dir) with open(output_file, "w", encoding="utf-8") as fd: fd.write(json.dumps(buildcache_copies)) class CDashHandler: """ Class for managing CDash data and processing. """ def __init__(self, ci_cdash): # start with the gitlab ci configuration self.url = ci_cdash.get("url") self.build_group = ci_cdash.get("build-group") self.project = ci_cdash.get("project") self.site = ci_cdash.get("site") # grab the authorization token when available self.auth_token = os.environ.get("SPACK_CDASH_AUTH_TOKEN") if self.auth_token: tty.verbose("Using CDash auth token from environment") # append runner description to the site if available runner = os.environ.get("CI_RUNNER_DESCRIPTION") if runner: self.site += f" ({runner})" def args(self): return [ "--cdash-upload-url", win_quote(self.upload_url), "--cdash-build", win_quote(self.build_name()), "--cdash-site", win_quote(self.site), "--cdash-buildstamp", win_quote(self.build_stamp), ] def build_name(self, spec: Optional[spack.spec.Spec] = None) -> Optional[str]: """Returns the CDash build name. A name will be generated if the ``spec`` is provided, otherwise, the value will be retrieved from the environment through the ``SPACK_CDASH_BUILD_NAME`` variable. Returns: (str) given spec's CDash build name.""" if spec: spec_str = spec.format("{name}{@version}{%compiler} hash={hash} arch={architecture}") build_name = f"{spec_str} ({self.build_group})" tty.debug(f"Generated CDash build name ({build_name}) from the {spec.name}") return build_name env_build_name = os.environ.get("SPACK_CDASH_BUILD_NAME") tty.debug(f"Using CDash build name ({env_build_name}) from the environment") return env_build_name @property # type: ignore def build_stamp(self): """Returns the CDash build stamp. The one defined by SPACK_CDASH_BUILD_STAMP environment variable is preferred due to the representation of timestamps; otherwise, one will be built. Returns: (str) current CDash build stamp""" build_stamp = os.environ.get("SPACK_CDASH_BUILD_STAMP") if build_stamp: tty.debug(f"Using build stamp ({build_stamp}) from the environment") return build_stamp build_stamp = cdash_build_stamp(self.build_group, time.time()) tty.debug(f"Generated new build stamp ({build_stamp})") return build_stamp @property # type: ignore @memoized def project_enc(self): tty.debug(f"Encoding project ({type(self.project)}): {self.project})") encode = urlencode({"project": self.project}) index = encode.find("=") + 1 return encode[index:] @property def upload_url(self): url_format = f"{self.url}/submit.php?project={self.project_enc}" return url_format def copy_test_results(self, source, dest): """Copy test results to artifacts directory.""" reports = fs.join_path(source, "*_Test*.xml") copy_files_to_artifacts(reports, dest) def create_buildgroup(self): """Create the CDash buildgroup if it does not already exist.""" headers = { "Authorization": f"Bearer {self.auth_token}", "Content-Type": "application/json", } data = {"newbuildgroup": self.build_group, "project": self.project, "type": "Daily"} enc_data = json.dumps(data).encode("utf-8") request = Request(f"{self.url}/api/v1/buildgroup.php", data=enc_data, headers=headers) response_text = None group_id = None try: with _urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response: response_text = response.read() except OSError as e: tty.warn(f"Failed to create CDash buildgroup: {e}") if response_text: try: response_json = json.loads(response_text) group_id = response_json["id"] except (json.JSONDecodeError, KeyError) as e: tty.warn(f"Failed to parse CDash response: {e}") if not group_id: tty.warn(f"Failed to create or retrieve buildgroup for {self.build_group}") def report_skipped(self, spec: spack.spec.Spec, report_dir: str, reason: Optional[str]): """Explicitly report skipping testing of a spec (e.g., it's CI configuration identifies it as known to have broken tests or the CI installation failed). Args: spec: spec being tested report_dir: directory where the report will be written reason: reason the test is being skipped """ configuration = CDashConfiguration( upload_url=self.upload_url, packages=[spec.name], build=self.build_name(), site=self.site, buildstamp=self.build_stamp, track=None, ) reporter = CDash(configuration=configuration) reporter.test_skipped_report(report_dir, spec, reason) class PipelineType(Enum): COPY_ONLY = 1 spack_copy_only = 1 PROTECTED_BRANCH = 2 spack_protected_branch = 2 PULL_REQUEST = 3 spack_pull_request = 3 class PipelineOptions: """A container for all pipeline options that can be specified (whether via cli, config/yaml, or environment variables)""" def __init__( self, env: ev.Environment, buildcache_destination: spack.mirrors.mirror.Mirror, artifacts_root: str = "jobs_scratch_dir", print_summary: bool = True, output_file: Optional[str] = None, check_index_only: bool = False, broken_specs_url: Optional[str] = None, rebuild_index: bool = True, untouched_pruning_dependent_depth: Optional[int] = None, prune_untouched: bool = False, prune_up_to_date: bool = True, prune_unaffected: bool = True, prune_external: bool = True, stack_name: Optional[str] = None, pipeline_type: Optional[PipelineType] = None, require_signing: bool = False, cdash_handler: Optional["CDashHandler"] = None, ): """ Args: env: Active spack environment buildcache_destination: The mirror where built binaries should be pushed artifacts_root: Path to location where artifacts should be stored print_summary: Print a summary of the scheduled pipeline output_file: Path where output file should be written check_index_only: Only fetch the index or fetch all spec files broken_specs_url: URL where broken specs (on develop) should be reported rebuild_index: Generate a job to rebuild mirror index after rebuilds untouched_pruning_dependent_depth: How many parents to traverse from changed pkg specs prune_untouched: Prune jobs for specs that were unchanged in git history prune_up_to_date: Prune specs from pipeline if binary exists on the mirror prune_external: Prune specs from pipeline if they are external stack_name: Name of spack stack pipeline_type: Type of pipeline running (optional) require_signing: Require buildcache to be signed (fail w/out signing key) cdash_handler: Object for communicating build information with CDash """ self.env = env self.buildcache_destination = buildcache_destination self.artifacts_root = artifacts_root self.print_summary = print_summary self.output_file = output_file self.check_index_only = check_index_only self.broken_specs_url = broken_specs_url self.rebuild_index = rebuild_index self.untouched_pruning_dependent_depth = untouched_pruning_dependent_depth self.prune_untouched = prune_untouched self.prune_up_to_date = prune_up_to_date self.prune_unaffected = prune_unaffected self.prune_external = prune_external self.stack_name = stack_name self.pipeline_type = pipeline_type self.require_signing = require_signing self.cdash_handler = cdash_handler self.forward_variables: List[str] = [] class PipelineNode: spec: spack.spec.Spec parents: Set[str] children: Set[str] def __init__(self, spec: spack.spec.Spec): self.spec = spec self.parents = set() self.children = set() @property def key(self): """Return key of the stored spec""" return PipelineDag.key(self.spec) class PipelineDag: """Turn a list of specs into a simple directed graph, that doesn't keep track of edge types.""" @classmethod def key(cls, spec: spack.spec.Spec) -> str: return spec.dag_hash() def __init__(self, specs: List[spack.spec.Spec]) -> None: # Build dictionary of nodes self.nodes: Dict[str, PipelineNode] = { PipelineDag.key(s): PipelineNode(s) for s in traverse.traverse_nodes(specs, deptype=dt.ALL_TYPES, root=True) } # Create edges for edge in traverse.traverse_edges( specs, deptype=dt.ALL_TYPES, root=False, cover="edges" ): parent_key = PipelineDag.key(edge.parent) child_key = PipelineDag.key(edge.spec) self.nodes[parent_key].children.add(child_key) self.nodes[child_key].parents.add(parent_key) def prune(self, node_key: str): """Remove a node from the graph, and reconnect its parents and children""" node = self.nodes[node_key] for parent in node.parents: self.nodes[parent].children.remove(node_key) self.nodes[parent].children |= node.children for child in node.children: self.nodes[child].parents.remove(node_key) self.nodes[child].parents |= node.parents del self.nodes[node_key] def traverse_nodes( self, direction: str = "children" ) -> Generator[Tuple[int, PipelineNode], None, None]: """Yields (depth, node) from the pipeline graph. Traversal is topologically ordered from the roots if ``direction`` is ``children``, or from the leaves if ``direction`` is ``parents``. The yielded depth is the length of the longest path from the starting point to the yielded node.""" if direction == "children": get_in_edges = lambda node: node.parents get_out_edges = lambda node: node.children else: get_in_edges = lambda node: node.children get_out_edges = lambda node: node.parents sort_key = lambda k: self.nodes[k].spec.name out_edges = {k: sorted(get_out_edges(n), key=sort_key) for k, n in self.nodes.items()} num_in_edges = {k: len(get_in_edges(n)) for k, n in self.nodes.items()} # Populate a queue with all the nodes that have no incoming edges nodes = deque( sorted( [(0, key) for key in self.nodes.keys() if num_in_edges[key] == 0], key=lambda item: item[1], ) ) while nodes: # Remove the next node, n, from the queue and yield it depth, n_key = nodes.pop() yield (depth, self.nodes[n_key]) # Remove an in-edge from every node, m, pointed to by an # out-edge from n. If any of those nodes are left with # 0 remaining in-edges, add them to the queue. for m in out_edges[n_key]: num_in_edges[m] -= 1 if num_in_edges[m] == 0: nodes.appendleft((depth + 1, m)) def get_dependencies(self, node: PipelineNode) -> List[PipelineNode]: """Returns a list of nodes corresponding to the direct dependencies of the given node.""" return [self.nodes[k] for k in node.children] class SpackCIConfig: """Spack CI object used to generate intermediate representation used by the CI generator(s). """ def __init__(self, ci_config): """Given the information from the ci section of the config and the staged jobs, set up meta data needed for generating Spack CI IR. """ self.ci_config = ci_config self.named_jobs = ["any", "build", "copy", "cleanup", "noop", "reindex", "signing"] self.ir = { "jobs": {}, "rebuild-index": self.ci_config.get("rebuild-index", True), "broken-specs-url": self.ci_config.get("broken-specs-url", None), "broken-tests-packages": self.ci_config.get("broken-tests-packages", []), "target": self.ci_config.get("target", "gitlab"), } jobs = self.ir["jobs"] for name in self.named_jobs: # Skip the special named jobs if name not in ["any", "build"]: jobs[name] = self.__init_job("") def __init_job(self, release_spec): """Initialize job object""" job_object = {"spec": release_spec, "attributes": {}} if release_spec: job_vars = job_object["attributes"].setdefault("variables", {}) job_vars["SPACK_JOB_SPEC_DAG_HASH"] = release_spec.dag_hash() job_vars["SPACK_JOB_SPEC_PKG_NAME"] = release_spec.name job_vars["SPACK_JOB_SPEC_PKG_VERSION"] = release_spec.format("{version}") job_vars["SPACK_JOB_SPEC_COMPILER_NAME"] = release_spec.format("{compiler.name}") job_vars["SPACK_JOB_SPEC_COMPILER_VERSION"] = release_spec.format("{compiler.version}") job_vars["SPACK_JOB_SPEC_ARCH"] = release_spec.format("{architecture}") job_vars["SPACK_JOB_SPEC_VARIANTS"] = release_spec.format("{variants}") return job_object def __is_named(self, section): """Check if a pipeline-gen configuration section is for a named job, and if so return the name otherwise return none. """ for _name in self.named_jobs: keys = [f"{_name}-job", f"{_name}-job-remove"] if any([key for key in keys if key in section]): return _name return None @staticmethod def __job_name(name, suffix=""): """Compute the name of a named job with appropriate suffix. Valid suffixes are either '-remove' or empty string or None """ assert isinstance(name, str) jname = name if suffix: jname = f"{name}-job{suffix}" else: jname = f"{name}-job" return jname def __apply_submapping(self, dest, spec, section): """Apply submapping section to the IR dict""" matched = False only_first = section.get("match_behavior", "first") == "first" for match_attrs in reversed(section["submapping"]): attrs = cfg.InternalConfigScope._process_dict_keyname_overrides(match_attrs) for match_string in match_attrs["match"]: if _spec_matches(spec, match_string): matched = True if "build-job-remove" in match_attrs: cfg.remove_yaml(dest, attrs["build-job-remove"]) if "build-job" in match_attrs: spack.schema.merge_yaml(dest, attrs["build-job"]) break if matched and only_first: break return dest # Create jobs for all the pipeline specs def init_pipeline_jobs(self, pipeline: PipelineDag): for _, node in pipeline.traverse_nodes(): dag_hash = node.spec.dag_hash() self.ir["jobs"][dag_hash] = self.__init_job(node.spec) # Generate IR from the configs def generate_ir(self): """Generate the IR from the Spack CI configurations.""" jobs = self.ir["jobs"] # Implicit job defaults defaults = [ { "build-job": { "script": [ "cd {env_dir}", "spack env activate --without-view .", "spack spec /$SPACK_JOB_SPEC_DAG_HASH", "spack ci rebuild", ] } }, {"noop-job": {"script": ['echo "All specs already up to date, nothing to rebuild."']}}, ] # Job overrides overrides = [ # Reindex script { "reindex-job": { "script:": ["spack -v buildcache update-index --keys {index_target_mirror}"] } }, # Cleanup script { "cleanup-job": { "script:": ["spack -d mirror destroy {mirror_prefix}/$CI_PIPELINE_ID"] } }, # Add signing job tags {"signing-job": {"tags": ["aws", "protected", "notary"]}}, # Remove reserved tags {"any-job-remove": {"tags": SPACK_RESERVED_TAGS}}, ] pipeline_gen = overrides + self.ci_config.get("pipeline-gen", []) + defaults for section in reversed(pipeline_gen): name = self.__is_named(section) has_submapping = "submapping" in section has_dynmapping = "dynamic-mapping" in section section = cfg.InternalConfigScope._process_dict_keyname_overrides(section) if name: remove_job_name = self.__job_name(name, suffix="-remove") merge_job_name = self.__job_name(name) do_remove = remove_job_name in section do_merge = merge_job_name in section def _apply_section(dest, src): if do_remove: dest = cfg.remove_yaml(dest, src[remove_job_name]) if do_merge: dest = copy.copy(spack.schema.merge_yaml(dest, src[merge_job_name])) if name == "build": # Apply attributes to all build jobs for _, job in jobs.items(): if job["spec"]: _apply_section(job["attributes"], section) elif name == "any": # Apply section attributes too all jobs for _, job in jobs.items(): _apply_section(job["attributes"], section) else: # Create a signing job if there is script and the job hasn't # been initialized yet if name == "signing" and name not in jobs: if "signing-job" in section: if "script" not in section["signing-job"]: continue else: jobs[name] = self.__init_job("") # Apply attributes to named job _apply_section(jobs[name]["attributes"], section) elif has_submapping: # Apply section jobs with specs to match for _, job in jobs.items(): if job["spec"]: job["attributes"] = self.__apply_submapping( job["attributes"], job["spec"], section ) elif has_dynmapping: mapping = section["dynamic-mapping"] dynmap_name = mapping.get("name") # Check if this section should be skipped dynmap_skip = os.environ.get("SPACK_CI_SKIP_DYNAMIC_MAPPING") if dynmap_name and dynmap_skip: if re.match(dynmap_skip, dynmap_name): continue # Get the endpoint endpoint = mapping["endpoint"] endpoint_url = urlparse(endpoint) # Configure the request header header = {"User-Agent": web_util.SPACK_USER_AGENT} header.update(mapping.get("header", {})) # Expand header environment variables # ie. if tokens are passed for value in header.values(): value = os.path.expandvars(value) required = mapping.get("require", []) allowed = mapping.get("allow", []) ignored = mapping.get("ignore", []) # required keys are implicitly allowed allowed = sorted(set(allowed + required)) ignored = sorted(set(ignored)) required = sorted(set(required)) # Make sure required things are not also ignored assert not any([ikey in required for ikey in ignored]) def job_query(job): job_vars = job["attributes"]["variables"] query = ( "{SPACK_JOB_SPEC_PKG_NAME}@{SPACK_JOB_SPEC_PKG_VERSION}" # The preceding spaces are required (ref. https://github.com/spack/spack-gantry/blob/develop/docs/api.md#allocation) " {SPACK_JOB_SPEC_VARIANTS}" " arch={SPACK_JOB_SPEC_ARCH}" "%{SPACK_JOB_SPEC_COMPILER_NAME}@{SPACK_JOB_SPEC_COMPILER_VERSION}" ).format_map(job_vars) return f"spec={quote(query)}" for job in jobs.values(): if not job["spec"]: continue # Create request for this job query = job_query(job) request = Request( endpoint_url._replace(query=query).geturl(), headers=header, method="GET" ) try: with _urlopen(request) as response: config = json.load(response) except Exception as e: # For now just ignore any errors from dynamic mapping and continue # This is still experimental, and failures should not stop CI # from running normally tty.warn(f"Failed to fetch dynamic mapping for query:\n\t{query}: {e}") continue # Strip ignore keys if ignored: for key in ignored: if key in config: config.pop(key) # Only keep allowed keys clean_config = {} if allowed: for key in allowed: if key in config: clean_config[key] = config[key] else: clean_config = config # Verify all of the required keys are present if required: missing_keys = [] for key in required: if key not in clean_config.keys(): missing_keys.append(key) if missing_keys: tty.warn(f"Response missing required keys: {missing_keys}") if clean_config: job["attributes"] = spack.schema.merge_yaml( job.get("attributes", {}), clean_config ) for _, job in jobs.items(): if job["spec"]: job["spec"] = job["spec"].name return self.ir class SpackCIError(spack.error.SpackError): def __init__(self, msg): super().__init__(msg) ================================================ FILE: lib/spack/spack/ci/generator_registry.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # Holds all known formatters """Generators that support writing out pipelines for various CI platforms, using a common pipeline graph definition. """ import spack.error _generators = {} def generator(name): """Decorator to register a pipeline generator method. A generator method should take PipelineDag, SpackCIConfig, and PipelineOptions arguments, and should produce a pipeline file. """ def _decorator(generate_method): _generators[name] = generate_method return generate_method return _decorator def get_generator(name): try: return _generators[name] except KeyError: raise UnknownGeneratorException(name) class UnknownGeneratorException(spack.error.SpackError): def __init__(self, generator_name): super().__init__(f"No registered generator for {generator_name}") ================================================ FILE: lib/spack/spack/ci/gitlab.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import copy import os import shutil import urllib from typing import List, Optional import spack.vendor.ruamel.yaml import spack import spack.binary_distribution import spack.config import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.schema import spack.spec import spack.util.path as path_util import spack.util.spack_yaml as syaml from .common import ( SPACK_RESERVED_TAGS, PipelineDag, PipelineOptions, PipelineType, SpackCIConfig, SpackCIError, ensure_expected_target_path, unpack_script, write_pipeline_manifest, ) from .generator_registry import generator # See https://docs.gitlab.com/ee/ci/yaml/#retry for descriptions of conditions JOB_RETRY_CONDITIONS = [ # "always", "unknown_failure", "script_failure", "api_failure", "stuck_or_timeout_failure", "runner_system_failure", "runner_unsupported", "stale_schedule", # "job_execution_timeout", "archived_failure", "unmet_prerequisites", "scheduler_failure", "data_integrity_failure", ] JOB_NAME_FORMAT = "{name}{@version} {/hash}" def _remove_reserved_tags(tags): """Convenience function to strip reserved tags from jobs""" return [tag for tag in tags if tag not in SPACK_RESERVED_TAGS] def get_job_name(spec: spack.spec.Spec, build_group: Optional[str] = None) -> str: """Given a spec and possibly a build group, return the job name. If the resulting name is longer than 255 characters, it will be truncated. Arguments: spec: Spec job will build build_group: Name of build group this job belongs to (a CDash notion) Returns: The job name """ job_name = spec.format(JOB_NAME_FORMAT) if build_group: job_name = f"{job_name} {build_group}" return job_name[:255] def maybe_generate_manifest(pipeline: PipelineDag, options: PipelineOptions, manifest_path): # TODO: Consider including only hashes of rebuilt specs in the manifest, # instead of full source and destination urls. Also, consider renaming # the variable that controls whether or not to write the manifest from # "SPACK_COPY_BUILDCACHE" to "SPACK_WRITE_PIPELINE_MANIFEST" or similar. spack_buildcache_copy = os.environ.get("SPACK_COPY_BUILDCACHE", None) if spack_buildcache_copy: buildcache_copy_src_prefix = options.buildcache_destination.fetch_url buildcache_copy_dest_prefix = spack_buildcache_copy if options.pipeline_type == PipelineType.COPY_ONLY: manifest_specs = [s for s in options.env.all_specs() if not s.external] else: manifest_specs = [n.spec for _, n in pipeline.traverse_nodes(direction="children")] write_pipeline_manifest( manifest_specs, buildcache_copy_src_prefix, buildcache_copy_dest_prefix, manifest_path ) @generator("gitlab") def generate_gitlab_yaml(pipeline: PipelineDag, spack_ci: SpackCIConfig, options: PipelineOptions): """Given a pipeline graph, job attributes, and pipeline options, write a pipeline that can be consumed by GitLab to the given output file. Arguments: pipeline: An already pruned graph of jobs representing all the specs to build spack_ci: An object containing the configured attributes of all jobs in the pipeline options: An object containing all the pipeline options gathered from yaml, env, etc... """ ci_project_dir = os.environ.get("CI_PROJECT_DIR") or os.getcwd() generate_job_name = os.environ.get("CI_JOB_NAME", "job-does-not-exist") generate_pipeline_id = os.environ.get("CI_PIPELINE_ID", "pipeline-does-not-exist") artifacts_root = options.artifacts_root if artifacts_root.startswith(ci_project_dir): artifacts_root = os.path.relpath(artifacts_root, ci_project_dir) pipeline_artifacts_dir = os.path.join(ci_project_dir, artifacts_root) output_file = options.output_file if not output_file: output_file = os.path.abspath(".gitlab-ci.yml") else: output_file_path = os.path.abspath(output_file) gen_ci_dir = os.path.dirname(output_file_path) if not os.path.exists(gen_ci_dir): os.makedirs(gen_ci_dir) spack_ci_ir = spack_ci.generate_ir() concrete_env_dir = os.path.join(pipeline_artifacts_dir, "concrete_environment") # Now that we've added the mirrors we know about, they should be properly # reflected in the environment manifest file, so copy that into the # concrete environment directory, along with the spack.lock file. if not os.path.exists(concrete_env_dir): os.makedirs(concrete_env_dir) # Copy the manifest and handle relative included paths with open(options.env.manifest_path, "r", encoding="utf-8") as fin, open( os.path.join(concrete_env_dir, "spack.yaml"), "w", encoding="utf-8" ) as fout: data = syaml.load(fin) if "spack" not in data: raise spack.config.ConfigSectionError( 'Missing top level "spack" section in environment' ) def _rewrite_include(path, orig_root, new_root): expanded_path = path_util.substitute_path_variables(path) # Skip non-local paths parsed = urllib.parse.urlparse(expanded_path) file_schemes = ["", "file"] if parsed.scheme not in file_schemes: return path if os.path.isabs(expanded_path): return path abs_path = path_util.canonicalize_path(path, orig_root) return os.path.relpath(abs_path, start=new_root) # If there are no includes, just copy if "include" in data["spack"]: includes = data["spack"]["include"] # If there are includes in the config, then we need to fix the relative paths # to be relative from the concrete env dir used by downstream pipelines env_root_path = os.path.dirname(os.path.abspath(options.env.manifest_path)) fixed_includes = [] for inc in includes: if isinstance(inc, dict): inc["path"] = _rewrite_include(inc["path"], env_root_path, concrete_env_dir) else: inc = _rewrite_include(inc, env_root_path, concrete_env_dir) fixed_includes.append(inc) data["spack"]["include"] = fixed_includes os.makedirs(concrete_env_dir, exist_ok=True) syaml.dump(data, fout) shutil.copyfile(options.env.lock_path, os.path.join(concrete_env_dir, "spack.lock")) job_log_dir = os.path.join(pipeline_artifacts_dir, "logs") job_repro_dir = os.path.join(pipeline_artifacts_dir, "reproduction") job_test_dir = os.path.join(pipeline_artifacts_dir, "tests") user_artifacts_dir = os.path.join(pipeline_artifacts_dir, "user_data") # We communicate relative paths to the downstream jobs to avoid issues in # situations where the CI_PROJECT_DIR varies between the pipeline # generation job and the rebuild jobs. This can happen when gitlab # checks out the project into a runner-specific directory, for example, # and different runners are picked for generate and rebuild jobs. rel_concrete_env_dir = os.path.relpath(concrete_env_dir, ci_project_dir) rel_job_log_dir = os.path.relpath(job_log_dir, ci_project_dir) rel_job_repro_dir = os.path.relpath(job_repro_dir, ci_project_dir) rel_job_test_dir = os.path.relpath(job_test_dir, ci_project_dir) rel_user_artifacts_dir = os.path.relpath(user_artifacts_dir, ci_project_dir) def main_script_replacements(cmd): return cmd.replace("{env_dir}", rel_concrete_env_dir) output_object = {} job_id = 0 stage_id = 0 stages: List[List] = [] stage_names = [] max_length_needs = 0 max_needs_job = "" if not options.pipeline_type == PipelineType.COPY_ONLY: for level, node in pipeline.traverse_nodes(direction="parents"): stage_id = level if len(stages) == stage_id: stages.append([]) stages[stage_id].append(node.spec) stage_name = f"stage-{level}" if stage_name not in stage_names: stage_names.append(stage_name) release_spec = node.spec release_spec_dag_hash = release_spec.dag_hash() job_object = spack_ci_ir["jobs"][release_spec_dag_hash]["attributes"] if not job_object: tty.warn(f"No match found for {release_spec}, skipping it") continue if options.pipeline_type is not None: # For spack pipelines "public" and "protected" are reserved tags job_object["tags"] = _remove_reserved_tags(job_object.get("tags", [])) if options.pipeline_type == PipelineType.PROTECTED_BRANCH: job_object["tags"].extend(["protected"]) elif options.pipeline_type == PipelineType.PULL_REQUEST: job_object["tags"].extend(["public"]) if "script" not in job_object: raise AttributeError job_object["script"] = unpack_script(job_object["script"], op=main_script_replacements) if "before_script" in job_object: job_object["before_script"] = unpack_script(job_object["before_script"]) if "after_script" in job_object: job_object["after_script"] = unpack_script(job_object["after_script"]) build_group = options.cdash_handler.build_group if options.cdash_handler else None job_name = get_job_name(release_spec, build_group) dep_nodes = pipeline.get_dependencies(node) job_object["needs"] = [ {"job": get_job_name(dep_node.spec, build_group), "artifacts": False} for dep_node in dep_nodes ] job_object["needs"].append( {"job": generate_job_name, "pipeline": f"{generate_pipeline_id}"} ) job_vars = job_object["variables"] # Let downstream jobs know whether the spec needed rebuilding, regardless # whether DAG pruning was enabled or not. already_built = spack.binary_distribution.get_mirrors_for_spec( spec=release_spec, index_only=True ) job_vars["SPACK_SPEC_NEEDS_REBUILD"] = "False" if already_built else "True" if options.cdash_handler: build_name = options.cdash_handler.build_name(release_spec) job_vars["SPACK_CDASH_BUILD_NAME"] = build_name build_stamp = options.cdash_handler.build_stamp job_vars["SPACK_CDASH_BUILD_STAMP"] = build_stamp job_object["artifacts"] = spack.schema.merge_yaml( job_object.get("artifacts", {}), { "when": "always", "paths": [ rel_job_log_dir, rel_job_repro_dir, rel_job_test_dir, rel_user_artifacts_dir, ], }, ) job_object["stage"] = stage_name job_object["retry"] = spack.schema.merge_yaml( {"max": 2, "when": JOB_RETRY_CONDITIONS}, job_object.get("retry", {}) ) job_object["interruptible"] = True length_needs = len(job_object["needs"]) if length_needs > max_length_needs: max_length_needs = length_needs max_needs_job = job_name output_object[job_name] = job_object job_id += 1 tty.debug(f"{job_id} build jobs generated in {stage_id} stages") if job_id > 0: tty.debug(f"The max_needs_job is {max_needs_job}, with {max_length_needs} needs") service_job_retries = { "max": 2, "when": ["runner_system_failure", "stuck_or_timeout_failure", "script_failure"], } # In some cases, pipeline generation should write a manifest. Currently # the only purpose is to specify a list of sources and destinations for # everything that should be copied. distinguish_stack = options.stack_name if options.stack_name else "rebuilt" manifest_path = os.path.join( pipeline_artifacts_dir, "specs_to_copy", f"copy_{distinguish_stack}_specs.json" ) maybe_generate_manifest(pipeline, options, manifest_path) relative_specs_url = spack.binary_distribution.buildcache_relative_specs_url() relative_keys_url = spack.binary_distribution.buildcache_relative_keys_url() if options.pipeline_type == PipelineType.COPY_ONLY: stage_names.append("copy") sync_job = copy.deepcopy(spack_ci_ir["jobs"]["copy"]["attributes"]) sync_job["stage"] = "copy" sync_job["needs"] = [{"job": generate_job_name, "pipeline": f"{generate_pipeline_id}"}] if "variables" not in sync_job: sync_job["variables"] = {} sync_job["variables"].update( { "SPACK_COPY_ONLY_DESTINATION": options.buildcache_destination.fetch_url, "SPACK_BUILDCACHE_RELATIVE_KEYS_URL": relative_keys_url, } ) pipeline_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True) if "buildcache-source" not in pipeline_mirrors: raise SpackCIError("Copy-only pipelines require a mirror named 'buildcache-source'") buildcache_source = pipeline_mirrors["buildcache-source"].fetch_url sync_job["variables"]["SPACK_BUILDCACHE_SOURCE"] = buildcache_source sync_job["dependencies"] = [] output_object["copy"] = sync_job job_id += 1 if job_id > 0: if ( "script" in spack_ci_ir["jobs"]["signing"]["attributes"] and options.pipeline_type == PipelineType.PROTECTED_BRANCH ): # External signing: generate a job to check and sign binary pkgs stage_names.append("stage-sign-pkgs") signing_job = spack_ci_ir["jobs"]["signing"]["attributes"] signing_job["script"] = unpack_script(signing_job["script"]) signing_job["stage"] = "stage-sign-pkgs" signing_job["when"] = "always" signing_job["retry"] = {"max": 2, "when": ["always"]} signing_job["interruptible"] = True if "variables" not in signing_job: signing_job["variables"] = {} signing_job["variables"].update( { "SPACK_BUILDCACHE_DESTINATION": options.buildcache_destination.push_url, "SPACK_BUILDCACHE_RELATIVE_SPECS_URL": relative_specs_url, "SPACK_BUILDCACHE_RELATIVE_KEYS_URL": relative_keys_url, } ) signing_job["dependencies"] = [] output_object["sign-pkgs"] = signing_job if options.rebuild_index: # Add a final job to regenerate the index stage_names.append("stage-rebuild-index") final_job = spack_ci_ir["jobs"]["reindex"]["attributes"] final_job["stage"] = "stage-rebuild-index" target_mirror = options.buildcache_destination.push_url final_job["script"] = unpack_script( final_job["script"], op=lambda cmd: cmd.replace("{index_target_mirror}", target_mirror), ) final_job["when"] = "always" final_job["retry"] = service_job_retries final_job["interruptible"] = True final_job["dependencies"] = [] output_object["rebuild-index"] = final_job output_object["stages"] = stage_names # Capture the version of Spack used to generate the pipeline, that can be # passed to `git checkout` for version consistency. If we aren't in a Git # repository, presume we are a Spack release and use the Git tag instead. spack_version = spack.get_version() version_to_clone = spack.get_spack_commit() or f"v{spack.spack_version}" rebuild_everything = not options.prune_up_to_date and not options.prune_untouched output_object["variables"] = { "SPACK_ARTIFACTS_ROOT": artifacts_root, "SPACK_CONCRETE_ENV_DIR": rel_concrete_env_dir, "SPACK_VERSION": spack_version, "SPACK_CHECKOUT_VERSION": version_to_clone, "SPACK_JOB_LOG_DIR": rel_job_log_dir, "SPACK_JOB_REPRO_DIR": rel_job_repro_dir, "SPACK_JOB_TEST_DIR": rel_job_test_dir, "SPACK_PIPELINE_TYPE": options.pipeline_type.name if options.pipeline_type else "None", "SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"), "SPACK_REBUILD_CHECK_UP_TO_DATE": str(options.prune_up_to_date), "SPACK_REBUILD_EVERYTHING": str(rebuild_everything), "SPACK_REQUIRE_SIGNING": str(options.require_signing), } output_object["variables"].update( dict([(v, os.environ[v]) for v in options.forward_variables if v in os.environ]) ) if options.stack_name: output_object["variables"]["SPACK_CI_STACK_NAME"] = options.stack_name output_vars = output_object["variables"] for item, val in output_vars.items(): output_vars[item] = ensure_expected_target_path(val) else: # No jobs were generated noop_job = spack_ci_ir["jobs"]["noop"]["attributes"] # If this job fails ignore the status and carry on noop_job["retry"] = 0 noop_job["allow_failure"] = True tty.debug("No specs to rebuild, generating no-op job") output_object = {"no-specs-to-rebuild": noop_job} # Ensure the child pipeline always runs output_object["workflow"] = {"rules": [{"when": "always"}]} sorted_output = {} for output_key, output_value in sorted(output_object.items()): sorted_output[output_key] = output_value # Minimize yaml output size through use of anchors syaml.anchorify(sorted_output) with open(output_file, "w", encoding="utf-8") as f: spack.vendor.ruamel.yaml.YAML().dump(sorted_output, f) ================================================ FILE: lib/spack/spack/cmd/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import difflib import importlib import os import re import subprocess import sys import textwrap from collections import Counter from typing import Generator, List, Optional, Sequence, Union import spack.concretize import spack.config import spack.environment as ev import spack.error import spack.extensions import spack.llnl.string import spack.llnl.util.tty as tty import spack.paths import spack.repo import spack.spec import spack.spec_parser import spack.store import spack.traverse as traverse import spack.user_environment as uenv import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml from spack.llnl.util.filesystem import join_path from spack.llnl.util.lang import attr_setdefault, index_by from spack.llnl.util.tty.colify import colify from spack.llnl.util.tty.color import colorize from ..enums import InstallRecordStatus # cmd has a submodule called "list" so preserve the python list module python_list = list # Patterns to ignore in the commands directory when looking for commands. ignore_files = r"^\.|^__init__.py$|^#" SETUP_PARSER = "setup_parser" DESCRIPTION = "description" def python_name(cmd_name): """Convert ``-`` to ``_`` in command name, to make a valid identifier.""" return cmd_name.replace("-", "_") def require_python_name(pname): """Require that the provided name is a valid python name (per python_name()). Useful for checking parameters for function prerequisites.""" if python_name(pname) != pname: raise PythonNameError(pname) def cmd_name(python_name): """Convert module name (with ``_``) to command name (with ``-``).""" return python_name.replace("_", "-") def require_cmd_name(cname): """Require that the provided name is a valid command name (per cmd_name()). Useful for checking parameters for function prerequisites. """ if cmd_name(cname) != cname: raise CommandNameError(cname) #: global, cached list of all commands -- access through all_commands() _all_commands = None def all_commands(): """Get a sorted list of all spack commands. This will list the lib/spack/spack/cmd directory and find the commands there to construct the list. It does not actually import the python files -- just gets the names. """ global _all_commands if _all_commands is None: _all_commands = [] command_paths = [spack.paths.command_path] # Built-in commands command_paths += spack.extensions.get_command_paths() # Extensions for path in command_paths: for file in os.listdir(path): if file.endswith(".py") and not re.search(ignore_files, file): cmd = re.sub(r".py$", "", file) _all_commands.append(cmd_name(cmd)) _all_commands.sort() return _all_commands def remove_options(parser, *options): """Remove some options from a parser.""" for option in options: for action in parser._actions: if vars(action)["option_strings"][0] == option: parser._handle_conflict_resolve(None, [(option, action)]) break def get_module(cmd_name): """Imports the module for a particular command name and returns it. Args: cmd_name (str): name of the command for which to get a module (contains ``-``, not ``_``). """ require_cmd_name(cmd_name) pname = python_name(cmd_name) try: # Try to import the command from the built-in directory module_name = f"{__name__}.{pname}" module = importlib.import_module(module_name) tty.debug("Imported {0} from built-in commands".format(pname)) except ImportError: module = spack.extensions.get_module(cmd_name) if not module: raise CommandNotFoundError(cmd_name) attr_setdefault(module, SETUP_PARSER, lambda *args: None) # null-op attr_setdefault(module, DESCRIPTION, "") if not hasattr(module, pname): tty.die( "Command module %s (%s) must define function '%s'." % (module.__name__, module.__file__, pname) ) return module def get_command(cmd_name): """Imports the command function associated with cmd_name. The function's name is derived from cmd_name using python_name(). Args: cmd_name (str): name of the command (contains ``-``, not ``_``). """ require_cmd_name(cmd_name) pname = python_name(cmd_name) return getattr(get_module(cmd_name), pname) def quote_kvp(string: str) -> str: """For strings like ``name=value`` or ``name==value``, quote and escape the value if needed. This is a compromise to respect quoting of key-value pairs on the CLI. The shell strips quotes from quoted arguments, so we cannot know *exactly* how CLI arguments were quoted. To compensate, we re-add quotes around anything staritng with ``name=`` or ``name==``, and we assume the rest of the argument is the value. This covers the common cases of passign flags, e.g., ``cflags="-O2 -g"`` on the command line. """ match = spack.spec_parser.SPLIT_KVP.match(string) if not match: return string key, delim, value = match.groups() return f"{key}{delim}{spack.spec_parser.quote_if_needed(value)}" def parse_specs( args: Union[str, List[str]], concretize: bool = False, tests: spack.concretize.TestsType = False, ) -> List[spack.spec.Spec]: """Convenience function for parsing arguments from specs. Handles common exceptions and dies if there are errors. """ args = [args] if isinstance(args, str) else args arg_string = " ".join([quote_kvp(arg) for arg in args]) toolchains = spack.config.CONFIG.get("toolchains", {}) specs = spack.spec_parser.parse(arg_string, toolchains=toolchains) if not concretize: return specs to_concretize: List[spack.concretize.SpecPairInput] = [(s, None) for s in specs] return _concretize_spec_pairs(to_concretize, tests=tests) def _concretize_spec_pairs( to_concretize: List[spack.concretize.SpecPairInput], tests: spack.concretize.TestsType = False ) -> List[spack.spec.Spec]: """Helper method that concretizes abstract specs from a list of abstract,concrete pairs. Any spec with a concrete spec associated with it will concretize to that spec. Any spec with ``None`` for its concrete spec will be newly concretized. This method respects unification rules from config.""" unify = spack.config.get("concretizer:unify", False) # Special case for concretizing a single spec if len(to_concretize) == 1: abstract, concrete = to_concretize[0] return [concrete or spack.concretize.concretize_one(abstract, tests=tests)] # Special case if every spec is either concrete or has an abstract hash if all( concrete or abstract.concrete or abstract.abstract_hash for abstract, concrete in to_concretize ): # Get all the concrete specs ret = [ concrete or (abstract if abstract.concrete else abstract.lookup_hash()) for abstract, concrete in to_concretize ] # If unify: true, check that specs don't conflict # Since all concrete, "when_possible" is not relevant if unify is True: # True, "when_possible", False are possible values runtimes = spack.repo.PATH.packages_with_tags("runtime") specs_per_name = Counter( spec.name for spec in traverse.traverse_nodes( ret, deptype=("link", "run"), key=traverse.by_dag_hash ) if spec.name not in runtimes # runtimes are allowed multiple times ) conflicts = sorted(name for name, count in specs_per_name.items() if count > 1) if conflicts: raise spack.error.SpecError( "Specs conflict and `concretizer:unify` is configured true.", f" specs depend on multiple versions of {', '.join(conflicts)}", ) return ret # Standard case concretize_method = spack.concretize.concretize_separately # unify: false if unify is True: concretize_method = spack.concretize.concretize_together elif unify == "when_possible": concretize_method = spack.concretize.concretize_together_when_possible concretized = concretize_method(to_concretize, tests=tests) return [concrete for _, concrete in concretized] def matching_spec_from_env(spec): """ Returns a concrete spec, matching what is available in the environment. If no matching spec is found in the environment (or if no environment is active), this will return the given spec but concretized. """ env = ev.active_environment() if env: return env.matching_spec(spec) or spack.concretize.concretize_one(spec) else: return spack.concretize.concretize_one(spec) def matching_specs_from_env(specs): """ Same as ``matching_spec_from_env`` but respects spec unification rules. For each spec, if there is a matching spec in the environment it is used. If no matching spec is found, this will return the given spec but concretized in the context of the active environment and other given specs, with unification rules applied. """ env = ev.active_environment() spec_pairs = [(spec, env.matching_spec(spec) if env else None) for spec in specs] additional_concrete_specs = ( [(concrete, concrete) for _, concrete in env.concretized_specs()] if env else [] ) return _concretize_spec_pairs(spec_pairs + additional_concrete_specs)[: len(spec_pairs)] def disambiguate_spec( spec: spack.spec.Spec, env: Optional[ev.Environment], local: bool = False, installed: Union[bool, InstallRecordStatus] = True, first: bool = False, ) -> spack.spec.Spec: """Given a spec, figure out which installed package it refers to. Args: spec: a spec to disambiguate env: a spack environment, if one is active, or None if no environment is active local: do not search chained spack instances installed: install status argument passed to database query. first: returns the first matching spec, even if more than one match is found """ hashes = env.all_hashes() if env else None return disambiguate_spec_from_hashes(spec, hashes, local, installed, first) def disambiguate_spec_from_hashes( spec: spack.spec.Spec, hashes: Optional[List[str]], local: bool = False, installed: Union[bool, InstallRecordStatus] = True, first: bool = False, ) -> spack.spec.Spec: """Given a spec and a list of hashes, get concrete spec the spec refers to. Arguments: spec: a spec to disambiguate hashes: a set of hashes of specs among which to disambiguate local: if True, do not search chained spack instances installed: install status argument passed to database query. first: returns the first matching spec, even if more than one match is found """ if local: matching_specs = spack.store.STORE.db.query_local(spec, hashes=hashes, installed=installed) else: matching_specs = spack.store.STORE.db.query(spec, hashes=hashes, installed=installed) if not matching_specs: tty.die(f"Spec '{spec}' matches no installed packages.") elif first: return matching_specs[0] ensure_single_spec_or_die(spec, matching_specs) return matching_specs[0] def ensure_single_spec_or_die(spec, matching_specs): if len(matching_specs) <= 1: return format_string = ( "{name}{@version}" "{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}" "{%compiler.name}{@compiler.version}" ) args = ["%s matches multiple packages." % spec, "Matching packages:"] args += [ colorize(" @K{%s} " % s.dag_hash(7)) + s.cformat(format_string) for s in matching_specs ] args += ["Use a more specific spec (e.g., prepend '/' to the hash)."] tty.die(*args) def gray_hash(spec, length): if not length: # default to maximum hash length length = 32 h = spec.dag_hash(length) if spec.concrete else "-" * length return colorize("@K{%s}" % h) def display_specs_as_json(specs, deps=False): """Convert specs to a list of json records.""" seen = set() records = [] for spec in specs: dag_hash = spec.dag_hash() if dag_hash in seen: continue records.append(spec.node_dict_with_hashes()) seen.add(dag_hash) if deps: for dep in spec.traverse(): dep_dag_hash = dep.dag_hash() if dep_dag_hash in seen: continue records.append(dep.node_dict_with_hashes()) seen.add(dep_dag_hash) sjson.dump(records, sys.stdout) def iter_groups(specs, indent, all_headers): """Break a list of specs into groups indexed by arch/compilers.""" # Make a dict with specs keyed by architecture and compilers. index = index_by(specs, ("architecture", "compilers")) ispace = indent * " " def _key(item): if item is None: return "" return str(item) # Traverse the index and print out each package for i, (architecture, compilers) in enumerate(sorted(index, key=_key)): if i > 0: print() # Drop the leading space from compilers to clean up output and aid checks. compilers_info = compilers.strip() or "no compilers" header = "%s{%s} / %s{%s}" % ( spack.spec.ARCHITECTURE_COLOR, architecture if architecture else "no arch", spack.spec.COMPILER_COLOR, compilers_info, ) # Sometimes we want to display specs that are not yet concretized. # If they don't have compilers / architecture attached to them, # then skip the header if all_headers or (architecture is not None or compilers_info): sys.stdout.write(ispace) tty.hline(colorize(header), char="-") specs = index[(architecture, compilers)] specs.sort() yield specs def display_specs(specs, args=None, **kwargs): """Display human readable specs with customizable formatting. Prints the supplied specs to the screen, formatted according to the arguments provided. Specs are grouped by architecture and compiler, and columnized if possible. Options can add more information to the default display. Options can be provided either as keyword arguments or as an argparse namespace. Keyword arguments take precedence over settings in the argparse namespace. Args: specs (list): the specs to display args (argparse.Namespace or None): namespace containing formatting arguments Keyword Args: paths (bool): Show paths with each displayed spec deps (bool): Display dependencies with specs long (bool): Display short hashes with specs very_long (bool): Display full hashes with specs (supersedes ``long``) namespaces (bool): Print namespaces along with names show_flags (bool): Show compiler flags with specs variants (bool): Show variants with specs indent (int): indent each line this much groups (bool): display specs grouped by arch/compiler (default True) decorator (typing.Callable): function to call to decorate specs all_headers (bool): show headers even when arch/compiler aren't defined status_fn (typing.Callable): if provided, prepend install-status info output (typing.IO): A file object to write to. Default is ``sys.stdout`` specfile_format (bool): specfile format of the current spec """ def get_arg(name, default=None): """Prefer kwargs, then args, then default.""" if name in kwargs: return kwargs.get(name) elif args is not None: return getattr(args, name, default) else: return default paths = get_arg("paths", False) deps = get_arg("deps", False) hashes = get_arg("long", False) namespaces = get_arg("namespaces", False) flags = get_arg("show_flags", False) variants = get_arg("variants", False) groups = get_arg("groups", True) all_headers = get_arg("all_headers", False) output = get_arg("output", sys.stdout) status_fn = get_arg("status_fn", None) specfile_format = get_arg("specfile_format", False) decorator = get_arg("decorator", None) if decorator is None: decorator = lambda s, f: f indent = get_arg("indent", 0) hlen = 7 if get_arg("very_long", False): hashes = True hlen = None format_string = get_arg("format", None) if format_string is None: nfmt = "{fullname}" if namespaces else "{name}" ffmt = "" if flags: ffmt += " {compiler_flags}" vfmt = "{variants}" if variants else "" hfmt = "{/abstract_hash}" format_string = nfmt + "{@version}" + vfmt + ffmt + hfmt if specfile_format: format_string = "[{specfile_version}] " + format_string def fmt(s, depth=0): """Formatter function for all output specs""" string = "" if status_fn: # This was copied from spec.tree's colorization logic # then shortened because it seems like status_fn should # always return an InstallStatus string += colorize(status_fn(s).value) if hashes: string += gray_hash(s, hlen) + " " string += depth * " " string += decorator(s, s.cformat(format_string)) return string def format_list(specs): """Display a single list of specs, with no groups""" # create the final, formatted versions of all specs formatted = [] for spec in specs: if deps: for depth, dep in traverse.traverse_tree([spec], depth_first=False): formatted.append((fmt(dep.spec, depth), dep.spec)) formatted.append(("", None)) # mark newlines else: formatted.append((fmt(spec), spec)) # unless any of these are set, we can just colify and be done. if not any((deps, paths)): colify((f[0] for f in formatted), indent=indent, output=output) return "" # otherwise, we'll print specs one by one max_width = max(len(f[0]) for f in formatted) path_fmt = "%%-%ds%%s" % (max_width + 2) out = "" # getting lots of prefixes requires DB lookups. Ensure # all spec.prefix calls are in one transaction. with spack.store.STORE.db.read_transaction(): for string, spec in formatted: if not string: # print newline from above out += "\n" continue if paths: out += path_fmt % (string, spec.prefix) + "\n" else: out += string + "\n" return out out = "" if groups: for specs in iter_groups(specs, indent, all_headers): output.write(format_list(specs)) else: out = format_list(sorted(specs)) output.write(out) output.flush() def filter_loaded_specs(specs): """Filter a list of specs returning only those that are currently loaded.""" hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(os.pathsep) return [x for x in specs if x.dag_hash() in hashes] def print_how_many_pkgs(specs, pkg_type="", suffix=""): """Given a list of specs, this will print a message about how many specs are in that list. Args: specs (list): depending on how many items are in this list, choose the plural or singular form of the word "package" pkg_type (str): the output string will mention this provided category, e.g. if pkg_type is "installed" then the message would be "3 installed packages" """ tty.msg("%s" % spack.llnl.string.plural(len(specs), pkg_type + " package") + suffix) def spack_is_git_repo(): """Ensure that this instance of Spack is a git clone.""" return is_git_repo(spack.paths.prefix) def is_git_repo(path): dotgit_path = join_path(path, ".git") if os.path.isdir(dotgit_path): # we are in a regular git repo return True if os.path.isfile(dotgit_path): # we might be in a git worktree try: with open(dotgit_path, "rb") as f: dotgit_content = syaml.load(f) return os.path.isdir(dotgit_content.get("gitdir", dotgit_path)) except syaml.SpackYAMLError: pass return False class PythonNameError(spack.error.SpackError): """Exception class thrown for impermissible python names""" def __init__(self, name): self.name = name super().__init__("{0} is not a permissible Python name.".format(name)) class CommandNameError(spack.error.SpackError): """Exception class thrown for impermissible command names""" def __init__(self, name): self.name = name super().__init__("{0} is not a permissible Spack command name.".format(name)) class MultipleSpecsMatch(Exception): """Raised when multiple specs match a constraint, in a context where this is not allowed. """ class NoSpecMatches(Exception): """Raised when no spec matches a constraint, in a context where this is not allowed. """ ######################################## # argparse types for argument validation ######################################## def extant_file(f): """ Argparse type for files that exist. """ if not os.path.isfile(f): raise argparse.ArgumentTypeError("%s does not exist" % f) return f def require_active_env(cmd_name): """Used by commands to get the active environment If an environment is not found, print an error message that says the calling command *needs* an active environment. Arguments: cmd_name (str): name of calling command Returns: (spack.environment.Environment): the active environment """ env = ev.active_environment() if env: return env tty.die( "`spack %s` requires an environment" % cmd_name, "activate an environment first:", " spack env activate ENV", "or use:", " spack -e ENV %s ..." % cmd_name, ) def find_environment(args: argparse.Namespace) -> Optional[ev.Environment]: """Find active environment from args or environment variable. Check for an environment in this order: 1. via ``spack -e ENV`` or ``spack -D DIR`` (arguments) 2. via a path in the spack.environment.spack_env_var environment variable. If an environment is found, read it in. If not, return None. Arguments: args: argparse namespace with command arguments Returns: a found environment, or ``None`` """ # treat env as a name env = args.env if env: if ev.exists(env): return ev.read(env) else: # if env was specified, see if it is a directory otherwise, look # at env_dir (env and env_dir are mutually exclusive) env = args.env_dir # if no argument, look for the environment variable if not env: env = os.environ.get(ev.spack_env_var) # nothing was set; there's no active environment if not env: return None # if we get here, env isn't the name of a spack environment; it has # to be a path to an environment, or there is something wrong. if ev.is_env_dir(env): return ev.Environment(env) raise ev.SpackEnvironmentError("no environment in %s" % env) def doc_first_line(function: object) -> Optional[str]: """Return the first line of the docstring.""" return function.__doc__.split("\n", 1)[0].strip() if function.__doc__ else None if sys.version_info >= (3, 13): # indent of __doc__ is automatically removed in 3.13+ # see https://github.com/python/cpython/commit/2566b74b26bcce24199427acea392aed644f4b17 def doc_dedented(function: object) -> Optional[str]: """Return the docstring with leading indentation removed.""" return function.__doc__ else: def doc_dedented(function: object) -> Optional[str]: """Return the docstring with leading indentation removed.""" return textwrap.dedent(function.__doc__) if function.__doc__ else None def converted_arg_length(arg: str): if sys.platform == "win32": # An argument may have extra characters inserted for a command # line invocation (e.g. on Windows, an argument with a space # is quoted) return len(subprocess.list2cmdline([arg])) else: return len(arg) def group_arguments( args: Sequence[str], *, max_group_size: int = 500, prefix_length: int = 0, max_group_length: Optional[int] = None, ) -> Generator[List[str], None, None]: """Splits the supplied list of arguments into groups for passing to CLI tools. When passing CLI arguments, we need to ensure that argument lists are no longer than the system command line size limit, and we may also need to ensure that groups are no more than some number of arguments long. This returns an iterator over lists of arguments that meet these constraints. Arguments are in the same order they appeared in the original argument list. If any argument's length is greater than the max_group_length, this will raise a ``ValueError``. Arguments: args: list of arguments to split into groups max_group_size: max number of elements in any group (default 500) prefix_length: length of any additional arguments (including spaces) to be passed before the groups from args; default is 0 characters max_group_length: max length of characters that if a group of args is joined by ``" "`` On unix, this defaults to SC_ARG_MAX from sysconf. On Windows the default is the max usable for CreateProcess (32,768 chars) """ if max_group_length is None: # Windows limit is 32767, including null terminator (not measured by len) # so max length is 32766 max_group_length = 32766 if hasattr(os, "sysconf"): # sysconf is only on unix try: # returns -1 if an option isn't present (some older POSIXes) sysconf_max = os.sysconf("SC_ARG_MAX") max_group_length = sysconf_max if sysconf_max != -1 else max_group_length except (ValueError, OSError): pass # keep windows default if SC_ARG_MAX isn't in sysconf_names group: List[str] = [] grouplen, space = prefix_length, 0 for arg in args: arglen = converted_arg_length(arg) if arglen > max_group_length: raise ValueError(f"Argument is longer than max command line size: '{arg}'") if arglen + prefix_length > max_group_length: raise ValueError(f"Argument with prefix is longer than max command line size: '{arg}'") next_grouplen = grouplen + arglen + space if len(group) == max_group_size or next_grouplen > max_group_length: yield group group, grouplen, space = [], prefix_length, 0 group.append(arg) grouplen += arglen + space space = 1 # add a space for elements 1, 2, etc. but not 0 if group: yield group class CommandNotFoundError(spack.error.SpackError): """Exception class thrown when a requested command is not recognized as such. """ def __init__(self, cmd_name): msg = ( f"{cmd_name} is not a recognized Spack command or extension command; " "check with `spack commands`." ) long_msg = None similar = difflib.get_close_matches(cmd_name, all_commands()) if 1 <= len(similar) <= 5: long_msg = "\nDid you mean one of the following commands?\n " long_msg += "\n ".join(similar) super().__init__(msg, long_msg) ================================================ FILE: lib/spack/spack/cmd/add.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd import spack.llnl.util.tty as tty from spack.cmd.common import arguments description = "add a spec to an environment" section = "environments" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-l", "--list-name", dest="list_name", default="specs", help="name of the list to add specs to", ) arguments.add_common_arguments(subparser, ["specs"]) def add(parser, args): env = spack.cmd.require_active_env(cmd_name="add") with env.write_transaction(): for spec in spack.cmd.parse_specs(args.specs): if not env.add(spec, args.list_name): tty.msg("Package {0} was already added to {1}".format(spec.name, env.name)) else: tty.msg("Adding %s to environment %s" % (spec, env.name)) env.write() ================================================ FILE: lib/spack/spack/cmd/arch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import collections import warnings import spack.vendor.archspec.cpu import spack.llnl.util.tty.colify as colify import spack.llnl.util.tty.color as color import spack.platforms import spack.spec description = "print architecture information about this machine" section = "config" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: # DEPRECATED: equivalent to --generic --target subparser.add_argument( "-g", "--generic-target", action="store_true", help="show the best generic target (deprecated)", ) subparser.add_argument( "--known-targets", action="store_true", help="show a list of all known targets and exit" ) target_type = subparser.add_mutually_exclusive_group() target_type.add_argument( "--family", action="store_true", help="print generic ISA (x86_64, aarch64, ppc64le, ...)" ) target_type.add_argument( "--generic", action="store_true", help="print feature level (x86_64_v3, armv8.4a, ...)" ) parts = subparser.add_mutually_exclusive_group() parts2 = subparser.add_mutually_exclusive_group() parts.add_argument( "-p", "--platform", action="store_true", default=False, help="print only the platform" ) parts.add_argument( "-o", "--operating-system", action="store_true", default=False, help="print only the operating system", ) parts.add_argument( "-t", "--target", action="store_true", default=False, help="print only the target" ) parts2.add_argument( "-f", "--frontend", action="store_true", default=False, help="print frontend (DEPRECATED)" ) parts2.add_argument( "-b", "--backend", action="store_true", default=False, help="print backend (DEPRECATED)" ) def display_targets(targets): """Prints a human readable list of the targets passed as argument.""" by_vendor = collections.defaultdict(list) for _, target in targets.items(): by_vendor[target.vendor].append(target) def display_target_group(header, target_group): print(header) colify.colify(target_group, indent=4) print("") generic_architectures = by_vendor.pop("generic", None) if generic_architectures: header = color.colorize(r"@*B{Generic architectures (families)}") group = sorted(generic_architectures, key=lambda x: str(x)) display_target_group(header, group) for vendor, vendor_targets in by_vendor.items(): by_family = collections.defaultdict(list) for t in vendor_targets: by_family[str(t.family)].append(t) for family, group in by_family.items(): vendor = color.colorize(r"@*B{" + vendor + r"}") family = color.colorize(r"@*B{" + family + r"}") header = " - ".join([vendor, family]) group = sorted(group, key=lambda x: len(x.ancestors)) display_target_group(header, group) def arch(parser, args): if args.generic_target: # TODO: add deprecation warning in 0.24 print(spack.vendor.archspec.cpu.host().generic) return if args.known_targets: display_targets(spack.vendor.archspec.cpu.TARGETS) return if args.frontend: warnings.warn("the argument --frontend is deprecated, and will be removed in Spack v1.0") elif args.backend: warnings.warn("the argument --backend is deprecated, and will be removed in Spack v1.0") host_platform = spack.platforms.host() host_os = host_platform.default_operating_system() host_target = host_platform.default_target() if args.family: host_target = host_target.family elif args.generic: host_target = host_target.generic architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target))) if args.platform: print(architecture.platform) elif args.operating_system: print(architecture.os) elif args.target: print(architecture.target) else: print(architecture) ================================================ FILE: lib/spack/spack/cmd/audit.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import warnings import spack.audit import spack.llnl.util.tty as tty import spack.llnl.util.tty.colify import spack.llnl.util.tty.color as cl import spack.repo description = "audit configuration files, packages, etc." section = "packaging" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: # Top level flags, valid for every audit class sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="subcommand") # Audit configuration files sp.add_parser("configs", help="audit configuration files") # Audit package recipes external_parser = sp.add_parser("externals", help="check external detection in packages") external_parser.add_argument( "--list", action="store_true", dest="list_externals", help="if passed, list which packages have detection tests", ) # Https and other linting https_parser = sp.add_parser("packages-https", help="check https in packages") https_parser.add_argument( "--all", action="store_true", default=False, dest="check_all", help="audit all packages" ) # Audit package recipes pkg_parser = sp.add_parser("packages", help="audit package recipes") for group in [pkg_parser, https_parser, external_parser]: group.add_argument( "name", metavar="PKG", nargs="*", help="package to be analyzed (if none all packages will be processed)", ) # List all checks sp.add_parser("list", help="list available checks and exits") def configs(parser, args): with warnings.catch_warnings(): warnings.simplefilter("ignore") reports = spack.audit.run_group(args.subcommand) _process_reports(reports) def packages(parser, args): pkgs = args.name or spack.repo.PATH.all_package_names() reports = spack.audit.run_group(args.subcommand, pkgs=pkgs) _process_reports(reports) def packages_https(parser, args): # Since packages takes a long time, --all is required without name if not args.check_all and not args.name: tty.die("Please specify one or more packages to audit, or --all.") pkgs = args.name or spack.repo.PATH.all_package_names() reports = spack.audit.run_group(args.subcommand, pkgs=pkgs) _process_reports(reports) def externals(parser, args): if args.list_externals: msg = "@*{The following packages have detection tests:}" tty.msg(cl.colorize(msg)) spack.llnl.util.tty.colify.colify(spack.audit.packages_with_detection_tests(), indent=2) return pkgs = args.name or spack.repo.PATH.all_package_names() reports = spack.audit.run_group(args.subcommand, pkgs=pkgs, debug_log=tty.debug) _process_reports(reports) def list(parser, args): for subcommand, check_tags in spack.audit.GROUPS.items(): print(cl.colorize("@*b{" + subcommand + "}:")) for tag in check_tags: audit_obj = spack.audit.CALLBACKS[tag] print(" " + audit_obj.description) if args.verbose: for idx, fn in enumerate(audit_obj.callbacks): print(" {0}. ".format(idx + 1) + fn.__doc__) print() print() def audit(parser, args): subcommands = { "configs": configs, "externals": externals, "packages": packages, "packages-https": packages_https, "list": list, } subcommands[args.subcommand](parser, args) def _process_reports(reports): for check, errors in reports: if errors: status = f"{len(errors)} issue{'' if len(errors) == 1 else 's'} found" print(cl.colorize(f"{check}: @*r{{{status}}}")) numdigits = len(str(len(errors))) for idx, error in enumerate(errors): print(f"{idx + 1:>{numdigits}}. {error}") raise SystemExit(1) else: print(cl.colorize(f"{check}: @*g{{passed}}")) ================================================ FILE: lib/spack/spack/cmd/blame.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import pathlib import re import sys from typing import Optional, Union import spack.config import spack.llnl.util.tty as tty import spack.repo import spack.util.git import spack.util.spack_json as sjson from spack.cmd import spack_is_git_repo from spack.llnl.util.filesystem import working_dir from spack.llnl.util.lang import pretty_date from spack.llnl.util.tty.colify import colify_table from spack.util.executable import ProcessError description = "show contributors to packages" section = "query" level = "long" git = spack.util.git.git(required=True) def setup_parser(subparser: argparse.ArgumentParser) -> None: view_group = subparser.add_mutually_exclusive_group() view_group.add_argument( "-t", "--time", dest="view", action="store_const", const="time", default="time", help="sort by last modification date (default)", ) view_group.add_argument( "-p", "--percent", dest="view", action="store_const", const="percent", help="sort by percent of code", ) view_group.add_argument( "-g", "--git", dest="view", action="store_const", const="git", help="show git blame output instead of summary", ) subparser.add_argument( "--json", action="store_true", default=False, help="output blame as machine-readable json records", ) subparser.add_argument( "package_or_file", help="name of package to show contributions for, or path to a file in the spack repo", ) def print_table(rows, last_mod, total_lines, emails): """ Given a set of rows with authors and lines, print a table. """ table = [["LAST_COMMIT", "LINES", "%", "AUTHOR", "EMAIL"]] for author, nlines in rows: table += [ [ pretty_date(last_mod[author]), nlines, round(nlines / float(total_lines) * 100, 1), author, emails[author], ] ] table += [[""] * 5] table += [[pretty_date(max(last_mod.values())), total_lines, "100.0"] + [""] * 3] colify_table(table) def dump_json(rows, last_mod, total_lines, emails): """ Dump the blame as a json object to the terminal. """ result = {} authors = [] for author, nlines in rows: authors.append( { "last_commit": pretty_date(last_mod[author]), "lines": nlines, "percentage": round(nlines / float(total_lines) * 100, 1), "author": author, "email": emails[author], } ) result["authors"] = authors result["totals"] = { "last_commit": pretty_date(max(last_mod.values())), "lines": total_lines, "percentage": "100.0", } sjson.dump(result, sys.stdout) def git_prefix(path: Union[str, pathlib.Path]) -> Optional[pathlib.Path]: """Return the top level directory if path is under a git repository. Args: path: path of the item presumably under a git repository Returns: path to the root of the git repository """ if not os.path.exists(path): return None work_dir = path if os.path.isdir(path) else os.path.dirname(path) with working_dir(work_dir): try: result = git("rev-parse", "--show-toplevel", output=str, error=str) return pathlib.Path(result.split("\n")[0]) except ProcessError: tty.die(f"'{path}' is not in a git repository.") def package_repo_root(path: Union[str, pathlib.Path]) -> Optional[pathlib.Path]: """Find the appropriate package repository's git root directory. Provides a warning for a remote package repository since there is a risk that the blame results are inaccurate. Args: path: path to an arbitrary file presumably in one of the spack package repos Returns: path to the package repository's git root directory or None """ descriptors = spack.repo.RepoDescriptors.from_config( lock=spack.repo.package_repository_lock(), config=spack.config.CONFIG ) path = pathlib.Path(path) prefix: Optional[pathlib.Path] = None for _, desc in descriptors.items(): # Handle the remote case, whose destination is by definition the git root if hasattr(desc, "destination"): repo_dest = pathlib.Path(desc.destination) if (repo_dest / ".git").exists(): prefix = repo_dest # TODO: replace check with `is_relative_to` once supported if prefix and str(path).startswith(str(prefix)): return prefix # Handle the local repository case, making sure it's a spack repository. if hasattr(desc, "path"): repo_path = pathlib.Path(desc.path) if "spack_repo" in repo_path.parts: prefix = git_prefix(repo_path) # TODO: replace check with `is_relative_to` once supported if prefix and str(path).startswith(str(prefix)): return prefix return None def git_supports_unshallow() -> bool: output = git("fetch", "--help", output=str, error=str) return "--unshallow" in output def ensure_full_history(prefix: str, path: str) -> None: """Ensure the git repository at the prefix has its full history. Args: prefix: the root directory of the git repository path: the package or file name under consideration (for messages) """ assert os.path.isdir(prefix) with working_dir(prefix): shallow_dir = os.path.join(prefix, ".git", "shallow") if os.path.isdir(shallow_dir): if git_supports_unshallow(): try: # Capture the error output (e.g., irrelevant for full repo) # to ensure the output is clean. git("fetch", "--unshallow", error=str) except ProcessError as e: tty.die( f"Cannot report blame for {path}.\n" "Unable to retrieve the full git history for " f'{prefix} due to "{str(e)}" error.' ) else: tty.die( f"Cannot report blame for {path}.\n" f"Unable to retrieve the full git history for {prefix}. " "Use a newer 'git' that supports 'git fetch --unshallow'." ) def blame(parser, args): # make sure this is a git repo if not spack_is_git_repo(): tty.die("This spack is not a git clone. You cannot use 'spack blame'.") # Get the name of the path to blame and its repository prefix # so we can honor any .git-blame-ignore-revs that may be present. blame_file = None prefix = None if os.path.exists(args.package_or_file): blame_file = os.path.realpath(args.package_or_file) prefix = package_repo_root(blame_file) # Get path to what we assume is a package (including to a cached version # of a remote package repository.) if not blame_file: try: blame_file = spack.repo.PATH.filename_for_package_name(args.package_or_file) except spack.repo.UnknownNamespaceError: # the argument is not a package (or does not exist) pass if blame_file and os.path.isfile(blame_file): prefix = package_repo_root(blame_file) if not blame_file or not os.path.exists(blame_file): tty.die(f"'{args.package_or_file}' does not exist.") if prefix is None: tty.msg(f"'{args.package_or_file}' is not within a spack package repository") path_prefix = git_prefix(blame_file) if path_prefix != prefix: # You are attempting to get 'blame' for a path outside of a configured # package repository (e.g., within a spack/spack clone). We'll use the # path's prefix instead to ensure working under the proper git # repository. prefix = path_prefix # Make sure we can get the full/known blame even when the repository # is remote. ensure_full_history(prefix, args.package_or_file) # Get blame information for the path EVEN when it is located in a different # spack repository (e.g., spack/spack-packages) or a different git # repository. with working_dir(prefix): # Now we can get the blame results. options = ["blame"] # ignore the great black reformatting of 2022 ignore_file = prefix / ".git-blame-ignore-revs" if ignore_file.exists(): options.extend(["--ignore-revs-file", str(ignore_file)]) try: if args.view == "git": options.append(str(blame_file)) git(*options) return else: options.extend(["--line-porcelain", str(blame_file)]) output = git(*options, output=str, error=str) lines = output.split("\n") except ProcessError as err: # e.g., blame information is not tracked if the path is a directory tty.die(f"Blame information is not tracked for '{blame_file}':\n{err.long_message}") # Histogram authors counts = {} emails = {} last_mod = {} total_lines = 0 for line in lines: match = re.match(r"^author (.*)", line) if match: author = match.group(1) match = re.match(r"^author-mail (.*)", line) if match: email = match.group(1) match = re.match(r"^author-time (.*)", line) if match: mod = int(match.group(1)) last_mod[author] = max(last_mod.setdefault(author, 0), mod) # ignore comments if re.match(r"^\t[^#]", line): counts[author] = counts.setdefault(author, 0) + 1 emails.setdefault(author, email) total_lines += 1 if args.view == "time": rows = sorted(counts.items(), key=lambda t: last_mod[t[0]], reverse=True) else: # args.view == 'percent' rows = sorted(counts.items(), key=lambda t: t[1], reverse=True) # Dump as json if args.json: dump_json(rows, last_mod, total_lines, emails) # Print a nice table with authors and emails else: print_table(rows, last_mod, total_lines, emails) ================================================ FILE: lib/spack/spack/cmd/bootstrap.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import pathlib import shutil import sys import tempfile import spack import spack.bootstrap import spack.bootstrap.config import spack.bootstrap.core import spack.cmd.mirror import spack.concretize import spack.config import spack.llnl.util.filesystem import spack.llnl.util.tty import spack.llnl.util.tty.color import spack.stage import spack.util.path import spack.util.spack_yaml from spack.cmd.common import arguments description = "manage bootstrap configuration" section = "admin" level = "long" # Tarball to be downloaded if binary packages are requested in a local mirror BINARY_TARBALL = "https://github.com/spack/spack-bootstrap-mirrors/releases/download/v2.2/bootstrap-buildcache.tar.gz" #: Subdirectory where to create the mirror LOCAL_MIRROR_DIR = "bootstrap_cache" # Metadata for a generated binary mirror BINARY_METADATA = { "type": "buildcache", "description": ( "Buildcache copied from a public tarball available on Github." "The sha256 checksum of binaries is checked before installation." ), "info": { # This is a mis-nomer since it's not a URL; but file urls cannot # represent relative paths, so we have to live with it for now. "url": os.path.join("..", "..", LOCAL_MIRROR_DIR), "homepage": "https://github.com/spack/spack-bootstrap-mirrors", "releases": "https://github.com/spack/spack-bootstrap-mirrors/releases", "tarball": BINARY_TARBALL, }, } CLINGO_JSON = "$spack/share/spack/bootstrap/github-actions-v2/clingo.json" GNUPG_JSON = "$spack/share/spack/bootstrap/github-actions-v2/gnupg.json" PATCHELF_JSON = "$spack/share/spack/bootstrap/github-actions-v2/patchelf.json" # Metadata for a generated source mirror SOURCE_METADATA = { "type": "install", "description": "Mirror with software needed to bootstrap Spack", "info": { # This is a mis-nomer since it's not a URL; but file urls cannot # represent relative paths, so we have to live with it for now. "url": os.path.join("..", "..", LOCAL_MIRROR_DIR) }, } def _add_scope_option(parser): parser.add_argument( "--scope", action=arguments.ConfigScope, help="configuration scope to read/modify" ) def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(dest="subcommand") now = sp.add_parser("now", help="Spack ready, right now!") now.add_argument("--dev", action="store_true", help="bootstrap dev dependencies too") status = sp.add_parser("status", help="get the status of Spack") status.add_argument( "--optional", action="store_true", default=False, help="show the status of rarely used optional dependencies", ) status.add_argument( "--dev", action="store_true", default=False, help="show the status of dependencies needed to develop Spack", ) enable = sp.add_parser("enable", help="enable bootstrapping") _add_scope_option(enable) enable.add_argument("name", help="name of the source to be enabled", nargs="?", default=None) disable = sp.add_parser("disable", help="disable bootstrapping") _add_scope_option(disable) disable.add_argument("name", help="name of the source to be disabled", nargs="?", default=None) reset = sp.add_parser("reset", help="reset bootstrapping configuration to Spack defaults") arguments.add_common_arguments(reset, ["yes_to_all"]) root = sp.add_parser("root", help="get/set the root bootstrap directory") _add_scope_option(root) root.add_argument( "path", nargs="?", default=None, help="set the bootstrap directory to this value" ) list = sp.add_parser("list", help="list all the sources of software to bootstrap Spack") list.add_argument( "--scope", action=arguments.ConfigScope, type=arguments.config_scope_readable_validator, help="configuration scope to read/modify", ) add = sp.add_parser("add", help="add a new source for bootstrapping") _add_scope_option(add) add.add_argument( "--trust", action="store_true", help="enable the source immediately upon addition" ) add.add_argument("name", help="name of the new source of software") add.add_argument("metadata_dir", help="directory where to find metadata files") remove = sp.add_parser("remove", help="remove a bootstrapping source") remove.add_argument("name", help="name of the source to be removed") mirror = sp.add_parser("mirror", help="create a local mirror to bootstrap Spack") mirror.add_argument( "--binary-packages", action="store_true", help="download public binaries in the mirror" ) mirror.add_argument("--dev", action="store_true", help="download dev dependencies too") mirror.add_argument( metavar="DIRECTORY", dest="root_dir", help="root directory in which to create the mirror and metadata", ) def _enable_or_disable(args): value = args.subcommand == "enable" if args.name is None: # Set to True if we called "enable", otherwise set to false old_value = spack.config.get("bootstrap:enable", scope=args.scope) if old_value == value: spack.llnl.util.tty.msg("Bootstrapping is already {}d".format(args.subcommand)) else: spack.config.set("bootstrap:enable", value, scope=args.scope) spack.llnl.util.tty.msg("Bootstrapping has been {}d".format(args.subcommand)) return if value is True: _enable_source(args) else: _disable_source(args) def _reset(args): if not args.yes_to_all: msg = [ "Bootstrapping configuration is being reset to Spack's defaults. " "Current configuration will be lost.\n", "Do you want to continue?", ] ok_to_continue = spack.llnl.util.tty.get_yes_or_no("".join(msg), default=True) if not ok_to_continue: raise RuntimeError("Aborting") for scope in spack.config.CONFIG.writable_scopes: # The default scope should stay untouched if scope.name == "defaults": continue # If we are in an env scope we can't delete a file, but the best we # can do is nullify the corresponding configuration if scope.name.startswith("env") and spack.config.get("bootstrap", scope=scope.name): spack.config.set("bootstrap", {}, scope=scope.name) continue # If we are outside of an env scope delete the bootstrap.yaml file bootstrap_yaml = os.path.join(scope.path, "bootstrap.yaml") backup_file = bootstrap_yaml + ".bkp" if os.path.exists(bootstrap_yaml): shutil.move(bootstrap_yaml, backup_file) spack.config.CONFIG.clear_caches() def _root(args): if args.path: spack.config.set("bootstrap:root", args.path, scope=args.scope) elif args.scope: if args.scope not in spack.config.existing_scope_names(): spack.llnl.util.tty.die( f"The argument --scope={args.scope} must refer to an existing scope." ) root = spack.config.get("bootstrap:root", default=None, scope=args.scope) if root: root = spack.util.path.canonicalize_path(root) print(root) def _list(args): sources = spack.bootstrap.core.bootstrapping_sources(scope=args.scope) if not sources: spack.llnl.util.tty.msg("No method available for bootstrapping Spack's dependencies") return def _print_method(source, trusted): color = spack.llnl.util.tty.color def fmt(header, content): header_fmt = "@*b{{{0}:}} {1}" color.cprint(header_fmt.format(header, content)) trust_str = "@*y{DISABLED}" if trusted is True: trust_str = "@*g{ENABLED}" elif trusted is False: trust_str = "@*r{DISABLED}" fmt("Name", source["name"] + " " + trust_str) print() if trusted is True or args.verbose: fmt(" Type", source["type"]) print() info_lines = ["\n"] for key, value in source.get("info", {}).items(): info_lines.append(" " * 4 + "@*{{{0}}}: {1}\n".format(key, value)) if len(info_lines) > 1: fmt(" Info", "".join(info_lines)) description_lines = ["\n"] for line in source["description"].split("\n"): description_lines.append(" " * 4 + line + "\n") fmt(" Description", "".join(description_lines)) trusted = spack.config.get("bootstrap:trusted", {}) def sort_fn(x): x_trust = trusted.get(x["name"], None) if x_trust is True: return 0 elif x_trust is None: return 1 return 2 sources = sorted(sources, key=sort_fn) for s in sources: _print_method(s, trusted.get(s["name"], None)) def _write_bootstrapping_source_status(name, enabled, scope=None): """Write if a bootstrapping source is enable or disabled to config file. Args: name (str): name of the bootstrapping source. enabled (bool): True if the source is enabled, False if it is disabled. scope (None or str): configuration scope to modify. If none use the default scope. """ sources = spack.config.get("bootstrap:sources") matches = [s for s in sources if s["name"] == name] if not matches: names = [s["name"] for s in sources] msg = 'there is no bootstrapping method named "{0}". Valid method names are: {1}'.format( name, ", ".join(names) ) raise RuntimeError(msg) if len(matches) > 1: msg = ( 'there is more than one bootstrapping method named "{0}". ' "Please delete all methods but one from bootstrap.yaml " "before proceeding" ).format(name) raise RuntimeError(msg) # Setting the scope explicitly is needed to not copy over to a new scope # the entire default configuration for bootstrap.yaml scope = scope or spack.config.default_modify_scope("bootstrap") spack.config.add("bootstrap:trusted:{0}:{1}".format(name, str(enabled)), scope=scope) def _enable_source(args): _write_bootstrapping_source_status(args.name, enabled=True, scope=args.scope) msg = '"{0}" is now enabled for bootstrapping' spack.llnl.util.tty.msg(msg.format(args.name)) def _disable_source(args): _write_bootstrapping_source_status(args.name, enabled=False, scope=args.scope) msg = '"{0}" is now disabled and will not be used for bootstrapping' spack.llnl.util.tty.msg(msg.format(args.name)) def _status(args): sections = ["core", "buildcache"] if args.optional: sections.append("optional") if args.dev: sections.append("develop") header = "@*b{{Spack v{0} - {1}}}".format( spack.spack_version, spack.bootstrap.config.spec_for_current_python() ) print(spack.llnl.util.tty.color.colorize(header)) print() # Use the context manager here to avoid swapping between user and # bootstrap config many times missing = False with spack.bootstrap.ensure_bootstrap_configuration(): for current_section in sections: status_msg, fail = spack.bootstrap.status_message(section=current_section) missing = missing or fail if status_msg: print(spack.llnl.util.tty.color.colorize(status_msg)) print() legend = ( "Spack will take care of bootstrapping any missing dependency marked" " as [@*y{B}]. Dependencies marked as [@*y{-}] are instead required" " to be found on the system." ) if missing: print(spack.llnl.util.tty.color.colorize(legend)) print() sys.exit(1) def _add(args): initial_sources = spack.bootstrap.core.bootstrapping_sources() names = [s["name"] for s in initial_sources] # If the name is already used error out if args.name in names: msg = 'a source named "{0}" already exist. Please choose a different name' raise RuntimeError(msg.format(args.name)) # Check that the metadata file exists metadata_dir = spack.util.path.canonicalize_path(args.metadata_dir) if not os.path.exists(metadata_dir) or not os.path.isdir(metadata_dir): raise RuntimeError('the directory "{0}" does not exist'.format(args.metadata_dir)) file = os.path.join(metadata_dir, "metadata.yaml") if not os.path.exists(file): raise RuntimeError('the file "{0}" does not exist'.format(file)) # Insert the new source as the highest priority one write_scope = args.scope or spack.config.default_modify_scope(section="bootstrap") sources = spack.config.get("bootstrap:sources", scope=write_scope) or [] sources = [{"name": args.name, "metadata": args.metadata_dir}] + sources spack.config.set("bootstrap:sources", sources, scope=write_scope) msg = 'New bootstrapping source "{0}" added in the "{1}" configuration scope' spack.llnl.util.tty.msg(msg.format(args.name, write_scope)) if args.trust: _enable_source(args) def _remove(args): initial_sources = spack.bootstrap.core.bootstrapping_sources() names = [s["name"] for s in initial_sources] if args.name not in names: msg = ( 'cannot find any bootstrapping source named "{0}". ' "Run `spack bootstrap list` to see available sources." ) raise RuntimeError(msg.format(args.name)) for current_scope in spack.config.scopes(): sources = spack.config.get("bootstrap:sources", scope=current_scope) or [] if args.name in [s["name"] for s in sources]: sources = [s for s in sources if s["name"] != args.name] spack.config.set("bootstrap:sources", sources, scope=current_scope) msg = ( 'Removed the bootstrapping source named "{0}" from the "{1}" configuration scope.' ) spack.llnl.util.tty.msg(msg.format(args.name, current_scope)) trusted = spack.config.get("bootstrap:trusted", scope=current_scope) or [] if args.name in trusted: trusted.pop(args.name) spack.config.set("bootstrap:trusted", trusted, scope=current_scope) msg = 'Deleting information on "{0}" from list of trusted sources' spack.llnl.util.tty.msg(msg.format(args.name)) def _mirror(args): mirror_dir = spack.util.path.canonicalize_path(os.path.join(args.root_dir, LOCAL_MIRROR_DIR)) # TODO: Here we are adding gnuconfig manually, but this can be fixed # TODO: as soon as we have an option to add to a mirror all the possible # TODO: dependencies of a spec root_specs = spack.bootstrap.all_core_root_specs() + ["gnuconfig"] if args.dev: root_specs += spack.bootstrap.BootstrapEnvironment.spack_dev_requirements() for spec_str in root_specs: msg = 'Adding "{0}" and dependencies to the mirror at {1}' spack.llnl.util.tty.msg(msg.format(spec_str, mirror_dir)) # Suppress tty from the call below for terser messages spack.llnl.util.tty.set_msg_enabled(False) spec = spack.concretize.concretize_one(spec_str) for node in spec.traverse(): if node.external: continue spack.cmd.mirror.create(mirror_dir, [node]) spack.llnl.util.tty.set_msg_enabled(True) if args.binary_packages: msg = 'Adding binary packages from "{0}" to the mirror at {1}' spack.llnl.util.tty.msg(msg.format(BINARY_TARBALL, mirror_dir)) spack.llnl.util.tty.set_msg_enabled(False) stage = spack.stage.Stage(BINARY_TARBALL, path=tempfile.mkdtemp()) stage.create() stage.fetch() stage.expand_archive() stage_dir = pathlib.Path(stage.source_path) for entry in stage_dir.iterdir(): shutil.move(str(entry), mirror_dir) spack.llnl.util.tty.set_msg_enabled(True) def write_metadata(subdir, metadata): metadata_rel_dir = os.path.join("metadata", subdir) metadata_yaml = os.path.join(args.root_dir, metadata_rel_dir, "metadata.yaml") spack.llnl.util.filesystem.mkdirp(os.path.dirname(metadata_yaml)) with open(metadata_yaml, mode="w", encoding="utf-8") as f: spack.util.spack_yaml.dump(metadata, stream=f) return os.path.dirname(metadata_yaml), metadata_rel_dir instructions = ( "\nTo register the mirror on the platform where it's supposed " 'to be used, move "{0}" to its final location and run the ' "following command(s):\n\n" ).format(args.root_dir) cmd = " % spack bootstrap add --trust {0} /{1}\n" _, rel_directory = write_metadata(subdir="sources", metadata=SOURCE_METADATA) instructions += cmd.format("local-sources", rel_directory) if args.binary_packages: abs_directory, rel_directory = write_metadata(subdir="binaries", metadata=BINARY_METADATA) shutil.copy(spack.util.path.canonicalize_path(CLINGO_JSON), abs_directory) shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory) shutil.copy(spack.util.path.canonicalize_path(PATCHELF_JSON), abs_directory) instructions += cmd.format("local-binaries", rel_directory) print(instructions) def _now(args): with spack.bootstrap.ensure_bootstrap_configuration(): spack.bootstrap.ensure_core_dependencies() if args.dev: spack.bootstrap.ensure_environment_dependencies() def bootstrap(parser, args): callbacks = { "status": _status, "enable": _enable_or_disable, "disable": _enable_or_disable, "reset": _reset, "root": _root, "list": _list, "add": _add, "remove": _remove, "mirror": _mirror, "now": _now, } callbacks[args.subcommand](args) ================================================ FILE: lib/spack/spack/cmd/build_env.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.cmd.common.env_utility as env_utility from spack.context import Context description = "dump the install environment for a spec,\nor run a command in that environment" section = "build" level = "long" setup_parser = env_utility.setup_parser def build_env(parser, args): env_utility.emulate_env_utility("build-env", Context.BUILD, args) ================================================ FILE: lib/spack/spack/cmd/buildcache.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import enum import glob import json import os import sys import tempfile from typing import List, Optional, Tuple import spack.binary_distribution import spack.cmd import spack.concretize import spack.config import spack.deptypes as dt import spack.environment as ev import spack.error import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.oci.image import spack.oci.oci import spack.spec import spack.stage import spack.store import spack.util.parallel import spack.util.timer as timer_mod import spack.util.web as web_util from spack import traverse from spack.binary_distribution import BINARY_INDEX from spack.cmd import display_specs from spack.cmd.common import arguments from spack.llnl.string import plural from spack.llnl.util.lang import elide_list, stable_partition from spack.spec import Spec, save_dependency_specfiles from ..buildcache_migrate import migrate from ..buildcache_prune import prune_buildcache from ..enums import InstallRecordStatus from ..url_buildcache import ( BuildcacheComponent, BuildcacheEntryError, URLBuildcacheEntry, check_mirror_for_layout, get_entries_from_cache, get_url_buildcache_class, ) description = "create, download and install binary packages" section = "packaging" level = "long" class ViewUpdateMode(enum.Enum): CREATE = enum.auto() OVERWRITE = enum.auto() APPEND = enum.auto() def setup_parser(subparser: argparse.ArgumentParser): setattr(setup_parser, "parser", subparser) subparsers = subparser.add_subparsers(help="buildcache sub-commands") push = subparsers.add_parser("push", aliases=["create"], help=push_fn.__doc__) push.add_argument("-f", "--force", action="store_true", help="overwrite tarball if it exists") push_sign = push.add_mutually_exclusive_group(required=False) push_sign.add_argument( "--unsigned", "-u", action="store_false", dest="signed", default=None, help="push unsigned buildcache tarballs", ) push_sign.add_argument( "--signed", action="store_true", dest="signed", default=None, help="push signed buildcache tarballs", ) push_sign.add_argument( "--key", "-k", metavar="key", type=str, default=None, help="key for signing" ) push.add_argument( "mirror", type=arguments.mirror_name_or_url, help="mirror name, path, or URL" ) push.add_argument( "--update-index", "--rebuild-index", action="store_true", default=False, help="regenerate buildcache index after building package(s)", ) push.add_argument( "--only", default="package,dependencies", dest="things_to_install", choices=["package", "dependencies"], help="select the buildcache mode. " "The default is to build a cache for the package along with all its dependencies. " "Alternatively, one can decide to build a cache for only the package or only the " "dependencies", ) with_or_without_build_deps = push.add_mutually_exclusive_group() with_or_without_build_deps.add_argument( "--with-build-dependencies", action="store_true", help="include build dependencies in the buildcache", ) with_or_without_build_deps.add_argument( "--without-build-dependencies", action="store_true", help="exclude build dependencies from the buildcache", ) push.add_argument( "--fail-fast", action="store_true", help="stop pushing on first failure (default is best effort)", ) push.add_argument( "--base-image", default=None, help="specify the base image for the buildcache" ) push.add_argument( "--tag", "-t", default=None, help="when pushing to an OCI registry, tag an image containing all root specs and their " "runtime dependencies", ) push.add_argument( "--private", action="store_true", help="for a private mirror, include non-redistributable packages", ) push.add_argument( "--group", action="append", default=None, dest="groups", metavar="GROUP", help="push only specs from the given environment group " "(can be specified multiple times, requires an active environment)", ) arguments.add_common_arguments(push, ["specs", "jobs"]) push.set_defaults(func=push_fn) install = subparsers.add_parser("install", help=install_fn.__doc__) install.add_argument( "-f", "--force", action="store_true", help="overwrite install directory if it exists" ) install.add_argument( "-m", "--multiple", action="store_true", help="allow all matching packages" ) install.add_argument( "-u", "--unsigned", action="store_true", help="install unsigned buildcache tarballs for testing", ) install.add_argument( "-o", "--otherarch", action="store_true", help="install specs from other architectures instead of default platform and OS", ) arguments.add_common_arguments(install, ["specs"]) install.set_defaults(func=install_fn) listcache = subparsers.add_parser("list", help=list_fn.__doc__) arguments.add_common_arguments(listcache, ["long", "very_long", "namespaces"]) listcache.add_argument( "-v", "--variants", action="store_true", dest="variants", help="show variants in output (can be long)", ) listcache.add_argument( "-a", "--allarch", action="store_true", help="list specs for all available architectures instead of default platform and OS", ) arguments.add_common_arguments(listcache, ["specs"]) listcache.set_defaults(func=list_fn) keys = subparsers.add_parser("keys", help=keys_fn.__doc__) keys.add_argument( "-i", "--install", action="store_true", help="install Keys pulled from mirror" ) keys.add_argument("-t", "--trust", action="store_true", help="trust all downloaded keys") keys.add_argument("-f", "--force", action="store_true", help="force new download of keys") keys.set_defaults(func=keys_fn) # Check if binaries need to be rebuilt on remote mirror check = subparsers.add_parser("check", help=check_fn.__doc__) check.add_argument( "-m", "--mirror-url", default=None, help="override any configured mirrors with this mirror URL", ) check.add_argument( "-o", "--output-file", default=None, help="file where rebuild info should be written" ) # used to construct scope arguments below check.add_argument( "--scope", action=arguments.ConfigScope, type=arguments.config_scope_readable_validator, default=lambda: spack.config.default_modify_scope(), help="configuration scope containing mirrors to check", ) arguments.add_common_arguments(check, ["specs"]) check.set_defaults(func=check_fn) # Download tarball and specfile download = subparsers.add_parser("download", help=download_fn.__doc__) download.add_argument("-s", "--spec", help="download built tarball for spec from mirror") download.add_argument( "-p", "--path", required=True, default=None, help="path to directory where tarball should be downloaded", ) download.set_defaults(func=download_fn) prune = subparsers.add_parser("prune", help=prune_fn.__doc__) prune.add_argument( "mirror", type=arguments.mirror_name_or_url, help="mirror name, path, or URL" ) prune.add_argument( "-k", "--keeplist", default=None, help="file containing newline-delimited list of package hashes to keep (optional)", ) prune.add_argument( "--dry-run", action="store_true", help="do not actually delete anything from the buildcache, but log what would be deleted", ) prune.set_defaults(func=prune_fn) # Given the root spec, save the yaml of the dependent spec to a file savespecfile = subparsers.add_parser("save-specfile", help=save_specfile_fn.__doc__) savespecfile_spec_or_specfile = savespecfile.add_mutually_exclusive_group(required=True) savespecfile_spec_or_specfile.add_argument("--root-spec", help="root spec of dependent spec") savespecfile.add_argument( "-s", "--specs", required=True, help="list of dependent specs for which saved yaml is desired", ) savespecfile.add_argument( "--specfile-dir", required=True, help="path to directory where spec yamls should be saved" ) savespecfile.set_defaults(func=save_specfile_fn) # Sync buildcache entries from one mirror to another sync = subparsers.add_parser("sync", help=sync_fn.__doc__) sync_manifest_source = sync.add_argument_group( "Manifest Source", "Specify a list of build cache objects to sync using manifest file(s)." 'This option takes the place of the "source mirror" for synchronization' 'and optionally takes a "destination mirror" ', ) sync_manifest_source.add_argument( "--manifest-glob", help="a quoted glob pattern identifying CI rebuild manifest files" ) sync_source_mirror = sync.add_argument_group( "Named Source", "Specify a single registered source mirror to synchronize from. This option requires" "the specification of a destination mirror.", ) sync_source_mirror.add_argument( "src_mirror", metavar="source mirror", nargs="?", type=arguments.mirror_name_or_url, help="source mirror name, path, or URL", ) sync.add_argument( "dest_mirror", metavar="destination mirror", nargs="?", type=arguments.mirror_name_or_url, help="destination mirror name, path, or URL", ) sync.set_defaults(func=sync_fn) # Check the validity of a buildcache check_index = subparsers.add_parser("check-index", help=check_index_fn.__doc__) check_index.add_argument( "--verify", nargs="+", choices=["exists", "manifests", "blobs", "all"], default=["exists"], help="List of items to verify along along with the index.", ) check_index.add_argument( "--name", "-n", action="store", help="Name of the view index to check" ) check_index.add_argument( "--output", "-o", action="store", help="File to write check details to" ) check_index.add_argument( "mirror", type=arguments.mirror_name_or_url, help="mirror name, path, or URL" ) check_index.set_defaults(func=check_index_fn) # Update buildcache index without copying any additional packages update_index = subparsers.add_parser( "update-index", aliases=["rebuild-index"], help=update_index_fn.__doc__ ) update_index.add_argument( "mirror", type=arguments.mirror_name_or_url, help="destination mirror name, path, or URL" ) update_index_view_args = update_index.add_argument_group("view arguments") update_index_view_args.add_argument( "sources", nargs="*", help="List of environments names or paths" ) update_index_view_args.add_argument( "--name", "-n", action="store", help="Name of the view index to update" ) update_index_view_mode_args = update_index_view_args.add_mutually_exclusive_group( required=False ) update_index_view_mode_args.add_argument( "--append", "-a", action="store_true", help="Append the listed specs to the current view index if it already exists. " "This operation does not guarantee atomic write and should be run with care.", ) update_index_view_mode_args.add_argument( "--force", "-f", action="store_true", help="If a view index already exists, overwrite it and " "suppress warnings (this is the default for non-view indices)", ) update_index.add_argument( "-k", "--keys", default=False, action="store_true", help="if provided, key index will be updated as well as package index", ) arguments.add_common_arguments(update_index, ["yes_to_all"]) update_index.set_defaults(func=update_index_fn) # Migrate a buildcache from layout_version 2 to version 3 migrate = subparsers.add_parser("migrate", help=migrate_fn.__doc__) migrate.add_argument("mirror", type=arguments.mirror_name, help="name of a configured mirror") migrate.add_argument( "-u", "--unsigned", default=False, action="store_true", help="Ignore signatures and do not resign, default is False", ) migrate.add_argument( "-d", "--delete-existing", default=False, action="store_true", help="Delete the previous layout, the default is to keep it.", ) arguments.add_common_arguments(migrate, ["yes_to_all"]) # TODO: add -y argument to prompt if user really means to delete existing migrate.set_defaults(func=migrate_fn) def _matching_specs(specs: List[Spec]) -> List[Spec]: """Disambiguate specs and return a list of matching specs""" return [ spack.cmd.disambiguate_spec(s, ev.active_environment(), installed=InstallRecordStatus.ANY) for s in specs ] def _format_spec(spec: Spec) -> str: return spec.cformat("{name}{@version}{/hash:7}") def _skip_no_redistribute_for_public(specs): remaining_specs = list() removed_specs = list() for spec in specs: if spec.package.redistribute_binary: remaining_specs.append(spec) else: removed_specs.append(spec) if removed_specs: colified_output = tty.colify.colified(list(s.name for s in removed_specs), indent=4) tty.debug( "The following specs will not be added to the binary cache" " because they cannot be redistributed:\n" f"{colified_output}\n" "You can use `--private` to include them." ) return remaining_specs class PackagesAreNotInstalledError(spack.error.SpackError): """Raised when a list of specs is not installed but picked to be packaged.""" def __init__(self, specs: List[Spec]): super().__init__( "Cannot push non-installed packages", ", ".join(elide_list([_format_spec(s) for s in specs], 5)), ) class PackageNotInstalledError(spack.error.SpackError): """Raised when a spec is not installed but picked to be packaged.""" def _specs_to_be_packaged( requested: List[Spec], things_to_install: str, build_deps: bool ) -> List[Spec]: """Collect all non-external with or without roots and dependencies""" if "dependencies" not in things_to_install: deptype = dt.NONE elif build_deps: deptype = dt.ALL else: deptype = dt.RUN | dt.LINK | dt.TEST specs = [ s for s in traverse.traverse_nodes( requested, root="package" in things_to_install, deptype=deptype, order="breadth", key=traverse.by_dag_hash, ) if not s.external ] specs.reverse() return specs def push_fn(args): """create a binary package and push it to a mirror""" if args.specs and args.groups: tty.die("--group and explicit specs are mutually exclusive") if args.groups: env = spack.cmd.require_active_env(cmd_name="buildcache push") available_groups = env.manifest.groups() if any(g not in available_groups for g in args.groups): tty.die( f"Some of the groups do not exist in the environment. " f"Available groups are: {', '.join(sorted(available_groups))}" ) roots = [c for g in args.groups for _, c in env.concretized_specs_by(group=g)] elif args.specs: roots = _matching_specs(spack.cmd.parse_specs(args.specs)) else: roots = spack.cmd.require_active_env(cmd_name="buildcache push").concrete_roots() mirror = args.mirror assert isinstance(mirror, spack.mirrors.mirror.Mirror) push_url = mirror.push_url # When neither --signed, --unsigned nor --key are specified, use the mirror's default. if args.signed is None and not args.key: unsigned = not mirror.signed else: unsigned = not (args.key or args.signed) # For OCI images, we require dependencies to be pushed for now. if spack.oci.image.is_oci_url(mirror.push_url) and not unsigned: tty.warn( "Code signing is currently not supported for OCI images. " "Use --unsigned to silence this warning." ) unsigned = True # Select a signing key, or None if unsigned. signing_key = ( None if unsigned else (args.key or spack.binary_distribution.select_signing_key()) ) specs = _specs_to_be_packaged( roots, things_to_install=args.things_to_install, build_deps=args.with_build_dependencies or not args.without_build_dependencies, ) if not args.private: specs = _skip_no_redistribute_for_public(specs) if len(specs) > 1: tty.info(f"Selected {len(specs)} specs to push to {push_url}") # Pushing not installed specs is an error. Either fail fast or populate the error list and # push installed package in best effort mode. failed: List[Tuple[Spec, BaseException]] = [] with spack.store.STORE.db.read_transaction(): if any(not s.installed for s in specs): specs, not_installed = stable_partition(specs, lambda s: s.installed) if args.fail_fast: raise PackagesAreNotInstalledError(not_installed) else: failed.extend( (s, PackageNotInstalledError("package not installed")) for s in not_installed ) # Warn about possible old binary mirror layout if not spack.oci.image.is_oci_url(mirror.push_url): check_mirror_for_layout(mirror) with spack.binary_distribution.make_uploader( mirror=mirror, force=args.force, update_index=args.update_index, signing_key=signing_key, base_image=args.base_image, ) as uploader: skipped, upload_errors = uploader.push(specs=specs) failed.extend(upload_errors) if skipped: if len(specs) == 1: tty.info("The spec is already in the buildcache. Use --force to overwrite it.") elif len(skipped) == len(specs): tty.info("All specs are already in the buildcache. Use --force to overwrite them.") else: tty.info( "The following {} specs were skipped as they already exist in the " "buildcache:\n" " {}\n" " Use --force to overwrite them.".format( len(skipped), ", ".join(elide_list([_format_spec(s) for s in skipped], 5)) ) ) if failed: if len(failed) == 1: raise failed[0][1] raise spack.error.SpackError( f"The following {len(failed)} errors occurred while pushing specs to the " "buildcache", "\n".join( elide_list( [ f" {_format_spec(spec)}: {e.__class__.__name__}: {e}" for spec, e in failed ], 5, ) ), ) # Finally tag all roots as a single image if requested. if args.tag: uploader.tag(args.tag, roots) def install_fn(args): """install from a binary package""" if not args.specs: tty.die("a spec argument is required to install from a buildcache") query = spack.binary_distribution.BinaryCacheQuery(all_architectures=args.otherarch) matches = spack.store.find(args.specs, multiple=args.multiple, query_fn=query) for match in matches: spack.binary_distribution.install_single_spec( match, unsigned=args.unsigned, force=args.force ) def list_fn(args): """list binary packages available from mirrors""" try: specs = spack.binary_distribution.update_cache_and_get_specs() except spack.binary_distribution.FetchCacheError as e: tty.die(e) if not args.allarch: arch = spack.spec.Spec.default_arch() specs = [s for s in specs if s.intersects(arch)] if args.specs: constraints = set(args.specs) specs = [s for s in specs if any(s.intersects(c) for c in constraints)] if sys.stdout.isatty(): builds = len(specs) tty.msg("%s." % plural(builds, "cached build")) if not builds and not args.allarch: tty.msg( "You can query all available architectures with:", "spack buildcache list --allarch", ) display_specs(specs, args, all_headers=True) def keys_fn(args): """get public keys available on mirrors""" spack.binary_distribution.get_keys(args.install, args.trust, args.force) def check_fn(args: argparse.Namespace): """check specs against remote binary mirror(s) to see if any need to be rebuilt this command uses the process exit code to indicate its result, specifically, if the exit code is non-zero, then at least one of the indicated specs needs to be rebuilt """ specs_arg = args.specs if specs_arg: specs = _matching_specs(spack.cmd.parse_specs(specs_arg)) else: specs = spack.cmd.require_active_env("buildcache check").all_specs() if not specs: tty.msg("No specs provided, exiting.") return specs = [spack.concretize.concretize_one(s) for s in specs] # Next see if there are any configured binary mirrors configured_mirrors = spack.config.get("mirrors", scope=args.scope) if args.mirror_url: configured_mirrors = {"additionalMirrorUrl": args.mirror_url} if not configured_mirrors: tty.msg("No mirrors provided, exiting.") return if ( spack.binary_distribution.check_specs_against_mirrors( configured_mirrors, specs, args.output_file ) == 1 ): sys.exit(1) def download_fn(args): """download buildcache entry from a remote mirror to local folder this command uses the process exit code to indicate its result, specifically, a non-zero exit code indicates that the command failed to download at least one of the required buildcache components """ specs = _matching_specs(spack.cmd.parse_specs(args.spec)) if len(specs) != 1: tty.die("a single spec argument is required to download from a buildcache") spack.binary_distribution.download_single_spec(specs[0], args.path) def save_specfile_fn(args): """get full spec for dependencies and write them to files in the specified output directory uses exit code to signal success or failure. an exit code of zero means the command was likely successful. if any errors or exceptions are encountered, or if expected command-line arguments are not provided, then the exit code will be non-zero """ specs = spack.cmd.parse_specs(args.root_spec) if len(specs) != 1: tty.die("a single spec argument is required to save specfile") root = specs[0] if not root.concrete: root = spack.concretize.concretize_one(root) save_dependency_specfiles( root, args.specfile_dir, dependencies=spack.cmd.parse_specs(args.specs) ) def copy_buildcache_entry(cache_entry: URLBuildcacheEntry, destination_url: str): """Download buildcache entry and copy it to the destination_url""" try: spec_dict = cache_entry.fetch_metadata() cache_entry.fetch_archive() except spack.binary_distribution.BuildcacheEntryError as e: tty.warn(f"Failed to retrieve buildcache for copying due to {e}") cache_entry.destroy() return spec_blob_record = cache_entry.get_blob_record(BuildcacheComponent.SPEC) local_spec_path = cache_entry.get_local_spec_path() tarball_blob_record = cache_entry.get_blob_record(BuildcacheComponent.TARBALL) local_tarball_path = cache_entry.get_local_archive_path() target_spec = spack.spec.Spec.from_dict(spec_dict) spec_label = f"{target_spec.name}/{target_spec.dag_hash()[:7]}" if not tarball_blob_record: cache_entry.destroy() raise BuildcacheEntryError(f"No source tarball blob record, failed to sync {spec_label}") # Try to push the tarball tarball_dest_url = cache_entry.get_blob_url(destination_url, tarball_blob_record) try: web_util.push_to_url(local_tarball_path, tarball_dest_url, keep_original=True) except Exception as e: tty.warn(f"Failed to push {local_tarball_path} to {tarball_dest_url} due to {e}") cache_entry.destroy() return if not spec_blob_record: cache_entry.destroy() raise BuildcacheEntryError(f"No source spec blob record, failed to sync {spec_label}") # Try to push the spec file spec_dest_url = cache_entry.get_blob_url(destination_url, spec_blob_record) try: web_util.push_to_url(local_spec_path, spec_dest_url, keep_original=True) except Exception as e: tty.warn(f"Failed to push {local_spec_path} to {spec_dest_url} due to {e}") cache_entry.destroy() return # Stage the manifest locally, since if it's signed, we don't want to try to # to reproduce that here. Instead just push the locally staged manifest to # the expected path at the destination url. manifest_src_url = cache_entry.remote_manifest_url manifest_dest_url = cache_entry.get_manifest_url(target_spec, destination_url) manifest_stage = spack.stage.Stage(manifest_src_url) try: manifest_stage.create() manifest_stage.fetch() except Exception as e: tty.warn(f"Failed to fetch manifest from {manifest_src_url} due to {e}") manifest_stage.destroy() cache_entry.destroy() return local_manifest_path = manifest_stage.save_filename try: web_util.push_to_url(local_manifest_path, manifest_dest_url, keep_original=True) except Exception as e: tty.warn(f"Failed to push manifest to {manifest_dest_url} due to {e}") manifest_stage.destroy() cache_entry.destroy() def sync_fn(args): """sync binaries (and associated metadata) from one mirror to another requires an active environment in order to know which specs to sync """ if args.manifest_glob: # Passing the args.src_mirror here because it is not possible to # have the destination be required when specifying a named source # mirror and optional for the --manifest-glob argument. In the case # of manifest glob sync, the source mirror positional argument is the # destination mirror if it is specified. If there are two mirrors # specified, the second is ignored and the first is the override # destination. if args.dest_mirror: tty.warn(f"Ignoring unused argument: {args.dest_mirror.name}") manifest_copy(glob.glob(args.manifest_glob), args.src_mirror) return 0 if args.src_mirror is None or args.dest_mirror is None: tty.die("Provide mirrors to sync from and to.") src_mirror = args.src_mirror dest_mirror = args.dest_mirror src_mirror_url = src_mirror.fetch_url dest_mirror_url = dest_mirror.push_url # Get the active environment env = spack.cmd.require_active_env(cmd_name="buildcache sync") tty.msg( "Syncing environment buildcache files from {0} to {1}".format( src_mirror_url, dest_mirror_url ) ) tty.debug("Syncing the following specs:") specs_to_sync = [s for s in env.all_specs() if not s.external] for s in specs_to_sync: tty.debug(" {0}{1}: {2}".format("* " if s in env.roots() else " ", s.name, s.dag_hash())) cache_class = get_url_buildcache_class( layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ) src_cache_entry = cache_class(src_mirror_url, s, allow_unsigned=True) src_cache_entry.read_manifest() copy_buildcache_entry(src_cache_entry, dest_mirror_url) def manifest_copy( manifest_file_list: List[str], dest_mirror: Optional[spack.mirrors.mirror.Mirror] = None ): """Read manifest files containing information about specific specs to copy from source to destination, remove duplicates since any binary package for a given hash should be the same as any other, and copy all files specified in the manifest files.""" deduped_manifest = {} for manifest_path in manifest_file_list: with open(manifest_path, encoding="utf-8") as fd: manifest = json.loads(fd.read()) for spec_hash, copy_obj in manifest.items(): # Last duplicate hash wins deduped_manifest[spec_hash] = copy_obj for spec_hash, copy_obj in deduped_manifest.items(): cache_class = get_url_buildcache_class( layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ) src_cache_entry = cache_class( cache_class.get_base_url(copy_obj["src"]), allow_unsigned=True ) src_cache_entry.read_manifest(manifest_url=copy_obj["src"]) if dest_mirror: destination_url = dest_mirror.push_url else: destination_url = cache_class.get_base_url(copy_obj["dest"]) tty.debug("copying {0} to {1}".format(copy_obj["src"], destination_url)) copy_buildcache_entry(src_cache_entry, destination_url) def update_index( mirror: spack.mirrors.mirror.Mirror, update_keys=False, timer=timer_mod.NULL_TIMER ): timer.start() # Special case OCI images for now. try: image_ref = spack.oci.oci.image_from_mirror(mirror) except ValueError: image_ref = None if image_ref: with tempfile.TemporaryDirectory( dir=spack.stage.get_stage_root() ) as tmpdir, spack.util.parallel.make_concurrent_executor() as executor: spack.binary_distribution._oci_update_index(image_ref, tmpdir, executor) return # Otherwise, assume a normal mirror. url = mirror.push_url with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: spack.binary_distribution._url_generate_package_index(url, tmpdir, timer=timer) if update_keys: mirror_update_keys(mirror) def mirror_update_keys(mirror: spack.mirrors.mirror.Mirror): url = mirror.push_url try: with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: spack.binary_distribution.generate_key_index(url, tmpdir) except spack.binary_distribution.CannotListKeys as e: # Do not error out if listing keys went wrong. This usually means that the _gpg path # does not exist. TODO: distinguish between this and other errors. tty.warn(f"did not update the key index: {e}") def update_view( mirror: spack.mirrors.mirror.Mirror, update_mode: ViewUpdateMode, *sources: str, name: Optional[str] = None, update_keys: bool = False, yes_to_all: bool = False, ): """update a buildcache view index""" # OCI images do not support views. try: spack.oci.oci.image_from_mirror(mirror) raise spack.error.SpackError("OCI build caches do not support index views") except ValueError: pass if update_mode == ViewUpdateMode.APPEND and not yes_to_all: tty.warn( "Appending to a view index does not guarantee idempotent write when contending " "with multiple writers. This feature is meant to be used by a single process." ) tty.get_yes_or_no("Do you want to proceed?", default=False) # Otherwise, assume a normal mirror. url = mirror.push_url if (name and mirror.push_view) and not name == mirror.push_view: tty.warn( ( f"Updating index view with name ({name}), which is different than " f"the configured name ({mirror.push_view}) for the mirror {mirror.name}" ) ) name = name or mirror.push_view if not name: tty.die( "Attempting to update a view but could not determine the view name.\n" " Either pass --name or configure the view name in mirrors.yaml" ) mirror_metadata = spack.binary_distribution.MirrorMetadata( url, spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION, name ) # Check if the index already exists, if it does make sure there is a copy in the # local cache. index_exists = True try: BINARY_INDEX._fetch_and_cache_index(mirror_metadata) except spack.binary_distribution.BuildcacheIndexNotExists: index_exists = False if index_exists and update_mode == ViewUpdateMode.CREATE: raise spack.error.SpackError( "Index already exists. To overwrite or update pass --force or --append respectively" ) hashes = [] if sources: for source in sources: tty.debug(f"reading specs from source: {source}") env = ev.environment_from_name_or_dir(source) hashes.extend(env.all_hashes()) else: # Get hashes in the current active environment hashes = spack.cmd.require_active_env(cmd_name="buildcache update-view").all_hashes() if not hashes: tty.warn("No specs found for view, creating an empty index") filter_fn = lambda x: x in hashes with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: # Initialize a database db = spack.binary_distribution.BuildCacheDatabase(tmpdir) db._write() if update_mode == ViewUpdateMode.APPEND: # Load the current state of the view index from the cache into the database cache_index = BINARY_INDEX._local_index_cache.get(str(mirror_metadata)) if cache_index: cache_key = cache_index["index_path"] with BINARY_INDEX._index_file_cache.read_transaction(cache_key) as f: if f is not None: db._read_from_stream(f) spack.binary_distribution._url_generate_package_index(url, tmpdir, db, name, filter_fn) if update_keys: mirror_update_keys(mirror) def check_index_fn(args): """Check if a build cache index, manifests, and blobs are consistent""" mirror = args.mirror verify = set(args.verify) checking_view_index = (args.name or mirror.fetch_view) is not None if "all" in verify: verify.update(["exists", "manifests", "blobs"]) try: spack.oci.oci.image_from_mirror(mirror) raise spack.error.SpackError("OCI build caches do not support index views") except ValueError: pass # Check if the index exists, and cache it locally for next operations mirror_metadata = spack.binary_distribution.MirrorMetadata( mirror.fetch_url, spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION, args.name or mirror.fetch_view, ) index_exists = True missing_index_blob = False try: BINARY_INDEX._fetch_and_cache_index(mirror_metadata) except spack.binary_distribution.BuildcacheIndexNotExists: index_exists = False except spack.binary_distribution.FetchIndexError: # Here the index manifest exists, but the index blob did not # We can still run some of the other validations here, so let's try index_exists = False missing_index_blob = True missing_specs = [] unindexed_specs = [] missing_blobs = {} cache_hash_list = [] index_hash_list = [] # List the manifests and verify with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: # Get listing of spec manifests in mirror manifest_files = [] if "manifests" in verify or "blobs" in verify: manifest_files, read_fn = get_entries_from_cache( mirror.fetch_url, tmpdir, BuildcacheComponent.SPEC ) if "manifests" in verify and index_exists: # Read the index file db = spack.binary_distribution.BuildCacheDatabase(tmpdir) cache_entry = BINARY_INDEX._local_index_cache[str(mirror_metadata)] cache_key = cache_entry["index_path"] with BINARY_INDEX._index_file_cache.read_transaction(cache_key) as f: if f is not None: db._read_from_stream(f) index_hash_list = set( [ s.dag_hash() for s in db.query_local(installed=InstallRecordStatus.ANY) if db._data[s.dag_hash()].in_buildcache ] ) for spec_manifest in manifest_files: # Spec manifests have a naming format # --.spec.manifest.json spec_hash = spec_manifest.rsplit("-", 1)[1].split(".", 1)[0] if checking_view_index and spec_hash not in index_hash_list: continue cache_hash_list.append(spec_hash) if spec_hash not in index_hash_list: unindexed_specs.append(spec_hash) if "blobs" in verify: entry = read_fn(spec_manifest) entry.read_manifest() for record in entry.manifest.data: if not entry.check_blob_exists(record): blobs = missing_blobs.get(spec_hash, []) blobs.append(record) missing_blobs[spec_hash] = blobs for h in index_hash_list: if h not in cache_hash_list: missing_specs.append(h) # Print summary summary_msg = "Build cache check:\n\t" if "exists" in verify: if index_exists: summary_msg = f"Index exists in mirror: {mirror.name}" else: summary_msg = f"Index does not exist in mirror: {mirror.name}" if mirror.fetch_view: summary_msg += f"@{mirror.fetch_view}" summary_msg += "\n" if missing_index_blob: tty.warn("The index blob is missing") if "manifests" in verify: if checking_view_index: count = "n/a" else: count = len(unindexed_specs) summary_msg += f"\tUnindexed specs: {count}\n" if "manifests" in verify: summary_msg += f"\tMissing specs: {len(missing_specs)}\n" if "blobs" in verify: summary_msg += f"\tMissing blobs: {len(missing_blobs)}\n" if args.output: os.makedirs(os.path.dirname(args.output), exist_ok=True) with open(args.output, "w", encoding="utf-8") as fd: json.dump( { "exists": index_exists, "manifests": {"missing": missing_specs, "unindexed": unindexed_specs}, "blobs": {"missing": missing_blobs}, }, fd, ) tty.info(summary_msg) def update_index_fn(args): """update a buildcache index or index view if extra arguments are provided.""" t = timer_mod.Timer() if tty.is_verbose() else timer_mod.NullTimer() update_view_index = ( args.append or args.force or args.name or args.sources or args.mirror.push_view ) if update_view_index: update_mode = ViewUpdateMode.CREATE if args.force: update_mode = ViewUpdateMode.OVERWRITE elif args.append: update_mode = ViewUpdateMode.APPEND return update_view( args.mirror, update_mode, *args.sources, name=args.name, update_keys=args.keys, yes_to_all=args.yes_to_all, ) else: update_index(args.mirror, update_keys=args.keys, timer=t) if tty.is_verbose(): tty.msg("Timing summary:") t.stop() t.write_tty() def migrate_fn(args): """perform in-place binary mirror migration (2 to 3) A mirror can contain both layout version 2 and version 3 simultaneously without interference. This command performs in-place migration of a binary mirror laid out according to version 2, to a binary mirror laid out according to layout version 3. Only indexed specs will be migrated, so consider updating the mirror index before running this command. Re-run the command to migrate any missing items. The default mode of operation is to perform a signed migration, that is, spack will attempt to verify the signatures on specs, and then re-sign them before migration, using whatever keys are already installed in your key ring. You can migrate a mirror of unsigned binaries (or convert a mirror of signed binaries to unsigned) by providing the ``--unsigned`` argument. By default spack will leave the original mirror contents (in the old layout) in place after migration. You can have spack remove the old contents by providing the ``--delete-existing`` argument. Because migrating a mostly-already-migrated mirror should be fast, consider a workflow where you perform a default migration, (i.e. preserve the existing layout rather than deleting it) then evaluate the state of the migrated mirror by attempting to install from it, and finally running the migration again with ``--delete-existing``.""" target_mirror = args.mirror unsigned = args.unsigned assert isinstance(target_mirror, spack.mirrors.mirror.Mirror) delete_existing = args.delete_existing proceed = True if delete_existing and not args.yes_to_all: msg = ( "Using --delete-existing will delete the entire contents \n" " of the old layout within the mirror. Because migrating a mirror \n" " that has already been migrated should be fast, consider a workflow \n" " where you perform a default migration (i.e. preserve the existing \n" " layout rather than deleting it), then evaluate the state of the \n" " migrated mirror by attempting to install from it, and finally, \n" " run the migration again with --delete-existing." ) tty.warn(msg) proceed = tty.get_yes_or_no("Do you want to proceed?", default=False) if not proceed: tty.die("Migration aborted.") migrate(target_mirror, unsigned=unsigned, delete_existing=delete_existing) def prune_fn(args): """prune buildcache entries from the mirror If a keeplist file is provided, performs direct pruning (deletes packages not in keeplist) followed by orphan pruning. If no keeplist is provided, only performs orphan pruning. """ mirror: spack.mirrors.mirror.Mirror = args.mirror keeplist: Optional[str] = args.keeplist dry_run: bool = args.dry_run assert isinstance(mirror, spack.mirrors.mirror.Mirror) prune_buildcache(mirror=mirror, keeplist=keeplist, dry_run=dry_run) def buildcache(parser, args): return args.func(args) ================================================ FILE: lib/spack/spack/cmd/cd.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd.common import spack.cmd.location description = "cd to spack directories in the shell" section = "user environment" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: """This is for decoration -- spack cd is used through spack's shell support. This allows spack cd to print a descriptive help message when called with -h.""" spack.cmd.location.setup_parser(subparser) def cd(parser, args): spec = " ".join(args.spec) if args.spec else "SPEC" spack.cmd.common.shell_init_instructions( "spack cd", "cd `spack location --install-dir %s`" % spec ) ================================================ FILE: lib/spack/spack/cmd/change.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import warnings import spack.cmd import spack.environment import spack.spec from spack.cmd.common import arguments description = "change an existing spec in an environment" section = "environments" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-l", "--list-name", dest="list_name", default="specs", help="name of the list to remove abstract specs from", ) subparser.add_argument( "--match-spec", dest="match_spec", help="change all specs matching match-spec (default is match by spec name)", ) subparser.add_argument( "-a", "--all", action="store_true", help="change all matching abstract specs (allow changing more than one abstract spec)", ) subparser.add_argument( "-c", "--concrete", action="store_true", default=False, help="change concrete specs in the environment", ) subparser.add_argument( "-C", "--concrete-only", action="store_true", default=False, help="change only concrete specs in the environment", ) arguments.add_common_arguments(subparser, ["specs"]) def change(parser, args): if args.all and args.concrete_only: warnings.warn("'spack change --all' argument is ignored with '--concrete-only'") if args.list_name != "specs" and args.concrete_only: warnings.warn("'spack change --list-name' argument is ignored with '--concrete-only'") env = spack.cmd.require_active_env(cmd_name="change") match_spec = None if args.match_spec: match_spec = spack.cmd.parse_specs([args.match_spec])[0] specs = spack.cmd.parse_specs(args.specs) with env.write_transaction(): if not args.concrete_only: try: for spec in specs: env.change_existing_spec( spec, list_name=args.list_name, match_spec=match_spec, allow_changing_multiple_specs=args.all, ) except (ValueError, spack.environment.SpackEnvironmentError) as e: msg = "Cannot change abstract specs." msg += " Try again with '--concrete-only' to change concrete specs only." raise ValueError(msg) from e if args.concrete or args.concrete_only: for spec in specs: env.mutate(selector=match_spec or spack.spec.Spec(spec.name), mutator=spec) env.write() ================================================ FILE: lib/spack/spack/cmd/checksum.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import re import sys from typing import Dict, Optional, Tuple import spack.llnl.string import spack.llnl.util.lang import spack.repo import spack.spec import spack.stage import spack.util.web as web_util from spack.cmd.common import arguments from spack.llnl.util import tty from spack.package_base import ( ManualDownloadRequiredError, PackageBase, deprecated_version, preferred_version, ) from spack.util.editor import editor from spack.util.format import get_version_lines from spack.version import StandardVersion, Version description = "checksum available versions of a package" section = "packaging" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--keep-stage", action="store_true", default=False, help="don't clean up staging area when command completes", ) subparser.add_argument( "--batch", "-b", action="store_true", default=False, help="don't ask which versions to checksum", ) subparser.add_argument( "--latest", "-l", action="store_true", default=False, help="checksum the latest available version", ) subparser.add_argument( "--preferred", "-p", action="store_true", default=False, help="checksum the known Spack preferred version", ) modes_parser = subparser.add_mutually_exclusive_group() modes_parser.add_argument( "--add-to-package", "-a", action="store_true", default=False, help="add new versions to package", ) modes_parser.add_argument( "--verify", action="store_true", default=False, help="verify known package checksums" ) subparser.add_argument("package", help="name or spec (e.g. ``cmake`` or ``cmake@3.18``)") subparser.add_argument( "versions", nargs="*", help="checksum these specific versions (if omitted, Spack searches for remote versions)", ) arguments.add_common_arguments(subparser, ["jobs"]) subparser.epilog = ( "examples:\n" " `spack checksum zlib@1.2` autodetects versions 1.2.0 to 1.2.13 from the remote\n" " `spack checksum zlib 1.2.13` checksums exact version 1.2.13 directly without search\n" ) def checksum(parser, args): spec = spack.spec.Spec(args.package) # Get the package we're going to generate checksums for pkg: PackageBase = spack.repo.PATH.get_pkg_class(spec.name)(spec) # Skip manually downloaded packages if pkg.manual_download: raise ManualDownloadRequiredError(pkg.download_instr) versions = [StandardVersion.from_string(v) for v in args.versions] # Define placeholder for remote versions. This'll help reduce redundant work if we need to # check for the existence of remote versions more than once. remote_versions: Optional[Dict[StandardVersion, str]] = None # Add latest version if requested if args.latest: remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs) if len(remote_versions) > 0: versions.append(max(remote_versions.keys())) # Add preferred version if requested (todo: exclude git versions) if args.preferred: versions.append(preferred_version(pkg)) # Store a dict of the form version -> URL url_dict: Dict[StandardVersion, str] = {} for version in versions: if deprecated_version(pkg, version): tty.warn(f"Version {version} is deprecated") url = pkg.find_valid_url_for_version(version) if url is not None: url_dict[version] = url continue # If we get here, it's because no valid url was provided by the package. Do expensive # fallback to try to recover if remote_versions is None: remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs) if version in remote_versions: url_dict[version] = remote_versions[version] if len(versions) <= 0: if remote_versions is None: remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs) url_dict = remote_versions # A spidered URL can differ from the package.py *computed* URL, pointing to different tarballs. # For example, GitHub release pages sometimes have multiple tarballs with different shasum: # - releases/download/1.0/-1.0.tar.gz (uploaded tarball) # - archive/refs/tags/1.0.tar.gz (generated tarball) # We wanna ensure that `spack checksum` and `spack install` ultimately use the same URL, so # here we check whether the crawled and computed URLs disagree, and if so, prioritize the # former if that URL exists (just sending a HEAD request that is). url_changed_for_version = set() for version, url in url_dict.items(): possible_urls = pkg.all_urls_for_version(version) if url not in possible_urls: for possible_url in possible_urls: if web_util.url_exists(possible_url): url_dict[version] = possible_url break else: url_changed_for_version.add(version) if not url_dict: tty.die(f"Could not find any remote versions for {pkg.name}") elif len(url_dict) > 1 and not args.batch and sys.stdin.isatty(): filtered_url_dict = spack.stage.interactive_version_filter( url_dict, pkg.versions, url_changes=url_changed_for_version, initial_verion_filter=spec.versions, ) if not filtered_url_dict: exit(0) url_dict = filtered_url_dict else: tty.info(f"Found {spack.llnl.string.plural(len(url_dict), 'version')} of {pkg.name}") version_hashes = spack.stage.get_checksums_for_versions( url_dict, pkg.name, keep_stage=args.keep_stage, fetch_options=pkg.fetch_options ) if args.verify: print_checksum_status(pkg, version_hashes) sys.exit(0) # convert dict into package.py version statements version_lines = get_version_lines(version_hashes) print() print(version_lines) print() if args.add_to_package: path = spack.repo.PATH.filename_for_package_name(pkg.name) num_versions_added = add_versions_to_pkg(path, version_lines) tty.msg(f"Added {num_versions_added} new versions to {pkg.name} in {path}") if not args.batch and sys.stdin.isatty(): editor(path) def print_checksum_status(pkg: PackageBase, version_hashes: dict): """ Verify checksums present in version_hashes against those present in the package's instructions. Args: pkg (spack.package_base.PackageBase): A package class for a given package in Spack. version_hashes (dict): A dictionary of the form: version -> checksum. """ results = [] num_verified = 0 failed = False max_len = max(len(str(v)) for v in version_hashes) num_total = len(version_hashes) for version, sha in version_hashes.items(): if version not in pkg.versions: msg = "No previous checksum" status = "-" elif sha == pkg.versions[version]["sha256"]: msg = "Correct" status = "=" num_verified += 1 else: msg = sha status = "x" failed = True results.append("{0:{1}} {2} {3}".format(str(version), max_len, f"[{status}]", msg)) # Display table of checksum results. tty.msg( f"Verified {num_verified} of {num_total}", "", *spack.llnl.util.lang.elide_list(results), "", ) # Terminate at the end of function to prevent additional output. if failed: print() tty.die("Invalid checksums found.") def _update_version_statements(package_src: str, version_lines: str) -> Tuple[int, str]: """Returns a tuple of number of versions added and the package's modified contents.""" num_versions_added = 0 version_statement_re = re.compile(r"([\t ]+version\([^\)]*\))") version_re = re.compile(r'[\t ]+version\(\s*"([^"]+)"[^\)]*\)') # Split rendered version lines into tuple of (version, version_line) # We reverse sort here to make sure the versions match the version_lines new_versions = [] for ver_line in version_lines.split("\n"): match = version_re.match(ver_line) if match: new_versions.append((Version(match.group(1)), ver_line)) split_contents = version_statement_re.split(package_src) for i, subsection in enumerate(split_contents): # If there are no more versions to add we should exit if len(new_versions) <= 0: break # Check if the section contains a version contents_version = version_re.match(subsection) if contents_version is not None: parsed_version = Version(contents_version.group(1)) if parsed_version < new_versions[0][0]: split_contents[i:i] = [new_versions.pop(0)[1], " # FIXME", "\n"] num_versions_added += 1 elif parsed_version == new_versions[0][0]: new_versions.pop(0) return num_versions_added, "".join(split_contents) def add_versions_to_pkg(path: str, version_lines: str) -> int: """Add new versions to a package.py file. Returns the number of versions added.""" with open(path, "r", encoding="utf-8") as f: package_src = f.read() num_versions_added, package_src = _update_version_statements(package_src, version_lines) if num_versions_added > 0: with open(path, "w", encoding="utf-8") as f: f.write(package_src) return num_versions_added ================================================ FILE: lib/spack/spack/cmd/ci.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import json import os import shutil import sys from typing import Dict, List from urllib.parse import urlparse, urlunparse import spack.binary_distribution import spack.ci as spack_ci import spack.cmd import spack.cmd.buildcache as buildcache import spack.cmd.common.arguments import spack.config as cfg import spack.environment as ev import spack.error import spack.fetch_strategy import spack.hash_types as ht import spack.llnl.util.filesystem as fs import spack.llnl.util.tty.color as clr import spack.mirrors.mirror import spack.package_base import spack.repo import spack.spec import spack.stage import spack.util.git import spack.util.gpg as gpg_util import spack.util.timer as timer import spack.util.url as url_util import spack.util.web as web_util from spack.llnl.util import tty from spack.version import StandardVersion from . import doc_dedented, doc_first_line description = "manage continuous integration pipelines" section = "build" level = "long" SPACK_COMMAND = "spack" INSTALL_FAIL_CODE = 1 FAILED_CREATE_BUILDCACHE_CODE = 100 def deindent(desc): return desc.replace(" ", "") def unicode_escape(path: str) -> str: """Returns transformed path with any unicode characters replaced with their corresponding escapes""" return path.encode("unicode-escape").decode("utf-8") def setup_parser(subparser: argparse.ArgumentParser) -> None: setattr(setup_parser, "parser", subparser) subparsers = subparser.add_subparsers(help="CI sub-commands") # Dynamic generation of the jobs yaml from a spack environment generate = subparsers.add_parser( "generate", description=doc_dedented(ci_generate), help=doc_first_line(ci_generate) ) generate.add_argument( "--output-file", default=None, help="pathname for the generated gitlab ci yaml file\n\n" "path to the file where generated jobs file should be written. " "default is .gitlab-ci.yml in the root of the repository", ) prune_dag_group = generate.add_mutually_exclusive_group() prune_dag_group.add_argument( "--prune-dag", action="store_true", dest="prune_dag", default=True, help="skip up-to-date specs\n\n" "do not generate jobs for specs that are up-to-date on the mirror", ) prune_dag_group.add_argument( "--no-prune-dag", action="store_false", dest="prune_dag", default=True, help="process up-to-date specs\n\n" "generate jobs for specs even when they are up-to-date on the mirror", ) prune_unaffected_group = generate.add_mutually_exclusive_group() prune_unaffected_group.add_argument( "--prune-unaffected", action="store_true", dest="prune_unaffected", default=False, help="skip up-to-date specs\n\n" "do not generate jobs for specs that are up-to-date on the mirror", ) prune_unaffected_group.add_argument( "--no-prune-unaffected", action="store_false", dest="prune_unaffected", default=False, help="process up-to-date specs\n\n" "generate jobs for specs even when they are up-to-date on the mirror", ) prune_ext_group = generate.add_mutually_exclusive_group() prune_ext_group.add_argument( "--prune-externals", action="store_true", dest="prune_externals", default=True, help="skip external specs\n\ndo not generate jobs for specs that are marked as external", ) prune_ext_group.add_argument( "--no-prune-externals", action="store_false", dest="prune_externals", default=True, help="process external specs\n\n" "generate jobs for specs even when they are marked as external", ) generate.add_argument( "--check-index-only", action="store_true", dest="index_only", default=False, help="only check spec state from buildcache indices\n\n" "Spack always checks specs against configured binary mirrors, regardless of the DAG " "pruning option. if enabled, Spack will assume all remote buildcache indices are " "up-to-date when assessing whether the spec on the mirror, if present, is up-to-date. " "this has the benefit of reducing pipeline generation time but at the potential cost of " "needlessly rebuilding specs when the indices are outdated. if not enabled, Spack will " "fetch remote spec files directly to assess whether the spec on the mirror is up-to-date", ) generate.add_argument( "--artifacts-root", default="jobs_scratch_dir", help="path to the root of the artifacts directory\n\n" "The spack ci module assumes it will normally be run from within your project " "directory, wherever that is checked out to run your ci. The artifacts root directory " "should specify a name that can safely be used for artifacts within your project " "directory.", ) generate.add_argument( "--forward-variable", action="append", help="Environment variables to forward from the generate environment " "to the generated jobs.", ) generate.set_defaults(func=ci_generate) spack.cmd.common.arguments.add_concretizer_args(generate) spack.cmd.common.arguments.add_common_arguments(generate, ["jobs"]) # Rebuild the buildcache index associated with the mirror in the # active, gitlab-enabled environment. index = subparsers.add_parser( "rebuild-index", description=doc_dedented(ci_reindex), help=doc_first_line(ci_reindex) ) index.set_defaults(func=ci_reindex) # Handle steps of a ci build/rebuild rebuild = subparsers.add_parser( "rebuild", description=doc_dedented(ci_rebuild), help=doc_first_line(ci_rebuild) ) rebuild.add_argument( "-t", "--tests", action="store_true", default=False, help="run stand-alone tests after the build", ) rebuild_ff_group = rebuild.add_mutually_exclusive_group() rebuild_ff_group.add_argument( "--no-fail-fast", action="store_false", default=True, dest="fail_fast", help="continue build/stand-alone tests after the first failure", ) rebuild_ff_group.add_argument( "--fail-fast", action="store_true", dest="fail_fast", help="stop build/stand-alone tests after the first failure", ) rebuild.add_argument( "--timeout", type=int, default=None, help="maximum time (in seconds) that tests are allowed to run", ) rebuild.set_defaults(func=ci_rebuild) spack.cmd.common.arguments.add_common_arguments(rebuild, ["jobs"]) # Facilitate reproduction of a failed CI build job reproduce = subparsers.add_parser( "reproduce-build", description=doc_dedented(ci_reproduce), help=doc_first_line(ci_reproduce), ) reproduce.add_argument( "job_url", help="URL of GitLab job web page or artifact", type=_gitlab_artifacts_url ) reproduce.add_argument( "--runtime", help="Container runtime to use.", default="docker", choices=["docker", "podman"], ) reproduce.add_argument( "--working-dir", help="where to unpack artifacts", default=os.path.join(os.getcwd(), "ci_reproduction"), ) reproduce.add_argument( "-s", "--autostart", help="Run docker reproducer automatically", action="store_true" ) reproduce.add_argument( "--use-local-head", help="Use the HEAD of the local Spack instead of reproducing a commit", action="store_true", ) gpg_group = reproduce.add_mutually_exclusive_group(required=False) gpg_group.add_argument( "--gpg-file", help="Path to public GPG key for validating binary cache installs" ) gpg_group.add_argument( "--gpg-url", help="URL to public GPG key for validating binary cache installs" ) reproduce.set_defaults(func=ci_reproduce) # Verify checksums inside of ci workflows verify_versions = subparsers.add_parser( "verify-versions", description=doc_dedented(ci_verify_versions), help=doc_first_line(ci_verify_versions), ) verify_versions.add_argument("from_ref", help="git ref from which start looking at changes") verify_versions.add_argument("to_ref", help="git ref to end looking at changes") verify_versions.set_defaults(func=ci_verify_versions) def ci_generate(args): """\ generate jobs file from a CI-aware spack file if you want to report the results on CDash, you will need to set the SPACK_CDASH_AUTH_TOKEN before invoking this command. the value must be the CDash authorization token needed to create a build group and register all generated jobs under it """ env = spack.cmd.require_active_env(cmd_name="ci generate") spack_ci.generate_pipeline(env, args) def ci_reindex(args): """\ rebuild the buildcache index for the remote mirror use the active, gitlab-enabled environment to rebuild the buildcache index for the associated mirror """ env = spack.cmd.require_active_env(cmd_name="ci rebuild-index") yaml_root = env.manifest[ev.TOP_LEVEL_KEY] if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1: tty.die("spack ci rebuild-index requires an env containing a mirror") ci_mirrors = yaml_root["mirrors"] mirror_urls = [url for url in ci_mirrors.values()] remote_mirror_url = mirror_urls[0] mirror = spack.mirrors.mirror.Mirror(remote_mirror_url) buildcache.update_index(mirror, update_keys=True) def ci_rebuild(args): """\ rebuild a spec if it is not on the remote mirror check a single spec against the remote mirror, and rebuild it from source if the mirror does not contain the hash """ rebuild_timer = timer.Timer() env = spack.cmd.require_active_env(cmd_name="ci rebuild") # Make sure the environment is "gitlab-enabled", or else there's nothing # to do. ci_config = cfg.get("ci") if not ci_config: tty.die("spack ci rebuild requires an env containing ci cfg") # Grab the environment variables we need. These either come from the # pipeline generation step ("spack ci generate"), where they were written # out as variables, or else provided by GitLab itself. pipeline_artifacts_dir = os.environ.get("SPACK_ARTIFACTS_ROOT") job_log_dir = os.environ.get("SPACK_JOB_LOG_DIR") job_test_dir = os.environ.get("SPACK_JOB_TEST_DIR") repro_dir = os.environ.get("SPACK_JOB_REPRO_DIR") concrete_env_dir = os.environ.get("SPACK_CONCRETE_ENV_DIR") ci_job_name = os.environ.get("CI_JOB_NAME") signing_key = os.environ.get("SPACK_SIGNING_KEY") job_spec_pkg_name = os.environ.get("SPACK_JOB_SPEC_PKG_NAME") job_spec_dag_hash = os.environ.get("SPACK_JOB_SPEC_DAG_HASH") spack_pipeline_type = os.environ.get("SPACK_PIPELINE_TYPE") spack_ci_stack_name = os.environ.get("SPACK_CI_STACK_NAME") rebuild_everything = os.environ.get("SPACK_REBUILD_EVERYTHING") require_signing = os.environ.get("SPACK_REQUIRE_SIGNING") # If signing key was provided via "SPACK_SIGNING_KEY", then try to import it. if signing_key: spack_ci.import_signing_key(signing_key) # Fail early if signing is required but we don't have a signing key sign_binaries = require_signing is not None and require_signing.lower() == "true" if sign_binaries and not spack_ci.can_sign_binaries(): gpg_util.list(False, True) tty.die("SPACK_REQUIRE_SIGNING=True => spack must have exactly one signing key") # Construct absolute paths relative to current $CI_PROJECT_DIR ci_project_dir = os.environ.get("CI_PROJECT_DIR") pipeline_artifacts_dir = os.path.join(ci_project_dir, pipeline_artifacts_dir) job_log_dir = os.path.join(ci_project_dir, job_log_dir) job_test_dir = os.path.join(ci_project_dir, job_test_dir) repro_dir = os.path.join(ci_project_dir, repro_dir) concrete_env_dir = os.path.join(ci_project_dir, concrete_env_dir) # Debug print some of the key environment variables we should have received tty.debug("pipeline_artifacts_dir = {0}".format(pipeline_artifacts_dir)) tty.debug("job_spec_pkg_name = {0}".format(job_spec_pkg_name)) # Query the environment manifest to find out whether we're reporting to a # CDash instance, and if so, gather some information from the manifest to # support that task. cdash_config = cfg.get("cdash") cdash_handler = None if "build-group" in cdash_config: cdash_handler = spack_ci.CDashHandler(cdash_config) tty.debug("cdash url = {0}".format(cdash_handler.url)) tty.debug("cdash project = {0}".format(cdash_handler.project)) tty.debug("cdash project_enc = {0}".format(cdash_handler.project_enc)) tty.debug("cdash build_name = {0}".format(cdash_handler.build_name)) tty.debug("cdash build_stamp = {0}".format(cdash_handler.build_stamp)) tty.debug("cdash site = {0}".format(cdash_handler.site)) tty.debug("cdash build_group = {0}".format(cdash_handler.build_group)) # Is this a pipeline run on a spack PR or a merge to develop? It might # be neither, e.g. a pipeline run on some environment repository. spack_is_pr_pipeline = spack_pipeline_type == "spack_pull_request" spack_is_develop_pipeline = spack_pipeline_type == "spack_protected_branch" tty.debug( "Pipeline type - PR: {0}, develop: {1}".format( spack_is_pr_pipeline, spack_is_develop_pipeline ) ) full_rebuild = True if rebuild_everything and rebuild_everything.lower() == "true" else False pipeline_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True) buildcache_destination = None if "buildcache-destination" not in pipeline_mirrors: tty.die("spack ci rebuild requires a mirror named 'buildcache-destination") buildcache_destination = pipeline_mirrors["buildcache-destination"] # Get the concrete spec to be built by this job. try: job_spec = env.get_one_by_hash(job_spec_dag_hash) except AssertionError: tty.die("Could not find environment spec with hash {0}".format(job_spec_dag_hash)) job_spec_json_file = "{0}.json".format(job_spec_pkg_name) job_spec_json_path = os.path.join(repro_dir, job_spec_json_file) # To provide logs, cdash reports, etc for developer download/perusal, # these things have to be put into artifacts. This means downstream # jobs that "need" this job will get those artifacts too. So here we # need to clean out the artifacts we may have got from upstream jobs. cdash_report_dir = os.path.join(pipeline_artifacts_dir, "cdash_report") if os.path.exists(cdash_report_dir): shutil.rmtree(cdash_report_dir) if os.path.exists(job_log_dir): shutil.rmtree(job_log_dir) if os.path.exists(job_test_dir): shutil.rmtree(job_test_dir) if os.path.exists(repro_dir): shutil.rmtree(repro_dir) # Now that we removed them if they existed, create the directories we # need for storing artifacts. The cdash_report directory will be # created internally if needed. os.makedirs(job_log_dir) os.makedirs(job_test_dir) os.makedirs(repro_dir) # Copy the concrete environment files to the repro directory so we can # expose them as artifacts and not conflict with the concrete environment # files we got as artifacts from the upstream pipeline generation job. # Try to cast a slightly wider net too, and hopefully get the generated # pipeline yaml. If we miss it, the user will still be able to go to the # pipeline generation job and get it from there. target_dirs = [concrete_env_dir, pipeline_artifacts_dir] for dir_to_list in target_dirs: for file_name in os.listdir(dir_to_list): src_file = os.path.join(dir_to_list, file_name) if os.path.isfile(src_file): dst_file = os.path.join(repro_dir, file_name) shutil.copyfile(src_file, dst_file) # Write this job's spec json into the reproduction directory, and it will # also be used in the generated "spack install" command to install the spec tty.debug("job concrete spec path: {0}".format(job_spec_json_path)) with open(job_spec_json_path, "w", encoding="utf-8") as fd: fd.write(job_spec.to_json(hash=ht.dag_hash)) # Write some other details to aid in reproduction into an artifact repro_file = os.path.join(repro_dir, "repro.json") repro_details = { "job_name": ci_job_name, "job_spec_json": job_spec_json_file, "ci_project_dir": ci_project_dir, } with open(repro_file, "w", encoding="utf-8") as fd: fd.write(json.dumps(repro_details)) # Write information about spack into an artifact in the repro dir spack_info = spack_ci.get_spack_info() spack_info_file = os.path.join(repro_dir, "spack_info.txt") with open(spack_info_file, "wb") as fd: fd.write(b"\n") fd.write(spack_info.encode("utf8")) fd.write(b"\n") matches = ( None if full_rebuild else spack.binary_distribution.get_mirrors_for_spec(job_spec, index_only=False) ) if matches: # Got a hash match on at least one configured mirror. All # matches represent the fully up-to-date spec, so should all be # equivalent. If artifacts mirror is enabled, we just pick one # of the matches and download the buildcache files from there to # the artifacts, so they're available to be used by dependent # jobs in subsequent stages. tty.msg("No need to rebuild {0}, found hash match at: ".format(job_spec_pkg_name)) for match in matches: tty.msg(" {0}".format(match.url)) # Now we are done and successful return 0 # No hash match anywhere means we need to rebuild spec # Start with spack arguments spack_cmd = [SPACK_COMMAND, "--color=always", "install"] config = cfg.get("config") if not config["verify_ssl"]: spack_cmd.append("-k") install_args = [ f"--use-buildcache={spack_ci.common.win_quote('package:never,dependencies:only')}" ] can_verify = spack_ci.can_verify_binaries() verify_binaries = can_verify and spack_is_pr_pipeline is False if not verify_binaries: install_args.append("--no-check-signature") if args.jobs: install_args.append(f"-j{args.jobs}") fail_fast = bool(os.environ.get("SPACK_CI_FAIL_FAST", str(args.fail_fast))) if fail_fast: install_args.append("--fail-fast") slash_hash = spack_ci.common.win_quote("/" + job_spec.dag_hash()) # Arguments when installing the root from sources deps_install_args = install_args + ["--only=dependencies"] root_install_args = install_args + ["--verbose", "--keep-stage", "--only=package"] if cdash_handler: # Add additional arguments to `spack install` for CDash reporting. root_install_args.extend(cdash_handler.args()) commands = [ # apparently there's a race when spack bootstraps? do it up front once [SPACK_COMMAND, "-e", unicode_escape(env.path), "bootstrap", "now"], spack_cmd + deps_install_args + [slash_hash], spack_cmd + root_install_args + [slash_hash], ] tty.debug("Installing {0} from source".format(job_spec.name)) install_exit_code = spack_ci.process_command("install", commands, repro_dir) # Now do the post-install tasks tty.debug("spack install exited {0}".format(install_exit_code)) # If a spec fails to build in a spack develop pipeline, we add it to a # list of known broken hashes. This allows spack PR pipelines to # avoid wasting compute cycles attempting to build those hashes. if install_exit_code == INSTALL_FAIL_CODE and spack_is_develop_pipeline: tty.debug("Install failed on develop") if "broken-specs-url" in ci_config: broken_specs_url = ci_config["broken-specs-url"] dev_fail_hash = job_spec.dag_hash() broken_spec_path = url_util.join(broken_specs_url, dev_fail_hash) tty.msg("Reporting broken develop build as: {0}".format(broken_spec_path)) spack_ci.write_broken_spec( broken_spec_path, job_spec_pkg_name, spack_ci_stack_name, os.environ.get("CI_JOB_URL"), os.environ.get("CI_PIPELINE_URL"), job_spec.to_dict(hash=ht.dag_hash), ) # Copy logs and archived files from the install metadata (.spack) directory to artifacts now spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) # Clear the stage directory spack.stage.purge() # If the installation succeeded and we're running stand-alone tests for # the package, run them and copy the output. Failures of any kind should # *not* terminate the build process or preclude creating the build cache. broken_tests = ( "broken-tests-packages" in ci_config and job_spec.name in ci_config["broken-tests-packages"] ) reports_dir = fs.join_path(os.getcwd(), "cdash_report") if args.tests and broken_tests: tty.warn("Unable to run stand-alone tests since listed in ci's 'broken-tests-packages'") if cdash_handler: msg = "Package is listed in ci's broken-tests-packages" cdash_handler.report_skipped(job_spec, reports_dir, reason=msg) cdash_handler.copy_test_results(reports_dir, job_test_dir) elif args.tests: if install_exit_code == 0: try: # First ensure we will use a reasonable test stage directory stage_root = os.path.dirname(str(job_spec.package.stage.path)) test_stage = fs.join_path(stage_root, "spack-standalone-tests") tty.debug("Configuring test_stage to {0}".format(test_stage)) config_test_path = "config:test_stage:{0}".format(test_stage) cfg.add(config_test_path, scope=cfg.default_modify_scope()) # Run the tests, resorting to junit results if not using cdash log_file = ( None if cdash_handler else fs.join_path(test_stage, "ci-test-results.xml") ) spack_ci.run_standalone_tests( cdash=cdash_handler, job_spec=job_spec, fail_fast=fail_fast, log_file=log_file, repro_dir=repro_dir, timeout=args.timeout, ) except Exception as err: # If there is any error, just print a warning. msg = "Error processing stand-alone tests: {0}".format(str(err)) tty.warn(msg) finally: # Copy the test log/results files spack_ci.copy_test_logs_to_artifacts(test_stage, job_test_dir) if cdash_handler: cdash_handler.copy_test_results(reports_dir, job_test_dir) elif log_file: spack_ci.copy_files_to_artifacts(log_file, job_test_dir) else: tty.warn("No recognized test results reporting option") else: tty.warn("Unable to run stand-alone tests due to unsuccessful installation") if cdash_handler: msg = "Failed to install the package" cdash_handler.report_skipped(job_spec, reports_dir, reason=msg) cdash_handler.copy_test_results(reports_dir, job_test_dir) if install_exit_code == 0: # If the install succeeded, push it to the buildcache destination. Failure to push # will result in a non-zero exit code. Pushing is best-effort. for result in spack_ci.create_buildcache( input_spec=job_spec, destination_mirror_urls=[buildcache_destination.push_url], sign_binaries=spack_ci.can_sign_binaries(), ): if not result.success: install_exit_code = FAILED_CREATE_BUILDCACHE_CODE (tty.msg if result.success else tty.error)( f"{'Pushed' if result.success else 'Failed to push'} " f"{job_spec.format('{name}{@version}{/hash:7}', color=clr.get_color_when())} " f"to {result.url}" ) # If this is a develop pipeline, check if the spec that we just built is # on the broken-specs list. If so, remove it. if spack_is_develop_pipeline and "broken-specs-url" in ci_config: broken_specs_url = ci_config["broken-specs-url"] just_built_hash = job_spec.dag_hash() broken_spec_path = url_util.join(broken_specs_url, just_built_hash) if web_util.url_exists(broken_spec_path): tty.msg("Removing {0} from the list of broken specs".format(broken_spec_path)) try: web_util.remove_url(broken_spec_path) except Exception as err: # If there is an S3 error (e.g., access denied or connection # error), the first non boto-specific class in the exception # hierarchy is Exception. Just print a warning and return. msg = "Error removing {0} from broken specs list: {1}" tty.warn(msg.format(broken_spec_path, err)) else: # If the install did not succeed, print out some instructions on how to reproduce this # build failure outside of the pipeline environment. tty.debug("spack install exited non-zero, will not create buildcache") api_root_url = os.environ.get("CI_API_V4_URL") ci_project_id = os.environ.get("CI_PROJECT_ID") ci_job_id = os.environ.get("CI_JOB_ID") repro_job_url = f"{api_root_url}/projects/{ci_project_id}/jobs/{ci_job_id}/artifacts" # Control characters cause this to be printed in blue so it stands out print( f""" \033[34mTo reproduce this build locally, run: spack ci reproduce-build {repro_job_url} [--working-dir ] [--autostart] If this project does not have public pipelines, you will need to first: export GITLAB_PRIVATE_TOKEN= ... then follow the printed instructions.\033[0;0m """ ) rebuild_timer.stop() try: with open("install_timers.json", "w", encoding="utf-8") as timelog: extra_attributes = {"name": ".ci-rebuild"} rebuild_timer.write_json(timelog, extra_attributes=extra_attributes) except Exception as e: tty.debug(str(e)) # Tie job success/failure to the success/failure of building the spec return install_exit_code def ci_reproduce(args): """\ generate instructions for reproducing the spec rebuild job artifacts of the provided gitlab pipeline rebuild job's URL will be used to derive instructions for reproducing the build locally """ # Allow passing GPG key for reprocuding protected CI jobs if args.gpg_file: gpg_key_url = url_util.path_to_file_url(args.gpg_file) elif args.gpg_url: gpg_key_url = args.gpg_url else: gpg_key_url = None return spack_ci.reproduce_ci_job( args.job_url, args.working_dir, args.autostart, gpg_key_url, args.runtime, args.use_local_head, ) def _gitlab_artifacts_url(url: str) -> str: """Take a URL either to the URL of the job in the GitLab UI, or to the artifacts zip file, and output the URL to the artifacts zip file.""" parsed = urlparse(url) if not parsed.scheme or not parsed.netloc: raise ValueError(url) parts = parsed.path.split("/") if len(parts) < 2: raise ValueError(url) # Just use API endpoints verbatim, they're probably generated by Spack. if parts[1] == "api": return url # If it's a URL to the job in the Gitlab UI, we may need to append the artifacts path. minus_idx = parts.index("-") # Remove repeated slashes in the remainder rest = [p for p in parts[minus_idx + 1 :] if p] # Now the format is jobs/X or jobs/X/artifacts/download if len(rest) < 2 or rest[0] != "jobs": raise ValueError(url) if len(rest) == 2: # replace jobs/X with jobs/X/artifacts/download rest.extend(("artifacts", "download")) # Replace the parts and unparse. parts[minus_idx + 1 :] = rest # Don't allow fragments / queries return urlunparse(parsed._replace(path="/".join(parts), fragment="", query="")) def validate_standard_versions( pkg: spack.package_base.PackageBase, versions: List[StandardVersion] ) -> bool: """Get and test the checksum of a package version based on a tarball. Args: pkg: Spack package for which to validate a version checksum versions: list of package versions to validate Returns: True if all versions are valid, False if any version is invalid. """ url_dict: Dict[StandardVersion, str] = {} for version in versions: url = pkg.find_valid_url_for_version(version) assert url is not None, ( f"Package {pkg.name} does not have a valid URL for version {version}" ) url_dict[version] = url version_hashes = spack.stage.get_checksums_for_versions( url_dict, pkg.name, fetch_options=pkg.fetch_options ) valid_checksums = True for version, sha in version_hashes.items(): if sha != pkg.versions[version]["sha256"]: tty.error( f"Invalid checksum found {pkg.name}@{version}\n" f" [package.py] {pkg.versions[version]['sha256']}\n" f" [Downloaded] {sha}" ) valid_checksums = False continue tty.info(f"Validated {pkg.name}@{version} --> {sha}") return valid_checksums def validate_git_versions( pkg: spack.package_base.PackageBase, versions: List[StandardVersion] ) -> bool: """Get and test the commit and tag of a package version based on a git repository. Args: pkg: Spack package for which to validate a version versions: list of package versions to validate Returns: True if all versions are valid, False if any version is invalid. """ valid_commit = True for version in versions: fetcher = spack.fetch_strategy.for_package_version(pkg, version) assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy) with spack.stage.Stage(fetcher) as stage: known_commit = pkg.versions[version]["commit"] try: stage.fetch() except spack.error.FetchError: tty.error( f"Invalid commit for {pkg.name}@{version}\n" f" {known_commit} could not be checked out in the git repository." ) valid_commit = False continue # Test if the specified tag matches the commit in the package.py # We retrieve the commit associated with a tag and compare it to the # commit that is located in the package.py file. if "tag" in pkg.versions[version]: tag = pkg.versions[version]["tag"] url = pkg.version_or_package_attr("git", version) found_commit = spack.util.git.get_commit_sha(url, tag) if not found_commit: tty.error( f"Invalid tag for {pkg.name}@{version}\n" f" {tag} could not be found in the git repository." ) valid_commit = False continue if found_commit != known_commit: tty.error( f"Mismatched tag <-> commit found for {pkg.name}@{version}\n" f" [package.py] {known_commit}\n" f" [Downloaded] {found_commit}" ) valid_commit = False continue # If we have downloaded the repository, found the commit, and compared # the tag (if specified) we can conclude that the version is pointing # at what we would expect. tty.info(f"Validated {pkg.name}@{version} --> {known_commit}") return valid_commit def ci_verify_versions(args): """\ validate version checksum & commits between git refs This command takes a from_ref and to_ref arguments and then parses the git diff between the two to determine which packages have been modified verifies the new checksums inside of them. """ # Get a list of all packages that have been changed or added # between from_ref and to_ref pkgs = spack.repo.get_all_package_diffs( "AC", spack.repo.builtin_repo(), args.from_ref, args.to_ref ) success = True for pkg_name in pkgs: spec = spack.spec.Spec(pkg_name) pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) path = spack.repo.PATH.package_path(pkg_name) # Skip checking manual download packages and trust the maintainers if pkg.manual_download: tty.warn(f"Skipping manual download package: {pkg_name}") continue # Store versions checksums / commits for future loop url_version_to_checksum: Dict[StandardVersion, str] = {} git_version_to_checksum: Dict[StandardVersion, str] = {} for version in pkg.versions: # If the package version defines a sha256 we'll use that as the high entropy # string to detect which versions have been added between from_ref and to_ref if "sha256" in pkg.versions[version]: url_version_to_checksum[version] = pkg.versions[version]["sha256"] # If a package version instead defines a commit we'll use that as a # high entropy string to detect new versions. elif "commit" in pkg.versions[version]: git_version_to_checksum[version] = pkg.versions[version]["commit"] # TODO: enforce every version have a commit or a sha256 defined if not # an infinite version (there are a lot of packages where this doesn't work yet.) def filter_added_versions(versions: Dict[StandardVersion, str]) -> List[StandardVersion]: added_checksums = spack_ci.filter_added_checksums( versions.values(), path, from_ref=args.from_ref, to_ref=args.to_ref ) return [v for v, c in versions.items() if c in added_checksums] with fs.working_dir(os.path.dirname(path)): new_url_versions = filter_added_versions(url_version_to_checksum) new_git_versions = filter_added_versions(git_version_to_checksum) if new_url_versions: success &= validate_standard_versions(pkg, new_url_versions) if new_git_versions: success &= validate_git_versions(pkg, new_git_versions) if not success: sys.exit(1) def ci(parser, args): if args.func: return args.func(args) ================================================ FILE: lib/spack/spack/cmd/clean.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import shutil import spack.caches import spack.cmd import spack.config import spack.llnl.util.filesystem import spack.llnl.util.tty as tty import spack.stage import spack.store import spack.util.path from spack.cmd.common import arguments from spack.paths import lib_path, var_path description = "remove temporary build files and/or downloaded archives" section = "build" level = "long" class AllClean(argparse.Action): """Activates flags -s -d -f -m -p and -b simultaneously""" def __call__(self, parser, namespace, values, option_string=None): parser.parse_args(["-sdfmpb"], namespace=namespace) def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-s", "--stage", action="store_true", help="remove all temporary build stages (default)" ) subparser.add_argument( "-d", "--downloads", action="store_true", help="remove cached downloads" ) subparser.add_argument( "-f", "--failures", action="store_true", help="force removal of all install failure tracking markers", ) subparser.add_argument( "-m", "--misc-cache", action="store_true", help="remove long-lived caches, like the virtual package index", ) subparser.add_argument( "-p", "--python-cache", action="store_true", help="remove .pyc, .pyo files and __pycache__ folders", ) subparser.add_argument( "-b", "--bootstrap", action="store_true", help="remove software and configuration needed to bootstrap Spack", ) subparser.add_argument( "-a", "--all", action=AllClean, help="equivalent to ``-sdfmpb``", nargs=0 ) arguments.add_common_arguments(subparser, ["specs"]) def remove_python_cache(): for directory in [lib_path, var_path]: for root, dirs, files in os.walk(directory): for f in files: if f.endswith(".pyc") or f.endswith(".pyo"): fname = os.path.join(root, f) tty.debug("Removing {0}".format(fname)) os.remove(fname) for d in dirs: if d == "__pycache__": dname = os.path.join(root, d) tty.debug("Removing {0}".format(dname)) shutil.rmtree(dname) def clean(parser, args): # If nothing was set, activate the default if not any( [ args.specs, args.stage, args.downloads, args.failures, args.misc_cache, args.python_cache, args.bootstrap, ] ): args.stage = True # Then do the cleaning falling through the cases if args.specs: specs = spack.cmd.parse_specs(args.specs, concretize=False) specs = spack.cmd.matching_specs_from_env(specs) for spec in specs: msg = "Cleaning build stage [{0}]" tty.msg(msg.format(spec.short_spec)) spec.package.do_clean() if args.stage: tty.msg("Removing all temporary build stages") spack.stage.purge() if args.downloads: tty.msg("Removing cached downloads") spack.caches.FETCH_CACHE.destroy() if args.failures: tty.msg("Removing install failure marks") spack.store.STORE.failure_tracker.clear_all() if args.misc_cache: tty.msg("Removing cached information on repositories") spack.caches.MISC_CACHE.destroy() if args.python_cache: tty.msg("Removing python cache files") remove_python_cache() if args.bootstrap: bootstrap_prefix = spack.util.path.canonicalize_path(spack.config.get("bootstrap:root")) msg = 'Removing bootstrapped software and configuration in "{0}"' tty.msg(msg.format(bootstrap_prefix)) spack.llnl.util.filesystem.remove_directory_contents(bootstrap_prefix) ================================================ FILE: lib/spack/spack/cmd/commands.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import copy import os import re import shlex import sys from argparse import ArgumentParser, Namespace from typing import IO, Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union import spack.cmd import spack.config import spack.llnl.util.tty as tty import spack.main import spack.paths import spack.platforms from spack.llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command from spack.llnl.util.tty.colify import colify from spack.main import SpackArgumentParser, section_descriptions description = "list available spack commands" section = "config" level = "long" #: list of command formatters formatters: Dict[str, Callable[[Namespace, IO], None]] = {} #: standard arguments for updating completion scripts #: we iterate through these when called with ``--update-completion`` update_completion_args: Dict[str, Dict[str, Any]] = { "bash": { "aliases": True, "format": "bash", "header": os.path.join(spack.paths.share_path, "bash", "spack-completion.bash"), "update": os.path.join(spack.paths.share_path, "spack-completion.bash"), }, "fish": { "aliases": True, "format": "fish", "header": os.path.join(spack.paths.share_path, "fish", "spack-completion.fish"), "update": os.path.join(spack.paths.share_path, "spack-completion.fish"), }, } def formatter(func: Callable[[Namespace, IO], None]) -> Callable[[Namespace, IO], None]: """Decorator used to register formatters. Args: func: Formatting function. Returns: The same function. """ formatters[func.__name__] = func return func def setup_parser(subparser: ArgumentParser) -> None: """Set up the argument parser. Args: subparser: Preliminary argument parser. """ subparser.add_argument( "--update-completion", action="store_true", default=False, help="regenerate spack's tab completion scripts", ) subparser.add_argument( "-a", "--aliases", action="store_true", default=False, help="include command aliases" ) subparser.add_argument( "--format", default="names", choices=formatters, help="format to be used to print the output (default: names)", ) subparser.add_argument( "--header", metavar="FILE", default=None, action="store", help="prepend contents of FILE to the output (useful for rst format)", ) subparser.add_argument( "--update", metavar="FILE", default=None, action="store", help="write output to the specified file, if any command is newer", ) subparser.add_argument( "rst_files", nargs=argparse.REMAINDER, help="list of rst files to search for `_cmd-spack-` cross-refs", ) class SpackArgparseRstWriter(ArgparseRstWriter): """RST writer tailored for spack documentation.""" def __init__( self, prog: str, out: IO = sys.stdout, aliases: bool = False, documented_commands: Set[str] = set(), rst_levels: Sequence[str] = ["-", "-", "^", "~", ":", "`"], ): """Initialize a new SpackArgparseRstWriter instance. Args: prog: Program name. out: File object to write to. aliases: Whether or not to include subparsers for aliases. documented_commands: Set of commands with additional documentation. rst_levels: List of characters for rst section headings. """ super().__init__(prog, out, aliases, rst_levels) self.documented = documented_commands def usage(self, usage: str) -> str: """Example usage of a command. Args: usage: Command usage. Returns: Usage of a command. """ string = super().usage(usage) cmd = self.parser.prog.replace(" ", "-") if cmd in self.documented: string = f"{string}\n:ref:`More documentation `\n" return string class SubcommandWriter(ArgparseWriter): """Write argparse output as a list of subcommands.""" def format(self, cmd: Command) -> str: """Return the string representation of a single node in the parser tree. Args: cmd: Parsed information about a command or subcommand. Returns: String representation of this subcommand. """ return " " * self.level + cmd.prog + "\n" _positional_to_subroutine: Dict[str, str] = { "package": "_all_packages", "spec": "_all_packages", "filter": "_all_packages", "installed": "_installed_packages", "compiler": "_installed_compilers", "section": "_config_sections", "env": "_environments", "extendable": "_extensions", "keys": "_keys", "help_command": "_subcommands", "mirror": "_mirrors", "virtual": "_providers", "namespace": "_repos", "hash": "_all_resource_hashes", "pytest": "_unit_tests", } class BashCompletionWriter(ArgparseWriter): """Write argparse output as bash programmable tab completion.""" def format(self, cmd: Command) -> str: """Return the string representation of a single node in the parser tree. Args: cmd: Parsed information about a command or subcommand. Returns: String representation of this subcommand. """ assert cmd.optionals # we should always at least have -h, --help assert not (cmd.positionals and cmd.subcommands) # one or the other # We only care about the arguments/flags, not the help messages positionals = cmd.positionals or () optionals, _, _, _, _ = zip(*cmd.optionals) subcommands: Tuple[str, ...] = () if cmd.subcommands: _, subcommands, _ = zip(*cmd.subcommands) # Flatten lists of lists optionals = [x for xx in optionals for x in xx] return ( self.start_function(cmd.prog) + self.body(positionals, optionals, subcommands) + self.end_function(cmd.prog) ) def start_function(self, prog: str) -> str: """Return the syntax needed to begin a function definition. Args: prog: Program name. Returns: Function definition beginning. """ name = prog.replace("-", "_").replace(" ", "_") return "\n_{0}() {{".format(name) def end_function(self, prog: str) -> str: """Return the syntax needed to end a function definition. Args: prog: Program name Returns: Function definition ending. """ return "}\n" def body( self, positionals: Sequence, optionals: Sequence[str], subcommands: Sequence[str] ) -> str: """Return the body of the function. Args: positionals: List of positional argument tuples (name, choices, nargs, help). optionals: List of optional arguments. subcommands: List of subcommand parsers. Returns: Function body. """ if positionals: return f""" if $list_options then {self.optionals(optionals)} else {self.positionals(positionals)} fi """ elif subcommands: return f""" if $list_options then {self.optionals(optionals)} else {self.subcommands(subcommands)} fi """ else: return f""" {self.optionals(optionals)} """ def positionals(self, positionals: Sequence) -> str: """Return the syntax for reporting positional arguments. Args: positionals: List of positional argument tuples (name, choices, nargs, help). Returns: Syntax for positional arguments. """ for name, choices, nargs, help in positionals: # Check for a predefined subroutine mapping for key, value in _positional_to_subroutine.items(): if name.startswith(key): return value # Use choices if available if choices is not None: if isinstance(choices, dict): choices = sorted(choices.keys()) elif isinstance(choices, (set, frozenset)): choices = sorted(choices) else: choices = sorted(choices) return 'SPACK_COMPREPLY="{}"'.format(" ".join(str(c) for c in choices)) # If no matches found, return empty list return 'SPACK_COMPREPLY=""' def optionals(self, optionals: Sequence[str]) -> str: """Return the syntax for reporting optional flags. Args: optionals: List of optional arguments. Returns: Syntax for optional flags. """ return f'SPACK_COMPREPLY="{" ".join(optionals)}"' def subcommands(self, subcommands: Sequence[str]) -> str: """Return the syntax for reporting subcommands. Args: subcommands: List of subcommand parsers. Returns: Syntax for subcommand parsers """ return f'SPACK_COMPREPLY="{" ".join(subcommands)}"' # Map argument destination names to their complete commands # Earlier items in the list have higher precedence _dest_to_fish_complete = { ("activate", "view"): "-f -a '(__fish_complete_directories)'", ("bootstrap root", "path"): "-f -a '(__fish_complete_directories)'", ("mirror add", "mirror"): "-f", ("repo add", "path"): "-f -a '(__fish_complete_directories)'", ("test find", "filter"): "-f -a '(__fish_spack_tests)'", ("bootstrap", "name"): "-f -a '(__fish_spack_bootstrap_names)'", ("buildcache create", "key"): "-f -a '(__fish_spack_gpg_keys)'", ("build-env", r"spec \[--\].*"): "-f -a '(__fish_spack_build_env_spec)'", ("checksum", "package"): "-f -a '(__fish_spack_packages)'", ( "checksum", "versions", ): "-f -a '(__fish_spack_package_versions $__fish_spack_argparse_argv[1])'", ("config", "path"): "-f -a '(__fish_spack_colon_path)'", ("config", "section"): "-f -a '(__fish_spack_config_sections)'", ("develop", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", ("diff", "specs?"): "-f -a '(__fish_spack_installed_specs)'", ("gpg sign", "output"): "-f -a '(__fish_complete_directories)'", ("gpg", "keys?"): "-f -a '(__fish_spack_gpg_keys)'", ("graph", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", ("help", "help_command"): "-f -a '(__fish_spack_commands)'", ("list", "filter"): "-f -a '(__fish_spack_packages)'", ("mirror", "mirror"): "-f -a '(__fish_spack_mirrors)'", ("pkg", "package"): "-f -a '(__fish_spack_pkg_packages)'", ("remove", "specs?"): "-f -a '(__fish_spack_installed_specs)'", ("repo", "namespace_or_path"): "$__fish_spack_force_files -a '(__fish_spack_repos)'", ("restage", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", ("rm", "specs?"): "-f -a '(__fish_spack_installed_specs)'", ("solve", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", ("spec", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", ("stage", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", ("test-env", r"spec \[--\].*"): "-f -a '(__fish_spack_build_env_spec)'", ("test", r"\[?name.*"): "-f -a '(__fish_spack_tests)'", ("undevelop", "specs?"): "-f -k -a '(__fish_spack_specs_or_id)'", ("verify", "specs_or_files"): "$__fish_spack_force_files -a '(__fish_spack_installed_specs)'", ("view", "path"): "-f -a '(__fish_complete_directories)'", ("", "comment"): "-f", ("", "compiler_spec"): "-f -a '(__fish_spack_installed_compilers)'", ("", "config_scopes"): "-f -a '(__fish_complete_directories)'", ("", "extendable"): "-f -a '(__fish_spack_extensions)'", ("", "installed_specs?"): "-f -a '(__fish_spack_installed_specs)'", ("", "job_url"): "-f", ("", "location_env"): "-f -a '(__fish_complete_directories)'", ("", "pytest_args"): "-f -a '(__fish_spack_unit_tests)'", ("", "package_or_file"): "$__fish_spack_force_files -a '(__fish_spack_packages)'", ("", "package_or_user"): "-f -a '(__fish_spack_packages)'", ("", "package"): "-f -a '(__fish_spack_packages)'", ("", "PKG"): "-f -a '(__fish_spack_packages)'", ("", "prefix"): "-f -a '(__fish_complete_directories)'", ("", r"rev\d?"): "-f -a '(__fish_spack_git_rev)'", ("", "specs?"): "-f -k -a '(__fish_spack_specs)'", ("", "tags?"): "-f -a '(__fish_spack_tags)'", ("", "virtual_package"): "-f -a '(__fish_spack_providers)'", ("", "working_dir"): "-f -a '(__fish_complete_directories)'", ("", r"(\w*_)?env"): "-f -a '(__fish_spack_environments)'", ("", r"(\w*_)?dir(ectory)?"): "-f -a '(__fish_spack_environments)'", ("", r"(\w*_)?mirror_name"): "-f -a '(__fish_spack_mirrors)'", } def _fish_dest_get_complete(prog: str, dest: str) -> Optional[str]: """Map from subcommand to autocompletion argument. Args: prog: Program name. dest: Destination. Returns: Autocompletion argument. """ s = prog.split(None, 1) subcmd = s[1] if len(s) == 2 else "" for (prog_key, pos_key), value in _dest_to_fish_complete.items(): if subcmd.startswith(prog_key) and re.match(f"^{pos_key}$", dest): return value return None class FishCompletionWriter(ArgparseWriter): """Write argparse output as bash programmable tab completion.""" def format(self, cmd: Command) -> str: """Return the string representation of a single node in the parser tree. Args: cmd: Parsed information about a command or subcommand. Returns: String representation of a node. """ assert cmd.optionals # we should always at least have -h, --help assert not (cmd.positionals and cmd.subcommands) # one or the other # We also need help messages and how arguments are used # So we pass everything to completion writer positionals = cmd.positionals optionals = cmd.optionals subcommands = cmd.subcommands return ( self.prog_comment(cmd.prog) + self.optspecs(cmd.prog, optionals) + self.complete(cmd.prog, positionals, optionals, subcommands) ) def optspecs( self, prog: str, optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]], ) -> str: """Read the optionals and return the command to set optspec. Args: prog: Program name. optionals: List of optional arguments. Returns: Command to set optspec variable. """ # Variables of optspecs optspec_var = "__fish_spack_optspecs_" + prog.replace(" ", "_").replace("-", "_") if optionals is None: return f"set -g {optspec_var}\n" # Build optspec by iterating over options args = [] for flags, dest, _, nargs, _ in optionals: if len(flags) == 0: continue required = "" # Because nargs '?' is treated differently in fish, we treat it as required. # Because multi-argument options are not supported, we treat it like one argument. required = "=" if nargs == 0: required = "" # Pair short options with long options # We need to do this because fish doesn't support multiple short # or long options. # However, since we are paring options only, this is fine short = [f[1:] for f in flags if f.startswith("-") and len(f) == 2] long = [f[2:] for f in flags if f.startswith("--")] while len(short) > 0 and len(long) > 0: arg = f"{short.pop()}/{long.pop()}{required}" while len(short) > 0: arg = f"{short.pop()}/{required}" while len(long) > 0: arg = f"{long.pop()}{required}" args.append(arg) # Even if there is no option, we still set variable. # In fish such variable is an empty array, we use it to # indicate that such subcommand exists. args = " ".join(args) return f"set -g {optspec_var} {args}\n" @staticmethod def complete_head( prog: str, index: Optional[int] = None, nargs: Optional[Union[int, str]] = None ) -> str: """Return the head of the completion command. Args: prog: Program name. index: Index of positional argument. nargs: Number of arguments. Returns: Head of the completion command. """ # Split command and subcommand s = prog.split(None, 1) subcmd = s[1] if len(s) == 2 else "" if index is None: return f"complete -c {s[0]} -n '__fish_spack_using_command {subcmd}'" elif nargs in [argparse.ZERO_OR_MORE, argparse.ONE_OR_MORE, argparse.REMAINDER]: return ( f"complete -c {s[0]} -n '__fish_spack_using_command_pos_remainder " f"{index} {subcmd}'" ) else: return f"complete -c {s[0]} -n '__fish_spack_using_command_pos {index} {subcmd}'" def complete( self, prog: str, positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]], optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]], subcommands: List[Tuple[ArgumentParser, str, str]], ) -> str: """Return all the completion commands. Args: prog: Program name. positionals: List of positional arguments. optionals: List of optional arguments. subcommands: List of subcommand parsers. Returns: Completion command. """ commands = [] if positionals: commands.append(self.positionals(prog, positionals)) if subcommands: commands.append(self.subcommands(prog, subcommands)) if optionals: commands.append(self.optionals(prog, optionals)) return "".join(commands) def positionals( self, prog: str, positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]], ) -> str: """Return the completion for positional arguments. Args: prog: Program name. positionals: List of positional arguments. Returns: Completion command. """ commands = [] for idx, (args, choices, nargs, help) in enumerate(positionals): # Make sure we always get same order of output if isinstance(choices, dict): choices = sorted(choices.keys()) elif isinstance(choices, (set, frozenset)): choices = sorted(choices) # Remove platform-specific choices to avoid hard-coding the platform. if choices is not None: valid_choices = [] for choice in choices: if spack.platforms.host().name not in choice: valid_choices.append(choice) choices = valid_choices head = self.complete_head(prog, idx, nargs) if choices is not None: # If there are choices, we provide a completion for all possible values. commands.append(f"{head} -f -a {shlex.quote(' '.join(choices))}") else: # Otherwise, we try to find a predefined completion for it value = _fish_dest_get_complete(prog, args) if value is not None: commands.append(f"{head} {value}") return "\n".join(commands) + "\n" def prog_comment(self, prog: str) -> str: """Return a comment line for the command.""" return f"\n# {prog}\n" def optionals( self, prog: str, optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]], ) -> str: """Return the completion for optional arguments. Args: prog: Program name. optionals: List of optional arguments. Returns: Completion command. """ commands = [] head = self.complete_head(prog) for flags, dest, _, nargs, help in optionals: # Make sure we always get same order of output if isinstance(dest, dict): dest = sorted(dest.keys()) elif isinstance(dest, (set, frozenset)): dest = sorted(dest) # Remove platform-specific choices to avoid hard-coding the platform. if dest is not None: valid_choices = [] for choice in dest: if spack.platforms.host().name not in choice: valid_choices.append(choice) dest = valid_choices # To provide description for optionals, and also possible values, # we need to use two split completion command. # Otherwise, each option will have same description. prefix = head # Add all flags to the completion for f in flags: if f.startswith("--"): long = f[2:] prefix = f"{prefix} -l {long}" elif f.startswith("-"): short = f[1:] assert len(short) == 1 prefix = f"{prefix} -s {short}" # Check if option require argument. # Currently multi-argument options are not supported, so we treat it like one argument. if nargs != 0: prefix = f"{prefix} -r" if dest is not None: # If there are choices, we provide a completion for all possible values. commands.append(f"{prefix} -f -a {shlex.quote(' '.join(dest))}") else: # Otherwise, we try to find a predefined completion for it value = _fish_dest_get_complete(prog, dest) if value is not None: commands.append(f"{prefix} {value}") if help: commands.append(f"{prefix} -d {shlex.quote(help)}") return "\n".join(commands) + "\n" def subcommands(self, prog: str, subcommands: List[Tuple[ArgumentParser, str, str]]) -> str: """Return the completion for subcommands. Args: prog: Program name. subcommands: List of subcommand parsers. Returns: Completion command. """ commands = [] head = self.complete_head(prog, 0) for _, subcommand, help in subcommands: command = f"{head} -f -a {shlex.quote(subcommand)}" if help is not None and len(help) > 0: help = help.split("\n")[0] command = f"{command} -d {shlex.quote(help)}" commands.append(command) return "\n".join(commands) + "\n" @formatter def subcommands(args: Namespace, out: IO) -> None: """Hierarchical tree of subcommands. args: args: Command-line arguments. out: File object to write to. """ parser = get_all_spack_commands(out) writer = SubcommandWriter(parser.prog, out, args.aliases) writer.write(parser) def rst_index(out: IO) -> None: """Generate an index of all commands. Args: out: File object to write to. """ out.write("\n") index = spack.main.index_commands() sections = index["long"] dmax = max(len(section_descriptions.get(s, s)) for s in sections) + 2 cmax = max(len(c) for _, c in sections.items()) + 60 row = "%s %s\n" % ("=" * dmax, "=" * cmax) line = "%%-%ds %%s\n" % dmax out.write(row) out.write(line % (" Category ", " Commands ")) out.write(row) for section, commands in sorted(sections.items()): description = section_descriptions.get(section, section) for i, cmd in enumerate(sorted(commands)): description = description.capitalize() if i == 0 else "" ref = f":ref:`{cmd} `" comma = "," if i != len(commands) - 1 else "" bar = "| " if i % 8 == 0 else " " out.write(line % (description, bar + ref + comma)) out.write(row) @formatter def rst(args: Namespace, out: IO) -> None: """ReStructuredText documentation of subcommands. args: args: Command-line arguments. out: File object to write to. """ # create a parser with all commands parser = get_all_spack_commands(out) # extract cross-refs of the form `_cmd-spack-:` from rst files documented_commands: Set[str] = set() for filename in args.rst_files: with open(filename, encoding="utf-8") as f: for line in f: match = re.match(r"\.\. _cmd-(spack-.*):", line) if match: documented_commands.add(match.group(1).strip()) # print an index to each command rst_index(out) out.write("\n") # print sections for each command and subcommand writer = SpackArgparseRstWriter(parser.prog, out, args.aliases, documented_commands) writer.write(parser) @formatter def names(args: Namespace, out: IO) -> None: """Simple list of top-level commands. args: args: Command-line arguments. out: File object to write to. """ commands = copy.copy(spack.cmd.all_commands()) if args.aliases: aliases = spack.config.get("config:aliases") if aliases: commands.extend(aliases.keys()) colify(commands, output=out) def get_all_spack_commands(out: IO) -> SpackArgumentParser: is_tty = hasattr(out, "isatty") and out.isatty() # Argparse python 3.14 adds a default color argument that # adds color control characters to argparse output # that breaks expected output format from spack formatters # when written to non tty IO # If 3.14 and newer and not tty, disable color parser = spack.main.make_argument_parser( **({"color": False} if sys.version_info[:2] >= (3, 14) and not is_tty else {}) ) spack.main.add_all_commands(parser) return parser @formatter def bash(args: Namespace, out: IO) -> None: """Bash tab-completion script. args: args: Command-line arguments. out: File object to write to. """ parser = get_all_spack_commands(out) aliases_config = spack.config.get("config:aliases") if aliases_config: aliases = ";".join(f"{key}:{val}" for key, val in aliases_config.items()) out.write(f'SPACK_ALIASES="{aliases}"\n\n') writer = BashCompletionWriter(parser.prog, out, args.aliases) writer.write(parser) @formatter def fish(args, out): parser = get_all_spack_commands(out) writer = FishCompletionWriter(parser.prog, out, args.aliases) writer.write(parser) def prepend_header(args: Namespace, out: IO) -> None: """Prepend header text at the beginning of a file. Args: args: Command-line arguments. out: File object to write to. """ if not args.header: return with open(args.header, encoding="utf-8") as header: out.write(header.read()) def _commands(parser: ArgumentParser, args: Namespace) -> None: """This is the 'regular' command, which can be called multiple times. See ``commands()`` below for ``--update-completion`` handling. Args: parser: Argument parser. args: Command-line arguments. """ formatter = formatters[args.format] # check header first so we don't open out files unnecessarily if args.header and not os.path.exists(args.header): tty.die(f"No such file: '{args.header}'") if args.update: tty.msg(f"Updating file: {args.update}") with open(args.update, "w", encoding="utf-8") as f: prepend_header(args, f) formatter(args, f) else: prepend_header(args, sys.stdout) formatter(args, sys.stdout) def update_completion(parser: ArgumentParser, args: Namespace) -> None: """Iterate through the shells and update the standard completion files. This is a convenience method to avoid calling this command many times, and to simplify completion update for developers. Args: parser: Argument parser. args: Command-line arguments. """ for shell, shell_args in update_completion_args.items(): for attr, value in shell_args.items(): setattr(args, attr, value) _commands(parser, args) def commands(parser: ArgumentParser, args: Namespace) -> None: """Main function that calls formatter functions. Args: parser: Argument parser. args: Command-line arguments. """ if args.update_completion: if args.format != "names" or any([args.aliases, args.update, args.header]): tty.die("--update-completion can only be specified alone.") # this runs the command multiple times with different arguments update_completion(parser, args) else: # run commands normally _commands(parser, args) ================================================ FILE: lib/spack/spack/cmd/common/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.paths def shell_init_instructions(cmd, equivalent): """Print out instructions for users to initialize shell support. Arguments: cmd (str): the command the user tried to run that requires shell support in order to work equivalent (str): a command they can run instead, without enabling shell support """ shell_specific = "{sh_arg}" in equivalent msg = [ "`%s` requires Spack's shell support." % cmd, "", "To set up shell support, run the command below for your shell.", "", color.colorize("@*c{For bash/zsh/sh:}"), " . %s/setup-env.sh" % spack.paths.share_path, "", color.colorize("@*c{For csh/tcsh:}"), " source %s/setup-env.csh" % spack.paths.share_path, "", color.colorize("@*c{For fish:}"), " source %s/setup-env.fish" % spack.paths.share_path, "", color.colorize("@*c{For Windows batch:}"), " %s\\spack_cmd.bat" % spack.paths.bin_path, "", color.colorize("@*c{For PowerShell:}"), " %s\\setup-env.ps1" % spack.paths.share_path, "", "Or, if you do not want to use shell support, run " + ("one of these" if shell_specific else "this") + " instead:", "", ] if shell_specific: msg += [ equivalent.format(sh_arg="--sh ") + " # bash/zsh/sh", equivalent.format(sh_arg="--csh ") + " # csh/tcsh", equivalent.format(sh_arg="--fish") + " # fish", equivalent.format(sh_arg="--bat ") + " # batch", equivalent.format(sh_arg="--pwsh") + " # powershell", ] else: msg += [" " + equivalent] msg += [ "", "If you have already set up Spack's shell support but still receive", "this message, please make sure to call Spack via the `spack` command", "without any path components (such as `bin/spack`).", ] msg += [""] tty.error(*msg) ================================================ FILE: lib/spack/spack/cmd/common/arguments.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import textwrap from typing import Any, Optional import spack.cmd import spack.config import spack.deptypes as dt import spack.environment as ev import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.mirrors.utils import spack.reporters import spack.spec import spack.store from spack.llnl.util.lang import stable_partition from spack.util.pattern import Args __all__ = ["add_common_arguments"] #: dictionary of argument-generating functions, keyed by name _arguments = {} def arg(fn): """Decorator for a function that generates a common argument. This ensures that argument bunches are created lazily. Decorate argument-generating functions below with @arg so that ``add_common_arguments()`` can find them. """ _arguments[fn.__name__] = fn return fn def add_common_arguments(parser, list_of_arguments): """Extend a parser with extra arguments Args: parser: parser to be extended list_of_arguments: arguments to be added to the parser """ for argument in list_of_arguments: if argument not in _arguments: message = 'Trying to add non existing argument "{0}" to a command' raise KeyError(message.format(argument)) x = _arguments[argument]() parser.add_argument(*x.flags, **x.kwargs) class ConstraintAction(argparse.Action): """Constructs a list of specs based on constraints from the command line An instance of this class is supposed to be used as an argument action in a parser. It will read a constraint and will attach a function to the arguments that accepts optional keyword arguments. To obtain the specs from a command the function must be called. """ def __call__(self, parser, namespace, values, option_string=None): # Query specs from command line self.constraint = namespace.constraint = values self.constraint_specs = namespace.constraint_specs = [] namespace.specs = self._specs def _specs(self, **kwargs): # store parsed specs in spec.constraint after a call to specs() self.constraint_specs[:] = spack.cmd.parse_specs(self.constraint) # If an environment is provided, we'll restrict the search to # only its installed packages. env = ev.active_environment() if env: kwargs["hashes"] = set(env.all_hashes()) # return everything for an empty query. if not self.constraint_specs: return spack.store.STORE.db.query(**kwargs) # Return only matching stuff otherwise. specs = {} for spec in self.constraint_specs: for s in spack.store.STORE.db.query(spec, **kwargs): # This is fast for already-concrete specs specs[s.dag_hash()] = s return sorted(specs.values()) class SetParallelJobs(argparse.Action): """Sets the correct value for parallel build jobs. The value is set in the command line configuration scope so that it can be retrieved using the spack.config API. """ def __call__(self, parser, namespace, jobs, option_string): # Jobs is a single integer, type conversion is already applied # see https://docs.python.org/3/library/argparse.html#action-classes if jobs < 1: msg = 'invalid value for argument "{0}" [expected a positive integer, got "{1}"]' raise ValueError(msg.format(option_string, jobs)) spack.config.set("config:build_jobs", jobs, scope="command_line") setattr(namespace, "jobs", jobs) class SetConcurrentPackages(argparse.Action): """Sets the value for maximum number of concurrent package builds The value is set in the command line configuration scope so that it can be retrieved using the spack.config API. """ def __call__(self, parser, namespace, concurrent_packages, option_string): if concurrent_packages < 1: msg = 'invalid value for argument "{0}" [expected a positive integer, got "{1}"]' raise ValueError(msg.format(option_string, concurrent_packages)) spack.config.set("config:concurrent_packages", concurrent_packages, scope="command_line") setattr(namespace, "concurrent_packages", concurrent_packages) class DeprecatedStoreTrueAction(argparse.Action): """Like the builtin store_true, but prints a deprecation warning.""" def __init__( self, option_strings, dest: str, default: Optional[Any] = False, required: bool = False, help: Optional[str] = None, removed_in: Optional[str] = None, instructions: Optional[str] = None, ): super().__init__( option_strings=option_strings, dest=dest, nargs=0, const=True, required=required, help=help, default=default, ) self.removed_in = removed_in self.instructions = instructions def __call__(self, parser, namespace, value, option_string=None): instructions = [] if not self.instructions else [self.instructions] tty.warn( f"{option_string} is deprecated and will be removed in {self.removed_in}.", *instructions, ) setattr(namespace, self.dest, self.const) class DeptypeAction(argparse.Action): """Creates a flag of valid dependency types from a deptype argument.""" def __call__(self, parser, namespace, values, option_string=None): if not values or values == "all": deptype = dt.ALL else: deptype = dt.canonicalize(values.split(",")) setattr(namespace, self.dest, deptype) class ConfigScope(argparse.Action): """Pick the currently configured config scopes.""" def __init__(self, *args, **kwargs) -> None: kwargs.setdefault("metavar", spack.config.SCOPES_METAVAR) super().__init__(*args, **kwargs) @property def default(self): return self._default() if callable(self._default) else self._default @default.setter def default(self, value): self._default = value @property def choices(self): return spack.config.scopes().keys() @choices.setter def choices(self, value): pass def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) def config_scope_readable_validator(value): if value not in spack.config.existing_scope_names(): raise ValueError( f"Invalid scope argument {value} " "for config read operation, scope context does not exist" ) return value def _cdash_reporter(namespace): """Helper function to create a CDash reporter. This function gets an early reference to the argparse namespace under construction, so it can later use it to create the object. """ def _factory(): def installed_specs(args): packages = [] if getattr(args, "spec", ""): packages = args.spec elif getattr(args, "specs", ""): packages = args.specs elif getattr(args, "package", ""): # Ensure CI 'spack test run' can output CDash results packages = args.package return [str(spack.spec.Spec(s)) for s in packages] configuration = spack.reporters.CDashConfiguration( upload_url=namespace.cdash_upload_url, packages=installed_specs(namespace), build=namespace.cdash_build, site=namespace.cdash_site, buildstamp=namespace.cdash_buildstamp, track=namespace.cdash_track, ) return spack.reporters.CDash(configuration=configuration) return _factory class CreateReporter(argparse.Action): """Create the correct object to generate reports for installation and testing.""" def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) if values == "junit": setattr(namespace, "reporter", spack.reporters.JUnit) elif values == "cdash": setattr(namespace, "reporter", _cdash_reporter(namespace)) @arg def log_format(): return Args( "--log-format", default=None, action=CreateReporter, choices=("junit", "cdash"), help="format to be used for log files", ) # TODO: merge constraint and installed_specs @arg def constraint(): return Args( "constraint", nargs=argparse.REMAINDER, action=ConstraintAction, help="constraint to select a subset of installed packages", metavar="installed_specs", ) @arg def package(): return Args("package", help="package name") @arg def packages(): return Args("packages", nargs="+", help="one or more package names", metavar="package") # Specs must use `nargs=argparse.REMAINDER` because a single spec can # contain spaces, and contain variants like '-mpi' that argparse thinks # are a collection of optional flags. @arg def spec(): return Args("spec", nargs=argparse.REMAINDER, help="package spec") @arg def specs(): return Args("specs", nargs=argparse.REMAINDER, help="one or more package specs") @arg def installed_spec(): return Args( "spec", nargs=argparse.REMAINDER, help="installed package spec", metavar="installed_spec" ) @arg def installed_specs(): return Args( "specs", nargs=argparse.REMAINDER, help="one or more installed package specs", metavar="installed_specs", ) @arg def yes_to_all(): return Args( "-y", "--yes-to-all", action="store_true", dest="yes_to_all", help='assume "yes" is the answer to every confirmation request', ) @arg def recurse_dependencies(): return Args( "-r", "--dependencies", action="store_true", dest="recurse_dependencies", help="recursively traverse spec dependencies", ) @arg def recurse_dependents(): return Args( "-R", "--dependents", action="store_true", dest="dependents", help="also uninstall any packages that depend on the ones given via command line", ) @arg def clean(): return Args( "--clean", action="store_false", default=spack.config.get("config:dirty"), dest="dirty", help="unset harmful variables in the build environment (default)", ) @arg def deptype(): return Args( "--deptype", action=DeptypeAction, default=dt.ALL, help="comma-separated list of deptypes to traverse (default=%s)" % ",".join(dt.ALL_TYPES), ) @arg def dirty(): return Args( "--dirty", action="store_true", default=spack.config.get("config:dirty"), dest="dirty", help="preserve user environment in spack's build environment (danger!)", ) @arg def long(): return Args( "-l", "--long", action="store_true", help="show dependency hashes as well as versions" ) @arg def very_long(): return Args( "-L", "--very-long", action="store_true", help="show full dependency hashes as well as versions", ) @arg def tags(): return Args( "-t", "--tag", action="append", dest="tags", metavar="TAG", help="filter a package query by tag (multiple use allowed)", ) @arg def namespaces(): return Args( "-N", "--namespaces", action="store_true", default=False, help="show fully qualified package names", ) @arg def jobs(): return Args( "-j", "--jobs", action=SetParallelJobs, type=int, dest="jobs", help="explicitly set number of parallel jobs", ) @arg def concurrent_packages(): return Args( "-p", "--concurrent-packages", action=SetConcurrentPackages, type=int, default=None, help="maximum number of packages to build concurrently", ) @arg def install_status(): return Args( "-I", "--install-status", action="store_true", default=True, help=( "show install status of packages\n" "[+] installed [^] installed in an upstream\n" " - not installed [-] missing dep of installed package\n" ), ) @arg def no_install_status(): return Args( "--no-install-status", dest="install_status", action="store_false", default=True, help="do not show install status annotations", ) @arg def show_non_defaults(): return Args( "--non-defaults", action="store_true", default=False, help="highlight non-default versions or variants", ) @arg def no_checksum(): return Args( "-n", "--no-checksum", action="store_true", default=False, help="do not use checksums to verify downloaded files (unsafe)", ) @arg def deprecated(): return Args( "--deprecated", action="store_true", default=False, help="fetch deprecated versions without warning", ) def add_cdash_args(subparser, add_help): cdash_help = {} if add_help: cdash_help["upload-url"] = "CDash URL where reports will be uploaded" cdash_help["build"] = ( "name of the build that will be reported to CDash\n\n" "defaults to spec of the package to operate on" ) cdash_help["site"] = ( "site name that will be reported to CDash\n\ndefaults to current system hostname" ) cdash_help["track"] = ( "results will be reported to this group on CDash\n\ndefaults to Experimental" ) cdash_help["buildstamp"] = ( "use custom buildstamp\n\n" "instead of letting the CDash reporter prepare the " "buildstamp which, when combined with build name, site and project, " "uniquely identifies the build, provide this argument to identify " "the build yourself. format: %%Y%%m%%d-%%H%%M-[cdash-track]" ) else: cdash_help["upload-url"] = argparse.SUPPRESS cdash_help["build"] = argparse.SUPPRESS cdash_help["site"] = argparse.SUPPRESS cdash_help["track"] = argparse.SUPPRESS cdash_help["buildstamp"] = argparse.SUPPRESS subparser.add_argument("--cdash-upload-url", default=None, help=cdash_help["upload-url"]) subparser.add_argument("--cdash-build", default=None, help=cdash_help["build"]) subparser.add_argument("--cdash-site", default=None, help=cdash_help["site"]) cdash_subgroup = subparser.add_mutually_exclusive_group() cdash_subgroup.add_argument("--cdash-track", default="Experimental", help=cdash_help["track"]) cdash_subgroup.add_argument("--cdash-buildstamp", default=None, help=cdash_help["buildstamp"]) def print_cdash_help(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, epilog=textwrap.dedent( """\ environment variables: SPACK_CDASH_AUTH_TOKEN authentication token to present to CDash """ ), ) add_cdash_args(parser, True) parser.print_help() def sanitize_reporter_options(namespace: argparse.Namespace): """Sanitize options that affect generation and configuration of reports, like CDash or JUnit. Args: namespace: options parsed from cli """ has_any_cdash_option = ( namespace.cdash_upload_url or namespace.cdash_build or namespace.cdash_site ) if namespace.log_format == "junit" and has_any_cdash_option: raise argparse.ArgumentTypeError("cannot pass any cdash option when --log-format=junit") # If any CDash option is passed, assume --log-format=cdash is implied if namespace.log_format is None and has_any_cdash_option: namespace.log_format = "cdash" namespace.reporter = _cdash_reporter(namespace) class ConfigSetAction(argparse.Action): """Generic action for setting spack config options from CLI. This works like a ``store_const`` action but you can set the ``dest`` to some Spack configuration path (like ``concretizer:reuse``) and the ``const`` will be stored there using ``spack.config.set()`` """ def __init__( self, option_strings, dest, const, default=None, required=False, help=None, metavar=None, require_environment=False, ): # save the config option we're supposed to set self.config_path = dest # save whether the option requires an active env self.require_environment = require_environment # destination is translated to a legal python identifier by # substituting '_' for ':'. dest = dest.replace(":", "_") super().__init__( option_strings=option_strings, dest=dest, nargs=0, const=const, default=default, required=required, help=help, ) def __call__(self, parser, namespace, values, option_string): if self.require_environment and not ev.active_environment(): raise argparse.ArgumentTypeError( f"argument '{self.option_strings[-1]}' requires an environment" ) # Retrieve the name of the config option and set it to # the const from the constructor or a value from the CLI. # Note that this is only called if the argument is actually # specified on the command line. spack.config.set(self.config_path, self.const, scope="command_line") def add_concretizer_args(subparser): """Add a subgroup of arguments for controlling concretization. These will appear in a separate group called 'concretizer arguments'. There's no need to handle them in your command logic -- they all use ``ConfigSetAction``, which automatically handles setting configuration options. If you *do* need to access a value passed on the command line, you can get at, e.g., the ``concretizer:reuse`` via ``args.concretizer_reuse``. Just substitute ``_`` for ``:``. """ subgroup = subparser.add_argument_group("concretizer arguments") subgroup.add_argument( "-f", "--force", action=ConfigSetAction, require_environment=True, dest="concretizer:force", const=True, default=False, help="allow changes to concretized specs in spack.lock (in an env)", ) subgroup.add_argument( "-U", "--fresh", action=ConfigSetAction, dest="concretizer:reuse", const=False, default=None, help="do not reuse installed deps; build newest configuration", ) subgroup.add_argument( "--reuse", action=ConfigSetAction, dest="concretizer:reuse", const=True, default=None, help="reuse installed packages/buildcaches when possible", ) subgroup.add_argument( "--fresh-roots", "--reuse-deps", action=ConfigSetAction, dest="concretizer:reuse", const="dependencies", default=None, help="concretize with fresh roots and reused dependencies", ) subgroup.add_argument( "--deprecated", action=ConfigSetAction, dest="config:deprecated", const=True, default=None, help="allow concretizer to select deprecated versions", ) def add_connection_args(subparser, add_help): def add_argument_string_or_variable(parser, arg: str, *, deprecate_str: bool = True, **kwargs): group = parser.add_mutually_exclusive_group() group.add_argument(arg, **kwargs) # Update help string if "help" in kwargs: kwargs["help"] = "environment variable containing " + kwargs["help"] group.add_argument(arg + "-variable", **kwargs) s3_connection_parser = subparser.add_argument_group("S3 Connection") add_argument_string_or_variable( s3_connection_parser, "--s3-access-key-id", help="ID string to use to connect to this S3 mirror", ) s3_connection_parser.add_argument( "--s3-access-key-secret-variable", help="environment variable containing secret string to use to connect to this S3 mirror", ) s3_connection_parser.add_argument( "--s3-access-token-variable", help="environment variable containing access token to use to connect to this S3 mirror", ) s3_connection_parser.add_argument( "--s3-profile", help="S3 profile name to use to connect to this S3 mirror", default=None ) s3_connection_parser.add_argument( "--s3-endpoint-url", help="endpoint URL to use to connect to this S3 mirror" ) oci_connection_parser = subparser.add_argument_group("OCI Connection") add_argument_string_or_variable( oci_connection_parser, "--oci-username", deprecate_str=False, help="username to use to connect to this OCI mirror", ) oci_connection_parser.add_argument( "--oci-password-variable", help="environment variable containing password to use to connect to this OCI mirror", ) def use_buildcache(cli_arg_value): """Translate buildcache related command line arguments into a pair of strings, representing whether the root or its dependencies can use buildcaches. Argument type that accepts comma-separated subargs: 1. auto|only|never 2. package:auto|only|never 3. dependencies:auto|only|never Args: cli_arg_value (str): command line argument value to be translated Return: Tuple of two strings """ valid_keys = frozenset(["package", "dependencies"]) valid_values = frozenset(["only", "never", "auto"]) # Split in args, split in key/value, and trim whitespace args = [tuple(map(lambda x: x.strip(), part.split(":"))) for part in cli_arg_value.split(",")] # Verify keys and values def is_valid(arg): if len(arg) == 1: return arg[0] in valid_values if len(arg) == 2: return arg[0] in valid_keys and arg[1] in valid_values return False valid, invalid = stable_partition(args, is_valid) # print first error if invalid: raise argparse.ArgumentTypeError("invalid argument `{}`".format(":".join(invalid[0]))) # Default values package = "auto" dependencies = "auto" # Override in order. for arg in valid: if len(arg) == 1: package = dependencies = arg[0] continue key, val = arg if key == "package": package = val else: dependencies = val return package, dependencies def mirror_name_or_url(m): # Look up mirror by name or use anonymous mirror with path/url. # We want to guard against typos in mirror names, to avoid pushing # accidentally to a dir in the current working directory. # If there's a \ or / in the name, it's interpreted as a path or url. if "/" in m or "\\" in m or m in (".", ".."): return spack.mirrors.mirror.Mirror(m) # Otherwise, the named mirror is required to exist. try: return spack.mirrors.utils.require_mirror_name(m) except ValueError as e: raise argparse.ArgumentTypeError(f"{e}. Did you mean {os.path.join('.', m)}?") from e def mirror_url(url): try: return spack.mirrors.mirror.Mirror.from_url(url) except ValueError as e: raise argparse.ArgumentTypeError(str(e)) from e def mirror_directory(path): try: return spack.mirrors.mirror.Mirror.from_local_path(path) except ValueError as e: raise argparse.ArgumentTypeError(str(e)) from e def mirror_name(name): try: return spack.mirrors.utils.require_mirror_name(name) except ValueError as e: raise argparse.ArgumentTypeError(str(e)) from e ================================================ FILE: lib/spack/spack/cmd/common/confirmation.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import sys from typing import List import spack.cmd import spack.llnl.util.tty as tty import spack.spec display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4} def confirm_action(specs: List[spack.spec.Spec], participle: str, noun: str): """Display the list of specs to be acted on and ask for confirmation. Args: specs: specs to be removed participle: action expressed as a participle, e.g. "uninstalled" noun: action expressed as a noun, e.g. "uninstallation" """ spack.cmd.display_specs(specs, **display_args) print() answer = tty.get_yes_or_no( f"{len(specs)} packages will be {participle}. Do you want to proceed?", default=False ) if not answer: tty.msg(f"Aborting {noun}") sys.exit(0) ================================================ FILE: lib/spack/spack/cmd/common/env_utility.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import spack.cmd import spack.deptypes as dt import spack.error import spack.llnl.util.tty as tty import spack.spec import spack.store from spack import build_environment, traverse from spack.cmd.common import arguments from spack.context import Context from spack.util.environment import dump_environment, pickle_environment def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["clean", "dirty"]) arguments.add_concretizer_args(subparser) subparser.add_argument("--dump", metavar="FILE", help="dump a source-able environment to FILE") subparser.add_argument( "--pickle", metavar="FILE", help="dump a pickled source-able environment to FILE" ) subparser.add_argument( "spec", nargs=argparse.REMAINDER, metavar="spec [--] [cmd]...", help="specs of package environment to emulate", ) subparser.epilog = ( "If a command is not specified, the environment will be printed " "to standard output (cf /usr/bin/env) unless --dump and/or --pickle " "are specified.\n\nIf a command is specified and spec is " "multi-word, then the -- separator is obligatory." ) class AreDepsInstalledVisitor: def __init__(self, context: Context = Context.BUILD): if context == Context.BUILD: # TODO: run deps shouldn't be required for build env. self.direct_deps = dt.BUILD | dt.LINK | dt.RUN elif context == Context.TEST: self.direct_deps = dt.BUILD | dt.TEST | dt.LINK | dt.RUN else: raise ValueError("context can only be Context.BUILD or Context.TEST") self.has_uninstalled_deps = False def accept(self, item): # The root may be installed or uninstalled. if item.depth == 0: return True # Early exit after we've seen an uninstalled dep. if self.has_uninstalled_deps: return False spec = item.edge.spec if not spec.external and not spec.installed: self.has_uninstalled_deps = True return False return True def neighbors(self, item): # Direct deps: follow build & test edges. # Transitive deps: follow link / run. depflag = self.direct_deps if item.depth == 0 else dt.LINK | dt.RUN return item.edge.spec.edges_to_dependencies(depflag=depflag) def emulate_env_utility(cmd_name, context: Context, args): if not args.spec: tty.die("spack %s requires a spec." % cmd_name) # Specs may have spaces in them, so if they do, require that the # caller put a '--' between the spec and the command to be # executed. If there is no '--', assume that the spec is the # first argument. sep = "--" if sep in args.spec: s = args.spec.index(sep) spec = args.spec[:s] cmd = args.spec[s + 1 :] else: spec = args.spec[0] cmd = args.spec[1:] if not spec: tty.die("spack %s requires a spec." % cmd_name) specs = spack.cmd.parse_specs(spec, concretize=False) if len(specs) > 1: tty.die("spack %s only takes one spec." % cmd_name) spec = specs[0] spec = spack.cmd.matching_spec_from_env(spec) # Require that dependencies are installed. visitor = AreDepsInstalledVisitor(context=context) # Mass install check needs read transaction. with spack.store.STORE.db.read_transaction(): traverse.traverse_breadth_first_with_visitor([spec], traverse.CoverNodesVisitor(visitor)) if visitor.has_uninstalled_deps: raise spack.error.SpackError( f"Not all dependencies of {spec.name} are installed. " f"Cannot setup {context} environment:", spec.tree( status_fn=spack.spec.Spec.install_status, hashlen=7, hashes=True, # This shows more than necessary, but we cannot dynamically change deptypes # in Spec.tree(...). deptypes="all" if context == Context.BUILD else ("build", "test", "link", "run"), ), ) build_environment.setup_package(spec.package, args.dirty, context) if args.dump: # Dump a source-able environment to a text file. tty.msg("Dumping a source-able environment to {0}".format(args.dump)) dump_environment(args.dump) if args.pickle: # Dump a source-able environment to a pickle file. tty.msg("Pickling a source-able environment to {0}".format(args.pickle)) pickle_environment(args.pickle) if cmd: # Execute the command with the new environment os.execvp(cmd[0], cmd) elif not bool(args.pickle or args.dump): # If no command or dump/pickle option then act like the "env" command # and print out env vars. for key, val in os.environ.items(): print("%s=%s" % (key, val)) ================================================ FILE: lib/spack/spack/cmd/common/spec_strings.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import ast import os import re import sys import warnings from typing import Callable, List, Optional import spack.llnl.util.tty as tty import spack.util.spack_yaml from spack.spec_parser import NAME, VERSION_LIST, SpecTokens from spack.tokenize import Token, TokenBase, Tokenizer IS_PROBABLY_COMPILER = re.compile(r"%[a-zA-Z_][a-zA-Z0-9\-]") class _LegacySpecTokens(TokenBase): """Reconstructs the tokens for previous specs, so we can reuse code to rotate them""" # Dependency START_EDGE_PROPERTIES = r"(?:\^\[)" END_EDGE_PROPERTIES = r"(?:\])" DEPENDENCY = r"(?:\^)" # Version VERSION_HASH_PAIR = SpecTokens.VERSION_HASH_PAIR.regex GIT_VERSION = SpecTokens.GIT_VERSION.regex VERSION = SpecTokens.VERSION.regex # Variants PROPAGATED_BOOL_VARIANT = SpecTokens.PROPAGATED_BOOL_VARIANT.regex BOOL_VARIANT = SpecTokens.BOOL_VARIANT.regex PROPAGATED_KEY_VALUE_PAIR = SpecTokens.PROPAGATED_KEY_VALUE_PAIR.regex KEY_VALUE_PAIR = SpecTokens.KEY_VALUE_PAIR.regex # Compilers COMPILER_AND_VERSION = rf"(?:%\s*(?:{NAME})(?:[\s]*)@\s*(?:{VERSION_LIST}))" COMPILER = rf"(?:%\s*(?:{NAME}))" # FILENAME FILENAME = SpecTokens.FILENAME.regex # Package name FULLY_QUALIFIED_PACKAGE_NAME = SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME.regex UNQUALIFIED_PACKAGE_NAME = SpecTokens.UNQUALIFIED_PACKAGE_NAME.regex # DAG hash DAG_HASH = SpecTokens.DAG_HASH.regex # White spaces WS = SpecTokens.WS.regex # Unexpected character(s) UNEXPECTED = SpecTokens.UNEXPECTED.regex def _spec_str_reorder_compiler(idx: int, blocks: List[List[Token]]) -> None: # only move the compiler to the back if it exists and is not already at the end if not 0 <= idx < len(blocks) - 1: return # if there's only whitespace after the compiler, don't move it if all(token.kind == _LegacySpecTokens.WS for block in blocks[idx + 1 :] for token in block): return # rotate left and always add at least one WS token between compiler and previous token compiler_block = blocks.pop(idx) if compiler_block[0].kind != _LegacySpecTokens.WS: compiler_block.insert(0, Token(_LegacySpecTokens.WS, " ")) # delete the WS tokens from the new first block if it was at the very start, to prevent leading # WS tokens. while idx == 0 and blocks[0][0].kind == _LegacySpecTokens.WS: blocks[0].pop(0) blocks.append(compiler_block) def _spec_str_format(spec_str: str) -> Optional[str]: """Given any string, try to parse as spec string, and rotate the compiler token to the end of each spec instance. Returns the formatted string if it was changed, otherwise None.""" # We parse blocks of tokens that include leading whitespace, and move the compiler block to # the end when we hit a dependency ^... or the end of a string. # [@3.1][ +foo][ +bar][ %gcc@3.1][ +baz] # [@3.1][ +foo][ +bar][ +baz][ %gcc@3.1] current_block: List[Token] = [] blocks: List[List[Token]] = [] compiler_block_idx = -1 in_edge_attr = False legacy_tokenizer = Tokenizer(_LegacySpecTokens) for token in legacy_tokenizer.tokenize(spec_str): if token.kind == _LegacySpecTokens.UNEXPECTED: # parsing error, we cannot fix this string. return None elif token.kind in (_LegacySpecTokens.COMPILER, _LegacySpecTokens.COMPILER_AND_VERSION): # multiple compilers are not supported in Spack v0.x, so early return if compiler_block_idx != -1: return None current_block.append(token) blocks.append(current_block) current_block = [] compiler_block_idx = len(blocks) - 1 elif token.kind in ( _LegacySpecTokens.START_EDGE_PROPERTIES, _LegacySpecTokens.DEPENDENCY, _LegacySpecTokens.UNQUALIFIED_PACKAGE_NAME, _LegacySpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, ): _spec_str_reorder_compiler(compiler_block_idx, blocks) compiler_block_idx = -1 if token.kind == _LegacySpecTokens.START_EDGE_PROPERTIES: in_edge_attr = True current_block.append(token) blocks.append(current_block) current_block = [] elif token.kind == _LegacySpecTokens.END_EDGE_PROPERTIES: in_edge_attr = False current_block.append(token) blocks.append(current_block) current_block = [] elif in_edge_attr: current_block.append(token) elif token.kind in ( _LegacySpecTokens.VERSION_HASH_PAIR, _LegacySpecTokens.GIT_VERSION, _LegacySpecTokens.VERSION, _LegacySpecTokens.PROPAGATED_BOOL_VARIANT, _LegacySpecTokens.BOOL_VARIANT, _LegacySpecTokens.PROPAGATED_KEY_VALUE_PAIR, _LegacySpecTokens.KEY_VALUE_PAIR, _LegacySpecTokens.DAG_HASH, ): current_block.append(token) blocks.append(current_block) current_block = [] elif token.kind == _LegacySpecTokens.WS: current_block.append(token) else: raise ValueError(f"unexpected token {token}") if current_block: blocks.append(current_block) _spec_str_reorder_compiler(compiler_block_idx, blocks) new_spec_str = "".join(token.value for block in blocks for token in block) return new_spec_str if spec_str != new_spec_str else None SpecStrHandler = Callable[[str, int, int, str, str], None] def _spec_str_default_handler(path: str, line: int, col: int, old: str, new: str): """A SpecStrHandler that prints formatted spec strings and their locations.""" print(f"{path}:{line}:{col}: `{old}` -> `{new}`") def _spec_str_fix_handler(path: str, line: int, col: int, old: str, new: str): """A SpecStrHandler that updates formatted spec strings in files.""" with open(path, "r", encoding="utf-8") as f: lines = f.readlines() new_line = lines[line - 1].replace(old, new) if new_line == lines[line - 1]: tty.warn(f"{path}:{line}:{col}: could not apply fix: `{old}` -> `{new}`") return lines[line - 1] = new_line print(f"{path}:{line}:{col}: fixed `{old}` -> `{new}`") with open(path, "w", encoding="utf-8") as f: f.writelines(lines) def _spec_str_ast(path: str, tree: ast.AST, handler: SpecStrHandler) -> None: """Walk the AST of a Python file and apply handler to formatted spec strings.""" for node in ast.walk(tree): if sys.version_info >= (3, 8): if isinstance(node, ast.Constant) and isinstance(node.value, str): current_str = node.value else: continue elif isinstance(node, ast.Str): current_str = node.s else: continue if not IS_PROBABLY_COMPILER.search(current_str): continue new = _spec_str_format(current_str) if new is not None: handler(path, node.lineno, node.col_offset, current_str, new) def _spec_str_json_and_yaml(path: str, data: dict, handler: SpecStrHandler) -> None: """Walk a YAML or JSON data structure and apply handler to formatted spec strings.""" queue = [data] seen = set() while queue: current = queue.pop(0) if id(current) in seen: continue seen.add(id(current)) if isinstance(current, dict): queue.extend(current.values()) queue.extend(current.keys()) elif isinstance(current, list): queue.extend(current) elif isinstance(current, str) and IS_PROBABLY_COMPILER.search(current): new = _spec_str_format(current) if new is not None: mark = getattr(current, "_start_mark", None) if mark: line, col = mark.line + 1, mark.column + 1 else: line, col = 0, 0 handler(path, line, col, current, new) def _check_spec_strings( paths: List[str], handler: SpecStrHandler = _spec_str_default_handler ) -> None: """Open Python, JSON and YAML files, and format their string literals that look like spec strings. A handler is called for each formatting, which can be used to print or apply fixes.""" for path in paths: is_json_or_yaml = path.endswith(".json") or path.endswith(".yaml") or path.endswith(".yml") is_python = path.endswith(".py") if not is_json_or_yaml and not is_python: continue try: with open(path, "r", encoding="utf-8") as f: # skip files that are likely too large to be user code or config if os.fstat(f.fileno()).st_size > 1024 * 1024: warnings.warn(f"skipping {path}: too large.") continue if is_json_or_yaml: _spec_str_json_and_yaml(path, spack.util.spack_yaml.load_config(f), handler) elif is_python: _spec_str_ast(path, ast.parse(f.read()), handler) except (OSError, spack.util.spack_yaml.SpackYAMLError, SyntaxError, ValueError): warnings.warn(f"skipping {path}") continue ================================================ FILE: lib/spack/spack/cmd/compiler.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys from typing import List, Optional import spack.binary_distribution import spack.compilers.config import spack.config import spack.llnl.util.tty as tty import spack.spec import spack.store from spack.cmd.common import arguments from spack.llnl.util.lang import index_by from spack.llnl.util.tty.colify import colify from spack.llnl.util.tty.color import colorize from spack.spec import Spec description = "manage compilers" section = "config" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="compiler_command") # Find find_parser = sp.add_parser( "find", aliases=["add"], help="search the system for compilers to add to Spack configuration", ) find_parser.add_argument("add_paths", nargs=argparse.REMAINDER) find_parser.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope("packages"), help="configuration scope to modify", ) arguments.add_common_arguments(find_parser, ["jobs"]) # Remove remove_parser = sp.add_parser("remove", aliases=["rm"], help="remove compiler by spec") remove_parser.add_argument( "-a", "--all", action="store_true", help="remove ALL compilers that match spec" ) remove_parser.add_argument("compiler_spec") remove_parser.add_argument( "--scope", action=arguments.ConfigScope, default=None, help="configuration scope to modify" ) # List list_parser = sp.add_parser("list", aliases=["ls"], help="list available compilers") list_parser.add_argument( "--scope", action=arguments.ConfigScope, type=arguments.config_scope_readable_validator, help="configuration scope to read from", ) list_parser.add_argument( "--remote", action="store_true", help="list also compilers from registered buildcaches" ) # Info info_parser = sp.add_parser("info", help="show compiler paths") info_parser.add_argument("compiler_spec") info_parser.add_argument( "--scope", action=arguments.ConfigScope, type=arguments.config_scope_readable_validator, help="configuration scope to read from", ) info_parser.add_argument( "--remote", action="store_true", help="list also compilers from registered buildcaches" ) def compiler_find(args): """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration. """ paths = args.add_paths or None new_compilers = spack.compilers.config.find_compilers( path_hints=paths, scope=args.scope, max_workers=args.jobs ) if new_compilers: n = len(new_compilers) s = "s" if n > 1 else "" filename = spack.config.CONFIG.get_config_filename(args.scope, "packages") tty.msg(f"Added {n:d} new compiler{s} to {filename}") compiler_strs = sorted(f"{spec.name}@{spec.versions}" for spec in new_compilers) colify(reversed(compiler_strs), indent=4) else: tty.msg("Found no new compilers") tty.msg("Compilers are defined in the following files:") colify(spack.compilers.config.compiler_config_files(), indent=4) def compiler_remove(args): remover = spack.compilers.config.CompilerRemover(spack.config.CONFIG) candidates = remover.mark_compilers(match=args.compiler_spec, scope=args.scope) if not candidates: tty.die(f"No compiler matches '{args.compiler_spec}'") compiler_strs = reversed(sorted(f"{spec.name}@{spec.versions}" for spec in candidates)) if not args.all and len(candidates) > 1: tty.error(f"multiple compilers match the spec '{args.compiler_spec}':") print() colify(compiler_strs, indent=4) print() print( "Either use a stricter spec to select only one, or use `spack compiler remove -a`" " to remove all of them." ) sys.exit(1) remover.flush() tty.msg("The following compilers have been removed:") print() colify(compiler_strs, indent=4) print() def compiler_info(args): """Print info about all compilers matching a spec.""" all_compilers = _all_available_compilers(scope=args.scope, remote=args.remote) query = spack.spec.Spec(args.compiler_spec) compilers = [x for x in all_compilers if x.satisfies(query)] if not compilers: tty.die(f"No compilers match spec {query.cformat()}") compilers.sort(key=lambda x: (not x.external, x.name, x.version)) for c in compilers: exes = { cname: getattr(c.package, cname) for cname in ("cc", "cxx", "fortran") if hasattr(c.package, cname) } if not exes: tty.debug( f"{__name__}: skipping {c.format()} from compiler list, " f"since it has no executables" ) continue print(f"{c.tree(recurse_dependencies=False, status_fn=spack.spec.Spec.install_status)}") print(f" prefix: {c.prefix}") print(" compilers:") for language, exe in exes.items(): print(f" {language}: {exe}") extra_attributes = getattr(c, "extra_attributes", {}) if "flags" in extra_attributes: print(" flags:") for flag, flag_value in extra_attributes["flags"].items(): print(f" {flag} = {flag_value}") if "environment" in extra_attributes: environment = extra_attributes["environment"] if len(environment.get("set", {})) != 0: print("\tenvironment:") print("\t set:") for key, value in environment["set"].items(): print(f"\t {key} = {value}") if "extra_rpaths" in extra_attributes: print(" extra rpaths:") for extra_rpath in extra_attributes["extra_rpaths"]: print(f" {extra_rpath}") if getattr(c, "external_modules", []): print(" modules: ") for module in c.external_modules: print(f" {module}") print() def compiler_list(args): compilers = _all_available_compilers(scope=args.scope, remote=args.remote) if not sys.stdout.isatty(): for c in sorted(compilers): # type: ignore print(c.format("{name}@{version}")) return # If there are no compilers in any scope, and we're outputting to a tty, give a # hint to the user. if len(compilers) == 0: msg = "No compilers available" if args.scope is None: msg += ". Run `spack compiler find` to autodetect compilers" tty.msg(msg) return index = index_by(compilers, spack.compilers.config.name_os_target) tty.msg("Available compilers") # For a container, take each element which does not evaluate to false and # convert it to a string. For elements which evaluate to False (e.g. None) # convert them to '' (in which case it still evaluates to False but is a # string type). Tuples produced by this are guaranteed to be comparable in # Python 3 convert_str = lambda tuple_container: tuple(str(x) if x else "" for x in tuple_container) index_str_keys = list((convert_str(x), y) for x, y in index.items()) ordered_sections = sorted(index_str_keys, key=lambda item: item[0]) for i, (key, compilers) in enumerate(ordered_sections): if i >= 1: print() name, os, target = key os_str = os if target: os_str += f"-{target}" cname = f"{spack.spec.COMPILER_COLOR}{{{name}}} {os_str}" tty.hline(colorize(cname), char="-") result = { colorize(c.install_status().value) + c.format("{name}@{version}") for c in compilers } colify(reversed(sorted(result))) def _all_available_compilers(scope: Optional[str], remote: bool) -> List[Spec]: supported_compilers = spack.compilers.config.supported_compilers() def _is_compiler(x): return x.name in supported_compilers and x.package.supported_languages and not x.external compilers_from_store = [x for x in spack.store.STORE.db.query() if _is_compiler(x)] compilers_from_yaml = spack.compilers.config.all_compilers(scope=scope, init_config=False) compilers = compilers_from_yaml + compilers_from_store if remote: compilers.extend( [x for x in spack.binary_distribution.update_cache_and_get_specs() if _is_compiler(x)] ) return compilers def compiler(parser, args): action = { "add": compiler_find, "find": compiler_find, "remove": compiler_remove, "rm": compiler_remove, "info": compiler_info, "list": compiler_list, "ls": compiler_list, } action[args.compiler_command](args) ================================================ FILE: lib/spack/spack/cmd/compilers.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse from spack.cmd.common import arguments from spack.cmd.compiler import compiler_list description = "list available compilers" section = "config" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--scope", action=arguments.ConfigScope, type=arguments.config_scope_readable_validator, help="configuration scope to read/modify", ) subparser.add_argument( "--remote", action="store_true", help="list also compilers from registered buildcaches" ) def compilers(parser, args): compiler_list(args) ================================================ FILE: lib/spack/spack/cmd/concretize.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd import spack.cmd.common.arguments import spack.environment as ev import spack.llnl.util.tty as tty from spack.llnl.string import plural description = "concretize an environment and write a lockfile" section = "environments" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--test", default=None, choices=["root", "all"], help="concretize with test dependencies of only root packages or all packages", ) subparser.add_argument( "-q", "--quiet", action="store_true", help="don't print concretized specs" ) spack.cmd.common.arguments.add_concretizer_args(subparser) spack.cmd.common.arguments.add_common_arguments(subparser, ["jobs", "show_non_defaults"]) def concretize(parser, args): env = spack.cmd.require_active_env(cmd_name="concretize") if args.test == "all": tests = True elif args.test == "root": tests = [spec.name for spec in env.user_specs] else: tests = False with env.write_transaction(): concretized_specs = env.concretize(tests=tests) if not args.quiet: if concretized_specs: tty.msg(f"Concretized {plural(len(concretized_specs), 'spec')}:") ev.display_specs( [concrete for _, concrete in concretized_specs], highlight_non_defaults=args.non_defaults, ) else: tty.msg("No new specs to concretize.") env.write() ================================================ FILE: lib/spack/spack/cmd/config.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import collections import os import shutil import sys from typing import List import spack.config import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.schema import spack.schema.env import spack.spec import spack.store import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml from spack.cmd.common import arguments from spack.llnl.util.tty.colify import colify_table from spack.util.editor import editor description = "get and set configuration options" section = "config" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: # User can only choose one subparser.add_argument( "--scope", action=arguments.ConfigScope, help="configuration scope to read/modify" ) sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="config_command") get_parser = sp.add_parser("get", help="print configuration values") get_parser.add_argument( "section", help="configuration section to print\n\noptions: %(choices)s", nargs="?", metavar="section", choices=spack.config.SECTION_SCHEMAS, ) get_parser.add_argument("--json", action="store_true", help="output configuration as JSON") get_parser.add_argument( "--group", metavar="group", default=None, help="show configuration as seen by this environment spec group (requires active env)", ) blame_parser = sp.add_parser( "blame", help="print configuration annotated with source file:line" ) blame_parser.add_argument( "section", help="configuration section to print\n\noptions: %(choices)s", nargs="?", metavar="section", choices=spack.config.SECTION_SCHEMAS, ) blame_parser.add_argument( "--group", metavar="group", default=None, help="show configuration as seen by this environment spec group (requires active env)", ) edit_parser = sp.add_parser("edit", help="edit configuration file") edit_parser.add_argument( "section", help="configuration section to edit\n\noptions: %(choices)s", metavar="section", nargs="?", choices=spack.config.SECTION_SCHEMAS, ) edit_parser.add_argument( "--print-file", action="store_true", help="print the file name that would be edited" ) sp.add_parser("list", help="list configuration sections") scopes_parser = sp.add_parser( "scopes", help="list defined scopes in descending order of precedence" ) scopes_parser.add_argument( "-p", "--paths", action="store_true", default=False, help="show associated paths for appropriate scopes", ) scopes_parser.add_argument( "-t", "--type", default=["all"], metavar="scope-type", nargs="+", choices=("all", "env", "include", "internal", "path"), help="list only scopes of the specified type(s)\n\noptions: %(choices)s", ) scopes_parser.add_argument( "-v", "--verbose", dest="scopes_verbose", # spack has -v as well action="store_true", default=False, help="show scope types and whether scopes are overridden", ) scopes_parser.add_argument( "section", help="tailor scope path information to the specified section (implies ``--paths``)" "\n\noptions: %(choices)s", metavar="section", nargs="?", choices=spack.config.SECTION_SCHEMAS, ) add_parser = sp.add_parser("add", help="add configuration parameters") add_parser.add_argument( "path", nargs="?", help="colon-separated path to config that should be added, e.g. 'config:default:true'", ) add_parser.add_argument("-f", "--file", help="file from which to set all config values") change_parser = sp.add_parser("change", help="swap variants etc. on specs in config") change_parser.add_argument("path", help="colon-separated path to config section with specs") change_parser.add_argument("--match-spec", help="only change constraints that match this") prefer_upstream_parser = sp.add_parser( "prefer-upstream", help="set package preferences from upstream" ) prefer_upstream_parser.add_argument( "--local", action="store_true", default=False, help="set packages preferences based on local installs, rather than upstream", ) remove_parser = sp.add_parser("remove", aliases=["rm"], help="remove configuration parameters") remove_parser.add_argument( "path", help="colon-separated path to config that should be removed, e.g. 'config:default:true'", ) # Make the add parser available later setattr(setup_parser, "add_parser", add_parser) update = sp.add_parser("update", help="update configuration files to the latest format") arguments.add_common_arguments(update, ["yes_to_all"]) update.add_argument("section", help="section to update") revert = sp.add_parser( "revert", help="revert configuration files to their state before update" ) arguments.add_common_arguments(revert, ["yes_to_all"]) revert.add_argument("section", help="section to update") def _get_scope_and_section(args): """Extract config scope and section from arguments.""" scope = args.scope section = getattr(args, "section", None) path = getattr(args, "path", None) # w/no args and an active environment, point to env manifest if not section and not scope: env = ev.active_environment() if env: scope = env.scope_name # set scope defaults elif not scope: scope = spack.config.default_modify_scope(section) # special handling for commands that take value instead of section if path: section = path[: path.find(":")] if ":" in path else path if not scope: scope = spack.config.default_modify_scope(section) return scope, section def print_configuration(args, *, blame: bool) -> None: if args.scope and args.scope not in spack.config.existing_scope_names(): tty.die(f"the argument --scope={args.scope} must refer to an existing scope.") if args.scope and args.section is None: tty.die(f"the argument --scope={args.scope} requires specifying a section.") group = getattr(args, "group", None) if group is not None: env = ev.active_environment() if env is None: tty.die("the argument --group requires an active environment") try: with env.config_override_for_group(group=group): _print_configuration_helper(args, blame=blame) except ValueError as e: tty.die(str(e)) return _print_configuration_helper(args, blame=blame) def _print_configuration_helper(args, *, blame: bool) -> None: yaml = blame or not args.json if args.section is not None: spack.config.CONFIG.print_section(args.section, yaml=yaml, blame=blame, scope=args.scope) return print_flattened_configuration(blame=blame, yaml=yaml) def print_flattened_configuration(*, blame: bool, yaml: bool) -> None: """Prints to stdout a flattened version of the configuration. Args: blame: if True, shows file provenance for each entry in the configuration. """ env = ev.active_environment() if env is not None: pristine = env.manifest.yaml_content flattened = pristine.copy() flattened[spack.schema.env.TOP_LEVEL_KEY] = pristine[spack.schema.env.TOP_LEVEL_KEY].copy() else: flattened = syaml.syaml_dict() flattened[spack.schema.env.TOP_LEVEL_KEY] = syaml.syaml_dict() for config_section in spack.config.SECTION_SCHEMAS: current = spack.config.get(config_section) flattened[spack.schema.env.TOP_LEVEL_KEY][config_section] = current if blame or yaml: syaml.dump_config(flattened, stream=sys.stdout, default_flow_style=False, blame=blame) else: sjson.dump(flattened, sys.stdout) sys.stdout.write("\n") def config_get(args): """Dump merged YAML configuration for a specific section. With no arguments and an active environment, print the contents of the environment's manifest file (spack.yaml). """ print_configuration(args, blame=False) def config_blame(args): """Print out line-by-line blame of merged YAML.""" print_configuration(args, blame=True) def config_edit(args): """Edit the configuration file for a specific scope and config section. With no arguments and an active environment, edit the spack.yaml for the active environment. """ spack_env = os.environ.get(ev.spack_env_var) env_error = ev.environment._active_environment_error if env_error and args.scope: # Cannot use scopes beyond the environment itself with a failed environment raise env_error elif env_error: # The rest of the config system wasn't set up fully, but spack.main was allowed # to progress so the user can open the malformed environment file config_file = env_error.filename elif spack_env and not args.scope: # Don't use the scope object for envs, as `config edit` can be called # for a malformed environment. Use SPACK_ENV to find spack.yaml. config_file = ev.manifest_file(spack_env) else: # If we aren't editing a spack.yaml file, get config path from scope. scope, section = _get_scope_and_section(args) if not scope and not section: tty.die("`spack config edit` requires a section argument or an active environment.") config_file = spack.config.CONFIG.get_config_filename(scope, section) if args.print_file: print(config_file) else: editor(config_file) def config_list(args): """List the possible configuration sections. Used primarily for shell tab completion scripts. """ print(" ".join(list(spack.config.SECTION_SCHEMAS))) def _config_scope_info(args, scope, active, included): result = [scope.name] # always print the name if args.scopes_verbose: result.append(",".join(_config_basic_scope_types(scope, included))) if scope.name not in active: scope_status = "override" elif args.section and not spack.config.CONFIG.get_config(args.section, scope=scope.name): scope_status = "absent" else: scope_status = "active" result.append(scope_status) section_path = None if args.section or args.paths: if hasattr(scope, "path"): section_path = scope.get_section_filename(args.section) if args.section else None result.append( section_path if section_path and os.path.exists(section_path) else f"{scope.path}{'' if os.path.isfile(scope.path) else os.sep}" ) else: result.append(" ") if args.scopes_verbose and scope_status in ("absent", "override"): result = [color.colorize(f"@k{{{elt}}}") for elt in result] return result def _config_basic_scope_types(scope, included): types = [] if isinstance(scope, spack.config.InternalConfigScope): types.append("internal") if hasattr(scope, "yaml_path") and scope.yaml_path == [spack.schema.env.TOP_LEVEL_KEY]: types.append("env") if hasattr(scope, "path"): types.append("path") if scope.name in included: types.append("include") return sorted(types) def config_scopes(args): """List configured scopes in descending order of precedence.""" included = list(i.name for s in spack.config.scopes().values() for i in s.included_scopes) active = [s.name for s in spack.config.CONFIG.active_scopes] scopes = [ s for s in spack.config.scopes().reversed_values() if ( "include" in args.type and s.name in included or any(i in ("all", *_config_basic_scope_types(s, included)) for i in args.type) ) and (s.name in active or args.scopes_verbose) ] if scopes: headers = ["Scope"] if args.scopes_verbose: headers += ["Type", "Status"] if args.section or args.paths: headers += ["Path"] table = [_config_scope_info(args, s, active, included) for s in scopes] # add headers if we have > 1 column if len(headers) > 1: table = [[color.colorize(f"@*C{{{colname}}}") for colname in headers]] + table colify_table(table) def config_add(args): """Add the given configuration to the specified config scope This is a stateful operation that edits the config files.""" if not (args.file or args.path): tty.error("No changes requested. Specify a file or value.") setup_parser.add_parser.print_help() exit(1) scope, section = _get_scope_and_section(args) if args.file: spack.config.add_from_file(args.file, scope=scope) if args.path: spack.config.add(args.path, scope=scope) def config_remove(args): """Remove the given configuration from the specified config scope This is a stateful operation that edits the config files.""" scope, _ = _get_scope_and_section(args) path, _, value = args.path.rpartition(":") existing = spack.config.get(path, scope=scope) if not isinstance(existing, (list, dict)): path, _, value = path.rpartition(":") existing = spack.config.get(path, scope=scope) value = syaml.load(value) if isinstance(existing, list): values = value if isinstance(value, list) else [value] for v in values: existing.remove(v) elif isinstance(existing, dict): existing.pop(value, None) else: # This should be impossible to reach raise spack.error.ConfigError("Config has nested non-dict values") spack.config.set(path, existing, scope) def _can_update_config_file(scope: spack.config.ConfigScope, cfg_file): if isinstance(scope, spack.config.SingleFileScope): return fs.can_access(cfg_file) elif isinstance(scope, spack.config.DirectoryConfigScope): return fs.can_write_to_dir(scope.path) and fs.can_access(cfg_file) return False def _config_change_requires_scope(path, spec, scope, match_spec=None): """Return whether or not anything changed.""" require = spack.config.get(path, scope=scope) if not require: return False changed = False def override_cfg_spec(spec_str): nonlocal changed init_spec = spack.spec.Spec(spec_str) # Overridden spec cannot be anonymous init_spec.name = spec.name if match_spec and not init_spec.satisfies(match_spec): # If there is a match_spec, don't change constraints that # don't match it return spec_str elif not init_spec.intersects(spec): changed = True return str(spack.spec.Spec.override(init_spec, spec)) else: # Don't override things if they intersect, otherwise we'd # be e.g. attaching +debug to every single version spec return spec_str if isinstance(require, str): new_require = override_cfg_spec(require) else: new_require = [] for item in require: if "one_of" in item: item["one_of"] = [override_cfg_spec(x) for x in item["one_of"]] elif "any_of" in item: item["any_of"] = [override_cfg_spec(x) for x in item["any_of"]] elif "spec" in item: item["spec"] = override_cfg_spec(item["spec"]) elif isinstance(item, str): item = override_cfg_spec(item) else: raise ValueError(f"Unexpected requirement: ({type(item)}) {str(item)}") new_require.append(item) spack.config.set(path, new_require, scope=scope) return changed def _config_change(config_path, match_spec_str=None): all_components = spack.config.process_config_path(config_path) key_components = all_components[:-1] key_path = ":".join(key_components) spec = spack.spec.Spec(syaml.syaml_str(all_components[-1])) match_spec = None if match_spec_str: match_spec = spack.spec.Spec(match_spec_str) if key_components[-1] == "require": # Extract the package name from the config path, which allows # args.spec to be anonymous if desired pkg_name = key_components[1] spec.name = pkg_name changed = False for scope in spack.config.writable_scope_names(): changed |= _config_change_requires_scope(key_path, spec, scope, match_spec=match_spec) if not changed: existing_requirements = spack.config.get(key_path) if isinstance(existing_requirements, str): raise spack.error.ConfigError( "'config change' needs to append a requirement," " but existing require: config is not a list" ) ideal_scope_to_modify = None for scope in spack.config.writable_scope_names(): if spack.config.get(key_path, scope=scope): ideal_scope_to_modify = scope break # If we find our key in a specific scope, that's the one we want # to modify. Otherwise we use the default write scope. write_scope = ideal_scope_to_modify or spack.config.default_modify_scope() update_path = f"{key_path}:[{str(spec)}]" spack.config.add(update_path, scope=write_scope) else: raise ValueError("'config change' can currently only change 'require' sections") def config_change(args): _config_change(args.path, args.match_spec) def config_update(args): # Read the configuration files spack.config.CONFIG.get_config(args.section, scope=args.scope) updates: List[spack.config.ConfigScope] = [ x for x in spack.config.CONFIG.updated_scopes_by_section[args.section] if not isinstance(x, spack.config.InternalConfigScope) and x.writable ] cannot_overwrite, skip_system_scope = [], False for scope in updates: cfg_file = spack.config.CONFIG.get_config_filename(scope.name, args.section) can_be_updated = _can_update_config_file(scope, cfg_file) if not can_be_updated: if scope.name == "system": skip_system_scope = True tty.warn( 'Not enough permissions to write to "system" scope. ' f"Skipping update at that location [cfg={cfg_file}]" ) continue cannot_overwrite.append((scope, cfg_file)) if cannot_overwrite: msg = "Detected permission issues with the following scopes:\n\n" for scope, cfg_file in cannot_overwrite: msg += "\t[scope={0}, cfg={1}]\n".format(scope.name, cfg_file) msg += ( "\nEither ensure that you have sufficient permissions to " "modify these files or do not include these scopes in the " "update." ) tty.die(msg) if skip_system_scope: updates = [x for x in updates if x.name != "system"] # Report if there are no updates to be done if not updates: msg = 'No updates needed for "{0}" section.' tty.msg(msg.format(args.section)) return proceed = True if not args.yes_to_all: msg = ( "The following configuration files are going to be updated to" " the latest schema format:\n\n" ) for scope in updates: cfg_file = spack.config.CONFIG.get_config_filename(scope.name, args.section) msg += "\t[scope={0}, file={1}]\n".format(scope.name, cfg_file) msg += ( "\nIf the configuration files are updated, versions of Spack " "that are older than this version may not be able to read " "them. Spack stores backups of the updated files which can " 'be retrieved with "spack config revert"' ) tty.msg(msg) proceed = tty.get_yes_or_no("Do you want to proceed?", default=False) if not proceed: tty.die("Operation aborted.") # Get a function to update the format update_fn = spack.config.ensure_latest_format_fn(args.section) for scope in updates: cfg_file = spack.config.CONFIG.get_config_filename(scope.name, args.section) data = scope.get_section(args.section) assert data is not None, f"Cannot find section {args.section} in {scope.name} scope" update_fn(data) # Make a backup copy and rewrite the file bkp_file = cfg_file + ".bkp" shutil.copy(cfg_file, bkp_file) spack.config.CONFIG.update_config( args.section, data[args.section], scope=scope.name, force=True ) tty.msg(f'File "{cfg_file}" update [backup={bkp_file}]') def _can_revert_update(scope_dir, cfg_file, bkp_file): dir_ok = fs.can_write_to_dir(scope_dir) cfg_ok = not os.path.exists(cfg_file) or fs.can_access(cfg_file) bkp_ok = fs.can_access(bkp_file) return dir_ok and cfg_ok and bkp_ok def config_revert(args): scopes = [args.scope] if args.scope else [x.name for x in spack.config.CONFIG.writable_scopes] # Search for backup files in the configuration scopes Entry = collections.namedtuple("Entry", ["scope", "cfg", "bkp"]) to_be_restored, cannot_overwrite = [], [] for scope in scopes: cfg_file = spack.config.CONFIG.get_config_filename(scope, args.section) bkp_file = cfg_file + ".bkp" # If the backup files doesn't exist move to the next scope if not os.path.exists(bkp_file): continue # If it exists and we don't have write access in this scope # keep track of it and report a comprehensive error later entry = Entry(scope, cfg_file, bkp_file) scope_dir = os.path.dirname(bkp_file) can_be_reverted = _can_revert_update(scope_dir, cfg_file, bkp_file) if not can_be_reverted: cannot_overwrite.append(entry) continue to_be_restored.append(entry) # Report errors if we can't revert a configuration if cannot_overwrite: msg = "Detected permission issues with the following scopes:\n\n" for e in cannot_overwrite: msg += "\t[scope={0.scope}, cfg={0.cfg}, bkp={0.bkp}]\n".format(e) msg += ( "\nEither ensure to have the right permissions before retrying" " or be more specific on the scope to revert." ) tty.die(msg) proceed = True if not args.yes_to_all: msg = "The following scopes will be restored from the corresponding backup files:\n" for entry in to_be_restored: msg += "\t[scope={0.scope}, bkp={0.bkp}]\n".format(entry) msg += "This operation cannot be undone." tty.msg(msg) proceed = tty.get_yes_or_no("Do you want to proceed?", default=False) if not proceed: tty.die("Operation aborted.") for _, cfg_file, bkp_file in to_be_restored: shutil.copy(bkp_file, cfg_file) os.unlink(bkp_file) msg = 'File "{0}" reverted to old state' tty.msg(msg.format(cfg_file)) def config_prefer_upstream(args): """Generate a packages config based on the configuration of all upstream installs.""" scope = args.scope if scope is None: scope = spack.config.default_modify_scope("packages") all_specs = set(spack.store.STORE.db.query(installed=True)) local_specs = set(spack.store.STORE.db.query_local(installed=True)) pref_specs = local_specs if args.local else all_specs - local_specs conflicting_variants = set() pkgs = {} for spec in pref_specs: # Collect all the upstream compilers and versions for this package. pkg = pkgs.get(spec.name, {"version": []}) pkgs[spec.name] = pkg # We have no existing variant if this is our first added version. existing_variants = pkg.get("variants", None if not pkg["version"] else "") version = spec.version.string if version not in pkg["version"]: pkg["version"].append(version) # Get and list all the variants that differ from the default. variants = [] for var_name, variant in spec.variants.items(): if var_name in ["patches"] or not spec.package.has_variant(var_name): continue vdef = spec.package.get_variant(var_name) if variant.value != vdef.default: variants.append(str(variant)) variants.sort() variants = " ".join(variants) if spec.name not in conflicting_variants: # Only specify the variants if there's a single variant # set across all versions/compilers. if existing_variants is not None and existing_variants != variants: conflicting_variants.add(spec.name) pkg.pop("variants", None) elif variants: pkg["variants"] = variants if conflicting_variants: tty.warn( "The following packages have multiple conflicting upstream " "specs. You may have to specify, by " "concretized hash, which spec you want when building " "packages that depend on them:\n - {0}".format( "\n - ".join(sorted(conflicting_variants)) ) ) # Simply write the config to the specified file. existing = spack.config.get("packages", scope=scope) new = spack.schema.merge_yaml(existing, pkgs) spack.config.set("packages", new, scope) config_file = spack.config.CONFIG.get_config_filename(scope, section) tty.msg("Updated config at {0}".format(config_file)) def config(parser, args): action = { "get": config_get, "blame": config_blame, "edit": config_edit, "list": config_list, "scopes": config_scopes, "add": config_add, "rm": config_remove, "remove": config_remove, "update": config_update, "revert": config_revert, "prefer-upstream": config_prefer_upstream, "change": config_change, } action[args.config_command](args) ================================================ FILE: lib/spack/spack/cmd/containerize.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import spack.container import spack.container.images import spack.llnl.util.tty description = "create a container build recipe from an environment" section = "environments" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--list-os", action="store_true", default=False, help="list all the OS that can be used in the bootstrap phase and exit", ) subparser.add_argument( "--last-stage", choices=("bootstrap", "build", "final"), default="final", help="last stage in the container recipe", ) def containerize(parser, args): if args.list_os: possible_os = spack.container.images.all_bootstrap_os() msg = "The following operating systems can be used to bootstrap Spack:" msg += "\n{0}".format(" ".join(possible_os)) spack.llnl.util.tty.msg(msg) return config_dir = args.env_dir or os.getcwd() config_file = os.path.abspath(os.path.join(config_dir, "spack.yaml")) if not os.path.exists(config_file): msg = "file not found: {0}" raise ValueError(msg.format(config_file)) config = spack.container.validate(config_file) recipe = spack.container.recipe(config, last_phase=args.last_stage) print(recipe) ================================================ FILE: lib/spack/spack/cmd/create.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import re import sys import urllib.parse from typing import List, Optional, Tuple import spack.llnl.util.tty as tty import spack.repo import spack.stage from spack.llnl.util.filesystem import mkdirp from spack.spec import Spec from spack.url import ( UndetectableNameError, UndetectableVersionError, find_versions_of_archive, parse_name, parse_version, ) from spack.util.editor import editor from spack.util.executable import which from spack.util.format import get_version_lines from spack.util.naming import pkg_name_to_class_name, simplify_name description = "create a new package file" section = "packaging" level = "short" package_template = '''\ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # ---------------------------------------------------------------------------- # If you submit this package back to Spack as a pull request, # please first remove this boilerplate and all FIXME comments. # # This is a template package file for Spack. We've put "FIXME" # next to all the things you'll want to change. Once you've handled # them, you can save this file and test your package like this: # # spack install {name} # # You can edit this file again by typing: # # spack edit {name} # # See the Spack documentation for more information on packaging. # ---------------------------------------------------------------------------- {package_class_import} from spack.package import * class {class_name}({base_class_name}): """FIXME: Put a proper description of your package here.""" # FIXME: Add a proper url for your package's homepage here. homepage = "https://www.example.com" {url_def} # FIXME: Add a list of GitHub accounts to # notify when the package is updated. # maintainers("github_user1", "github_user2") # FIXME: Add the SPDX identifier of the project's license below. # See https://spdx.org/licenses/ for a list. Upon manually verifying # the license, set checked_by to your Github username. license("UNKNOWN", checked_by="github_user1") {versions} {dependencies} {body_def} ''' class BundlePackageTemplate: """ Provides the default values to be used for a bundle package file template. """ base_class_name = "BundlePackage" package_class_import = "from spack_repo.builtin.build_systems.bundle import BundlePackage" dependencies = """\ # FIXME: Add dependencies if required. # depends_on("foo")""" url_def = " # There is no URL since there is no code to download." body_def = " # There is no need for install() since there is no code." def __init__(self, name: str, versions, languages: List[str]): self.name = name self.class_name = pkg_name_to_class_name(name) self.versions = versions self.languages = languages def write(self, pkg_path): """Writes the new package file.""" all_deps = [f' depends_on("{lang}", type="build")' for lang in self.languages] if all_deps and self.dependencies: all_deps.append("") all_deps.append(self.dependencies) # Write out a template for the file with open(pkg_path, "w", encoding="utf-8") as pkg_file: pkg_file.write( package_template.format( name=self.name, class_name=self.class_name, base_class_name=self.base_class_name, package_class_import=self.package_class_import, url_def=self.url_def, versions=self.versions, dependencies="\n".join(all_deps), body_def=self.body_def, ) ) class PackageTemplate(BundlePackageTemplate): """Provides the default values to be used for the package file template""" base_class_name = "Package" package_class_import = "from spack_repo.builtin.build_systems.generic import Package" body_def = """\ def install(self, spec, prefix): # FIXME: Unknown build system make() make("install")""" url_line = ' url = "{url}"' def __init__(self, name, url, versions, languages: List[str]): super().__init__(name, versions, languages) self.url_def = self.url_line.format(url=url) class AutotoolsPackageTemplate(PackageTemplate): """Provides appropriate overrides for Autotools-based packages that *do* come with a ``configure`` script""" base_class_name = "AutotoolsPackage" package_class_import = ( "from spack_repo.builtin.build_systems.autotools import AutotoolsPackage" ) body_def = """\ def configure_args(self): # FIXME: Add arguments other than --prefix # FIXME: If not needed delete this function args = [] return args""" class AutoreconfPackageTemplate(PackageTemplate): """Provides appropriate overrides for Autotools-based packages that *do not* come with a ``configure`` script""" base_class_name = "AutotoolsPackage" package_class_import = ( "from spack_repo.builtin.build_systems.autotools import AutotoolsPackage" ) dependencies = """\ with default_args(type="build"): depends_on("autoconf") depends_on("automake") depends_on("libtool") depends_on("m4") # FIXME: Add additional dependencies if required. # depends_on("foo")""" body_def = """\ def autoreconf(self, spec, prefix): # FIXME: Modify the autoreconf method as necessary autoreconf("--install", "--verbose", "--force") def configure_args(self): # FIXME: Add arguments other than --prefix # FIXME: If not needed delete this function args = [] return args""" class CargoPackageTemplate(PackageTemplate): """Provides appropriate overrides for cargo-based packages""" base_class_name = "CargoPackage" package_class_import = "from spack_repo.builtin.build_systems.cargo import CargoPackage" body_def = "" class CMakePackageTemplate(PackageTemplate): """Provides appropriate overrides for CMake-based packages""" base_class_name = "CMakePackage" package_class_import = "from spack_repo.builtin.build_systems.cmake import CMakePackage" body_def = """\ def cmake_args(self): # FIXME: Add arguments other than # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE # FIXME: If not needed delete this function args = [] return args""" class GoPackageTemplate(PackageTemplate): """Provides appropriate overrides for Go-module-based packages""" base_class_name = "GoPackage" package_class_import = "from spack_repo.builtin.build_systems.go import GoPackage" body_def = "" class LuaPackageTemplate(PackageTemplate): """Provides appropriate overrides for LuaRocks-based packages""" base_class_name = "LuaPackage" package_class_import = "from spack_repo.builtin.build_systems.lua import LuaPackage" body_def = """\ def luarocks_args(self): # FIXME: Add arguments to `luarocks make` other than rockspec path # FIXME: If not needed delete this function args = [] return args""" def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name lua-lpeg`, don't rename it lua-lua-lpeg if not name.startswith("lua-"): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to lua-{0}".format(name)) name = "lua-{0}".format(name) super().__init__(name, url, versions, languages) class MesonPackageTemplate(PackageTemplate): """Provides appropriate overrides for meson-based packages""" base_class_name = "MesonPackage" package_class_import = "from spack_repo.builtin.build_systems.meson import MesonPackage" body_def = """\ def meson_args(self): # FIXME: If not needed delete this function args = [] return args""" class QMakePackageTemplate(PackageTemplate): """Provides appropriate overrides for QMake-based packages""" base_class_name = "QMakePackage" package_class_import = "from spack_repo.builtin.build_systems.qmake import QMakePackage" body_def = """\ def qmake_args(self): # FIXME: If not needed delete this function args = [] return args""" class MavenPackageTemplate(PackageTemplate): """Provides appropriate overrides for Maven-based packages""" base_class_name = "MavenPackage" package_class_import = "from spack_repo.builtin.build_systems.maven import MavenPackage" body_def = """\ def build(self, spec, prefix): # FIXME: If not needed delete this function pass""" class SconsPackageTemplate(PackageTemplate): """Provides appropriate overrides for SCons-based packages""" base_class_name = "SConsPackage" package_class_import = "from spack_repo.builtin.build_systems.scons import SConsPackage" body_def = """\ def build_args(self, spec, prefix): # FIXME: Add arguments to pass to build. # FIXME: If not needed delete this function args = [] return args""" class WafPackageTemplate(PackageTemplate): """Provides appropriate override for Waf-based packages""" base_class_name = "WafPackage" package_class_import = "from spack_repo.builtin.build_systems.waf import WafPackage" body_def = """\ # FIXME: Override configure_args(), build_args(), # or install_args() if necessary.""" class BazelPackageTemplate(PackageTemplate): """Provides appropriate overrides for Bazel-based packages""" dependencies = """\ # FIXME: Add additional dependencies if required. with default_args(type="build"): depends_on("bazel")""" body_def = """\ def install(self, spec, prefix): # FIXME: Add logic to build and install here. bazel()""" class RacketPackageTemplate(PackageTemplate): """Provides appropriate overrides for Racket extensions""" base_class_name = "RacketPackage" package_class_import = "from spack_repo.builtin.build_systems.racket import RacketPackage" url_line = """\ # FIXME: set the proper location from which to fetch your package git = "git@github.com:example/example.git" """ dependencies = """\ # FIXME: Add dependencies if required. Only add the racket dependency # if you need specific versions. A generic racket dependency is # added implicitly by the RacketPackage class. # with default_args(type=("build", "run")): # depends_on("racket@8.3:")""" body_def = """\ # FIXME: specify the name of the package, # as it should appear to ``raco pkg install`` name = "{0}" # FIXME: set to true if published on pkgs.racket-lang.org # pkgs = False # FIXME: specify path to the root directory of the # package, if not the base directory # subdirectory = None """ def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name rkt-scribble`, don't rename it rkt-rkt-scribble if not name.startswith("rkt-"): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to rkt-{0}".format(name)) name = "rkt-{0}".format(name) self.body_def = self.body_def.format(name[4:]) super().__init__(name, url, versions, languages) class PythonPackageTemplate(PackageTemplate): """Provides appropriate overrides for python extensions""" base_class_name = "PythonPackage" package_class_import = "from spack_repo.builtin.build_systems.python import PythonPackage" dependencies = """\ # FIXME: Only add the python/pip/wheel dependencies if you need specific versions # or need to change the dependency type. Generic python/pip/wheel dependencies are # added implicitly by the PythonPackage base class. # depends_on("python@2.X:2.Y,3.Z:", type=("build", "run")) # depends_on("py-pip@X.Y:", type="build") # depends_on("py-wheel@X.Y:", type="build") # FIXME: Add a build backend, usually defined in pyproject.toml. If no such file # exists, use setuptools. # with default_args(type="build"): # depends_on("py-setuptools") # depends_on("py-hatchling") # depends_on("py-flit-core") # depends_on("py-poetry-core") # FIXME: Add additional dependencies if required. # with default_args(type=("build", "run")): # depends_on("py-foo")""" body_def = """\ def config_settings(self, spec, prefix): # FIXME: Add configuration settings to be passed to the build backend # FIXME: If not needed, delete this function settings = {} return settings""" def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name py-numpy`, don't rename it py-py-numpy if not name.startswith("py-"): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to py-{0}".format(name)) name = "py-{0}".format(name) # Simple PyPI URLs: # https:///packages//// # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip # PyPI URLs containing hash: # https:///packages//// # noqa: E501 # e.g. https://pypi.io/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512 # PyPI URLs for wheels: # https://pypi.io/packages/py3/a/azureml_core/azureml_core-1.11.0-py3-none-any.whl # https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-macosx_10_9_x86_64.whl # https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-manylinux1_x86_64.whl # https://files.pythonhosted.org/packages/cp35.cp36.cp37.cp38.cp39/s/shiboken2/shiboken2-5.15.2-5.15.2-cp35.cp36.cp37.cp38.cp39-abi3-manylinux1_x86_64.whl # https://files.pythonhosted.org/packages/f4/99/ad2ef1aeeb395ee2319bb981ea08dbbae878d30dd28ebf27e401430ae77a/azureml_core-1.36.0.post2-py3-none-any.whl#sha256=60bcad10b4380d78a8280deb7365de2c2cd66527aacdcb4a173f613876cbe739 match = re.search(r"(?:pypi|pythonhosted)[^/]+/packages" + "/([^/#]+)" * 4, url) if match: # PyPI URLs for wheels are too complicated, ignore them for now # https://www.python.org/dev/peps/pep-0427/#file-name-convention if not match.group(4).endswith(".whl"): if len(match.group(2)) == 1: # Simple PyPI URL url = "/".join(match.group(3, 4)) else: # PyPI URL containing hash # Project name doesn't necessarily match download name, but it # usually does, so this is the best we can do project = parse_name(url) url = "/".join([project, match.group(4)]) self.url_line = ' pypi = "{url}"' else: # Add a reminder about spack preferring PyPI URLs self.url_line = ( """ # FIXME: ensure the package is not available through PyPI. If it is, # re-run `spack create --force` with the PyPI URL. """ + self.url_line ) super().__init__(name, url, versions, languages) class RPackageTemplate(PackageTemplate): """Provides appropriate overrides for R extensions""" base_class_name = "RPackage" package_class_import = "from spack_repo.builtin.build_systems.r import RPackage" dependencies = """\ # FIXME: Add dependencies if required. # with default_args(type=("build", "run")): # depends_on("r-foo")""" body_def = """\ def configure_args(self): # FIXME: Add arguments to pass to install via --configure-args # FIXME: If not needed delete this function args = [] return args""" def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name r-rcpp`, don't rename it r-r-rcpp if not name.startswith("r-"): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to r-{0}".format(name)) name = "r-{0}".format(name) r_name = parse_name(url) cran = re.search(r"(?:r-project|rstudio)[^/]+/src" + "/([^/]+)" * 2, url) if cran: url = r_name self.url_line = ' cran = "{url}"' bioc = re.search(r"(?:bioconductor)[^/]+/packages" + "/([^/]+)" * 5, url) if bioc: self.url_line = ' url = "{0}"\n bioc = "{1}"'.format(url, r_name) super().__init__(name, url, versions, languages) class PerlmakePackageTemplate(PackageTemplate): """Provides appropriate overrides for Perl extensions that come with a Makefile.PL""" base_class_name = "PerlPackage" package_class_import = "from spack_repo.builtin.build_systems.perl import PerlPackage" dependencies = """\ # FIXME: Add dependencies if required: # with default_args(type=("build", "run")): # depends_on("perl-foo")""" body_def = """\ def configure_args(self): # FIXME: Add non-standard arguments # FIXME: If not needed delete this function args = [] return args""" def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name perl-cpp`, don't rename it perl-perl-cpp if not name.startswith("perl-"): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to perl-{0}".format(name)) name = "perl-{0}".format(name) super().__init__(name, url, versions, languages) class PerlbuildPackageTemplate(PerlmakePackageTemplate): """Provides appropriate overrides for Perl extensions that come with a Build.PL instead of a Makefile.PL""" dependencies = """\ depends_on("perl-module-build", type="build") # FIXME: Add additional dependencies if required: # with default_args(type=("build", "run")): # depends_on("perl-foo")""" class OctavePackageTemplate(PackageTemplate): """Provides appropriate overrides for octave packages""" base_class_name = "OctavePackage" package_class_import = "from spack_repo.builtin.build_systems.octave import OctavePackage" dependencies = """\ extends("octave") # FIXME: Add additional dependencies if required. # with default_args(type=("build", "run")): # depends_on("octave-foo")""" def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name octave-splines`, don't rename it # octave-octave-splines if not name.startswith("octave-"): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to octave-{0}".format(name)) name = "octave-{0}".format(name) super().__init__(name, url, versions, languages) class RubyPackageTemplate(PackageTemplate): """Provides appropriate overrides for Ruby packages""" base_class_name = "RubyPackage" package_class_import = "from spack_repo.builtin.build_systems.ruby import RubyPackage" dependencies = """\ # FIXME: Add dependencies if required. Only add the ruby dependency # if you need specific versions. A generic ruby dependency is # added implicitly by the RubyPackage class. # with default_args(type=("build", "run")): # depends_on("ruby@X.Y.Z:") # depends_on("ruby-foo")""" body_def = """\ def build(self, spec, prefix): # FIXME: If not needed delete this function pass""" def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name ruby-numpy`, don't rename it # ruby-ruby-numpy if not name.startswith("ruby-"): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to ruby-{0}".format(name)) name = "ruby-{0}".format(name) super().__init__(name, url, versions, languages) class MakefilePackageTemplate(PackageTemplate): """Provides appropriate overrides for Makefile packages""" base_class_name = "MakefilePackage" package_class_import = "from spack_repo.builtin.build_systems.makefile import MakefilePackage" body_def = """\ def edit(self, spec, prefix): # FIXME: Edit the Makefile if necessary # FIXME: If not needed delete this function # makefile = FileFilter("Makefile") # makefile.filter("CC = .*", "CC = cc") pass""" class IntelPackageTemplate(PackageTemplate): """Provides appropriate overrides for licensed Intel software""" base_class_name = "IntelOneApiPackage" package_class_import = "from spack_repo.builtin.build_systems.oneapi import IntelOneApiPackage" body_def = """\ # FIXME: Override `setup_environment` if necessary.""" class SIPPackageTemplate(PackageTemplate): """Provides appropriate overrides for SIP packages.""" base_class_name = "SIPPackage" package_class_import = "from spack_repo.builtin.build_systems.sip import SIPPackage" body_def = """\ def configure_args(self, spec, prefix): # FIXME: Add arguments other than --bindir and --destdir # FIXME: If not needed delete this function args = [] return args""" def __init__(self, name, url, versions, languages: List[str]): # If the user provided `--name py-pyqt4`, don't rename it py-py-pyqt4 if not name.startswith("py-"): # Make it more obvious that we are renaming the package tty.msg("Changing package name from {0} to py-{0}".format(name)) name = "py-{0}".format(name) super().__init__(name, url, versions, languages) templates = { "autoreconf": AutoreconfPackageTemplate, "autotools": AutotoolsPackageTemplate, "bazel": BazelPackageTemplate, "bundle": BundlePackageTemplate, "cargo": CargoPackageTemplate, "cmake": CMakePackageTemplate, "generic": PackageTemplate, "go": GoPackageTemplate, "intel": IntelPackageTemplate, "lua": LuaPackageTemplate, "makefile": MakefilePackageTemplate, "maven": MavenPackageTemplate, "meson": MesonPackageTemplate, "octave": OctavePackageTemplate, "perlbuild": PerlbuildPackageTemplate, "perlmake": PerlmakePackageTemplate, "python": PythonPackageTemplate, "qmake": QMakePackageTemplate, "r": RPackageTemplate, "racket": RacketPackageTemplate, "ruby": RubyPackageTemplate, "scons": SconsPackageTemplate, "sip": SIPPackageTemplate, "waf": WafPackageTemplate, } def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument("url", nargs="?", help="url of package archive") subparser.add_argument( "--keep-stage", action="store_true", help="don't clean up staging area when command completes", ) subparser.add_argument("-n", "--name", help="name of the package to create") subparser.add_argument( "-t", "--template", metavar="TEMPLATE", choices=sorted(templates.keys()), help="build system template to use\n\noptions: %(choices)s", ) subparser.add_argument( "-r", "--repo", help="path to a repository where the package should be created" ) subparser.add_argument( "-N", "--namespace", help="specify a namespace for the package\n\nmust be the namespace of " "a repository registered with Spack", ) subparser.add_argument( "-f", "--force", action="store_true", help="overwrite any existing package file with the same name", ) subparser.add_argument( "--skip-editor", action="store_true", help="skip the edit session for the package (e.g., automation)", ) subparser.add_argument( "-b", "--batch", action="store_true", help="don't ask which versions to checksum" ) #: C file extensions C_EXT = {".c"} #: C++ file extensions CXX_EXT = { ".C", ".c++", ".cc", ".ccm", ".cpp", ".CPP", ".cxx", ".h++", ".hh", ".hpp", ".hxx", ".inl", ".ipp", ".ixx", ".tcc", ".tpp", } #: Fortran file extensions FORTRAN_EXT = { ".f77", ".F77", ".f90", ".F90", ".f95", ".F95", ".f", ".F", ".for", ".FOR", ".ftn", ".FTN", } class BuildSystemAndLanguageGuesser: """An instance of BuildSystemAndLanguageGuesser provides a callable object to be used during ``spack create``. By passing this object to ``spack checksum``, we can take a peek at the fetched tarball and discern the build system it uses """ def __init__(self): """Sets the default build system.""" self.build_system = "generic" self._c = False self._cxx = False self._fortran = False # List of files in the archive ordered by their depth in the directory tree. self._file_entries: List[str] = [] def __call__(self, archive: str, url: str) -> None: """Try to guess the type of build system used by a project based on the contents of its archive or the URL it was downloaded from.""" # Peek inside the compressed file. if archive.endswith(".zip") or ".zip#" in archive: try: unzip = which("unzip") assert unzip is not None output = unzip("-lq", archive, output=str) except Exception: output = "" else: try: tar = which("tar") assert tar is not None output = tar("tf", archive, output=str) except Exception: output = "" self._file_entries[:] = output.splitlines() # Files closest to the root should be considered first when determining build system. self._file_entries.sort(key=lambda p: p.count("/")) self._determine_build_system(url) self._determine_language() def _determine_build_system(self, url: str) -> None: # Most octave extensions are hosted on Octave-Forge: # https://octave.sourceforge.net/index.html # They all have the same base URL. if "downloads.sourceforge.net/octave/" in url: self.build_system = "octave" elif url.endswith(".gem"): self.build_system = "ruby" elif url.endswith(".whl") or ".whl#" in url: self.build_system = "python" elif url.endswith(".rock"): self.build_system = "lua" elif self._file_entries: # A list of clues that give us an idea of the build system a package # uses. If the regular expression matches a file contained in the # archive, the corresponding build system is assumed. # NOTE: Order is important here. If a package supports multiple # build systems, we choose the first match in this list. clues = [ (re.compile(pattern), build_system) for pattern, build_system in ( (r"/CMakeLists\.txt$", "cmake"), (r"/NAMESPACE$", "r"), (r"/Cargo\.toml$", "cargo"), (r"/go\.mod$", "go"), (r"/configure$", "autotools"), (r"/configure\.(in|ac)$", "autoreconf"), (r"/Makefile\.am$", "autoreconf"), (r"/pom\.xml$", "maven"), (r"/SConstruct$", "scons"), (r"/waf$", "waf"), (r"/pyproject.toml", "python"), (r"/setup\.(py|cfg)$", "python"), (r"/WORKSPACE$", "bazel"), (r"/Build\.PL$", "perlbuild"), (r"/Makefile\.PL$", "perlmake"), (r"/.*\.gemspec$", "ruby"), (r"/Rakefile$", "ruby"), (r"/setup\.rb$", "ruby"), (r"/.*\.pro$", "qmake"), (r"/.*\.rockspec$", "lua"), (r"/(GNU)?[Mm]akefile$", "makefile"), (r"/DESCRIPTION$", "octave"), (r"/meson\.build$", "meson"), (r"/configure\.py$", "sip"), ) ] # Determine the build system based on the files contained in the archive. for file in self._file_entries: for pattern, build_system in clues: if pattern.search(file): self.build_system = build_system return def _determine_language(self): for entry in self._file_entries: _, ext = os.path.splitext(entry) if not self._c and ext in C_EXT: self._c = True elif not self._cxx and ext in CXX_EXT: self._cxx = True elif not self._fortran and ext in FORTRAN_EXT: self._fortran = True if self._c and self._cxx and self._fortran: return @property def languages(self) -> List[str]: langs: List[str] = [] if self._c: langs.append("c") if self._cxx: langs.append("cxx") if self._fortran: langs.append("fortran") return langs def get_name(name, url): """Get the name of the package based on the supplied arguments. If a name was provided, always use that. Otherwise, if a URL was provided, extract the name from that. Otherwise, use a default. Args: name (str): explicit ``--name`` argument given to ``spack create`` url (str): ``url`` argument given to ``spack create`` Returns: str: The name of the package """ # Default package name result = "example" if name is not None: # Use a user-supplied name if one is present result = name if len(name.strip()) > 0: tty.msg("Using specified package name: '{0}'".format(result)) else: tty.die("A package name must be provided when using the option.") elif url is not None: # Try to guess the package name based on the URL try: result = parse_name(url) if result != url: desc = "URL" else: desc = "package name" tty.msg("This looks like a {0} for {1}".format(desc, result)) except UndetectableNameError: tty.die( "Couldn't guess a name for this package.", " Please report this bug. In the meantime, try running:", " `spack create --name `", ) result = simplify_name(result) if not re.match(r"^[a-z0-9-]+$", result): tty.die("Package name can only contain a-z, 0-9, and '-'") return result def get_url(url: Optional[str]) -> str: """Get the URL to use. Use a default URL if none is provided. Args: url: ``url`` argument to ``spack create`` Returns: The URL of the package """ # Use the user-supplied URL or a default URL if none is present. return url or "https://www.example.com/example-1.2.3.tar.gz" def get_versions(args: argparse.Namespace, name: str) -> Tuple[str, BuildSystemAndLanguageGuesser]: """Returns a list of versions and hashes for a package. Also returns a BuildSystemAndLanguageGuesser object. Returns default values if no URL is provided. Args: args: The arguments given to ``spack create`` name: The name of the package Returns: Tuple of versions and hashes, and a BuildSystemAndLanguageGuesser object """ # Default version with hash hashed_versions = """\ # FIXME: Add proper versions and checksums here. # version("1.2.3", md5="0123456789abcdef0123456789abcdef")""" # Default version without hash unhashed_versions = """\ # FIXME: Add proper versions here. # version("1.2.4")""" # Default guesser guesser = BuildSystemAndLanguageGuesser() valid_url = True try: parsed = urllib.parse.urlparse(args.url) if not parsed.scheme or parsed.scheme == "file": valid_url = False # No point in spidering these except (ValueError, TypeError): valid_url = False if args.url is not None and args.template != "bundle" and valid_url: # Find available versions try: url_dict = find_versions_of_archive(args.url) if len(url_dict) > 1 and not args.batch and sys.stdin.isatty(): url_dict_filtered = spack.stage.interactive_version_filter(url_dict) if url_dict_filtered is None: exit(0) url_dict = url_dict_filtered except UndetectableVersionError: # Use fake versions tty.warn("Couldn't detect version in: {0}".format(args.url)) return hashed_versions, guesser if not url_dict: # If no versions were found, revert to what the user provided version = parse_version(args.url) url_dict = {version: args.url} version_hashes = spack.stage.get_checksums_for_versions( url_dict, name, first_stage_function=guesser, keep_stage=args.keep_stage ) versions = get_version_lines(version_hashes) else: versions = unhashed_versions return versions, guesser def get_build_system( template: Optional[str], url: str, guesser: BuildSystemAndLanguageGuesser ) -> str: """Determine the build system template. If a template is specified, always use that. Otherwise, if a URL is provided, download the tarball and peek inside to guess what build system it uses. Otherwise, use a generic template by default. Args: template: ``--template`` argument given to ``spack create`` url: ``url`` argument given to ``spack create`` guesser: The first_stage_function given to ``spack checksum`` which records the build system it detects Returns: str: The name of the build system template to use """ # Default template selected_template = "generic" if template is not None: selected_template = template # Use a user-supplied template if one is present tty.msg("Using specified package template: '{0}'".format(selected_template)) elif url is not None: # Use whatever build system the guesser detected selected_template = guesser.build_system if selected_template == "generic": tty.warn("Unable to detect a build system. Using a generic package template.") else: msg = "This package looks like it uses the {0} build system" tty.msg(msg.format(selected_template)) return selected_template def get_repository(args: argparse.Namespace, name: str) -> spack.repo.Repo: """Returns a Repo object that will allow us to determine the path where the new package file should be created. Args: args: The arguments given to ``spack create`` name: The name of the package to create Returns: A Repo object capable of determining the path to the package file """ spec = Spec(name) # Figure out namespace for spec if spec.namespace and args.namespace and spec.namespace != args.namespace: tty.die("Namespaces '{0}' and '{1}' do not match.".format(spec.namespace, args.namespace)) if not spec.namespace and args.namespace: spec.namespace = args.namespace # Figure out where the new package should live repo_path = args.repo if repo_path is not None: repo = spack.repo.from_path(repo_path) if spec.namespace and spec.namespace != repo.namespace: tty.die( "Can't create package with namespace {0} in repo with namespace {1}".format( spec.namespace, repo.namespace ) ) else: if spec.namespace: repo = spack.repo.PATH.get_repo(spec.namespace) else: _repo = spack.repo.PATH.first_repo() assert _repo is not None, "No package repository found" repo = _repo # Set the namespace on the spec if it's not there already if not spec.namespace: spec.namespace = repo.namespace return repo def create(parser, args): # Gather information about the package to be created name = get_name(args.name, args.url) url = get_url(args.url) versions, guesser = get_versions(args, name) build_system = get_build_system(args.template, url, guesser) # Create the package template object constr_args = {"name": name, "versions": versions, "languages": guesser.languages} package_class = templates[build_system] if package_class != BundlePackageTemplate: constr_args["url"] = url package = package_class(**constr_args) tty.msg("Created template for {0} package".format(package.name)) # Create a directory for the new package repo = get_repository(args, name) pkg_path = repo.filename_for_package_name(package.name) if os.path.exists(pkg_path) and not args.force: tty.die( "{0} already exists.".format(pkg_path), " Try running `spack create --force` to overwrite it.", ) else: mkdirp(os.path.dirname(pkg_path)) # Write the new package file package.write(pkg_path) tty.msg("Created package file: {0}".format(pkg_path)) # Optionally open up the new package file in your $EDITOR if not args.skip_editor: editor(pkg_path) ================================================ FILE: lib/spack/spack/cmd/debug.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import platform import re from typing import Optional import spack import spack.config import spack.platforms import spack.repo import spack.spec import spack.util.git description = "debugging commands for troubleshooting Spack" section = "developer" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="debug_command") sp.add_parser("report", help="print information useful for bug reports") def _format_repo_info(source, commit): if source.endswith(".git"): return f"{source[:-4]}/commit/{commit}" return f"{source} ({commit[:7]})" def _get_builtin_repo_info() -> Optional[str]: """Get the builtin package repository git commit sha.""" # Get builtin from config descriptors = spack.repo.RepoDescriptors.from_config( spack.repo.package_repository_lock(), spack.config.CONFIG ) if "builtin" not in descriptors: return None builtin = descriptors["builtin"] source = None if isinstance(builtin, spack.repo.RemoteRepoDescriptor) and builtin.fetched(): destination = builtin.destination source = builtin.repository elif isinstance(builtin, spack.repo.LocalRepoDescriptor): destination = builtin.path source = builtin.path else: return None # no git info git = spack.util.git.git(required=False) if not git: return None rev = git( "-C", destination, "rev-parse", "HEAD", output=str, error=os.devnull, fail_on_error=False ) if git.returncode != 0: return None match = re.match(r"[a-f\d]{7,}$", rev) return _format_repo_info(source, match.group(0)) if match else None def _get_spack_repo_info() -> str: """Get the spack package repository git info.""" commit = spack.get_spack_commit() if not commit: return spack.spack_version repo_info = _format_repo_info("https://github.com/spack/spack.git", commit) return f"{spack.spack_version} ({repo_info})" def report(args): host_platform = spack.platforms.host() host_os = host_platform.default_operating_system() host_target = host_platform.default_target() architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target))) print("* **Spack:**", _get_spack_repo_info()) print("* **Builtin repo:**", _get_builtin_repo_info() or "not available") print("* **Python:**", platform.python_version()) print("* **Platform:**", architecture) def debug(parser, args): if args.debug_command == "report": report(args) ================================================ FILE: lib/spack/spack/cmd/deconcretize.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys from typing import List import spack.cmd import spack.cmd.common.confirmation as confirmation import spack.environment as ev import spack.llnl.util.tty as tty import spack.spec from spack.cmd.common import arguments description = "remove specs from the lockfile of an environment" section = "environments" level = "long" # Arguments for display_specs when we find ambiguity display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4} def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--root", action="store_true", help="deconcretize only specific environment roots" ) arguments.add_common_arguments(subparser, ["yes_to_all", "specs"]) subparser.add_argument( "-a", "--all", action="store_true", dest="all", help="deconcretize ALL specs that match each supplied spec", ) def get_deconcretize_list( args: argparse.Namespace, specs: List[spack.spec.Spec], env: ev.Environment ) -> List[spack.spec.Spec]: """ Get list of environment roots to deconcretize """ env_specs = [s for _, s in env.concretized_specs()] to_deconcretize = [] errors = [] for s in specs: if args.root: # find all roots matching given spec to_deconc = [e for e in env_specs if e.satisfies(s)] else: # find all roots matching or depending on a matching spec to_deconc = [e for e in env_specs if any(d.satisfies(s) for d in e.traverse())] if len(to_deconc) < 1: tty.warn(f"No matching specs to deconcretize for {s}") elif len(to_deconc) > 1 and not args.all: errors.append((s, to_deconc)) to_deconcretize.extend(to_deconc) if errors: for spec, matching in errors: tty.error(f"{spec} matches multiple concrete specs:") sys.stderr.write("\n") spack.cmd.display_specs(matching, output=sys.stderr, **display_args) sys.stderr.write("\n") sys.stderr.flush() tty.die("Use '--all' to deconcretize all matching specs, or be more specific") return to_deconcretize def deconcretize_specs(args, specs): env = spack.cmd.require_active_env(cmd_name="deconcretize") if args.specs: deconcretize_list = get_deconcretize_list(args, specs, env) else: deconcretize_list = [s for _, s in env.concretized_specs()] if not args.yes_to_all: confirmation.confirm_action(deconcretize_list, "deconcretized", "deconcretization") with env.write_transaction(): for spec in deconcretize_list: env.deconcretize_by_hash(spec.dag_hash()) env.write() def deconcretize(parser, args): if not args.specs and not args.all: tty.die( "deconcretize requires at least one spec argument.", " Use `spack deconcretize --all` to deconcretize ALL specs.", ) specs = spack.cmd.parse_specs(args.specs) if args.specs else [None] deconcretize_specs(args, specs) ================================================ FILE: lib/spack/spack/cmd/dependencies.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys import spack.cmd import spack.environment as ev import spack.llnl.util.tty as tty import spack.store from spack.cmd.common import arguments from spack.llnl.util.tty.colify import colify from spack.solver.input_analysis import create_graph_analyzer description = "show dependencies of a package" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-i", "--installed", action="store_true", default=False, help="list installed dependencies of an installed spec " "instead of possible dependencies of a package", ) subparser.add_argument( "-t", "--transitive", action="store_true", default=False, help="show all transitive dependencies", ) arguments.add_common_arguments(subparser, ["deptype"]) subparser.add_argument( "-V", "--no-expand-virtuals", action="store_false", default=True, dest="expand_virtuals", help="do not expand virtual dependencies", ) arguments.add_common_arguments(subparser, ["spec"]) def dependencies(parser, args): specs = spack.cmd.parse_specs(args.spec) if len(specs) != 1: tty.die("spack dependencies takes only one spec.") if args.installed: env = ev.active_environment() spec = spack.cmd.disambiguate_spec(specs[0], env) format_string = "{name}{@version}{/hash:7}{%compiler}" if sys.stdout.isatty(): tty.msg("Dependencies of %s" % spec.format(format_string, color=True)) deps = spack.store.STORE.db.installed_relatives( spec, "children", args.transitive, deptype=args.deptype ) if deps: spack.cmd.display_specs(deps, long=True) else: print("No dependencies") else: spec = specs[0] dependencies, virtuals, _ = create_graph_analyzer().possible_dependencies( spec, transitive=args.transitive, expand_virtuals=args.expand_virtuals, allowed_deps=args.deptype, ) if not args.expand_virtuals: dependencies.update(virtuals) if spec.name in dependencies: dependencies.remove(spec.name) if dependencies: colify(sorted(dependencies)) else: print("No dependencies") ================================================ FILE: lib/spack/spack/cmd/dependents.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import collections import sys import spack.cmd import spack.environment as ev import spack.llnl.util.tty as tty import spack.repo import spack.store from spack.cmd.common import arguments from spack.llnl.util.tty.colify import colify description = "show packages that depend on another" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-i", "--installed", action="store_true", default=False, help="list installed dependents of an installed spec " "instead of possible dependents of a package", ) subparser.add_argument( "-t", "--transitive", action="store_true", default=False, help="show all transitive dependents", ) arguments.add_common_arguments(subparser, ["spec"]) def inverted_dependencies(): """Iterate through all packages and return a dictionary mapping package names to possible dependencies. Virtual packages are included as sources, so that you can query dependents of, e.g., ``mpi``, but virtuals are not included as actual dependents. """ dag = collections.defaultdict(set) for pkg_cls in spack.repo.PATH.all_package_classes(): for _, deps_by_name in pkg_cls.dependencies.items(): for dep in deps_by_name: deps = [dep] # expand virtuals if necessary if spack.repo.PATH.is_virtual(dep): deps += [s.name for s in spack.repo.PATH.providers_for(dep)] for d in deps: dag[d].add(pkg_cls.name) return dag def get_dependents(pkg_name, ideps, transitive=False, dependents=None): """Get all dependents for a package. Args: pkg_name (str): name of the package whose dependents should be returned ideps (dict): dictionary of dependents, from inverted_dependencies() transitive (bool or None): return transitive dependents when True """ if dependents is None: dependents = set() if pkg_name in dependents: return set() dependents.add(pkg_name) direct = ideps[pkg_name] if transitive: for dep_name in direct: get_dependents(dep_name, ideps, transitive, dependents) dependents.update(direct) return dependents def dependents(parser, args): specs = spack.cmd.parse_specs(args.spec) if len(specs) != 1: tty.die("spack dependents takes only one spec.") if args.installed: env = ev.active_environment() spec = spack.cmd.disambiguate_spec(specs[0], env) format_string = "{name}{@version}{/hash:7}{%compiler}" if sys.stdout.isatty(): tty.msg("Dependents of %s" % spec.cformat(format_string)) deps = spack.store.STORE.db.installed_relatives(spec, "parents", args.transitive) if deps: spack.cmd.display_specs(deps, long=True) else: print("No dependents") else: spec = specs[0] ideps = inverted_dependencies() dependents = get_dependents(spec.name, ideps, args.transitive) dependents.remove(spec.name) if dependents: colify(sorted(dependents)) else: print("No dependents") ================================================ FILE: lib/spack/spack/cmd/deprecate.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Deprecate one Spack install in favor of another Spack packages of different configurations cannot be installed to the same location. However, in some circumstances (e.g. security patches) old installations should never be used again. In these cases, we will mark the old installation as deprecated, remove it, and link another installation into its place. It is up to the user to ensure binary compatibility between the deprecated installation and its deprecator. """ import argparse import spack.cmd import spack.concretize import spack.environment as ev import spack.installer import spack.llnl.util.tty as tty import spack.store from spack.cmd.common import arguments from spack.error import SpackError from spack.llnl.util.filesystem import symlink from ..enums import InstallRecordStatus description = "replace one package with another via symlinks" section = "admin" level = "long" # Arguments for display_specs when we find ambiguity display_args = {"long": True, "show_flags": True, "variants": True, "indent": 4} def setup_parser(sp: argparse.ArgumentParser) -> None: setattr(setup_parser, "parser", sp) arguments.add_common_arguments(sp, ["yes_to_all"]) deps = sp.add_mutually_exclusive_group() deps.add_argument( "-d", "--dependencies", action="store_true", default=True, dest="dependencies", help="deprecate dependencies (default)", ) deps.add_argument( "-D", "--no-dependencies", action="store_false", default=True, dest="dependencies", help="do not deprecate dependencies", ) install = sp.add_mutually_exclusive_group() install.add_argument( "-i", "--install-deprecator", action="store_true", default=False, dest="install", help="concretize and install deprecator spec", ) install.add_argument( "-I", "--no-install-deprecator", action="store_false", default=False, dest="install", help="deprecator spec must already be installed (default)", ) sp.add_argument( "-l", "--link-type", type=str, default=None, choices=["soft", "hard"], help="(deprecated)" ) sp.add_argument( "specs", nargs=argparse.REMAINDER, help="spec to deprecate and spec to use as deprecator" ) def deprecate(parser, args): """Deprecate one spec in favor of another""" if args.link_type is not None: tty.warn("The --link-type option is deprecated and will be removed in a future release.") env = ev.active_environment() specs = spack.cmd.parse_specs(args.specs) if len(specs) != 2: raise SpackError("spack deprecate requires exactly two specs") deprecate = spack.cmd.disambiguate_spec( specs[0], env, local=True, installed=(InstallRecordStatus.INSTALLED | InstallRecordStatus.DEPRECATED), ) if args.install: deprecator = spack.concretize.concretize_one(specs[1]) else: deprecator = spack.cmd.disambiguate_spec(specs[1], env, local=True) # calculate all deprecation pairs for errors and warning message all_deprecate = [] all_deprecators = [] generator = ( deprecate.traverse(order="post", deptype="link", root=True) if args.dependencies else [deprecate] ) for spec in generator: all_deprecate.append(spec) all_deprecators.append(deprecator[spec.name]) # This will throw a key error if deprecator does not have a dep # that matches the name of a dep of the spec if not args.yes_to_all: tty.msg("The following packages will be deprecated:\n") spack.cmd.display_specs(all_deprecate, **display_args) tty.msg("In favor of (respectively):\n") spack.cmd.display_specs(all_deprecators, **display_args) print() already_deprecated = [] already_deprecated_for = [] for spec in all_deprecate: deprecated_for = spack.store.STORE.db.deprecator(spec) if deprecated_for: already_deprecated.append(spec) already_deprecated_for.append(deprecated_for) tty.msg("The following packages are already deprecated:\n") spack.cmd.display_specs(already_deprecated, **display_args) tty.msg("In favor of (respectively):\n") spack.cmd.display_specs(already_deprecated_for, **display_args) answer = tty.get_yes_or_no("Do you want to proceed?", default=False) if not answer: tty.die("Will not deprecate any packages.") for dcate, dcator in zip(all_deprecate, all_deprecators): spack.installer.deprecate(dcate, dcator, symlink) ================================================ FILE: lib/spack/spack/cmd/dev_build.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import sys import spack.build_environment import spack.cmd import spack.cmd.common.arguments import spack.concretize import spack.config import spack.installer_dispatch import spack.llnl.util.tty as tty import spack.repo from spack.cmd.common import arguments description = "build package from code in current working directory" section = "build" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["jobs", "no_checksum", "spec"]) subparser.add_argument( "-d", "--source-path", dest="source_path", default=None, help="path to source directory (defaults to the current directory)", ) subparser.add_argument( "-i", "--ignore-dependencies", action="store_true", dest="ignore_deps", help="do not try to install dependencies of requested packages", ) subparser.add_argument( "--keep-prefix", action="store_true", help="do not remove the install prefix if installation fails", ) subparser.add_argument( "--skip-patch", action="store_true", help="skip patching for the developer build" ) subparser.add_argument( "-q", "--quiet", action="store_true", dest="quiet", help="do not display verbose build output while installing", ) subparser.add_argument( "--drop-in", type=str, dest="shell", default=None, help="drop into a build environment in a new shell, e.g., bash", ) subparser.add_argument( "--test", default=None, choices=["root", "all"], help="run tests on only root packages or all packages", ) stop_group = subparser.add_mutually_exclusive_group() stop_group.add_argument( "-b", "--before", type=str, dest="before", default=None, help="phase to stop before when installing (default None)", ) stop_group.add_argument( "-u", "--until", type=str, dest="until", default=None, help="phase to stop after when installing (default None)", ) cd_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ["clean", "dirty"]) spack.cmd.common.arguments.add_concretizer_args(subparser) def dev_build(self, args): if not args.spec: tty.die("spack dev-build requires a package spec argument.") specs = spack.cmd.parse_specs(args.spec) if len(specs) > 1: tty.die("spack dev-build only takes one spec.") spec = specs[0] if not spack.repo.PATH.exists(spec.name): raise spack.repo.UnknownPackageError(spec.name) if not spec.versions.concrete_range_as_version: tty.die( "spack dev-build spec must have a single, concrete version. " "Did you forget a package version number?" ) source_path = args.source_path if source_path is None: source_path = os.getcwd() source_path = os.path.abspath(source_path) # Forces the build to run out of the source directory. spec.constrain(f'dev_path="{source_path}"') spec = spack.concretize.concretize_one(spec) if spec.installed: tty.error("Already installed in %s" % spec.prefix) tty.msg("Uninstall or try adding a version suffix for this dev build.") sys.exit(1) # disable checksumming if requested if args.no_checksum: spack.config.set("config:checksum", False, scope="command_line") tests = False if args.test == "all": tests = True elif args.test == "root": tests = [spec.name for spec in specs] spack.installer_dispatch.create_installer( [spec.package], tests=tests, keep_prefix=args.keep_prefix, install_deps=not args.ignore_deps, verbose=not args.quiet, dirty=args.dirty, stop_before=args.before, skip_patch=args.skip_patch, stop_at=args.until, ).install() # drop into the build environment of the package? if args.shell is not None: spack.build_environment.setup_package(spec.package, dirty=False) os.execvp(args.shell, [args.shell]) ================================================ FILE: lib/spack/spack/cmd/develop.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import shutil from typing import Optional import spack.cmd import spack.config import spack.environment import spack.fetch_strategy import spack.llnl.util.tty as tty import spack.repo import spack.spec import spack.stage import spack.util.path import spack.version from spack.cmd.common import arguments from spack.error import SpackError description = "add a spec to an environment's dev-build information" section = "environments" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument("-p", "--path", help="source location of package") subparser.add_argument("-b", "--build-directory", help="build directory for the package") clone_group = subparser.add_mutually_exclusive_group() clone_group.add_argument( "--no-clone", action="store_false", dest="clone", help="do not clone, the package already exists at the source path", ) clone_group.add_argument( "--clone", action="store_true", dest="clone", default=True, help=( "(default) clone the package unless the path already exists, " "use ``--force`` to overwrite" ), ) subparser.add_argument( "--no-modify-concrete-specs", action="store_false", default=True, dest="apply_changes", help=( "do not mutate concrete specs to have dev_path provenance." " This requires a later `spack concretize --force` command to use develop specs" ), ) subparser.add_argument( "-f", "--force", action="store_true", default=False, help="remove any files or directories that block cloning source code", ) subparser.add_argument( "-r", "--recursive", action="store_true", help="traverse nodes of the graph to mark everything up to the root as a develop spec", ) arguments.add_common_arguments(subparser, ["spec"]) def _retrieve_develop_source(spec: spack.spec.Spec, abspath: str) -> None: # "steal" the source code via staging API. We ask for a stage # to be created, then copy it afterwards somewhere else. It would be # better if we can create the `source_path` directly into its final # destination. pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) # We construct a package class ourselves, rather than asking for # Spec.package, since Spec only allows this when it is concrete package = pkg_cls(spec) source_stage: spack.stage.Stage = package.stage[0] if isinstance(source_stage.fetcher, spack.fetch_strategy.GitFetchStrategy): source_stage.fetcher.get_full_repo = True # If we retrieved this version before and cached it, we may have # done so without cloning the full git repo; likewise, any # mirror might store an instance with truncated history. source_stage.default_fetcher_only = True source_stage.fetcher.set_package(package) package.stage.steal_source(abspath) def assure_concrete_spec(env: spack.environment.Environment, spec: spack.spec.Spec): version = spec.versions.concrete_range_as_version if not version: # first check environment for a matching concrete spec matching_specs = env.all_matching_specs(spec) if matching_specs: version = matching_specs[0].version test_spec = spack.spec.Spec(f"{spec}@{version}") for m_spec in matching_specs: if not m_spec.satisfies(test_spec): raise SpackError( f"{spec.name}: has multiple concrete instances in the graph that can't be" " satisfied by a single develop spec. To use `spack develop` ensure one" " of the following:" f"\n a) {spec.name} nodes can satisfy the same develop spec (minimally " "this means they all share the same version)" f"\n b) Provide a concrete develop spec ({spec.name}@[version]) to clearly" " indicate what should be developed" ) else: # look up the maximum version so infintiy versions are preferred for develop version = max(spack.repo.PATH.get_pkg_class(spec.fullname).versions.keys()) tty.msg(f"Defaulting to highest version: {spec.name}@{version}") spec.versions = spack.version.VersionList([version]) def setup_src_code(spec: spack.spec.Spec, src_path: str, clone: bool = True, force: bool = False): """ Handle checking, cloning or overwriting source code """ assert spec.versions if clone: _clone(spec, src_path, force) if not clone and not os.path.exists(src_path): raise SpackError(f"Provided path {src_path} does not exist") version = spec.versions.concrete_range_as_version if not version: # look up the maximum version so infintiy versions are preferred for develop version = max(spack.repo.PATH.get_pkg_class(spec.fullname).versions.keys()) tty.msg(f"Defaulting to highest version: {spec.name}@{version}") spec.versions = spack.version.VersionList([version]) def _update_config(spec, path): find_fn = lambda section: spec.name in section entry = {"spec": str(spec)} if path and path != spec.name: entry["path"] = path def change_fn(section): section[spec.name] = entry spack.config.change_or_add("develop", find_fn, change_fn) def update_env( env: spack.environment.Environment, spec: spack.spec.Spec, specified_path: Optional[str] = None, build_dir: Optional[str] = None, apply_changes: bool = True, ): """ Update the spack.yaml file with additions or changes from a develop call """ tty.debug(f"Updating develop config for {env.name} transactionally") if not specified_path: dev_entry = env.dev_specs.get(spec.name) if dev_entry: specified_path = dev_entry.get("path", None) with env.write_transaction(): if build_dir is not None: spack.config.add( f"packages:{spec.name}:package_attributes:build_directory:{build_dir}", env.scope_name, ) # add develop spec and update path _update_config(spec, specified_path) # If we are automatically mutating the concrete specs for dev provenance, do so if apply_changes: env.apply_develop(spec, _abs_code_path(env, spec, specified_path)) def _clone(spec: spack.spec.Spec, abspath: str, force: bool = False): if os.path.exists(abspath): if force: shutil.rmtree(abspath) else: msg = f"Skipping developer download of {spec.name}" msg += f" because its path {abspath} already exists." tty.msg(msg) return # cloning can take a while and it's nice to get a message for the longer clones tty.msg(f"Cloning source code for {spec}") _retrieve_develop_source(spec, abspath) def _abs_code_path( env: spack.environment.Environment, spec: spack.spec.Spec, path: Optional[str] = None ): src_path = path if path else spec.name return spack.util.path.canonicalize_path(src_path, default_wd=env.path) def _dev_spec_generator(args, env): """ Generator function to loop over all the develop specs based on how the command is called If no specs are supplied then loop over the develop specs listed in the environment. """ if not args.spec: if args.clone is False: raise SpackError("No spec provided to spack develop command") for name, entry in env.dev_specs.items(): path = entry.get("path", name) abspath = spack.util.path.canonicalize_path(path, default_wd=env.path) # Both old syntax `spack develop pkg@x` and new syntax `spack develop pkg@=x` # are currently supported. spec = spack.spec.parse_with_version_concrete(entry["spec"]) yield spec, abspath else: specs = spack.cmd.parse_specs(args.spec) if (args.path or args.build_directory) and len(specs) > 1: raise SpackError( "spack develop requires at most one named spec when using the --path or" " --build-directory arguments" ) for spec in specs: if args.recursive: concrete_specs = env.all_matching_specs(spec) if not concrete_specs: tty.warn( f"{spec.name} has no matching concrete specs in the environment and " "will be skipped. `spack develop --recursive` requires a concretized" " environment" ) else: for s in concrete_specs: for node_spec in s.traverse(direction="parents", root=True): tty.debug(f"Recursive develop for {node_spec.name}") dev_spec = spack.spec.Spec(node_spec.format("{name}@{versions}")) yield dev_spec, _abs_code_path(env, node_spec, args.path) else: yield spec, _abs_code_path(env, spec, args.path) def develop(parser, args): env = spack.cmd.require_active_env(cmd_name="develop") for spec, abspath in _dev_spec_generator(args, env): assure_concrete_spec(env, spec) setup_src_code(spec, abspath, clone=args.clone, force=args.force) update_env(env, spec, args.path, args.build_directory, args.apply_changes) ================================================ FILE: lib/spack/spack/cmd/diff.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys import spack.cmd import spack.environment as ev import spack.llnl.util.tty as tty import spack.solver.asp as asp import spack.util.spack_json as sjson from spack.cmd.common import arguments from spack.llnl.util.tty.color import cprint, get_color_when description = "compare two specs" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["specs"]) subparser.add_argument( "--json", action="store_true", default=False, dest="dump_json", help="dump json output instead of pretty printing", ) subparser.add_argument( "--first", action="store_true", default=False, dest="load_first", help="load the first match if multiple packages match the spec", ) subparser.add_argument( "-a", "--attribute", action="append", help="select the attributes to show (defaults to all)", ) subparser.add_argument( "--ignore", action="append", help="omit diffs related to these dependencies" ) def shift(asp_function: asp.AspFunction) -> asp.AspFunction: """Transforms ``attr("foo", "bar")`` into ``foo("bar")``.""" args = asp_function.args if not args: raise ValueError(f"Can't shift ASP function with no arguments: {str(asp_function)}") return asp.AspFunction(args[0], args[1:]) def compare_specs(a, b, to_string=False, color=None, ignore_packages=None): """ Generate a comparison, including diffs (for each side) and an intersection. We can either print the result to the console, or parse into a json object for the user to save. We return an object that shows the differences, intersection, and names for a pair of specs a and b. Arguments: a (spack.spec.Spec): the first spec to compare b (spack.spec.Spec): the second spec to compare a_name (str): the name of spec a b_name (str): the name of spec b to_string (bool): return an object that can be json dumped color (bool): whether to format the names for the console """ if color is None: color = get_color_when() a = a.copy() b = b.copy() if ignore_packages: for pkg_name in ignore_packages: a.trim(pkg_name) b.trim(pkg_name) # Prepare a solver setup to parse differences setup = asp.SpackSolverSetup() # get facts for specs, making sure to include build dependencies of concrete # specs and to descend into dependency hashes so we include all facts. a_facts = set( shift(func) for func in setup.spec_clauses( a, body=True, expand_hashes=True, concrete_build_deps=True, include_runtimes=True ) if func.name == "attr" ) b_facts = set( shift(func) for func in setup.spec_clauses( b, body=True, expand_hashes=True, concrete_build_deps=True, include_runtimes=True ) if func.name == "attr" ) # We want to present them to the user as simple key: values intersect = sorted(a_facts.intersection(b_facts)) spec1_not_spec2 = sorted(a_facts.difference(b_facts)) spec2_not_spec1 = sorted(b_facts.difference(a_facts)) # Format the spec names to be colored fmt = "{name}{@version}{/hash}" a_name = a.format(fmt, color=color) b_name = b.format(fmt, color=color) # We want to show what is the same, and then difference for each return { "intersect": flatten(intersect) if to_string else intersect, "a_not_b": flatten(spec1_not_spec2) if to_string else spec1_not_spec2, "b_not_a": flatten(spec2_not_spec1) if to_string else spec2_not_spec1, "a_name": a_name, "b_name": b_name, } def flatten(functions): """ Given a list of ASP functions, convert into a list of key: value tuples. We are squashing whatever is after the first index into one string for easier parsing in the interface """ updated = [] for fun in functions: updated.append([fun.name, " ".join(str(a) for a in fun.args)]) return updated def print_difference(c, attributes="all", out=None): """ Print the difference. Given a diffset for A and a diffset for B, print red/green diffs to show the differences. """ # Default to standard out unless another stream is provided out = out or sys.stdout A = c["b_not_a"] B = c["a_not_b"] cprint("@R{--- %s}" % c["a_name"]) # bright red cprint("@G{+++ %s}" % c["b_name"]) # bright green # Cut out early if we don't have any differences! if not A and not B: print("No differences\n") return def group_by_type(diffset): grouped = {} for entry in diffset: if entry[0] not in grouped: grouped[entry[0]] = [] grouped[entry[0]].append(entry[1]) # Sort by second value to make comparison slightly closer for key, values in grouped.items(): values.sort() return grouped A = group_by_type(A) B = group_by_type(B) # print a directionally relevant diff keys = list(A) + list(B) category = None for key in keys: if "all" not in attributes and key not in attributes: continue # Write the attribute, B is subtraction A is addition subtraction = [] if key not in B else B[key] addition = [] if key not in A else A[key] # Bail out early if we don't have any entries if not subtraction and not addition: continue # If we have a new category, create a new section if category != key: category = key # print category in bold, colorized cprint("@*b{@@ %s @@}" % category) # bold blue # Print subtractions first while subtraction: cprint("@R{- %s}" % subtraction.pop(0)) # bright red if addition: cprint("@G{+ %s}" % addition.pop(0)) # bright green # Any additions left? while addition: cprint("@G{+ %s}" % addition.pop(0)) def diff(parser, args): env = ev.active_environment() if len(args.specs) != 2: tty.die("You must provide two specs to diff.") specs = [] for spec in spack.cmd.parse_specs(args.specs): # If the spec has a hash, check it before disambiguating spec.replace_hash() if spec.concrete: specs.append(spec) else: specs.append(spack.cmd.disambiguate_spec(spec, env, first=args.load_first)) # Calculate the comparison (c) color = False if args.dump_json else get_color_when() c = compare_specs(specs[0], specs[1], to_string=True, color=color, ignore_packages=args.ignore) # Default to all attributes attributes = args.attribute or ["all"] if args.dump_json: print(sjson.dump(c)) else: tty.warn("This interface is subject to change.\n") print_difference(c, attributes) ================================================ FILE: lib/spack/spack/cmd/docs.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import webbrowser description = "open spack documentation in a web browser" section = "help" level = "short" def docs(parser, args): webbrowser.open("https://spack.readthedocs.io") ================================================ FILE: lib/spack/spack/cmd/edit.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import errno import glob import os from typing import Optional, Union import spack.cmd import spack.llnl.util.tty as tty import spack.paths import spack.repo import spack.util.editor description = "open package files in ``$EDITOR``" section = "packaging" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: excl_args = subparser.add_mutually_exclusive_group() # Various types of Spack files that can be edited # Edits package files by default # build systems require separate logic to find excl_args.add_argument( "-b", "--build-system", dest="path", action="store_const", const="BUILD_SYSTEM", # placeholder for path that requires computing late help="edit the build system with the supplied name or fullname", ) excl_args.add_argument( "-c", "--command", dest="path", action="store_const", const=spack.paths.command_path, help="edit the command with the supplied name", ) excl_args.add_argument( "-d", "--docs", dest="path", action="store_const", const=os.path.join(spack.paths.lib_path, "docs"), help="edit the docs with the supplied name", ) excl_args.add_argument( "-t", "--test", dest="path", action="store_const", const=spack.paths.test_path, help="edit the test with the supplied name", ) excl_args.add_argument( "-m", "--module", dest="path", action="store_const", const=spack.paths.module_path, help="edit the main spack module with the supplied name", ) # Options for editing packages and build systems subparser.add_argument( "-r", "--repo", default=None, help="path to repo to edit package or build system in" ) subparser.add_argument( "-N", "--namespace", default=None, help="namespace of package or build system to edit" ) subparser.add_argument("package", nargs="*", default=None, help="package name") def locate_package(name: str, repo: Optional[spack.repo.Repo]) -> str: # if not given a repo, use the full repo path to choose one repo_like: Union[spack.repo.Repo, spack.repo.RepoPath] = repo or spack.repo.PATH path: str = repo_like.filename_for_package_name(name) try: with open(path, "r", encoding="utf-8"): return path except OSError as e: if e.errno == errno.ENOENT: raise spack.repo.UnknownPackageError(name) from e tty.die(f"Cannot edit package: {e}") def locate_build_system(name: str, repo: Optional[spack.repo.Repo]) -> str: # If given a fullname for a build system, split it into namespace and name namespace = None if "." in name: namespace, name = name.rsplit(".", 1) # If given a namespace and a repo, they better match if namespace and repo: if repo.namespace != namespace: msg = f"{namespace}.{name}: namespace conflicts with repo '{repo.namespace}'" msg += " specified from --repo or --namespace argument" raise ValueError(msg) if namespace: repo = spack.repo.PATH.get_repo(namespace) # If not given a namespace, use the default if not repo: repo = spack.repo.PATH.first_repo() assert repo return locate_file(name, repo.build_systems_path) def locate_file(name: str, path: str) -> str: # convert command names to python module name if path == spack.paths.command_path: name = spack.cmd.python_name(name) file_path = os.path.join(path, name) # Try to open direct match. try: with open(file_path, "r", encoding="utf-8"): return file_path except OSError as e: if e.errno != errno.ENOENT: tty.die(f"Cannot edit file: {e}") pass # Otherwise try to find a file that starts with the name candidates = glob.glob(file_path + "*") exclude_list = [".pyc", "~"] # exclude binaries and backups files = [f for f in candidates if not any(f.endswith(ext) for ext in exclude_list)] if len(files) > 1: tty.die( f"Multiple files start with `{name}`:\n" + "\n".join(f" {os.path.basename(f)}" for f in files) ) elif not files: tty.die(f"No file for '{name}' was found in {path}") return files[0] def edit(parser, args): names = args.package # If `--command`, `--test`, `--docs`, or `--module` is chosen, edit those instead if args.path and args.path != "BUILD_SYSTEM": paths = [locate_file(name, args.path) for name in names] if names else [args.path] spack.util.editor.editor(*paths) return # Cannot set repo = spack.repo.PATH.first_repo() as default because packages and build_systems # can include repo information as part of their fullname repo = None if args.namespace: repo = spack.repo.PATH.get_repo(args.namespace) elif args.repo: repo = spack.repo.from_path(args.repo) # default_repo used when no name provided default_repo = repo or spack.repo.PATH.first_repo() if args.path == "BUILD_SYSTEM": if names: paths = [locate_build_system(n, repo) for n in names] else: paths = [default_repo.build_systems_path] spack.util.editor.editor(*paths) return paths = [locate_package(n, repo) for n in names] if names else [default_repo.packages_path] spack.util.editor.editor(*paths) ================================================ FILE: lib/spack/spack/cmd/env.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import shlex import shutil import sys import tempfile from pathlib import Path from typing import List, Optional, Set, Tuple, Union import spack.cmd import spack.cmd.common import spack.cmd.common.arguments import spack.cmd.modules import spack.config import spack.environment as ev import spack.environment.depfile as depfile import spack.environment.environment import spack.environment.shell import spack.llnl.string as string import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.tengine from spack.cmd.common import arguments from spack.llnl.util.filesystem import islink, symlink from spack.llnl.util.tty.colify import colify from spack.llnl.util.tty.color import cescape, colorize from spack.traverse import traverse_nodes from spack.util.environment import EnvironmentModifications description = "manage environments" section = "environments" level = "short" #: List of subcommands of ``spack env`` subcommands: List[Tuple[str, ...]] = [ ("activate",), ("deactivate",), ("create",), ("remove", "rm"), ("rename", "mv"), ("list", "ls"), ("status", "st"), ("loads",), ("view",), ("update",), ("revert",), ("depfile",), ("track",), ("untrack",), ] # # env create # def env_create_setup_parser(subparser): """\ create a new environment create a new environment or, optionally, copy an existing environment a manifest file results in a new abstract environment while a lock file creates a new concrete environment """ subparser.add_argument( "env_name", metavar="env", help="name or directory of the new environment" ) subparser.add_argument( "-d", "--dir", action="store_true", help="create an environment in a specific directory" ) subparser.add_argument( "--keep-relative", action="store_true", help="copy envfile's relative develop paths verbatim", ) view_opts = subparser.add_mutually_exclusive_group() view_opts.add_argument( "--without-view", action="store_true", help="do not maintain a view for this environment" ) view_opts.add_argument( "--with-view", help="maintain view at WITH_VIEW (vs. environment's directory)" ) subparser.add_argument( "envfile", nargs="?", default=None, help="manifest or lock file (ends with '.json' or '.lock') or an environment name or path", ) subparser.add_argument( "--include-concrete", action="append", help="copy concrete specs from INCLUDE_CONCRETE's environment", ) def env_create(args): if args.with_view: # Expand relative paths provided on the command line to the current working directory # This way we interpret `spack env create --with-view ./view --dir ./env` as # a view in $PWD/view, not $PWD/env/view. This is different from specifying a relative # path in the manifest, which is resolved relative to the manifest file's location. with_view = os.path.abspath(args.with_view) elif args.without_view: with_view = False else: # Note that 'None' means unspecified, in which case the Environment # object could choose to enable a view by default. False means that # the environment should not include a view. with_view = None include_concrete = None if hasattr(args, "include_concrete"): include_concrete = args.include_concrete env = _env_create( args.env_name, init_file=args.envfile, dir=args.dir or os.path.sep in args.env_name or args.env_name in (".", ".."), with_view=with_view, keep_relative=args.keep_relative, include_concrete=include_concrete, ) # Generate views, only really useful for environments created from spack.lock files. if args.envfile: env.regenerate_views() def _env_create( name_or_path: str, *, init_file: Optional[str] = None, dir: bool = False, with_view: Optional[Union[bool, str]] = None, keep_relative: bool = False, include_concrete: Optional[List[str]] = None, ): """Create a new environment, with an optional yaml description. Arguments: name_or_path: name of the environment to create, or path to it init_file: optional initialization file -- can be a JSON lockfile (*.lock, *.json), YAML manifest file, or env dir dir: if True, create an environment in a directory instead of a named environment keep_relative: if True, develop paths are copied verbatim into the new environment file, otherwise they may be made absolute if the new environment is in a different location include_concrete: list of the included concrete environments """ if not dir: env = ev.create( name_or_path, init_file=init_file, with_view=with_view, keep_relative=keep_relative, include_concrete=include_concrete, ) tty.msg( colorize( f"Created environment @c{{{cescape(name_or_path)}}} in: @c{{{cescape(env.path)}}}" ) ) else: env = ev.create_in_dir( name_or_path, init_file=init_file, with_view=with_view, keep_relative=keep_relative, include_concrete=include_concrete, ) tty.msg(colorize(f"Created independent environment in: @c{{{cescape(env.path)}}}")) tty.msg(f"Activate with: {colorize(f'@c{{spack env activate {cescape(name_or_path)}}}')}") return env # # env activate # def env_activate_setup_parser(subparser): """set the active environment""" shells = subparser.add_mutually_exclusive_group() shells.add_argument( "--sh", action="store_const", dest="shell", const="sh", help="print sh commands to activate the environment", ) shells.add_argument( "--csh", action="store_const", dest="shell", const="csh", help="print csh commands to activate the environment", ) shells.add_argument( "--fish", action="store_const", dest="shell", const="fish", help="print fish commands to activate the environment", ) shells.add_argument( "--bat", action="store_const", dest="shell", const="bat", help="print bat commands to activate the environment", ) shells.add_argument( "--pwsh", action="store_const", dest="shell", const="pwsh", help="print powershell commands to activate environment", ) view_options = subparser.add_mutually_exclusive_group() view_options.add_argument( "-v", "--with-view", metavar="name", help="set runtime environment variables for the named view", ) view_options.add_argument( "-V", "--without-view", action="store_true", help="do not set runtime environment variables for any view", ) subparser.add_argument( "-p", "--prompt", action="store_true", default=False, help="add the active environment to the command line prompt", ) subparser.add_argument( "--temp", action="store_true", default=False, help="create and activate in a temporary directory", ) subparser.add_argument( "--create", action="store_true", default=False, help="create and activate the environment if it doesn't exist", ) subparser.add_argument( "--envfile", nargs="?", default=None, help="manifest or lock file (ends with '.json' or '.lock')", ) subparser.add_argument( "--keep-relative", action="store_true", help="copy envfile's relative develop paths verbatim when create", ) subparser.add_argument( "-d", "--dir", default=False, action="store_true", help="activate environment based on the directory supplied", ) subparser.add_argument( metavar="env", dest="env_name", nargs="?", default=None, help=("name or directory of the environment being activated"), ) def create_temp_env_directory(): """ Returns the path of a temporary directory in which to create an environment """ return tempfile.mkdtemp(prefix="spack-") def _tty_info(msg): """tty.info like function that prints the equivalent printf statement for eval.""" decorated = f"{colorize('@*b{==>}')} {msg}\n" executor = "echo" if sys.platform == "win32" else "printf" print(f"{executor} {shlex.quote(decorated)};") def env_activate(args): if not args.shell: spack.cmd.common.shell_init_instructions( "spack env activate", " eval `spack env activate {sh_arg} [...]`" ) return 1 # Error out when -e, -E, -D flags are given, cause they are ambiguous. if args.env or args.no_env or args.env_dir: tty.die("Calling spack env activate with --env, --env-dir and --no-env is ambiguous") # special parser error handling relative to the --temp flag temp_conflicts = iter([args.keep_relative, args.dir, args.env_name, args.with_view]) if args.temp and any(temp_conflicts): tty.die( "spack env activate --temp cannot be combined with managed environments, --with-view," " --keep-relative, or --dir." ) # When executing `spack env activate` without further arguments, activate # the default environment. It's created when it doesn't exist yet. if not args.env_name and not args.temp: short_name = "default" if not ev.exists(short_name): ev.create(short_name) action = "Created and activated" else: action = "Activated" env_path = ev.root(short_name) _tty_info(f"{action} default environment in {env_path}") # Temporary environment elif args.temp: env = create_temp_env_directory() env_path = os.path.abspath(env) short_name = os.path.basename(env_path) view = not args.without_view ev.create_in_dir(env, with_view=view).write(regenerate=False) _tty_info(f"Created and activated temporary environment in {env_path}") # Managed environment elif ev.exists(args.env_name) and not args.dir: env_path = ev.root(args.env_name) short_name = args.env_name # Environment directory elif ev.is_env_dir(args.env_name): env_path = os.path.abspath(args.env_name) short_name = os.path.basename(env_path) # create if user requested, and then recall recursively elif args.create: tty.set_msg_enabled(False) env_create(args) tty.set_msg_enabled(True) env_activate(args) return else: tty.die("No such environment: '%s'" % args.env_name) env_prompt = "[%s]" % short_name # We only support one active environment at a time, so deactivate the current one. if ev.active_environment() is None: cmds = "" env_mods = EnvironmentModifications() else: cmds = spack.environment.shell.deactivate_header(shell=args.shell) env_mods = spack.environment.shell.deactivate() # Activate new environment active_env = ev.Environment(env_path) # Check if runtime environment variables are requested, and if so, for what view. view: Optional[str] = None if args.with_view: view = args.with_view if not active_env.has_view(view): tty.die(f"The environment does not have a view named '{view}'") elif not args.without_view and active_env.has_view(ev.default_view_name): view = ev.default_view_name cmds += spack.environment.shell.activate_header( env=active_env, shell=args.shell, prompt=env_prompt if args.prompt else None, view=view ) env_mods.extend(spack.environment.shell.activate(env=active_env, view=view)) cmds += env_mods.shell_modifications(args.shell) sys.stdout.write(cmds) # # env deactivate # def env_deactivate_setup_parser(subparser): """deactivate the active environment""" shells = subparser.add_mutually_exclusive_group() shells.add_argument( "--sh", action="store_const", dest="shell", const="sh", help="print sh commands to deactivate the environment", ) shells.add_argument( "--csh", action="store_const", dest="shell", const="csh", help="print csh commands to deactivate the environment", ) shells.add_argument( "--fish", action="store_const", dest="shell", const="fish", help="print fish commands to activate the environment", ) shells.add_argument( "--bat", action="store_const", dest="shell", const="bat", help="print bat commands to activate the environment", ) shells.add_argument( "--pwsh", action="store_const", dest="shell", const="pwsh", help="print pwsh commands to activate the environment", ) def env_deactivate(args): if not args.shell: spack.cmd.common.shell_init_instructions( "spack env deactivate", " eval `spack env deactivate {sh_arg}`" ) return 1 # Error out when -e, -E, -D flags are given, cause they are ambiguous. if args.env or args.no_env or args.env_dir: tty.die("Calling spack env deactivate with --env, --env-dir and --no-env is ambiguous") if ev.active_environment() is None: tty.die("No environment is currently active.") cmds = spack.environment.shell.deactivate_header(args.shell) env_mods = spack.environment.shell.deactivate() cmds += env_mods.shell_modifications(args.shell) sys.stdout.write(cmds) # # env track # def env_track_setup_parser(subparser): """track an environment from a directory in Spack""" subparser.add_argument("-n", "--name", help="custom environment name") subparser.add_argument("dir", help="path to environment") arguments.add_common_arguments(subparser, ["yes_to_all"]) def env_track(args): src_path = os.path.abspath(args.dir) if not ev.is_env_dir(src_path): tty.die("Cannot track environment. Path doesn't contain an environment") if args.name: name = args.name else: name = os.path.basename(src_path) try: dst_path = ev.environment_dir_from_name(name, exists_ok=False) except ev.SpackEnvironmentError: tty.die( f"An environment named {name} already exists. Set a name with:" "\n\n" f" spack env track --name NAME {src_path}\n" ) symlink(src_path, dst_path) tty.msg(f"Tracking environment in {src_path}") tty.msg( "You can now activate this environment with the following command:\n\n" f" spack env activate {name}\n" ) # # env remove & untrack helpers # def filter_managed_env_names(env_names: Set[str]) -> Set[str]: tracked_env_names = {e for e in env_names if islink(ev.environment_dir_from_name(e))} managed_env_names = env_names - set(tracked_env_names) num_managed_envs = len(managed_env_names) managed_envs_str = " ".join(managed_env_names) if num_managed_envs >= 2: tty.error( f"The following are not tracked environments. " "To remove them completely run," "\n\n" f" spack env rm {managed_envs_str}\n" ) elif num_managed_envs > 0: tty.error( f"'{managed_envs_str}' is not a tracked env. " "To remove it completely run," "\n\n" f" spack env rm {managed_envs_str}\n" ) return tracked_env_names def get_valid_envs(env_names: Set[str]) -> Set[ev.Environment]: valid_envs = set() for env_name in env_names: try: env = ev.read(env_name) valid_envs.add(env) except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError): pass return valid_envs def _env_untrack_or_remove( env_names: List[str], remove: bool = False, force: bool = False, yes_to_all: bool = False ): all_env_names = set(ev.all_environment_names()) known_env_names = set(env_names).intersection(all_env_names) unknown_env_names = set(env_names) - known_env_names # print error for unknown environments for env_name in unknown_env_names: tty.error(f"Environment '{env_name}' does not exist") # if only unlinking is allowed, remove all environments # which do not point internally at symlinks if not remove: env_names_to_remove = filter_managed_env_names(known_env_names) else: env_names_to_remove = known_env_names # initialize all environments with valid spack.yaml configs all_valid_envs = get_valid_envs(all_env_names) # build a task list of environments and bad env names to remove envs_to_remove = [e for e in all_valid_envs if e.name in env_names_to_remove] bad_env_names_to_remove = env_names_to_remove - {e.name for e in envs_to_remove} for remove_env in envs_to_remove: for env in all_valid_envs: # don't check if an environment is included to itself if env.name == remove_env.name: continue # check if an environment is included in another if remove_env.path in env.included_concrete_env_root_dirs: msg = f"Environment '{remove_env.name}' is used by environment '{env.name}'" if force: tty.warn(msg) else: tty.error(msg) envs_to_remove.remove(remove_env) # ask the user if they really want to remove the known environments # force should do the same as yes to all here following the semantics of rm if not (yes_to_all or force) and (envs_to_remove or bad_env_names_to_remove): environments = string.plural(len(env_names_to_remove), "environment", show_n=False) envs = string.comma_and(list(env_names_to_remove)) answer = tty.get_yes_or_no( f"Really {'remove' if remove else 'untrack'} {environments} {envs}?", default=False ) if not answer: tty.msg(f"Will not remove environment(s) {envs}") return # keep track of the environments we remove for later printing the exit code removed_env_names = [] for env in envs_to_remove: name = env.name if not force and env.active: tty.error( f"Environment '{name}' can't be " f"{'removed' if remove else 'untracked'} while activated." ) continue # Get path to check if environment is a tracked / symlinked environment if islink(env.path): real_env_path = os.path.realpath(env.path) os.unlink(env.path) tty.msg( f"Successfully untracked environment '{name}', " "but it can still be found at:\n\n" f" {real_env_path}\n" ) else: env.destroy() tty.msg(f"Successfully removed environment '{name}'") removed_env_names.append(env.name) for bad_env_name in bad_env_names_to_remove: shutil.rmtree( spack.environment.environment.environment_dir_from_name(bad_env_name, exists_ok=True) ) tty.msg(f"Successfully removed environment '{bad_env_name}'") removed_env_names.append(bad_env_name) # Following the design of linux rm we should exit with a status of 1 # anytime we cannot delete every environment the user asks for. # However, we should still process all the environments we know about # and delete them instead of failing on the first unknown environment. if len(removed_env_names) < len(known_env_names): sys.exit(1) # # env untrack # def env_untrack_setup_parser(subparser): """untrack an environment from a directory in Spack""" subparser.add_argument("env", nargs="+", help="tracked environment name") subparser.add_argument( "-f", "--force", action="store_true", help="force unlink even when environment is active" ) arguments.add_common_arguments(subparser, ["yes_to_all"]) def env_untrack(args): _env_untrack_or_remove( env_names=args.env, force=args.force, yes_to_all=args.yes_to_all, remove=False ) # # env remove # def env_remove_setup_parser(subparser): """\ remove managed environment(s) remove existing environment(s) managed by Spack directory environments and manifests embedded in repositories must be removed manually """ subparser.add_argument( "rm_env", metavar="env", nargs="+", help="name(s) of the environment(s) being removed" ) arguments.add_common_arguments(subparser, ["yes_to_all"]) subparser.add_argument( "-f", "--force", action="store_true", help="force removal even when included in other environment(s)", ) def env_remove(args): """remove existing environment(s)""" _env_untrack_or_remove( env_names=args.rm_env, remove=True, force=args.force, yes_to_all=args.yes_to_all ) # # env rename # def env_rename_setup_parser(subparser): """\ rename an existing environment rename a managed environment or move an independent/directory environment operation cannot be performed to or from an active environment """ subparser.add_argument( "mv_from", metavar="from", help="current name or directory of the environment" ) subparser.add_argument("mv_to", metavar="to", help="new name or directory for the environment") subparser.add_argument( "-d", "--dir", action="store_true", help="positional arguments are environment directory paths", ) subparser.add_argument( "-f", "--force", action="store_true", help="force renaming even if overwriting an existing environment", ) def env_rename(args): """rename or move an existing environment""" # Directory option has been specified if args.dir: if not ev.is_env_dir(args.mv_from): tty.die("The specified path does not correspond to a valid spack environment") from_path = Path(args.mv_from) if not args.force: if ev.is_env_dir(args.mv_to): tty.die( "The new path corresponds to an existing environment;" " specify the --force flag to overwrite it." ) if Path(args.mv_to).exists(): tty.die("The new path already exists; specify the --force flag to overwrite it.") to_path = Path(args.mv_to) # Name option being used elif ev.exists(args.mv_from): from_path = ev.environment.environment_dir_from_name(args.mv_from) if not args.force and ev.exists(args.mv_to): tty.die( "The new name corresponds to an existing environment;" " specify the --force flag to overwrite it." ) to_path = ev.environment.root(args.mv_to) # Neither else: tty.die("The specified name does not correspond to a managed spack environment") # Guard against renaming from or to an active environment active_env = ev.active_environment() if active_env: from_env = ev.Environment(from_path) if from_env.path == active_env.path: tty.die("Cannot rename active environment") if to_path == active_env.path: tty.die(f"{args.mv_to} is an active environment") shutil.rmtree(to_path, ignore_errors=True) fs.rename(from_path, to_path) tty.msg(f"Successfully renamed environment {args.mv_from} to {args.mv_to}") # # env list # def env_list_setup_parser(subparser): """list all managed environments""" def env_list(args): names = ev.all_environment_names() color_names = [] for name in names: if ev.active(name): name = colorize("@*g{%s}" % name) color_names.append(name) # say how many there are if writing to a tty if sys.stdout.isatty(): if not names: tty.msg("No environments") else: tty.msg("%d environments" % len(names)) colify(color_names, indent=4) class ViewAction: regenerate = "regenerate" enable = "enable" disable = "disable" @staticmethod def actions(): return [ViewAction.regenerate, ViewAction.enable, ViewAction.disable] # # env view # def env_view_setup_parser(subparser): """\ manage the environment's view provide the path when enabling a view with a non-default path """ subparser.add_argument( "action", choices=ViewAction.actions(), help="action to take for the environment's view" ) subparser.add_argument("view_path", nargs="?", help="view's non-default path when enabling it") def env_view(args): env = ev.active_environment() if not env: tty.msg("No active environment") return if args.action == ViewAction.regenerate: env.regenerate_views() elif args.action == ViewAction.enable: if args.view_path: view_path = args.view_path else: view_path = env.view_path_default env.update_default_view(view_path) env.write() elif args.action == ViewAction.disable: env.update_default_view(path_or_bool=False) env.write() # # env status # def env_status_setup_parser(subparser): """print active environment status""" def env_status(args): env = ev.active_environment() if env: if env.path == os.getcwd(): tty.msg("Using %s in current directory: %s" % (ev.manifest_name, env.path)) else: tty.msg("In environment %s" % env.name) # Check if environment views can be safely activated env.check_views() else: tty.msg("No active environment") # # env loads # def env_loads_setup_parser(subparser): """list modules for an installed environment '(see spack module loads)'""" subparser.add_argument( "-n", "--module-set-name", default="default", help="module set for which to generate load operations", ) subparser.add_argument( "-m", "--module-type", choices=("tcl", "lmod"), help="type of module system to generate loads for", ) spack.cmd.modules.add_loads_arguments(subparser) def env_loads(args): env = spack.cmd.require_active_env(cmd_name="env loads") # Set the module types that have been selected module_type = args.module_type if module_type is None: # If no selection has been made select all of them module_type = "tcl" recurse_dependencies = args.recurse_dependencies args.recurse_dependencies = False loads_file = fs.join_path(env.path, "loads") with open(loads_file, "w", encoding="utf-8") as f: if not recurse_dependencies: specs = [env.specs_by_hash[x.hash] for x in env.concretized_roots] else: specs = list(traverse_nodes(env.concrete_roots(), deptype=("link", "run"))) spack.cmd.modules.loads(module_type, specs, args, f) print("To load this environment, type:") print(" source %s" % loads_file) def env_update_setup_parser(subparser): """\ update the environment manifest to the latest schema format update the environment to the latest schema format, which may not be readable by older versions of spack a backup copy of the manifest is retained in case there is a need to revert this operation """ subparser.add_argument( metavar="env", dest="update_env", help="name or directory of the environment" ) spack.cmd.common.arguments.add_common_arguments(subparser, ["yes_to_all"]) def env_update(args): """update the manifest to the latest format""" manifest_file = ev.manifest_file(args.update_env) backup_file = manifest_file + ".bkp" needs_update = not ev.is_latest_format(manifest_file) if not needs_update: tty.msg('No update needed for the environment "{0}"'.format(args.update_env)) return proceed = True if not args.yes_to_all: msg = ( 'The environment "{0}" is going to be updated to the latest ' "schema format.\nIf the environment is updated, versions of " "Spack that are older than this version may not be able to " "read it. Spack stores backups of the updated environment " 'which can be retrieved with "spack env revert"' ) tty.msg(msg.format(args.update_env)) proceed = tty.get_yes_or_no("Do you want to proceed?", default=False) if not proceed: tty.die("Operation aborted.") ev.update_yaml(manifest_file, backup_file=backup_file) msg = 'Environment "{0}" has been updated [backup={1}]' tty.msg(msg.format(args.update_env, backup_file)) def env_revert_setup_parser(subparser): """\ restore the environment manifest to its previous format revert the environment's manifest to the schema format from its last 'spack env update' the current manifest will be overwritten by the backup copy and the backup copy will be removed """ subparser.add_argument( metavar="env", dest="revert_env", help="name or directory of the environment" ) spack.cmd.common.arguments.add_common_arguments(subparser, ["yes_to_all"]) def env_revert(args): """restore the environment manifest to its previous format""" manifest_file = ev.manifest_file(args.revert_env) backup_file = manifest_file + ".bkp" # Check that both the spack.yaml and the backup exist, the inform user # on what is going to happen and ask for confirmation if not os.path.exists(manifest_file): msg = "cannot find the manifest file of the environment [file={0}]" tty.die(msg.format(manifest_file)) if not os.path.exists(backup_file): msg = "cannot find the old manifest file to be restored [file={0}]" tty.die(msg.format(backup_file)) proceed = True if not args.yes_to_all: msg = ( "Spack is going to overwrite the current manifest file" " with a backup copy [manifest={0}, backup={1}]" ) tty.msg(msg.format(manifest_file, backup_file)) proceed = tty.get_yes_or_no("Do you want to proceed?", default=False) if not proceed: tty.die("Operation aborted.") shutil.copy(backup_file, manifest_file) os.remove(backup_file) msg = 'Environment "{0}" reverted to old state' tty.msg(msg.format(manifest_file)) def env_depfile_setup_parser(subparser): """\ generate a depfile to exploit parallel builds across specs requires the active environment to be concrete """ subparser.add_argument( "--make-prefix", "--make-target-prefix", default=None, metavar="TARGET", help="prefix Makefile targets/variables with /,\n" "which can be an empty string (--make-prefix '')\n" "defaults to the absolute path of the environment's makedeps\n" "environment metadata dir\n", ) subparser.add_argument( "--make-disable-jobserver", default=True, action="store_false", dest="jobserver", help="disable POSIX jobserver support", ) subparser.add_argument( "--use-buildcache", dest="use_buildcache", type=arguments.use_buildcache, default="package:auto,dependencies:auto", metavar="[{auto,only,never},][package:{auto,only,never},][dependencies:{auto,only,never}]", help="use `only` to prune redundant build dependencies\n" "option is also passed to generated spack install commands", ) subparser.add_argument( "-o", "--output", default=None, metavar="FILE", help="write the depfile to FILE rather than to stdout", ) subparser.add_argument( "-G", "--generator", default="make", choices=("make",), help="specify the depfile type (only supports `make`)", ) subparser.add_argument( metavar="specs", dest="specs", nargs=argparse.REMAINDER, default=None, help="limit the generated file to matching specs", ) def env_depfile(args): # Currently only make is supported. spack.cmd.require_active_env(cmd_name="env depfile") env = ev.active_environment() # What things do we build when running make? By default, we build the # root specs. If specific specs are provided as input, we build those. filter_specs = spack.cmd.parse_specs(args.specs) if args.specs else None template = spack.tengine.make_environment().get_template(os.path.join("depfile", "Makefile")) model = depfile.MakefileModel.from_env( env, filter_specs=filter_specs, pkg_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[0]), dep_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[1]), make_prefix=args.make_prefix, jobserver=args.jobserver, ) # Warn in case we're generating a depfile for an empty environment. We don't automatically # concretize; the user should do that explicitly. Could be changed in the future if requested. if model.empty: if not env.user_specs: tty.warn("no specs in the environment") elif filter_specs is not None: tty.warn("no concrete matching specs found in environment") else: tty.warn("environment is not concretized. Run `spack concretize` first") makefile = template.render(model.to_dict()) # Finally write to stdout/file. if args.output: with open(args.output, "w", encoding="utf-8") as f: f.write(makefile) else: sys.stdout.write(makefile) #: Dictionary mapping subcommand names and aliases to functions subcommand_functions = {} # # spack env # def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="env_command") _globals = globals() for name_and_aliases in subcommands: name, aliases = name_and_aliases[0], name_and_aliases[1:] # add commands to subcommands dict for alias in name_and_aliases: subcommand_functions[alias] = _globals[f"env_{name}"] # make a subparser and run the command's setup function on it setup_parser_cmd = _globals[f"env_{name}_setup_parser"] subsubparser = sp.add_parser( name, aliases=aliases, description=spack.cmd.doc_dedented(setup_parser_cmd), help=spack.cmd.doc_first_line(setup_parser_cmd), ) setup_parser_cmd(subsubparser) def env(parser, args): """Look for a function called environment_ and call it.""" action = subcommand_functions[args.env_command] action(args) ================================================ FILE: lib/spack/spack/cmd/extensions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys import spack.cmd as cmd import spack.environment as ev import spack.llnl.util.tty as tty import spack.repo import spack.store from spack.cmd.common import arguments from spack.llnl.util.tty.colify import colify description = "list extensions for package" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.epilog = ( "If called without argument returns the list of all valid extendable packages" ) arguments.add_common_arguments(subparser, ["long", "very_long"]) subparser.add_argument( "-d", "--deps", action="store_true", help="output dependencies along with found specs" ) subparser.add_argument( "-p", "--paths", action="store_true", help="show paths to package install directories" ) subparser.add_argument( "-s", "--show", action="store", default="all", choices=("packages", "installed", "all"), help="show only part of output", ) subparser.add_argument( "spec", nargs=argparse.REMAINDER, help="spec of package to list extensions for", metavar="extendable", ) def extensions(parser, args): if not args.spec: # If called without arguments, list all the extendable packages isatty = sys.stdout.isatty() if isatty: tty.info("Extendable packages:") extendable_pkgs = [] for name in spack.repo.all_package_names(): pkg_cls = spack.repo.PATH.get_pkg_class(name) if pkg_cls.extendable: extendable_pkgs.append(name) colify(extendable_pkgs, indent=4) return # Checks spec = cmd.parse_specs(args.spec) if len(spec) > 1: tty.die("Can only list extensions for one package.") env = ev.active_environment() spec = cmd.disambiguate_spec(spec[0], env) if not spec.package.extendable: tty.die("%s is not an extendable package." % spec.name) if not spec.package.extendable: tty.die("%s does not have extensions." % spec.short_spec) if args.show in ("packages", "all"): # List package names of extensions extensions = spack.repo.PATH.extensions_for(spec) if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) else: tty.msg(spec.cshort_spec) tty.msg("%d extensions:" % len(extensions)) colify(ext.name for ext in extensions) if args.show in ("installed", "all"): # List specs of installed extensions. installed = [s.spec for s in spack.store.STORE.db.installed_extensions_for(spec)] if args.show == "all": print if not installed: tty.msg("None installed.") else: tty.msg("%d installed:" % len(installed)) cmd.display_specs(installed, args) ================================================ FILE: lib/spack/spack/cmd/external.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import errno import os import re import sys from typing import List, Optional, Set import spack import spack.cmd import spack.config import spack.cray_manifest as cray_manifest import spack.detection import spack.error import spack.llnl.util.tty as tty import spack.llnl.util.tty.colify as colify import spack.package_base import spack.repo import spack.spec from spack.cmd.common import arguments description = "manage external packages in Spack configuration" section = "config" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="external_command") find_parser = sp.add_parser("find", help="add external packages to packages.yaml") find_parser.add_argument( "--not-buildable", action="store_true", default=False, help="packages with detected externals won't be built with Spack", ) find_parser.add_argument("--exclude", action="append", help="packages to exclude from search") find_parser.add_argument( "-p", "--path", default=None, action="append", help="one or more alternative search paths for finding externals", ) find_parser.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope("packages"), help="configuration scope to modify", ) find_parser.add_argument( "--all", action="store_true", help="search for all packages that Spack knows about" ) arguments.add_common_arguments(find_parser, ["tags", "jobs"]) find_parser.add_argument("packages", nargs=argparse.REMAINDER) find_parser.epilog = ( 'The search is by default on packages tagged with the "build-tools" or ' '"core-packages" tags. Use the --all option to search for every possible ' "package Spack knows how to find." ) sp.add_parser("list", aliases=["ls"], help="list detectable packages, by repository and name") read_cray_manifest = sp.add_parser( "read-cray-manifest", help="consume a Spack-compatible description of externally-installed packages, including " "dependency relationships", ) read_cray_manifest.add_argument( "--file", default=None, help="specify a location other than the default" ) read_cray_manifest.add_argument( "--directory", default=None, help="specify a directory storing a group of manifest files" ) read_cray_manifest.add_argument( "--ignore-default-dir", action="store_true", default=False, help="ignore the default directory of manifest files", ) read_cray_manifest.add_argument( "--dry-run", action="store_true", default=False, help="don't modify DB with files that are read", ) read_cray_manifest.add_argument( "--fail-on-error", action="store_true", help="if a manifest file cannot be parsed, fail and report the full stack trace", ) def external_find(args): if args.all or not (args.tags or args.packages): # If the user calls 'spack external find' with no arguments, and # this system has a description of installed packages, then we should # consume it automatically. try: _collect_and_consume_cray_manifest_files() except NoManifestFileError: # It's fine to not find any manifest file if we are doing the # search implicitly (i.e. as part of 'spack external find') pass except Exception as e: # For most exceptions, just print a warning and continue. # Note that KeyboardInterrupt does not subclass Exception # (so CTRL-C will terminate the program as expected). skip_msg = "Skipping manifest and continuing with other external checks" if isinstance(e, OSError) and e.errno in (errno.EPERM, errno.EACCES): # The manifest file does not have sufficient permissions enabled: # print a warning and keep going tty.warn("Unable to read manifest due to insufficient permissions.", skip_msg) else: tty.warn("Unable to read manifest, unexpected error: {0}".format(str(e)), skip_msg) # Outside the Cray manifest, the search is done by tag for performance reasons, # since tags are cached. # If the user specified both --all and --tag, then --all has precedence if args.all or args.packages: # Each detectable package has at least the detectable tag args.tags = ["detectable"] elif not args.tags: # If the user didn't specify anything, search for build tools by default args.tags = ["core-packages", "build-tools"] candidate_packages = packages_to_search_for( names=args.packages, tags=args.tags, exclude=args.exclude ) detected_packages = spack.detection.by_path( candidate_packages, path_hints=args.path, max_workers=args.jobs ) new_specs = spack.detection.update_configuration( detected_packages, scope=args.scope, buildable=not args.not_buildable ) # If the user runs `spack external find --not-buildable mpich` we also mark `mpi` non-buildable # to avoid that the concretizer picks a different mpi provider. if new_specs and args.not_buildable: virtuals: Set[str] = { virtual.name for new_spec in new_specs for virtual_specs in spack.repo.PATH.get_pkg_class(new_spec.name).provided.values() for virtual in virtual_specs } new_virtuals = spack.detection.set_virtuals_nonbuildable(virtuals, scope=args.scope) new_specs.extend(spack.spec.Spec(name) for name in new_virtuals) if new_specs: path = spack.config.CONFIG.get_config_filename(args.scope, "packages") tty.msg(f"The following specs have been detected on this system and added to {path}") spack.cmd.display_specs(new_specs) else: tty.msg("No new external packages detected") def packages_to_search_for( *, names: Optional[List[str]], tags: List[str], exclude: Optional[List[str]] ): result = list( {pkg for tag in tags for pkg in spack.repo.PATH.packages_with_tags(tag, full=True)} ) if names: # Match both fully qualified and unqualified parts = [rf"(^{x}$|[.]{x}$)" for x in names] select_re = re.compile("|".join(parts)) result = [x for x in result if select_re.search(x)] if exclude: # Match both fully qualified and unqualified parts = [rf"(^{x}$|[.]{x}$)" for x in exclude] select_re = re.compile("|".join(parts)) result = [x for x in result if not select_re.search(x)] return result def external_read_cray_manifest(args): _collect_and_consume_cray_manifest_files( manifest_file=args.file, manifest_directory=args.directory, dry_run=args.dry_run, fail_on_error=args.fail_on_error, ignore_default_dir=args.ignore_default_dir, ) def _collect_and_consume_cray_manifest_files( manifest_file=None, manifest_directory=None, dry_run=False, fail_on_error=False, ignore_default_dir=False, ): manifest_files = [] if manifest_file: manifest_files.append(manifest_file) manifest_dirs = [] if manifest_directory: manifest_dirs.append(manifest_directory) if not ignore_default_dir and os.path.isdir(cray_manifest.default_path): tty.debug( "Cray manifest path {0} exists: collecting all files to read.".format( cray_manifest.default_path ) ) manifest_dirs.append(cray_manifest.default_path) else: tty.debug( "Default Cray manifest directory {0} does not exist.".format( cray_manifest.default_path ) ) for directory in manifest_dirs: for fname in os.listdir(directory): if fname.endswith(".json"): fpath = os.path.join(directory, fname) tty.debug("Adding manifest file: {0}".format(fpath)) manifest_files.append(os.path.join(directory, fpath)) if not manifest_files: raise NoManifestFileError( "--file/--directory not specified, and no manifest found at {0}".format( cray_manifest.default_path ) ) for path in manifest_files: tty.debug("Reading manifest file: " + path) try: cray_manifest.read(path, not dry_run) except spack.error.SpackError as e: if fail_on_error: raise else: tty.warn("Failure reading manifest file: {0}\n\t{1}".format(path, str(e))) def external_list(args): # Trigger a read of all packages, might take a long time. list(spack.repo.PATH.all_package_classes()) # Print all the detectable packages tty.msg("Detectable packages per repository") for namespace, pkgs in sorted(spack.package_base.detectable_packages.items()): print("Repository:", namespace) colify.colify(pkgs, indent=4, output=sys.stdout) def external(parser, args): action = { "find": external_find, "list": external_list, "ls": external_list, "read-cray-manifest": external_read_cray_manifest, } action[args.external_command](args) class NoManifestFileError(spack.error.SpackError): pass ================================================ FILE: lib/spack/spack/cmd/fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd import spack.config import spack.environment as ev import spack.llnl.util.tty as tty import spack.traverse from spack.cmd.common import arguments description = "fetch archives for packages" section = "build" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["no_checksum", "specs"]) subparser.add_argument( "-m", "--missing", action="store_true", help="fetch only missing (not yet installed) dependencies", ) subparser.add_argument( "-D", "--dependencies", action="store_true", help="also fetch all dependencies" ) arguments.add_concretizer_args(subparser) subparser.epilog = ( "With an active environment, the specs " "parameter can be omitted. In this case all (uninstalled" ", in case of --missing) specs from the environment are fetched" ) def fetch(parser, args): if args.no_checksum: spack.config.set("config:checksum", False, scope="command_line") if args.specs: specs = spack.cmd.parse_specs(args.specs, concretize=True) else: # No specs were given explicitly, check if we are in an # environment. If yes, check the missing argument, if yes # fetch all uninstalled specs from it otherwise fetch all. # If we are also not in an environment, complain to the # user that we don't know what to do. env = ev.active_environment() if env: if args.missing: specs = env.uninstalled_specs() else: specs = env.all_specs() if specs == []: tty.die("No uninstalled specs in environment. Did you run `spack concretize` yet?") else: tty.die("fetch requires at least one spec argument") if args.dependencies or args.missing: to_be_fetched = spack.traverse.traverse_nodes(specs, key=spack.traverse.by_dag_hash) else: to_be_fetched = specs for spec in to_be_fetched: if args.missing and spec.installed: continue pkg = spec.package pkg.stage.keep = True with pkg.stage: pkg.do_fetch() ================================================ FILE: lib/spack/spack/cmd/find.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import copy import sys import spack.cmd as cmd import spack.config import spack.environment as ev import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.repo import spack.solver.reuse import spack.spec import spack.store from spack.cmd.common import arguments from spack.llnl.util.tty.color import colorize from spack.solver.reuse import create_external_parser from spack.solver.runtimes import external_config_with_implicit_externals from ..enums import InstallRecordStatus description = "list and search installed packages" section = "query" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: format_group = subparser.add_mutually_exclusive_group() format_group.add_argument( "--format", action="store", default=None, help="output specs with the specified format string", ) format_group.add_argument( "-H", "--hashes", action="store_const", dest="format", const="{/hash}", help="same as ``--format {/hash}``; use with ``xargs`` or ``$()``", ) format_group.add_argument( "--json", action="store_true", default=False, help="output specs as machine-readable json records", ) subparser.add_argument( "-I", "--install-status", action="store_true", help="show install status of packages" ) subparser.add_argument( "--specfile-format", action="store_true", help="show the specfile format for installed deps ", ) subparser.add_argument( "-d", "--deps", action="store_true", help="output dependencies along with found specs" ) subparser.add_argument( "-p", "--paths", action="store_true", help="show paths to package install directories" ) subparser.add_argument( "--groups", action="store_true", default=None, dest="groups", help="display specs in arch/compiler groups (default on)", ) subparser.add_argument( "--no-groups", action="store_false", default=None, dest="groups", help="do not group specs by arch/compiler", ) arguments.add_common_arguments(subparser, ["long", "very_long", "tags", "namespaces"]) subparser.add_argument( "-r", "--only-roots", action="store_true", help="don't show full list of installed specs in an environment", ) concretized_vs_packages = subparser.add_mutually_exclusive_group() concretized_vs_packages.add_argument( "-c", "--show-concretized", action="store_true", help="show concretized specs in an environment", ) concretized_vs_packages.add_argument( "--show-configured-externals", action="store_true", help="show externals defined in the 'packages' section of the configuration", ) subparser.add_argument( "-f", "--show-flags", action="store_true", dest="show_flags", help="show spec compiler flags", ) subparser.add_argument( "--show-full-compiler", action="store_true", dest="show_full_compiler", help="(DEPRECATED) show full compiler specs. Currently it's a no-op", ) implicit_explicit = subparser.add_mutually_exclusive_group() implicit_explicit.add_argument( "-x", "--explicit", action="store_true", help="show only specs that were installed explicitly", ) implicit_explicit.add_argument( "-X", "--implicit", action="store_true", help="show only specs that were installed as dependencies", ) subparser.add_argument( "-e", "--external", action="store_true", help="show only specs that are marked as externals", ) subparser.add_argument( "-u", "--unknown", action="store_true", dest="unknown", help="show only specs Spack does not have a package for", ) subparser.add_argument( "-m", "--missing", action="store_true", dest="missing", help="show missing dependencies as well as installed specs", ) subparser.add_argument( "-v", "--variants", action="store_true", dest="variants", help="show variants in output (can be long)", ) subparser.add_argument( "--loaded", action="store_true", help="show only packages loaded in the user environment" ) only_missing_or_deprecated = subparser.add_mutually_exclusive_group() only_missing_or_deprecated.add_argument( "-M", "--only-missing", action="store_true", dest="only_missing", help="show only missing dependencies", ) only_missing_or_deprecated.add_argument( "--only-deprecated", action="store_true", help="show only deprecated packages" ) subparser.add_argument( "--deprecated", action="store_true", help="show deprecated packages as well as installed specs", ) subparser.add_argument( "--install-tree", action="store", default="all", help="Install trees to query: 'all' (default), 'local', 'upstream', upstream name or path", ) subparser.add_argument("--start-date", help="earliest date of installation [YYYY-MM-DD]") subparser.add_argument("--end-date", help="latest date of installation [YYYY-MM-DD]") arguments.add_common_arguments(subparser, ["constraint"]) def query_arguments(args): if args.only_missing and (args.deprecated or args.missing): raise RuntimeError("cannot use --only-missing with --deprecated, or --missing") if args.only_deprecated and (args.deprecated or args.missing): raise RuntimeError("cannot use --only-deprecated with --deprecated, or --missing") installed = InstallRecordStatus.INSTALLED if args.only_missing: installed = InstallRecordStatus.MISSING elif args.only_deprecated: installed = InstallRecordStatus.DEPRECATED if args.missing: installed |= InstallRecordStatus.MISSING if args.deprecated: installed |= InstallRecordStatus.DEPRECATED predicate_fn = None if args.unknown: predicate_fn = lambda x: not spack.repo.PATH.exists(x.spec.name) explicit = None if args.explicit: explicit = True if args.implicit: explicit = False q_args = {"installed": installed, "predicate_fn": predicate_fn, "explicit": explicit} install_tree = args.install_tree upstreams = spack.config.get("upstreams", {}) if install_tree in upstreams.keys(): install_tree = upstreams[install_tree]["install_tree"] q_args["install_tree"] = install_tree # Time window of installation for attribute in ("start_date", "end_date"): date = getattr(args, attribute) if date: q_args[attribute] = spack.llnl.util.lang.pretty_string_to_date(date) return q_args def make_env_decorator(env): """Create a function for decorating specs when in an environment.""" roots = set(env.roots()) removed = set(env.removed_specs()) def decorator(spec, fmt): # add +/-/* to show added/removed/root specs if any(spec.dag_hash() == r.dag_hash() for r in roots): return color.colorize(f"@*{{{fmt}}}") elif spec in removed: return color.colorize(f"@K{{{fmt}}}") else: return fmt return decorator def display_env(env, args, decorator, results): """Display extra find output when running in an environment. In an environment, ``spack find`` outputs a preliminary section showing the root specs of the environment (this is in addition to the section listing out specs matching the query parameters). """ total_roots = sum(len(env.user_specs_by(group=g)) for g in env.manifest.groups()) root_spec_str = f"{total_roots or 'no'} root {'spec' if total_roots == 1 else 'specs'}" tty.msg(f"In environment {env.name} ({root_spec_str})") concrete_specs = {x.root: env.specs_by_hash[x.hash] for x in env.concretized_roots} def root_decorator(spec, string): """Decorate root specs with their install status if needed""" concrete = concrete_specs.get(spec) if concrete: status = color.colorize(concrete.install_status().value) hash = concrete.dag_hash() else: status = color.colorize(spack.spec.InstallStatus.absent.value) hash = "-" * 32 # TODO: status has two extra spaces on the end of it, but fixing this and other spec # TODO: space format idiosyncrasies is complicated. Fix this eventually status = status[:-2] if args.long or args.very_long: hash = color.colorize(f"@K{{{hash[: 7 if args.long else None]}}}") return f"{status} {hash} {string}" else: return f"{status} {string}" with spack.store.STORE.db.read_transaction(): for group in env.manifest.groups(): group_specs = env.user_specs_by(group=group) if not group_specs: continue if env.has_groups(): header = ( f"{spack.spec.ARCHITECTURE_COLOR}{{root specs}} / " f"{spack.spec.COMPILER_COLOR}{{{group}}}" ) tty.hline(colorize(header), char="-") cmd.display_specs( group_specs, args, # these are overrides of CLI args paths=False, long=False, very_long=False, # these enforce details in the root specs to show what the user asked for groups=False, namespaces=True, show_flags=True, decorator=root_decorator, variants=True, specfile_format=args.specfile_format, ) print() if env.included_concrete_env_root_dirs: tty.msg("Included specs") # Root specs cannot be displayed with prefixes, since those are not # set for abstract specs. Same for hashes root_args = copy.copy(args) root_args.paths = False # Roots are displayed with variants, etc. so that we can see # specifically what the user asked for. cmd.display_specs( env.included_user_specs, root_args, decorator=lambda s, f: color.colorize("@*{%s}" % f), namespace=True, show_flags=True, variants=True, specfile_format=args.specfile_format, ) print() def _find_query(args, env): q_args = query_arguments(args) concretized_but_not_installed = [] if args.show_configured_externals: packages_with_externals = external_config_with_implicit_externals(spack.config.CONFIG) completion_mode = spack.config.CONFIG.get("concretizer:externals:completion") results = spack.solver.reuse.spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], exclude=[], ).selected_specs() elif env: all_env_specs = env.all_specs() if args.constraint: init_specs = cmd.parse_specs(args.constraint) env_specs = env.all_matching_specs(*init_specs) else: env_specs = all_env_specs spec_hashes = set(x.dag_hash() for x in env_specs) specs_meeting_q_args = set(spack.store.STORE.db.query(hashes=spec_hashes, **q_args)) results = list() with spack.store.STORE.db.read_transaction(): for spec in env_specs: if not spec.installed: concretized_but_not_installed.append(spec) if spec in specs_meeting_q_args: results.append(spec) else: results = args.specs(**q_args) if args.external: results = [s for s in results if s.external] # use groups by default except with format. if args.groups is None: args.groups = not args.format # Exit early with an error code if no package matches the constraint if concretized_but_not_installed and args.show_concretized: pass elif results: pass elif args.constraint: raise cmd.NoSpecMatches() # If tags have been specified on the command line, filter by tags if args.tags: packages_with_tags = spack.repo.PATH.packages_with_tags(*args.tags) results = [x for x in results if x.name in packages_with_tags] concretized_but_not_installed = [ x for x in concretized_but_not_installed if x.name in packages_with_tags ] if args.loaded: results = cmd.filter_loaded_specs(results) return results, concretized_but_not_installed def find(parser, args): env = ev.active_environment() if not env and args.only_roots: tty.die("-r / --only-roots requires an active environment") if not env and args.show_concretized: tty.die("-c / --show-concretized requires an active environment") try: results, concretized_but_not_installed = _find_query(args, env) except cmd.NoSpecMatches: # Note: this uses args.constraint vs. args.constraint_specs because # the latter only exists if you call args.specs() tty.die(f"No package matches the query: {' '.join(args.constraint)}") if args.install_status or args.show_concretized: status_fn = spack.spec.Spec.install_status else: status_fn = None # Display the result if args.json: cmd.display_specs_as_json(results, deps=args.deps) else: decorator = make_env_decorator(env) if env else lambda s, f: f if not args.format: if env: display_env(env, args, decorator, results) if not args.only_roots: display_results = list(results) if args.show_concretized: display_results += concretized_but_not_installed cmd.display_specs( display_results, args, decorator=decorator, all_headers=True, status_fn=status_fn, specfile_format=args.specfile_format, ) # print number of installed packages last (as the list may be long) if sys.stdout.isatty() and args.groups: installed_suffix = "" concretized_suffix = " to be installed" if args.only_roots: installed_suffix += " (not shown)" concretized_suffix += " (not shown)" else: if env and not args.show_concretized: concretized_suffix += " (show with `spack find -c`)" pkg_type = "loaded" if args.loaded else "installed" cmd.print_how_many_pkgs(results, pkg_type, suffix=installed_suffix) if env: cmd.print_how_many_pkgs( concretized_but_not_installed, "concretized", suffix=concretized_suffix ) ================================================ FILE: lib/spack/spack/cmd/gc.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd.common.arguments import spack.cmd.common.confirmation import spack.cmd.uninstall import spack.deptypes as dt import spack.environment as ev import spack.llnl.util.tty as tty import spack.store description = "remove specs that are now no longer needed" section = "build" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-E", "--except-any-environment", action="store_true", help="remove everything unless needed by an environment", ) subparser.add_argument( "-e", "--except-environment", metavar="ENV", action="append", default=[], help="remove everything unless needed by specified environment\n" "you can list multiple environments, or specify directory\n" "environments by path.", ) subparser.add_argument( "-b", "--keep-build-dependencies", action="store_true", help="do not remove installed build-only dependencies of roots\n" "(default is to keep only link & run dependencies)", ) spack.cmd.common.arguments.add_common_arguments(subparser, ["yes_to_all", "constraint"]) def roots_from_environments(args, active_env): # if we're using -E or -e, make a list of environments whose roots we should consider. all_environments = [] # -E will garbage collect anything not needed by any env, including the current one if args.except_any_environment: all_environments += list(ev.all_environments()) if active_env: all_environments.append(active_env) # -e says "also preserve things needed by this particular env" for env_name_or_dir in args.except_environment: if ev.exists(env_name_or_dir): env = ev.read(env_name_or_dir) elif ev.is_env_dir(env_name_or_dir): env = ev.Environment(env_name_or_dir) else: tty.die(f"No such environment: '{env_name_or_dir}'") all_environments.append(env) # add root hashes from all considered environments to list of roots root_hashes = set() for env in all_environments: root_hashes |= {x.hash for x in env.explicit_roots()} return root_hashes def gc(parser, args): deptype = dt.LINK | dt.RUN if args.keep_build_dependencies: deptype |= dt.BUILD active_env = ev.active_environment() # wrap the whole command with a read transaction to avoid multiple with spack.store.STORE.db.read_transaction(): if args.except_environment or args.except_any_environment: # if either of these is specified, we ignore the active environment and garbage # collect anything NOT in specified environments. root_hashes = roots_from_environments(args, active_env) elif active_env: # only gc what's in current environment tty.msg(f"Restricting garbage collection to environment '{active_env.name}'") root_hashes = set(spack.store.STORE.db.all_hashes()) # keep everything root_hashes -= set(active_env.all_hashes()) # except this env # but keep its explicit roots root_hashes |= {x.hash for x in active_env.explicit_roots()} else: # consider all explicit specs roots (the default for db.unused_specs()) root_hashes = None specs = spack.store.STORE.db.unused_specs(root_hashes=root_hashes, deptype=deptype) # limit search to constraint specs if provided if args.constraint: hashes = set(spec.dag_hash() for spec in args.specs()) specs = [spec for spec in specs if spec.dag_hash() in hashes] if not specs: tty.msg("There are no unused specs. Spack's store is clean.") return if not args.yes_to_all: spack.cmd.common.confirmation.confirm_action(specs, "uninstalled", "uninstall") spack.cmd.uninstall.do_uninstall(specs, force=False) ================================================ FILE: lib/spack/spack/cmd/gpg.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import tempfile import spack.binary_distribution import spack.mirrors.mirror import spack.paths import spack.stage import spack.util.gpg import spack.util.url from spack.cmd.common import arguments description = "handle GPG actions for spack" section = "packaging" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: setattr(setup_parser, "parser", subparser) subparsers = subparser.add_subparsers(help="GPG sub-commands") verify = subparsers.add_parser("verify", help=gpg_verify.__doc__) arguments.add_common_arguments(verify, ["installed_spec"]) verify.add_argument("signature", type=str, nargs="?", help="the signature file") verify.set_defaults(func=gpg_verify) trust = subparsers.add_parser("trust", help=gpg_trust.__doc__) trust.add_argument("keyfile", type=str, help="add a key to the trust store") trust.set_defaults(func=gpg_trust) untrust = subparsers.add_parser("untrust", help=gpg_untrust.__doc__) untrust.add_argument("--signing", action="store_true", help="allow untrusting signing keys") untrust.add_argument("keys", nargs="+", type=str, help="remove keys from the trust store") untrust.set_defaults(func=gpg_untrust) sign = subparsers.add_parser("sign", help=gpg_sign.__doc__) sign.add_argument( "--output", metavar="DEST", type=str, help="the directory to place signatures" ) sign.add_argument("--key", metavar="KEY", type=str, help="the key to use for signing") sign.add_argument( "--clearsign", action="store_true", help="if specified, create a clearsign signature" ) arguments.add_common_arguments(sign, ["installed_spec"]) sign.set_defaults(func=gpg_sign) create = subparsers.add_parser("create", help=gpg_create.__doc__) create.add_argument("name", type=str, help="the name to use for the new key") create.add_argument("email", type=str, help="the email address to use for the new key") create.add_argument( "--comment", metavar="COMMENT", type=str, default="GPG created for Spack", help="a description for the intended use of the key", ) create.add_argument( "--expires", metavar="EXPIRATION", type=str, default="0", help="when the key should expire" ) create.add_argument( "--export", metavar="DEST", type=str, help="export the public key to a file" ) create.add_argument( "--export-secret", metavar="DEST", type=str, dest="secret", help="export the private key to a file", ) create.set_defaults(func=gpg_create) list = subparsers.add_parser("list", help=gpg_list.__doc__) list.add_argument("--trusted", action="store_true", default=True, help="list trusted keys") list.add_argument( "--signing", action="store_true", help="list keys which may be used for signing" ) list.set_defaults(func=gpg_list) init = subparsers.add_parser("init", help=gpg_init.__doc__) init.add_argument("--from", metavar="DIR", type=str, dest="import_dir", help=argparse.SUPPRESS) init.set_defaults(func=gpg_init) export = subparsers.add_parser("export", help=gpg_export.__doc__) export.add_argument("location", type=str, help="where to export keys") export.add_argument( "keys", nargs="*", help="the keys to export (all public keys if unspecified)" ) export.add_argument("--secret", action="store_true", help="export secret keys") export.set_defaults(func=gpg_export) publish = subparsers.add_parser("publish", help=gpg_publish.__doc__) output = publish.add_mutually_exclusive_group(required=True) output.add_argument( "-d", "--directory", metavar="directory", type=str, help="local directory where keys will be published", ) output.add_argument( "-m", "--mirror-name", metavar="mirror-name", type=str, help="name of the mirror where keys will be published", ) output.add_argument( "--mirror-url", metavar="mirror-url", type=str, help="URL of the mirror where keys will be published", ) publish.add_argument( "--update-index", "--rebuild-index", action="store_true", default=False, help="regenerate buildcache key index after publishing key(s)", ) publish.add_argument( "keys", nargs="*", help="keys to publish (all public keys if unspecified)" ) publish.set_defaults(func=gpg_publish) def gpg_create(args): """create a new key""" if args.export or args.secret: old_sec_keys = spack.util.gpg.signing_keys() # Create the new key spack.util.gpg.create( name=args.name, email=args.email, comment=args.comment, expires=args.expires ) if args.export or args.secret: new_sec_keys = set(spack.util.gpg.signing_keys()) new_keys = new_sec_keys.difference(old_sec_keys) if args.export: spack.util.gpg.export_keys(args.export, new_keys) if args.secret: spack.util.gpg.export_keys(args.secret, new_keys, secret=True) def gpg_export(args): """export a gpg key, optionally including secret key""" keys = args.keys if not keys: keys = spack.util.gpg.signing_keys() spack.util.gpg.export_keys(args.location, keys, args.secret) def gpg_list(args): """list keys available in the keyring""" spack.util.gpg.list(args.trusted, args.signing) def gpg_sign(args): """sign a package""" key = args.key if key is None: keys = spack.util.gpg.signing_keys() if len(keys) == 1: key = keys[0] elif not keys: raise RuntimeError("no signing keys are available") else: raise RuntimeError("multiple signing keys are available; please choose one") output = args.output if not output: output = args.spec[0] + ".asc" # TODO: Support the package format Spack creates. spack.util.gpg.sign(key, " ".join(args.spec), output, args.clearsign) def gpg_trust(args): """add a key to the keyring""" spack.util.gpg.trust(args.keyfile) def gpg_init(args): """add the default keys to the keyring""" import_dir = args.import_dir if import_dir is None: import_dir = spack.paths.gpg_keys_path for root, _, filenames in os.walk(import_dir): for filename in filenames: if not filename.endswith(".key"): continue spack.util.gpg.trust(os.path.join(root, filename)) def gpg_untrust(args): """remove a key from the keyring""" spack.util.gpg.untrust(args.signing, *args.keys) def gpg_verify(args): """verify a signed package""" # TODO: Support the package format Spack creates. signature = args.signature if signature is None: signature = args.spec[0] + ".asc" spack.util.gpg.verify(signature, " ".join(args.spec)) def gpg_publish(args): """publish public keys to a build cache""" mirror = None if args.directory: url = spack.util.url.path_to_file_url(args.directory) mirror = spack.mirrors.mirror.Mirror(url, url) elif args.mirror_name: mirror = spack.mirrors.mirror.MirrorCollection(binary=True).lookup(args.mirror_name) elif args.mirror_url: mirror = spack.mirrors.mirror.Mirror(args.mirror_url, args.mirror_url) with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: spack.binary_distribution._url_push_keys( mirror, keys=args.keys, tmpdir=tmpdir, update_index=args.update_index ) def gpg(parser, args): if args.func: args.func(args) ================================================ FILE: lib/spack/spack/cmd/graph.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd import spack.config import spack.environment as ev import spack.store from spack.cmd.common import arguments from spack.graph import DAGWithDependencyTypes, SimpleDAG, graph_ascii, graph_dot, static_graph_dot from spack.llnl.util import tty description = "generate graphs of package dependency relationships" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: setattr(setup_parser, "parser", subparser) subparser.epilog = """ Outside of an environment, the command concretizes specs and graphs them, unless the --installed option is given. In that case specs are matched from the current DB. If an environment is active, specs are matched from the currently available concrete specs in the lockfile. """ method = subparser.add_mutually_exclusive_group() method.add_argument( "-a", "--ascii", action="store_true", help="draw graph as ascii to stdout (default)" ) method.add_argument( "-d", "--dot", action="store_true", help="generate graph in dot format and print to stdout" ) subparser.add_argument( "-s", "--static", action="store_true", help="graph static (possible) deps, don't concretize (implies ``--dot``)", ) subparser.add_argument( "-c", "--color", action="store_true", help="use different colors for different dependency types", ) subparser.add_argument( "-i", "--installed", action="store_true", help="graph specs from the DB" ) arguments.add_common_arguments(subparser, ["deptype", "specs"]) def graph(parser, args): env = ev.active_environment() if args.installed and env: tty.die("cannot use --installed with an active environment") if args.color and not args.dot: tty.die("the --color option can be used only with --dot") if args.installed: if not args.specs: specs = spack.store.STORE.db.query() else: result = [] for item in args.specs: result.extend(spack.store.STORE.db.query(item)) specs = list(set(result)) elif env: specs = env.concrete_roots() if args.specs: specs = env.all_matching_specs(*args.specs) else: specs = spack.cmd.parse_specs(args.specs, concretize=not args.static) if not specs: tty.die("no spec matching the query") if args.static: static_graph_dot(specs, depflag=args.deptype) return if args.dot: builder = SimpleDAG() if args.color: builder = DAGWithDependencyTypes() graph_dot(specs, builder=builder, depflag=args.deptype) return # ascii is default: user doesn't need to provide it explicitly debug = spack.config.get("config:debug") graph_ascii(specs[0], debug=debug, depflag=args.deptype) for spec in specs[1:]: print() # extra line bt/w independent graphs graph_ascii(spec, debug=debug) ================================================ FILE: lib/spack/spack/cmd/help.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys from spack.llnl.util.tty.color import colorize description = "get help on spack and its commands" section = "help" level = "short" # # These are longer guides on particular aspects of Spack. Currently there # is only one on spec syntax. # spec_guide = """\ @*B{spec expression syntax:} package [constraints] [^dependency [constraints] ...] package any package from 'spack list', or @K{/hash} unique prefix or full hash of installed package @*B{constraints:} @*c{versions:} @c{@version} single version @c{@min:max} version range (inclusive) @c{@min:} version or higher @c{@:max} up to version (inclusive) @c{@=version} exact version @*c{compilers:} @g{%compiler} build with @g{%compiler@version} build with specific compiler version @g{%compiler@min:max} specific version range (see above) @*c{compiler flags:} @g{cflags="flags"} cppflags, cflags, cxxflags, fflags, ldflags, ldlibs @g{==} propagate flags to package dependencies @*c{variants:} @B{+variant} enable @r{-variant} or @r{~variant} disable @B{variant=value} set non-boolean to @B{variant=value1,value2,value3} set multi-value values @B{++}, @r{--}, @r{~~}, @B{==} propagate variants to package dependencies @*c{architecture variants:} @m{platform=platform} linux, darwin, freebsd, windows @m{os=operating_system} specific @m{target=target} specific processor @m{arch=platform-os-target} shortcut for all three above @*c{dependencies:} ^dependency [constraints] specify constraints on dependencies ^@K{/hash} build with a specific installed dependency @*B{examples:} hdf5 any hdf5 configuration hdf5 @c{@1.10.1} hdf5 version 1.10.1 hdf5 @c{@1.8:} hdf5 1.8 or higher hdf5 @c{@1.8:} @g{%gcc} hdf5 1.8 or higher built with gcc hdf5 @B{+mpi} hdf5 with mpi enabled hdf5 @r{~mpi} hdf5 with mpi disabled hdf5 @B{++mpi} hdf5 with mpi enabled and propagates hdf5 @r{~~mpi} hdf5 with mpi disabled and propagates hdf5 @B{+mpi} ^mpich hdf5 with mpi, using mpich hdf5 @B{+mpi} ^openmpi@c{@1.7} hdf5 with mpi, using openmpi 1.7 boxlib @B{dim=2} boxlib built for 2 dimensions libdwarf @g{%intel} ^libelf@g{%gcc} libdwarf, built with intel compiler, linked to libelf built with gcc mvapich2 @B{fabrics=psm,mrail,sock} @g{%gcc} mvapich2, built with gcc compiler, with support for multiple fabrics """ guides = {"spec": spec_guide} def setup_parser(subparser: argparse.ArgumentParser) -> None: help_cmd_group = subparser.add_mutually_exclusive_group() help_cmd_group.add_argument( "help_command", nargs="?", default=None, help="command to get help on" ) help_all_group = subparser.add_mutually_exclusive_group() help_all_group.add_argument( "-a", "--all", action="store_const", const="long", default="short", help="list all available commands and options", ) help_spec_group = subparser.add_mutually_exclusive_group() help_spec_group.add_argument( "--spec", action="store_const", dest="guide", const="spec", default=None, help="help on the package specification syntax", ) def help(parser, args): if args.guide: print(colorize(guides[args.guide])) return 0 if args.help_command: parser.add_command(args.help_command) parser.parse_args([args.help_command, "-h"]) else: sys.stdout.write(parser.format_help(level=args.all)) ================================================ FILE: lib/spack/spack/cmd/info.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # mypy: disallow-untyped-defs import argparse import collections import shutil import sys import textwrap from argparse import Namespace from typing import Any, Callable, Dict, Iterable, List, Optional, TextIO, Tuple import spack.builder import spack.cmd import spack.dependency import spack.deptypes as dt import spack.fetch_strategy as fs import spack.install_test import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.package_base import spack.repo import spack.spec import spack.variant import spack.version from spack.cmd.common import arguments from spack.llnl.util.tty.colify import colify from spack.package_base import PackageBase from spack.util.typing import SupportsRichComparison description = "get detailed information on a particular package" section = "query" level = "short" header_color = "@*b" plain_format = "@." #: Allow at least this much room for values when formatting definitions #: Wrap after a long variant name/condition if we need to do so to preserve this width. MIN_VALUES_WIDTH = 30 class Formatter: """Generic formatter for elements displayed by `spack info`. Elements have four parts: name, values, when condition, and description. They can be formatted two ways (shown here for variants): Grouped by when (default):: when +cuda cuda_arch [none] none, 10, 100, 100a, 101, 101a, 11, 12, 120, 120a, 13 CUDA architecture Or, by name (each name has a when nested under it):: cuda_arch [none] none, 10, 100, 100a, 101, 101a, 11, 12, 120, 120a, 13 when +cuda CUDA architecture The values and description will be wrapped if needed. the name (and any additional info) will not (so they should be kept short). Subclasses are responsible for generating colorized text, but not wrapping, indentation, or other formatting, for the name, values, and description. """ def format_name(self, element: Any) -> str: return str(element) def format_values(self, element: Any) -> str: return "" def format_description(self, element: Any) -> str: return "" def padder(str_list: Iterable, extra: int = 0) -> Callable: """Return a function to pad elements of a list.""" length = max(len(str(s)) for s in str_list) + extra def pad(string: str) -> str: string = str(string) padding = max(0, length - len(string)) return string + (padding * " ") return pad def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-a", "--all", action="store_true", default=False, help="output all package information" ) by = subparser.add_mutually_exclusive_group() by.add_argument( "--by-name", dest="by_name", action="store_true", default=True, help="list variants, dependency, etc. in name order, then by when condition", ) by.add_argument( "--by-when", dest="by_name", action="store_false", default=False, help="group variants, dependencies, etc. first by when condition, then by name", ) options = [ ("--detectable", print_detectable.__doc__), ("--maintainers", print_maintainers.__doc__), ("--namespace", print_namespace.__doc__), ("--no-dependencies", f"do not {print_dependencies.__doc__}"), ("--no-variants", f"do not {print_variants.__doc__}"), ("--no-versions", f"do not {print_versions.__doc__}"), ("--phases", print_phases.__doc__), ("--tags", print_tags.__doc__), ("--tests", print_tests.__doc__), ("--virtuals", print_virtuals.__doc__), ] for opt, help_comment in options: subparser.add_argument(opt, action="store_true", help=help_comment) # deprecated for the more generic --by-name, but still here until we can remove it subparser.add_argument( "--variants-by-name", dest="by_name", action=arguments.DeprecatedStoreTrueAction, help=argparse.SUPPRESS, removed_in="a future Spack release", instructions="use --by-name instead", ) arguments.add_common_arguments(subparser, ["spec"]) def section_title(s: str) -> str: return header_color + s + plain_format def version(s: str) -> str: return spack.spec.VERSION_COLOR + s + plain_format def format_deptype(depflag: int) -> str: color_flags = zip("gcbm", dt.ALL_FLAGS) return ", ".join( color.colorize(f"@{c}{{{dt.flag_to_string(depflag & flag)}}}") for c, flag in color_flags if depflag & flag ) class DependencyFormatter(Formatter): def format_name(self, dep: spack.dependency.Dependency) -> str: return dep.spec._long_spec(color=color.get_color_when()) def format_values(self, dep: spack.dependency.Dependency) -> str: return str(format_deptype(dep.depflag)) def count_bool_variant_conditions( when_indexed_dictionary: Dict[spack.spec.Spec, Any], ) -> List[Tuple[int, Tuple[str, bool]]]: """Counts boolean variants in whens in a dictionary. Returns a list of the most used when conditions for boolean variants along with their value. """ top: Dict = collections.defaultdict(int) for when, _ in when_indexed_dictionary.items(): for v, variant in when.variants.items(): if type(variant.value) is bool: top[(variant.name, variant.value)] += 1 # sorted by frequency, highest first return list(reversed(sorted((n, t) for t, n in top.items()))) def print_dependencies(pkg: PackageBase, args: Namespace) -> None: """output build, link, and run package dependencies""" print_definitions(pkg, "Dependencies", pkg.dependencies, DependencyFormatter(), args.by_name) def print_dependency_suggestion(pkg: PackageBase) -> None: variant_counts = count_bool_variant_conditions(pkg.dependencies) big_variants = [ (name, val) for n, (name, val) in variant_counts # make a note of variants with large counts that aren't already toggled by the user. if n >= 20 and not (name in pkg.spec.variants and pkg.spec.variants[name].value != val) ] if big_variants: spec = spack.spec.Spec(pkg.name) for name, val in big_variants: # skip if user specified, or already saw a value (e.g. many +mpi and ~mpi) if name in spec.variants or name in pkg.spec.variants: continue spec.variants[name] = spack.variant.BoolValuedVariant(name, not val) # if there is new stuff to add beyond the input if spec.variants: spec.constrain(pkg.spec) # include already specified constraints print() tty.info( f"{pkg.name} has many complex dependencies; consider this for a simpler view:", f"spack info {spec.format(color=tty.color.get_color_when())}", format="y", ) def print_detectable(pkg: PackageBase, args: Namespace) -> None: """output information on external detection""" color.cprint("") color.cprint(section_title("Externally Detectable:")) # If the package has an 'executables' of 'libraries' field, it # can detect an installation if hasattr(pkg, "executables") or hasattr(pkg, "libraries"): find_attributes = [] if hasattr(pkg, "determine_version"): find_attributes.append("version") if hasattr(pkg, "determine_variants"): find_attributes.append("variants") # If the package does not define 'determine_version' nor # 'determine_variants', then it must use some custom detection # mechanism. In this case, just inform the user it's detectable somehow. color.cprint( " True{0}".format( " (" + ", ".join(find_attributes) + ")" if find_attributes else "" ) ) else: color.cprint(" False") def print_maintainers(pkg: PackageBase, args: Namespace) -> None: """output package maintainers""" if len(pkg.maintainers) > 0: mnt = " ".join(["@@" + m for m in pkg.maintainers]) color.cprint("") color.cprint(section_title("Maintainers: ") + mnt) def print_namespace(pkg: PackageBase, args: Namespace) -> None: """output package namespace""" repo = spack.repo.PATH.get_repo(pkg.namespace) color.cprint("") color.cprint(section_title("Namespace:")) color.cprint(f" @c{{{repo.namespace}}} at {repo.root}") def print_phases(pkg: PackageBase, args: Namespace) -> None: """output installation phases""" builder = spack.builder.create(pkg) if hasattr(builder, "phases") and builder.phases: color.cprint("") color.cprint(section_title("Installation Phases:")) phase_str = "" for phase in builder.phases: phase_str += " {0}".format(phase) color.cprint(phase_str) def print_tags(pkg: PackageBase, args: Namespace) -> None: """output package tags""" color.cprint("") color.cprint(section_title("Tags: ")) if hasattr(pkg, "tags"): tags = sorted(pkg.tags) colify(tags, indent=4) else: color.cprint(" None") def print_tests(pkg: PackageBase, args: Namespace) -> None: """output relevant build-time and stand-alone tests""" # Some built-in base packages (e.g., Autotools) define callback (e.g., # check) inherited by descendant packages. These checks may not result # in build-time testing if the package's build does not implement the # expected functionality (e.g., a 'check' or 'test' targets). # # So the presence of a callback in Spack does not necessarily correspond # to the actual presence of built-time tests for a package. for callbacks, phase in [ (getattr(pkg, "build_time_test_callbacks", None), "Build"), (getattr(pkg, "install_time_test_callbacks", None), "Install"), ]: color.cprint("") color.cprint(section_title("Available {0} Phase Test Methods:".format(phase))) names = [] if callbacks: for name in callbacks: if getattr(pkg, name, False): names.append(name) if names: colify(sorted(names), indent=4) else: color.cprint(" None") # PackageBase defines an empty install/smoke test but we want to know # if it has been overridden and, therefore, assumed to be implemented. color.cprint("") color.cprint(section_title("Stand-Alone/Smoke Test Methods:")) names = spack.install_test.test_function_names(pkg, add_virtuals=True) if names: colify(sorted(names), indent=4) else: color.cprint(" None") def _fmt_when(when: "spack.spec.Spec", indent: int) -> str: return color.colorize( f"{indent * ' '}@B{{when}} {color.cescape(when._long_spec(color=color.get_color_when()))}" ) def _fmt_variant_value(v: Any) -> str: return str(v).lower() if v is None or isinstance(v, bool) else str(v) def _print_definition( name_field: str, values_field: str, description: str, max_name_len: int, indent: int, when: Optional[spack.spec.Spec] = None, out: Optional[TextIO] = None, ) -> None: """Print a definition entry for `spack info` output. Arguments: name_field: name and optional info, e.g. a default; should be short. values_field: possible values for the entry; Wrapped if long. description: description of the field (wrapped if overly long) max_name_len: max length of any definition to be printed indent: size of leading indent for entry when: optional when condition out: stream to print to Caller is expected to calculate the max name length in advance and pass it to ``_print_definition``. """ out = out or sys.stdout cols = shutil.get_terminal_size().columns # prevent values from being compressed by really long names name_col_width = min(max_name_len, cols - MIN_VALUES_WIDTH - indent) name_len = color.clen(name_field) pad = 4 # min padding between name and values value_indent = (indent + name_col_width + pad) * " " # left edge of values formatted_name_and_values = f"{indent * ' '}{name_field}" if values_field: formatted_values = "\n".join( color.cwrap( values_field, width=cols - 2, initial_indent=value_indent, subsequent_indent=value_indent, ) ) if name_len > name_col_width: # for overlong names, values appear aligned on next line formatted_name_and_values += f"\n{formatted_values}" else: # for regular names, trim indentation to make room for name on same line formatted_values = formatted_values[indent + name_len + pad :] # e.g,. name [default] value1, value2, value3, ... formatted_name_and_values += f"{pad * ' '}{formatted_values}" out.write(f"{formatted_name_and_values}\n") # when description_indent = indent + 4 if when is not None and when != spack.spec.Spec(): out.write(_fmt_when(when, description_indent - 2)) out.write("\n") # description, preserving explicit line breaks from the way it's written in the # package file, but still wrapoing long lines for small terminals. This allows # descriptions to provide detailed help in descriptions (see, e.g., gasnet's variants). if description: formatted_description = "\n".join( textwrap.fill( line, width=cols - 2, initial_indent=description_indent * " ", subsequent_indent=description_indent * " ", ) for line in description.split("\n") ) out.write(formatted_description) out.write("\n") def print_header(header: str, when_indexed_dictionary: Dict, formatter: Formatter) -> bool: color.cprint("") color.cprint(section_title(f"{header}:")) if not when_indexed_dictionary: print(" None") return False return True def max_name_length(when_indexed_dictionary: Dict, formatter: Formatter) -> int: # Calculate the max length of the first field of the definition. Lets us know how # much to pad other fields on the first line. return max( color.clen(formatter.format_name(definition)) for subkey in spack.package_base._subkeys(when_indexed_dictionary) for _, definition in spack.package_base._definitions(when_indexed_dictionary, subkey) ) def print_grouped_by_when( pkg: PackageBase, header: str, when_indexed_dictionary: Dict, formatter: Formatter ) -> None: """Generic method to print metadata grouped by when conditions.""" if not print_header(header, when_indexed_dictionary, formatter): return max_name_len = max_name_length(when_indexed_dictionary, formatter) # ensure that items without conditions come first unconditional_first = lambda item: (item[0] != spack.spec.Spec(), item) indent = 4 for when, by_name in sorted(when_indexed_dictionary.items(), key=unconditional_first): if not pkg.intersects(when): continue start_indent = indent values_indent = max_name_len + 4 if when != spack.spec.Spec(): sys.stdout.write("\n") sys.stdout.write(_fmt_when(when, indent)) sys.stdout.write("\n") # indent names slightly inside 'when', but line up values start_indent += 2 values_indent -= 2 for subkey, definition in sorted(by_name.items()): _print_definition( formatter.format_name(definition), formatter.format_values(definition), formatter.format_description(definition), values_indent, start_indent, when=None, out=sys.stdout, ) def print_by_name( pkg: PackageBase, header: str, when_indexed_dictionary: Dict, formatter: Formatter ) -> None: if not print_header(header, when_indexed_dictionary, formatter): return max_name_len = max_name_length(when_indexed_dictionary, formatter) max_name_len += 4 indent = 4 def unconditional_first(definition: Any) -> SupportsRichComparison: spec = getattr(definition, "spec", None) if spec: return (spec != spack.spec.Spec(spec.name), spec) else: return getattr(definition, "name", None) # type: ignore[return-value] for subkey in spack.package_base._subkeys(when_indexed_dictionary): for when, definition in sorted( spack.package_base._definitions(when_indexed_dictionary, subkey), key=lambda t: unconditional_first(t[1]), ): if not pkg.intersects(when): continue _print_definition( formatter.format_name(definition), formatter.format_values(definition), formatter.format_description(definition), max_name_len, indent, when=when, out=sys.stdout, ) sys.stdout.write("\n") def print_definitions( pkg: PackageBase, header: str, when_indexed_dictionary: Dict, formatter: Formatter, by_name: bool, ) -> None: # convert simple dictionaries to dicts of dicts before formatting. # subkeys are ignored in formatting, so use stringified numbers. values = when_indexed_dictionary.values() if when_indexed_dictionary and not isinstance(next(iter(values)), dict): when_indexed_dictionary = { when: {str(i): element} for i, (when, element) in enumerate(when_indexed_dictionary.items()) } if by_name: print_by_name(pkg, header, when_indexed_dictionary, formatter) else: print_grouped_by_when(pkg, header, when_indexed_dictionary, formatter) class VariantFormatter(Formatter): def format_name(self, variant: spack.variant.Variant) -> str: return color.colorize( f"@c{{{variant.name}}} @C{{[{_fmt_variant_value(variant.default)}]}}" ) def format_values(self, variant: spack.variant.Variant) -> str: values = ( [variant.values] if not isinstance(variant.values, (tuple, list, spack.variant.DisjointSetsOfValues)) else variant.values ) # put 'none' first, sort the rest by value sorted_values = sorted(values, key=lambda v: (v != "none", v)) return color.colorize(f"@c{{{', '.join(_fmt_variant_value(v) for v in sorted_values)}}}") def format_description(self, variant: spack.variant.Variant) -> str: return variant.description def print_variants(pkg: PackageBase, args: Namespace) -> None: """output variants""" print_definitions(pkg, "Variants", pkg.variants, VariantFormatter(), args.by_name) def print_licenses(pkg: PackageBase, args: Namespace) -> None: """Output the licenses of the project.""" print_definitions(pkg, "Licenses", pkg.licenses, Formatter(), args.by_name) def print_versions(pkg: PackageBase, args: Namespace) -> None: """output versions""" color.cprint("") color.cprint(section_title("Preferred version: ")) versions = [v for v in pkg.versions if pkg.spec.versions.intersects(v)] if not versions: color.cprint(version(" None")) color.cprint("") color.cprint(section_title("Safe versions: ")) color.cprint(version(" None")) color.cprint("") color.cprint(section_title("Deprecated versions: ")) color.cprint(version(" None")) else: pad = padder(versions, 4) preferred = spack.package_base.preferred_version(pkg) def get_url(version: spack.version.VersionType) -> str: try: return str(fs.for_package_version(pkg, version)) except fs.InvalidArgsError: return "No URL" url = get_url(preferred) if pkg.has_code else "" line = version(" {0}".format(pad(preferred))) + color.cescape(str(url)) color.cwrite(line) print() safe = [] deprecated = [] for v in reversed(sorted(versions)): if pkg.has_code: url = get_url(v) if pkg.versions[v].get("deprecated", False): deprecated.append((v, url)) else: safe.append((v, url)) for title, vers in [("Safe", safe), ("Deprecated", deprecated)]: color.cprint("") color.cprint(section_title("{0} versions: ".format(title))) if not vers: color.cprint(version(" None")) continue for v, url in vers: line = version(" {0}".format(pad(v))) + color.cescape(str(url)) color.cprint(line) def print_virtuals(pkg: PackageBase, args: Namespace) -> None: """output virtual packages""" color.cprint("") color.cprint(section_title("Virtual Packages: ")) if pkg.provided: for when, specs in reversed(sorted(pkg.provided.items())): line = " %s provides %s" % (when.cformat(), ", ".join(s.cformat() for s in specs)) print(line) else: color.cprint(" None") def info(parser: argparse.ArgumentParser, args: Namespace) -> None: specs = spack.cmd.parse_specs(args.spec) if len(specs) > 1: tty.die(f"`spack info` requires exactly one spec. Parsed {len(specs)}") if len(specs) == 0: tty.die("`spack info` requires a spec.") spec = specs[0] pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname) pkg_cls.validate_variant_names(spec) pkg = pkg_cls(spec) # Output core package information header = section_title("{0}: ").format(pkg.build_system_class) + pkg.name color.cprint(header) color.cprint("") color.cprint(section_title("Description:")) if pkg.__doc__: color.cprint(color.cescape(pkg.format_doc(indent=4))) else: color.cprint(" None") if getattr(pkg, "homepage"): color.cprint(section_title("Homepage: ") + str(pkg.homepage)) # Now output optional information in expected order sections = [ (args.all or args.maintainers, print_maintainers), (args.all or args.namespace, print_namespace), (args.all or args.detectable, print_detectable), (args.all or args.tags, print_tags), (args.all or not args.no_versions, print_versions), (args.all or not args.no_variants, print_variants), (args.all or args.phases, print_phases), (args.all or not args.no_dependencies, print_dependencies), (args.all or args.virtuals, print_virtuals), (args.all or args.tests, print_tests), (True, print_licenses), ] for print_it, func in sections: if print_it: func(pkg, args) print_dependency_suggestion(pkg) color.cprint("") ================================================ FILE: lib/spack/spack/cmd/install.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import shutil import sys from typing import List import spack.cmd import spack.config import spack.environment as ev import spack.installer_dispatch import spack.llnl.util.filesystem as fs import spack.paths import spack.spec import spack.store from spack.cmd.common import arguments from spack.error import InstallError, SpackError from spack.installer import InstallPolicy from spack.llnl.string import plural from spack.llnl.util import tty description = "build and install packages" section = "build" level = "short" def cache_opt(use_buildcache: str, default: InstallPolicy) -> InstallPolicy: if use_buildcache == "only": return "cache_only" elif use_buildcache == "never": return "source_only" return default def install_kwargs_from_args(args): """Translate command line arguments into a dictionary that will be passed to the package installer. """ pkg_use_bc, dep_use_bc = args.use_buildcache if args.cache_only: default = "cache_only" elif args.use_cache: default = "auto" else: default = "source_only" return { "fail_fast": args.fail_fast, "keep_prefix": args.keep_prefix, "keep_stage": args.keep_stage, "restage": not args.dont_restage, "install_source": args.install_source, "verbose": args.verbose or args.install_verbose, "fake": args.fake, "dirty": args.dirty, "root_policy": cache_opt(pkg_use_bc, default), "dependencies_policy": cache_opt(dep_use_bc, default), "include_build_deps": args.include_build_deps, "stop_at": args.until, "unsigned": args.unsigned, "install_deps": ("dependencies" in args.things_to_install), "install_package": ("package" in args.things_to_install), "concurrent_packages": args.concurrent_packages, } def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--only", default="package,dependencies", dest="things_to_install", choices=["package", "dependencies"], help="select the mode of installation\n\n" "default is to install the package along with all its dependencies. " "alternatively, one can decide to install only the package or only the dependencies", ) subparser.add_argument( "-u", "--until", type=str, dest="until", default=None, help="phase to stop after when installing (default None)", ) arguments.add_common_arguments(subparser, ["concurrent_packages"]) arguments.add_common_arguments(subparser, ["jobs"]) subparser.add_argument( "--overwrite", action="store_true", help="reinstall an existing spec, even if it has dependents", ) subparser.add_argument( "--fail-fast", action="store_true", help="stop all builds if any build fails (default is best effort)", ) subparser.add_argument( "--keep-prefix", action="store_true", help="don't remove the install prefix if installation fails", ) subparser.add_argument( "--keep-stage", action="store_true", help="don't remove the build stage if installation succeeds", ) subparser.add_argument( "--dont-restage", action="store_true", help="if a partial install is detected, don't delete prior state", ) cache_group = subparser.add_mutually_exclusive_group() cache_group.add_argument( "--use-cache", action="store_true", dest="use_cache", default=True, help="check for pre-built Spack packages in mirrors (default)", ) cache_group.add_argument( "--no-cache", action="store_false", dest="use_cache", default=True, help="do not check for pre-built Spack packages in mirrors", ) cache_group.add_argument( "--cache-only", action="store_true", dest="cache_only", default=False, help="only install package from binary mirrors", ) cache_group.add_argument( "--use-buildcache", dest="use_buildcache", type=arguments.use_buildcache, default="package:auto,dependencies:auto", metavar="[{auto,only,never},][package:{auto,only,never},][dependencies:{auto,only,never}]", help="select the mode of buildcache for the 'package' and 'dependencies'\n\n" "default: package:auto,dependencies:auto\n\n" "- `auto` behaves like --use-cache\n" "- `only` behaves like --cache-only\n" "- `never` behaves like --no-cache", ) subparser.add_argument( "--include-build-deps", action="store_true", dest="include_build_deps", default=False, help="include build deps when installing from cache, " "useful for CI pipeline troubleshooting", ) subparser.add_argument( "--no-check-signature", action="store_true", dest="unsigned", default=None, help="do not check signatures of binary packages (override mirror config)", ) subparser.add_argument( "--show-log-on-error", action="store_true", help="print full build log to stderr if build fails", ) subparser.add_argument( "--source", action="store_true", dest="install_source", help="install source files in prefix", ) arguments.add_common_arguments(subparser, ["no_checksum"]) subparser.add_argument( "-v", "--verbose", action="store_true", dest="install_verbose", help="display verbose build output while installing", ) subparser.add_argument("--fake", action="store_true", help="fake install for debug purposes") subparser.add_argument( "--only-concrete", action="store_true", default=False, help="(with environment) only install already concretized specs", ) updateenv_group = subparser.add_mutually_exclusive_group() updateenv_group.add_argument( "--add", action="store_true", default=False, help="(with environment) add spec to the environment as a root", ) updateenv_group.add_argument( "--no-add", action="store_false", dest="add", help="(with environment) do not add spec to the environment as a root", ) cd_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ["clean", "dirty"]) testing = subparser.add_mutually_exclusive_group() testing.add_argument( "--test", default=None, choices=["root", "all"], help="run tests on only root packages or all packages", ) arguments.add_common_arguments(subparser, ["log_format"]) subparser.add_argument("--log-file", default=None, help="filename for the log file") subparser.add_argument( "--help-cdash", action="store_true", help="show usage instructions for CDash reporting" ) arguments.add_cdash_args(subparser, False) arguments.add_common_arguments(subparser, ["yes_to_all", "spec"]) arguments.add_concretizer_args(subparser) def default_log_file(spec): """Computes the default filename for the log file and creates the corresponding directory if not present """ basename = spec.format_path("test-{name}-{version}-{hash}.xml") dirname = fs.os.path.join(spack.paths.reports_path, "junit") fs.mkdirp(dirname) return fs.os.path.join(dirname, basename) def report_filename(args: argparse.Namespace, specs: List[spack.spec.Spec]) -> str: """Return the filename to be used for reporting to JUnit or CDash format.""" result = args.log_file or default_log_file(specs[0]) return result def compute_tests_install_kwargs(specs, cli_test_arg): """Translate the test cli argument into the proper install argument""" if cli_test_arg == "all": return True elif cli_test_arg == "root": return [spec.name for spec in specs] return False def require_user_confirmation_for_overwrite(concrete_specs, args): if args.yes_to_all: return installed = list(filter(lambda x: x, map(spack.store.STORE.db.query_one, concrete_specs))) display_args = {"long": True, "show_flags": True, "variants": True} if installed: tty.msg("The following package specs will be reinstalled:\n") spack.cmd.display_specs(installed, **display_args) not_installed = list(filter(lambda x: x not in installed, concrete_specs)) if not_installed: tty.msg( "The following package specs are not installed and" " the --overwrite flag was given. The package spec" " will be newly installed:\n" ) spack.cmd.display_specs(not_installed, **display_args) # We have some specs, so one of the above must have been true answer = tty.get_yes_or_no("Do you want to proceed?", default=False) if not answer: tty.die("Reinstallation aborted.") def _dump_log_on_error(e: InstallError): e.print_context() assert e.pkg, "Expected InstallError to include the associated package" if not os.path.exists(e.pkg.log_path): tty.error("'spack install' created no log.") else: sys.stderr.write("Full build log:\n") with open(e.pkg.log_path, errors="replace", encoding="utf-8") as log: shutil.copyfileobj(log, sys.stderr) def _die_require_env(): msg = "install requires a package argument or active environment" if "spack.yaml" in os.listdir(os.getcwd()): # There's a spack.yaml file in the working dir, the user may # have intended to use that msg += ( "\n\n" "Did you mean to install using the `spack.yaml`" " in this directory? Try: \n" " spack env activate .\n" " spack install\n" " OR\n" " spack --env . install" ) tty.die(msg) def install(parser, args): # TODO: unify args.verbose? tty.set_verbose(args.verbose or args.install_verbose) if args.help_cdash: arguments.print_cdash_help() return if args.no_checksum: spack.config.set("config:checksum", False, scope="command_line") if args.log_file and not args.log_format: msg = "the '--log-format' must be specified when using '--log-file'" tty.die(msg) arguments.sanitize_reporter_options(args) reporter = args.reporter() if args.log_format else None install_kwargs = install_kwargs_from_args(args) env = ev.active_environment() if not env and not args.spec: _die_require_env() try: if env: install_with_active_env(env, args, install_kwargs, reporter) else: install_without_active_env(args, install_kwargs, reporter) except InstallError as e: if args.show_log_on_error: _dump_log_on_error(e) raise def _maybe_add_and_concretize(args, env, specs): """Handle the overloaded spack install behavior of adding and automatically concretizing specs""" # Users can opt out of accidental concretizations with --only-concrete if args.only_concrete: return # Otherwise, we will modify the environment. with env.write_transaction(): # `spack add` adds these specs. if args.add: for spec in specs: env.add(spec) # `spack concretize` tests = compute_tests_install_kwargs(env.user_specs, args.test) concretized_specs = env.concretize(tests=tests) if concretized_specs: tty.msg(f"Concretized {plural(len(concretized_specs), 'spec')}") ev.display_specs([concrete for _, concrete in concretized_specs]) # save view regeneration for later, so that we only do it # once, as it can be slow. env.write(regenerate=False) def install_with_active_env(env: ev.Environment, args, install_kwargs, reporter): specs = spack.cmd.parse_specs(args.spec) # The following two commands are equivalent: # 1. `spack install --add x y z` # 2. `spack add x y z && spack concretize && spack install --only-concrete` # here we do the `add` and `concretize` part. _maybe_add_and_concretize(args, env, specs) # Now we're doing `spack install --only-concrete`. if args.add or not specs: specs_to_install = env.concrete_roots() if not specs_to_install: tty.msg(f"{env.name} environment has no specs to install") return # `spack install x y z` without --add is installing matching specs in the env. else: specs_to_install = env.all_matching_specs(*specs) if not specs_to_install: msg = ( "Cannot install '{0}' because no matching specs are in the current environment.\n" " Specs can be added to the environment with 'spack add {0}',\n" " or as part of the install command with 'spack install --add {0}'" ).format(" ".join(args.spec)) tty.die(msg) install_kwargs["tests"] = compute_tests_install_kwargs(specs_to_install, args.test) if args.overwrite: require_user_confirmation_for_overwrite(specs_to_install, args) install_kwargs["overwrite"] = [spec.dag_hash() for spec in specs_to_install] try: report_file = report_filename(args, specs_to_install) install_kwargs["report_file"] = report_file install_kwargs["reporter"] = reporter env.install_specs(specs_to_install, **install_kwargs) finally: if env.views: with env.write_transaction(): env.write(regenerate=True) def concrete_specs_from_cli(args, install_kwargs): """Return abstract and concrete spec parsed from the command line.""" abstract_specs = spack.cmd.parse_specs(args.spec) install_kwargs["tests"] = compute_tests_install_kwargs(abstract_specs, args.test) try: concrete_specs = spack.cmd.parse_specs( args.spec, concretize=True, tests=install_kwargs["tests"] ) except SpackError as e: tty.debug(e) if args.log_format is not None: reporter = args.reporter() reporter.concretization_report(report_filename(args, abstract_specs), e.message) raise return concrete_specs def install_without_active_env(args, install_kwargs, reporter): concrete_specs = concrete_specs_from_cli(args, install_kwargs) if len(concrete_specs) == 0: tty.die("The `spack install` command requires a spec to install.") if args.overwrite: require_user_confirmation_for_overwrite(concrete_specs, args) install_kwargs["overwrite"] = [spec.dag_hash() for spec in concrete_specs] installs = [s.package for s in concrete_specs] install_kwargs["explicit"] = [s.dag_hash() for s in concrete_specs] try: builder = spack.installer_dispatch.create_installer( installs, create_reports=reporter is not None, **install_kwargs ) builder.install() finally: if reporter: report_file = report_filename(args, concrete_specs) reporter.build_report(report_file, list(builder.reports.values())) ================================================ FILE: lib/spack/spack/cmd/installer/CMakeLists.txt ================================================ cmake_minimum_required (VERSION 3.13) project(spack_installer NONE) set(PYTHON_VERSION "3.9.0" CACHE STRING "Version of Python to build.") set(PY_DOWNLOAD_LINK "https://www.paraview.org/files/dependencies") set(PY_FILENAME "Python-${PYTHON_VERSION}-win64.tar.xz") set(PYTHON_DIR "Python-${PYTHON_VERSION}") if (SPACK_VERSION) set(SPACK_DL "https://github.com/spack/spack/releases/download/v${SPACK_VERSION}") set(SPACK_FILENAME "spack-${SPACK_VERSION}.tar.gz") set(SPACK_DIR "spack-${SPACK_VERSION}") # SPACK DOWNLOAD AND EXTRACTION----------------------------------- file(DOWNLOAD "${SPACK_DL}/${SPACK_FILENAME}" "${CMAKE_CURRENT_BINARY_DIR}/${SPACK_FILENAME}" STATUS download_status ) list(GET download_status 0 res) if(res) list(GET download_status 1 err) message(FATAL_ERROR "Failed to download ${SPACK_FILENAME} ${err}") endif() message(STATUS "Successfully downloaded ${SPACK_FILENAME}") execute_process(COMMAND ${CMAKE_COMMAND} -E tar xfz "${CMAKE_CURRENT_BINARY_DIR}/${SPACK_FILENAME}" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" RESULT_VARIABLE res) if(NOT res EQUAL 0) message(FATAL_ERROR "Extraction of ${SPACK_FILENAME} failed.") endif() message(STATUS "Extracted ${SPACK_DIR}") SET(SPACK_SOURCE "${CMAKE_CURRENT_BINARY_DIR}/${SPACK_DIR}") elseif(SPACK_SOURCE) get_filename_component(SPACK_DIR ${SPACK_SOURCE} NAME) else() message(FATAL_ERROR "Must specify SPACK_VERSION or SPACK_SOURCE") endif() # GIT DOWNLOAD---------------------------------------------------- set(GIT_FILENAME "Git-2.31.1-64-bit.exe") file(DOWNLOAD "https://github.com/git-for-windows/git/releases/download/v2.31.1.windows.1/Git-2.31.1-64-bit.exe" "${CMAKE_CURRENT_BINARY_DIR}/${GIT_FILENAME}" STATUS download_status EXPECTED_HASH "SHA256=c43611eb73ad1f17f5c8cc82ae51c3041a2e7279e0197ccf5f739e9129ce426e" ) list(GET download_status 0 res) if(res) list(GET download_status 1 err) message(FATAL_ERROR "Failed to download ${GIT_FILENAME} ${err}") endif() message(STATUS "Successfully downloaded ${GIT_FILENAME}") # PYTHON DOWNLOAD AND EXTRACTION----------------------------------- file(DOWNLOAD "${PY_DOWNLOAD_LINK}/${PY_FILENAME}" "${CMAKE_CURRENT_BINARY_DIR}/${PY_FILENAME}" STATUS download_status EXPECTED_HASH "SHA256=f6aeebc6d1ff77418678ed5612b64ce61be6bc9ef3ab9c291ac557abb1783420" ) list(GET download_status 0 res) if(res) list(GET download_status 1 err) message(FATAL_ERROR "Failed to download ${PY_FILENAME} ${err}") endif() message(STATUS "Successfully downloaded ${PY_FILENAME}") execute_process(COMMAND ${CMAKE_COMMAND} -E tar xfz "${CMAKE_CURRENT_BINARY_DIR}/${PY_FILENAME}" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" RESULT_VARIABLE res) if(NOT res EQUAL 0) message(FATAL_ERROR "Extraction of ${PY_FILENAME} failed.") endif() message(STATUS "Extracted ${PY_FILENAME}.") # license must be a .txt or .rtf file configure_file("${SPACK_LICENSE}" "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.rtf" COPYONLY) #INSTALLATION COMMANDS--------------------------------------------------- install(DIRECTORY "${SPACK_SOURCE}/" DESTINATION "${SPACK_DIR}") install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/Python-${PYTHON_VERSION}-win64/" DESTINATION "${PYTHON_DIR}") # CPACK Installer Instructions set(CPACK_PACKAGE_NAME "Spack") set(CPACK_PACKAGE_VENDOR "Lawrence Livermore National Laboratories") set(CPACK_PACKAGE_VERSION "0.16.0") set(CPACK_PACKAGE_DESCRIPTION "A flexible package manager designed to support multiple versions, configurations, platforms, and compilers.") set(CPACK_PACKAGE_HOMEPAGE_URL "https://spack.io") set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}") set(CPACK_PACKAGE_ICON "${SPACK_LOGO}") set(CPACK_RESOURCE_FILE_README "${CMAKE_CURRENT_SOURCE_DIR}/README.md") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.rtf") #set(CPACK_RESOURCE_FILE_WELCOME "${CMAKE_CURRENT_SOURCE_DIR}/NOTICE") # WIX options (the default) set(CPACK_GENERATOR "WIX") set(CPACK_WIX_PRODUCT_ICON "${SPACK_LOGO}") set(CPACK_WIX_UI_BANNER "${CMAKE_CURRENT_SOURCE_DIR}/banner493x58.bmp") set(CPACK_WIX_PATCH_FILE "${CMAKE_CURRENT_SOURCE_DIR}/patch.xml") set(CPACK_WIX_UPGRADE_GUID "D2C703E4-721D-44EC-8016-BCB96BB64E0B") set(CPACK_WIX_SKIP_PROGRAM_FOLDER TRUE) set(SHORTCUT_GUID "099213BC-0D37-4F29-B758-60CA2A7E6DDA") # Set full path to icon, shortcut in spack.wxs set(SPACK_SHORTCUT "spack_cmd.bat") configure_file("spack.wxs.in" "${CMAKE_CURRENT_BINARY_DIR}/spack.wxs") configure_file("bundle.wxs.in" "${CMAKE_CURRENT_BINARY_DIR}/bundle.wxs") set(CPACK_WIX_EXTRA_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/spack.wxs") include(CPack) ================================================ FILE: lib/spack/spack/cmd/installer/README.md ================================================ This README is a guide for creating a Spack installer for Windows using the ``make-installer`` command. The installer is an executable file that users can run to install Spack like any other Windows binary. Before proceeding, follow the setup instructions in Steps 1 and 2 of [Getting Started on Windows](https://spack.readthedocs.io/en/latest/getting_started.html#windows_support). # Step 1: Install prerequisites The only additional prerequisite for making the installer is Wix. Wix is a utility used for .msi creation and can be downloaded and installed at https://wixtoolset.org/releases/. The Visual Studio extensions are not necessary. # Step 2: Make the installer To use Spack, run ``spack_cmd.bat``. This will provide a Windows command prompt with an environment properly set up with Spack and its prerequisites. Ensure that Python and CMake are on your PATH. If needed, you may add the CMake executable provided by Visual Studio to your path, which will look something like: ``C:\Program Files (x86)\Microsoft Visual Studio\\\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake`` **IMPORTANT**: If you use Tab to complete any part of this path, the console will automatically add quotation marks to the start and the end since it will see the spaces and want to parse the whole of it as a string. This is incorrect for our purposes so before submitting the command, ensure that the quotes are removed. You will encounter configuration errors if you fail to do this. There are two ways to create the installer using Spack's ``make-installer`` command. The recommended method is to build the installer using a local checkout of Spack source (release or development), using the `-s` flag to specify the directory where the local checkout is. For example, if the local checkout is in a directory called ``spack-develop`` and want to generate an installer with the source there, you can use: ``spack make-installer -s spack-develop tmp`` Both the Spack source directory (e.g. ``spack-develop``) and installer destination directory (e.g. ``tmp``) may be an absolute path or relative to the current working directory. The entire contents of the specified directory will be included in the installer (e.g. .git files or local changes). Alternatively, if you would like to create an installer from a release version of Spack, say, 0.16.0, and store it in ``tmp``, you can use the following command: ``spack make-installer -v 0.16.0 tmp`` **IMPORTANT**: Windows features are not currently supported in Spack's official release branches, so an installer created using this method will *not* run on Windows. Regardless of your method, a file called ``Spack.exe`` will be created inside the destination directory. This executable bundles the Spack installer (``Spack.msi`` also located in destination directory) and the git installer. # Step 3: Run the installer After accepting the terms of service, select where on your computer you would like Spack installed, and after a few minutes Spack, Python and git will be installed and ready for use. **IMPORTANT**: To avoid permissions issues, it is recommended to select an install location other than ``C:\Program Files``. **IMPORTANT**: There is a specific option that must be chosen when letting Git install. When given the option of adjusting your ``PATH``, choose the ``Git from the command line and also from 3rd-party software`` option. This will automatically update your ``PATH`` variable to include the ``git`` command. Certain Spack commands expect ``git`` to be part of the ``PATH``. If this step is not performed properly, certain Spack commands will not work. If your Spack installation needs to be modified, repaired, or uninstalled, you can do any of these things by rerunning ``Spack.exe``. Running the installer creates a shortcut on your desktop that, when launched, will run ``spack_cmd.bat`` and launch a console with its initial directory being wherever Spack was installed on your computer. If Python is found on your PATH, that will be used. If not, the Python included with the installer will be used when running Spack. ================================================ FILE: lib/spack/spack/cmd/installer/bundle.wxs.in ================================================ ================================================ FILE: lib/spack/spack/cmd/installer/patch.xml ================================================  ================================================ FILE: lib/spack/spack/cmd/installer/spack.wxs.in ================================================ ================================================ FILE: lib/spack/spack/cmd/license.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import enum import os import re from collections import defaultdict from typing import Dict, Generator import spack.llnl.util.tty as tty import spack.paths description = "list and check license headers on files in spack" section = "query" level = "long" #: SPDX license id must appear in the first lines of a file license_lines = 6 #: Spack's license identifier apache2_mit_spdx = "(Apache-2.0 OR MIT)" subdirs = ("bin", "lib", "share", ".github") #: regular expressions for licensed files. licensed_files_patterns = [ # spack scripts r"^bin/spack$", r"^bin/spack\.bat$", r"^bin/spack\.ps1$", r"^bin/spack_pwsh\.ps1$", r"^bin/sbang$", r"^bin/spack-python$", r"^bin/haspywin\.py$", # all of spack core except unparse r"^lib/spack/spack/(?!vendor/|util/unparse|util/ctest_log_parser|test/util/unparse).*\.py$", r"^lib/spack/spack/.*\.sh$", r"^lib/spack/spack/.*\.lp$", r"^lib/spack/llnl/.*\.py$", # 1 file in vendored packages r"^lib/spack/spack/vendor/__init__.py$", # special case some test data files that have license headers r"^lib/spack/spack/test/data/style/broken.dummy", r"^lib/spack/spack/test/data/unparse/.*\.txt", # rst files in documentation r"^lib/spack/docs/(?!command_index|spack).*\.rst$", r"^lib/spack/docs/(?!\.spack/|\.spack-env/).*\.py$", r"^lib/spack/docs/spack.yaml$", # shell scripts in share r"^share/spack/.*\.sh$", r"^share/spack/.*\.bash$", r"^share/spack/.*\.csh$", r"^share/spack/.*\.fish$", r"^share/spack/setup-env\.ps1$", r"^share/spack/qa/run-[^/]*$", r"^share/spack/qa/*.py$", r"^share/spack/bash/spack-completion.in$", # action workflows r"^.github/actions/.*\.py$", ] def _licensed_files(root: str = spack.paths.prefix) -> Generator[str, None, None]: """Generates paths of licensed files.""" licensed_files = re.compile("|".join(f"(?:{pattern})" for pattern in licensed_files_patterns)) dirs = [ os.path.join(root, subdir) for subdir in subdirs if os.path.isdir(os.path.join(root, subdir)) ] while dirs: with os.scandir(dirs.pop()) as it: for entry in it: if entry.is_dir(follow_symlinks=False): dirs.append(entry.path) elif entry.is_file(follow_symlinks=False): relpath = os.path.relpath(entry.path, root) if licensed_files.match(relpath): yield relpath def list_files(args): """list files in spack that should have license headers""" for relpath in sorted(_licensed_files(args.root)): print(os.path.join(spack.paths.spack_root, relpath)) # Error codes for license verification. All values are chosen such that # bool(value) evaluates to True class ErrorType(enum.Enum): SPDX_MISMATCH = 1 NOT_IN_FIRST_N_LINES = 2 GENERAL_MISMATCH = 3 #: regexes for valid license lines at tops of files license_line_regexes = [ r"Copyright (Spack|sbang) [Pp]roject [Dd]evelopers\. See COPYRIGHT file for details.", r"", r"SPDX-License-Identifier: \(Apache-2\.0 OR MIT\)", ] class LicenseError: error_counts: Dict[ErrorType, int] def __init__(self): self.error_counts = defaultdict(int) def add_error(self, error): self.error_counts[error] += 1 def has_errors(self): return sum(self.error_counts.values()) > 0 def error_messages(self): total = sum(self.error_counts.values()) missing = self.error_counts[ErrorType.GENERAL_MISMATCH] lines = self.error_counts[ErrorType.NOT_IN_FIRST_N_LINES] spdx_mismatch = self.error_counts[ErrorType.SPDX_MISMATCH] return ( f"{total} improperly licensed files", f"files with wrong SPDX-License-Identifier: {spdx_mismatch}", f"files without license in first {license_lines} lines: {lines}", f"files not containing expected license: {missing}", ) def _check_license(lines, path): def sanitize(line): return re.sub(r"^[\s#\%\.\:]*", "", line).rstrip() for i, line in enumerate(lines): if all( re.match(regex, sanitize(lines[i + j])) for j, regex in enumerate(license_line_regexes) ): return if i >= (license_lines - len(license_line_regexes)): print(f"{path}: License not found in first {license_lines} lines") return ErrorType.NOT_IN_FIRST_N_LINES # If the SPDX identifier is present, then there is a mismatch (since it # did not match the above regex) def wrong_spdx_identifier(line, path): m = re.search(r"SPDX-License-Identifier: ([^\n]*)", line) if m and m.group(1) != apache2_mit_spdx: print( f"{path}: SPDX license identifier mismatch " f"(expecting {apache2_mit_spdx}, found {m.group(1)})" ) return ErrorType.SPDX_MISMATCH checks = [wrong_spdx_identifier] for line in lines: for check in checks: error = check(line, path) if error: return error print(f"{path}: the license header at the top of the file does not match the expected format") return ErrorType.GENERAL_MISMATCH def verify(args): """verify that files in spack have the right license header""" license_errors = LicenseError() for relpath in _licensed_files(args.root): path = os.path.join(args.root, relpath) with open(path, encoding="utf-8") as f: lines = [line for line in f][:license_lines] error = _check_license(lines, path) if error: license_errors.add_error(error) if license_errors.has_errors(): tty.die(*license_errors.error_messages()) else: tty.msg("No license issues found.") def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--root", action="store", default=spack.paths.prefix, help="scan a different prefix for license issues", ) sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="license_command") sp.add_parser("list-files", help=list_files.__doc__) sp.add_parser("verify", help=verify.__doc__) def license(parser, args): commands = {"list-files": list_files, "verify": verify} return commands[args.license_command](args) ================================================ FILE: lib/spack/spack/cmd/list.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import fnmatch import json import math import os import re import sys from html import escape from typing import Optional, Type import spack.deptypes as dt import spack.llnl.util.tty as tty import spack.package_base import spack.repo import spack.util.git from spack.cmd.common import arguments from spack.llnl.util.filesystem import working_dir from spack.llnl.util.tty.colify import colify from spack.util.url import path_to_file_url from spack.version import VersionList description = "list and search available packages" section = "query" level = "short" formatters = {} def formatter(func): """Decorator used to register formatters""" formatters[func.__name__] = func return func def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "filter", nargs=argparse.REMAINDER, help="optional case-insensitive glob patterns to filter results", ) subparser.add_argument( "-r", "--repo", "-N", "--namespace", dest="repos", action="append", default=[], help="only list packages from the specified repo/namespace", ) subparser.add_argument( "-d", "--search-description", action="store_true", default=False, help="filtering will also search the description for a match", ) subparser.add_argument( "--format", default="name_only", choices=formatters, help="format to be used to print the output [default: name_only]", ) subparser.add_argument( "-v", "--virtuals", action="store_true", default=False, help="include virtual packages in list", ) arguments.add_common_arguments(subparser, ["tags"]) # Doesn't really make sense to update in count mode. count_or_update = subparser.add_mutually_exclusive_group() count_or_update.add_argument( "--count", action="store_true", default=False, help="display the number of packages that would be listed", ) count_or_update.add_argument( "--update", metavar="FILE", default=None, action="store", help="write output to the specified file, if any package is newer", ) def filter_by_name(pkgs, args): """ Filters the sequence of packages according to user prescriptions Args: pkgs: sequence of packages args: parsed command line arguments Returns: filtered and sorted list of packages """ if args.filter: res = [] for f in args.filter: if "*" not in f and "?" not in f: r = fnmatch.translate("*" + f + "*") else: r = fnmatch.translate(f) rc = re.compile(r, flags=re.IGNORECASE) res.append(rc) if args.search_description: def match(p, f): if f.match(p): return True pkg_cls = spack.repo.PATH.get_pkg_class(p) if pkg_cls.__doc__: return f.match(pkg_cls.__doc__) return False else: def match(p, f): return f.match(p) pkgs = [p for p in pkgs if any(match(p, f) for f in res)] return sorted(pkgs, key=lambda s: s.lower()) @formatter def name_only(pkgs, out): indent = 0 colify(pkgs, indent=indent, output=out) if out.isatty(): tty.msg("%d packages" % len(pkgs)) def github_url(pkg: Type[spack.package_base.PackageBase]) -> Optional[str]: """Link to a package file in spack package's github or the path to the file. Args: pkg: package instance Returns: URL to the package file on github or the local file path; otherwise, ``None``. """ git = None module_path = f"{pkg.__module__.replace('.', '/')}.py" for repo in spack.repo.PATH.repos: if not repo.python_path: continue path = os.path.join(repo.python_path, module_path) if not os.path.exists(path): continue git = git or spack.util.git.git() if not git: tty.debug("Cannot determine package URL for {pkg} without 'git', using path URL") return path_to_file_url(path) tty.debug(f"Checking git for repository path '{path}'") with working_dir(os.path.dirname(path)): origin_url = git( "config", "--get", "remote.origin.url", output=str, error=os.devnull, fail_on_error=False, ) if not origin_url: tty.debug("Cannot determine remote origin url, using path URL") return path_to_file_url(path) # Handle spack repositories cloned with any scheme (e.g., ssh) by # ignoring the scheme designation. if any([name in origin_url for name in ["spack.git", "spack-packages.git"]]): git_repo = (origin_url.split("/")[-1]).replace(".git", "").strip() prefix = git( "rev-parse", "--show-prefix", output=str, error=os.devnull, fail_on_error=False ) return ( f"https://github.com/spack/{git_repo}/blob/develop/{prefix.strip()}package.py" ) tty.debug(f"Unrecognized repository for {pkg}, using path URL") return path_to_file_url(path) tty.debug(f"Unable to determine the package repository URL for {pkg}") return None def rows_for_ncols(elts, ncols): """Print out rows in a table with ncols of elts laid out vertically.""" clen = int(math.ceil(len(elts) / ncols)) for r in range(clen): row = [] for c in range(ncols): i = c * clen + r row.append(elts[i] if i < len(elts) else None) yield row def get_dependencies(pkg): all_deps = {} for deptype in dt.ALL_TYPES: deps = pkg.dependencies_of_type(dt.flag_from_string(deptype)) all_deps[deptype] = [d for d in deps] return all_deps @formatter def version_json(pkg_names, out): """Print all packages with their latest versions.""" pkg_classes = [spack.repo.PATH.get_pkg_class(name) for name in pkg_names] out.write("[\n") # output name and latest version for each package pkg_latest = ",\n".join( [ ' {{"name": "{0}",\n' ' "latest_version": "{1}",\n' ' "versions": {2},\n' ' "homepage": "{3}",\n' ' "file": "{4}",\n' ' "maintainers": {5},\n' ' "dependencies": {6}' "}}".format( pkg_cls.name, VersionList(pkg_cls.versions).preferred(), json.dumps([str(v) for v in reversed(sorted(pkg_cls.versions))]), pkg_cls.homepage, github_url(pkg_cls), json.dumps(pkg_cls.maintainers), json.dumps(get_dependencies(pkg_cls)), ) for pkg_cls in pkg_classes ] ) out.write(pkg_latest) # important: no trailing comma in JSON arrays out.write("\n]\n") @formatter def html(pkg_names, out): """Print out information on all packages in Sphinx HTML. This is intended to be inlined directly into Sphinx documentation. We write HTML instead of RST for speed; generating RST from *all* packages causes the Sphinx build to take forever. Including this as raw HTML is much faster. """ # Read in all packages pkg_classes = [spack.repo.PATH.get_pkg_class(name) for name in pkg_names] # Start at 2 because the title of the page from Sphinx is id1. span_id = 2 # HTML header with an increasing id span def head(n, span_id, title, anchor=None): if anchor is None: anchor = title out.write( ( '' '

%s' "

\n" ) % (span_id, title, anchor) ) # Start with the number of packages, skipping the title and intro # blurb, which we maintain in the RST file. out.write("

\n") out.write("Spack currently has %d mainline packages:\n" % len(pkg_classes)) out.write("

\n") # Table of links to all packages out.write('\n') out.write('\n') for i, row in enumerate(rows_for_ncols(pkg_names, 3)): out.write('\n' if i % 2 == 0 else '\n') for name in row: out.write("\n' % (name, name)) out.write("\n") out.write("\n") out.write("\n") out.write("
\n") out.write('%s
\n") out.write('
\n') # Output some text for each package. for pkg_cls in pkg_classes: out.write('
\n' % pkg_cls.name) head(2, span_id, pkg_cls.name) span_id += 1 out.write('
\n') out.write("
Homepage:
\n") out.write('
    \n') if pkg_cls.homepage: out.write( ('
  • %s
    • \n') % (pkg_cls.homepage, escape(pkg_cls.homepage, True)) ) else: out.write("No homepage\n") out.write("
\n") out.write("
Spack package:
\n") out.write('
    \n') out.write( ('
  • %s/package.py
    • \n') % (github_url(pkg_cls), pkg_cls.name) ) out.write("
\n") if pkg_cls.versions: out.write("
Versions:
\n") out.write("
\n") out.write(", ".join(str(v) for v in reversed(sorted(pkg_cls.versions)))) out.write("\n") out.write("
\n") for deptype in dt.ALL_TYPES: deps = pkg_cls.dependencies_of_type(dt.flag_from_string(deptype)) if deps: out.write("
%s Dependencies:
\n" % deptype.capitalize()) out.write("
\n") out.write( ", ".join( ( d if d not in pkg_names else '%s' % (d, d) ) for d in deps ) ) out.write("\n") out.write("
\n") out.write("
Description:
\n") out.write("
\n") out.write(escape(pkg_cls.format_doc(indent=2), True)) out.write("\n") out.write("
\n") out.write("
\n") out.write('
\n') out.write("
\n") def list(parser, args): # retrieve the formatter to use from args formatter = formatters[args.format] # Retrieve the names of all the packages repos = [spack.repo.PATH] if args.repos: repos = [spack.repo.PATH.get_repo(name) for name in args.repos] pkgs = {name for repo in repos for name in repo.all_package_names(args.virtuals)} # Filter the set appropriately sorted_packages = filter_by_name(pkgs, args) # If tags have been specified on the command line, filter by tags if args.tags: packages_with_tags = spack.repo.PATH.packages_with_tags(*args.tags) sorted_packages = [p for p in sorted_packages if p in packages_with_tags] if args.update: # change output stream if user asked for update if os.path.exists(args.update): if os.path.getmtime(args.update) > spack.repo.PATH.last_mtime(): tty.msg("File is up to date: %s" % args.update) return tty.msg("Updating file: %s" % args.update) with open(args.update, "w", encoding="utf-8") as f: formatter(sorted_packages, f) elif args.count: # just print the number of packages in the result print(len(sorted_packages)) else: # print formatted package list formatter(sorted_packages, sys.stdout) ================================================ FILE: lib/spack/spack/cmd/load.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys import spack.cmd import spack.cmd.common import spack.environment as ev import spack.store import spack.user_environment as uenv from spack.cmd.common import arguments description = "add package to the user environment" section = "user environment" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: """Parser is only constructed so that this prints a nice help message with -h.""" arguments.add_common_arguments(subparser, ["constraint"]) shells = subparser.add_mutually_exclusive_group() shells.add_argument( "--sh", action="store_const", dest="shell", const="sh", help="print sh commands to load the package", ) shells.add_argument( "--csh", action="store_const", dest="shell", const="csh", help="print csh commands to load the package", ) shells.add_argument( "--fish", action="store_const", dest="shell", const="fish", help="print fish commands to load the package", ) shells.add_argument( "--bat", action="store_const", dest="shell", const="bat", help="print bat commands to load the package", ) shells.add_argument( "--pwsh", action="store_const", dest="shell", const="pwsh", help="print pwsh commands to load the package", ) subparser.add_argument( "--first", action="store_true", default=False, dest="load_first", help="load the first match if multiple packages match the spec", ) subparser.add_argument( "--list", action="store_true", default=False, help="show loaded packages: same as ``spack find --loaded``", ) def load(parser, args): env = ev.active_environment() if args.list: results = spack.cmd.filter_loaded_specs(args.specs()) if sys.stdout.isatty(): spack.cmd.print_how_many_pkgs(results, "loaded") spack.cmd.display_specs(results) return constraint_specs = spack.cmd.parse_specs(args.constraint) specs = [ spack.cmd.disambiguate_spec(spec, env, first=args.load_first) for spec in constraint_specs ] if not args.shell: specs_str = " ".join(str(s) for s in constraint_specs) or "SPECS" spack.cmd.common.shell_init_instructions( "spack load", f" eval `spack load {{sh_arg}} {specs_str}`" ) return 1 with spack.store.STORE.db.read_transaction(): env_mod = uenv.environment_modifications_for_specs(*specs) for spec in specs: env_mod.prepend_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) cmds = env_mod.shell_modifications(args.shell) sys.stdout.write(cmds) ================================================ FILE: lib/spack/spack/cmd/location.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import spack.builder import spack.cmd import spack.environment as ev import spack.llnl.util.tty as tty import spack.paths import spack.repo import spack.stage from spack.cmd.common import arguments description = "print out locations of packages and spack directories" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: directories = subparser.add_mutually_exclusive_group() directories.add_argument( "-m", "--module-dir", action="store_true", help="spack python module directory" ) directories.add_argument( "-r", "--spack-root", action="store_true", help="spack installation root" ) directories.add_argument( "-i", "--install-dir", action="store_true", help="install prefix for spec (spec need not be installed)", ) directories.add_argument( "-p", "--package-dir", action="store_true", help="directory enclosing a spec's package.py file", ) directories.add_argument( "--repo", # for backwards compatibility "--packages", "-P", nargs="?", default=False, metavar="repo", help="package repository root (defaults to first configured repository)", ) directories.add_argument( "-s", "--stage-dir", action="store_true", help="stage directory for a spec" ) directories.add_argument( "-S", "--stages", action="store_true", help="top level stage directory" ) directories.add_argument( "-c", "--source-dir", action="store_true", help="source directory for a spec (requires it to be staged first)", ) directories.add_argument( "-b", "--build-dir", action="store_true", help="build directory for a spec (requires it to be staged first)", ) directories.add_argument( "-e", "--env", action="store", dest="location_env", nargs="?", metavar="name", default=False, help="location of the named or current environment", ) subparser.add_argument( "--first", action="store_true", default=False, dest="find_first", help="use the first match if multiple packages match the spec", ) arguments.add_common_arguments(subparser, ["spec"]) def location(parser, args): if args.module_dir: print(spack.paths.module_path) return if args.spack_root: print(spack.paths.prefix) return # no -e corresponds to False, -e without arg to None, -e name to the string name. if args.location_env is not False: if args.location_env is None: # Get current environment path spack.cmd.require_active_env("location -e") path = ev.active_environment().path else: # Get path of requested environment if not ev.exists(args.location_env): tty.die("no such environment: '%s'" % args.location_env) path = ev.root(args.location_env) print(path) return if args.repo is not False: if args.repo is None: print(spack.repo.PATH.first_repo().root) return try: print(spack.repo.PATH.get_repo(args.repo).root) except spack.repo.UnknownNamespaceError: tty.die(f"no such repository: '{args.repo}'") return if args.stages: print(spack.stage.get_stage_root()) return specs = spack.cmd.parse_specs(args.spec) if not specs: tty.die("You must supply a spec.") if len(specs) != 1: tty.die("Too many specs. Supply only one.") # install_dir command matches against installed specs. if args.install_dir: env = ev.active_environment() spec = spack.cmd.disambiguate_spec(specs[0], env, first=args.find_first) print(spec.prefix) return spec = specs[0] # Package dir just needs the spec name if args.package_dir: print(spack.repo.PATH.dirname_for_package_name(spec.name)) return # Either concretize or filter from already concretized environment spec = spack.cmd.matching_spec_from_env(spec) pkg = spec.package builder = spack.builder.create(pkg) if args.stage_dir: print(pkg.stage.path) return if args.build_dir: # Out of source builds have build_directory defined if hasattr(builder, "build_directory"): # build_directory can be either absolute or relative to the stage path # in either case os.path.join makes it absolute print(os.path.normpath(os.path.join(pkg.stage.path, builder.build_directory))) return # Otherwise assume in-source builds print(pkg.stage.source_path) return # source dir remains, which requires the spec to be staged if not pkg.stage.expanded: tty.die( "Source directory does not exist yet. Run this to create it:", "spack stage " + " ".join(args.spec), ) # Default to source dir. print(pkg.stage.source_path) ================================================ FILE: lib/spack/spack/cmd/log_parse.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import io import sys import warnings import spack.llnl.util.tty as tty from spack.util.log_parse import make_log_context, parse_log_events description = "filter errors and warnings from build logs" section = "developer" level = "long" event_types = ("errors", "warnings") def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--show", action="store", default="errors", help="comma-separated list of what to show; options: errors, warnings", ) subparser.add_argument( "-c", "--context", action="store", type=int, default=3, help="lines of context to show around lines of interest", ) subparser.add_argument( "-p", "--profile", action="store_true", help="print out a profile of time spent in regexes during parse", ) subparser.add_argument( "-w", "--width", action="store", type=int, default=None, help=argparse.SUPPRESS ) subparser.add_argument( "-j", "--jobs", action="store", type=int, default=None, help=argparse.SUPPRESS ) subparser.add_argument("file", help="a log file containing build output, or - for stdin") def log_parse(parser, args): input = args.file if args.file == "-": input = io.TextIOWrapper( sys.stdin.buffer, encoding="utf-8", errors="replace", closefd=False ) if args.width is not None: warnings.warn("The --width option is deprecated and will be removed in Spack v1.3") if args.jobs is not None: warnings.warn("The --jobs option is deprecated and will be removed in Spack v1.3") log_errors, log_warnings = parse_log_events(input, args.context, args.profile) if args.profile: return types = [s.strip() for s in args.show.split(",")] for e in types: if e not in event_types: tty.die("Invalid event type: %s" % e) events = [] if "errors" in types: events.extend(log_errors) print("%d errors" % len(log_errors)) if "warnings" in types: events.extend(log_warnings) print("%d warnings" % len(log_warnings)) print(make_log_context(events), end="") ================================================ FILE: lib/spack/spack/cmd/logs.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import errno import gzip import io import os import shutil import sys import spack.cmd import spack.spec import spack.util.compression as compression from spack.cmd.common import arguments from spack.error import SpackError description = "print out logs for packages" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["spec"]) def _dump_byte_stream_to_stdout(instream: io.BufferedIOBase) -> None: # Reopen stdout in binary mode so we don't have to worry about encoding outstream = os.fdopen(sys.stdout.fileno(), "wb", closefd=False) shutil.copyfileobj(instream, outstream) def _logs(cmdline_spec: spack.spec.Spec, concrete_spec: spack.spec.Spec): if concrete_spec.installed: log_path = concrete_spec.package.install_log_path elif os.path.exists(concrete_spec.package.stage.path): # TODO: `spack logs` can currently not show the logs while a package is being built, as the # combined log file is only written after the build is finished. log_path = concrete_spec.package.log_path else: raise SpackError(f"{cmdline_spec} is not installed or staged") try: stream = open(log_path, "rb") except OSError as e: if e.errno == errno.ENOENT: raise SpackError(f"No logs are available for {cmdline_spec}") from e raise SpackError(f"Error reading logs for {cmdline_spec}: {e}") from e with stream as f: ext = compression.extension_from_magic_numbers_by_stream(f, decompress=False) if ext and ext != "gz": raise SpackError(f"Unsupported storage format for {log_path}: {ext}") # If the log file is gzip compressed, wrap it with a decompressor _dump_byte_stream_to_stdout(gzip.GzipFile(fileobj=f) if ext == "gz" else f) def logs(parser, args): specs = spack.cmd.parse_specs(args.spec) if not specs: raise SpackError("You must supply a spec.") if len(specs) != 1: raise SpackError("Too many specs. Supply only one.") concrete_spec = spack.cmd.matching_spec_from_env(specs[0]) _logs(specs[0], concrete_spec) ================================================ FILE: lib/spack/spack/cmd/maintainers.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse from collections import defaultdict import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.repo from spack.llnl.util.tty.colify import colify description = "get information about package maintainers" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: maintained_group = subparser.add_mutually_exclusive_group() maintained_group.add_argument( "--maintained", action="store_true", default=False, help="show names of maintained packages", ) maintained_group.add_argument( "--unmaintained", action="store_true", default=False, help="show names of unmaintained packages", ) subparser.add_argument( "-a", "--all", action="store_true", default=False, help="show maintainers for all packages" ) subparser.add_argument( "--by-user", action="store_true", default=False, help="show packages for users instead of users for packages", ) # options for commands that take package arguments subparser.add_argument( "package_or_user", nargs=argparse.REMAINDER, help="names of packages or users to get info for", ) def packages_to_maintainers(package_names=None): if not package_names: package_names = spack.repo.PATH.all_package_names() pkg_to_users = defaultdict(lambda: set()) for name in package_names: cls = spack.repo.PATH.get_pkg_class(name) for user in cls.maintainers: pkg_to_users[name].add(user) return pkg_to_users def maintainers_to_packages(users=None): user_to_pkgs = defaultdict(lambda: []) for name in spack.repo.PATH.all_package_names(): cls = spack.repo.PATH.get_pkg_class(name) for user in cls.maintainers: lower_users = [u.lower() for u in users] if not users or user.lower() in lower_users: user_to_pkgs[user].append(cls.name) return user_to_pkgs def maintained_packages(): maintained = [] unmaintained = [] for name in spack.repo.PATH.all_package_names(): cls = spack.repo.PATH.get_pkg_class(name) if cls.maintainers: maintained.append(name) else: unmaintained.append(name) return maintained, unmaintained def union_values(dictionary): """Given a dictionary with values that are Collections, return their union. Arguments: dictionary (dict): dictionary whose values are all collections. Return: (set): the union of all collections in the dictionary's values. """ sets = [set(p) for p in dictionary.values()] return sorted(set.union(*sets)) if sets else set() def maintainers(parser, args): if args.maintained or args.unmaintained: maintained, unmaintained = maintained_packages() pkgs = maintained if args.maintained else unmaintained colify(pkgs) return 0 if pkgs else 1 if args.all: if args.by_user: maintainers = maintainers_to_packages(args.package_or_user) for user, packages in sorted(maintainers.items()): color.cprint("@c{%s}: %s" % (user, ", ".join(sorted(packages)))) return 0 if maintainers else 1 else: packages = packages_to_maintainers(args.package_or_user) for pkg, maintainers in sorted(packages.items()): color.cprint("@c{%s}: %s" % (pkg, ", ".join(sorted(maintainers)))) return 0 if packages else 1 if args.by_user: if not args.package_or_user: tty.die("spack maintainers --by-user requires a user or --all") packages = union_values(maintainers_to_packages(args.package_or_user)) colify(packages) return 0 if packages else 1 else: if not args.package_or_user: tty.die("spack maintainers requires a package or --all") users = union_values(packages_to_maintainers(args.package_or_user)) colify(users) return 0 if users else 1 ================================================ FILE: lib/spack/spack/cmd/make_installer.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import posixpath import sys import spack.concretize import spack.paths import spack.util.executable from spack.llnl.path import convert_to_posix_path description = "generate Windows installer" section = "admin" level = "long" def txt_to_rtf(file_path): rtf_header = r"""{{\rtf1\ansi\deff0\nouicompat {{\fonttbl{{\f0\\fnil\fcharset0 Courier New;}}}} {{\colortbl ;\red0\green0\blue255;}} {{\*\generator Riched20 10.0.19041}}\viewkind4\uc1 \f0\fs22\lang1033 {} }} """ def line_to_rtf(str): return str.replace("\n", "\\par") contents = "" with open(file_path, "r+", encoding="utf-8") as f: for line in f.readlines(): contents += line_to_rtf(line) return rtf_header.format(contents) def setup_parser(subparser: argparse.ArgumentParser) -> None: spack_source_group = subparser.add_mutually_exclusive_group(required=True) spack_source_group.add_argument( "-v", "--spack-version", default="", help="download given spack version" ) spack_source_group.add_argument( "-s", "--spack-source", default="", help="full path to spack source" ) subparser.add_argument( "-g", "--git-installer-verbosity", default="", choices=["SILENT", "VERYSILENT"], help="level of verbosity provided by bundled git installer (default is fully verbose)", required=False, action="store", dest="git_verbosity", ) subparser.add_argument("output_dir", help="output directory") def make_installer(parser, args): """ Use CMake to generate WIX installer in newly created build directory """ if sys.platform == "win32": output_dir = args.output_dir cmake_spec = spack.concretize.concretize_one("cmake") cmake_path = os.path.join(cmake_spec.prefix, "bin", "cmake.exe") cpack_path = os.path.join(cmake_spec.prefix, "bin", "cpack.exe") spack_source = args.spack_source git_verbosity = "" if args.git_verbosity: git_verbosity = "/" + args.git_verbosity if spack_source: if not os.path.exists(spack_source): print("%s does not exist" % spack_source) return else: if not os.path.isabs(spack_source): spack_source = posixpath.abspath(spack_source) spack_source = convert_to_posix_path(spack_source) spack_version = args.spack_version here = os.path.dirname(os.path.abspath(__file__)) source_dir = os.path.join(here, "installer") posix_root = convert_to_posix_path(spack.paths.spack_root) spack_license = posixpath.join(posix_root, "LICENSE-APACHE") rtf_spack_license = txt_to_rtf(spack_license) spack_license = posixpath.join(source_dir, "LICENSE.rtf") with open(spack_license, "w", encoding="utf-8") as rtf_license: written = rtf_license.write(rtf_spack_license) if written == 0: raise RuntimeError("Failed to generate properly formatted license file") spack_logo = posixpath.join(posix_root, "share/spack/logo/favicon.ico") try: spack.util.executable.Executable(cmake_path)( "-S", source_dir, "-B", output_dir, "-DSPACK_VERSION=%s" % spack_version, "-DSPACK_SOURCE=%s" % spack_source, "-DSPACK_LICENSE=%s" % spack_license, "-DSPACK_LOGO=%s" % spack_logo, "-DSPACK_GIT_VERBOSITY=%s" % git_verbosity, ) except spack.util.executable.ProcessError: print("Failed to generate installer") return spack.util.executable.ProcessError.returncode try: spack.util.executable.Executable(cpack_path)( "--config", "%s/CPackConfig.cmake" % output_dir, "-B", "%s/" % output_dir ) except spack.util.executable.ProcessError: print("Failed to generate installer") return spack.util.executable.ProcessError.returncode try: spack.util.executable.Executable(os.environ.get("WIX") + "/bin/candle.exe")( "-ext", "WixBalExtension", "%s/bundle.wxs" % output_dir, "-out", "%s/bundle.wixobj" % output_dir, ) except spack.util.executable.ProcessError: print("Failed to generate installer chain") return spack.util.executable.ProcessError.returncode try: spack.util.executable.Executable(os.environ.get("WIX") + "/bin/light.exe")( "-sw1134", "-ext", "WixBalExtension", "%s/bundle.wixobj" % output_dir, "-out", "%s/Spack.exe" % output_dir, ) except spack.util.executable.ProcessError: print("Failed to generate installer chain") return spack.util.executable.ProcessError.returncode print("Successfully generated Spack.exe in %s" % (output_dir)) else: print("The make-installer command is currently only supported on Windows.") ================================================ FILE: lib/spack/spack/cmd/mark.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys from typing import List, Union import spack.cmd import spack.spec import spack.store from spack.cmd.common import arguments from spack.llnl.util import tty from ..enums import InstallRecordStatus description = "mark packages as explicitly or implicitly installed" section = "build" level = "long" error_message = """You can either: a) use a more specific spec, or b) use `spack mark --all` to mark ALL matching specs. """ # Arguments for display_specs when we find ambiguity display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4} def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["installed_specs"]) subparser.add_argument( "-a", "--all", action="store_true", dest="all", help="mark ALL installed packages that match each supplied spec", ) exim = subparser.add_mutually_exclusive_group(required=True) exim.add_argument( "-e", "--explicit", action="store_true", dest="explicit", help="mark packages as explicitly installed", ) exim.add_argument( "-i", "--implicit", action="store_true", dest="implicit", help="mark packages as implicitly installed", ) def find_matching_specs( specs: List[Union[str, spack.spec.Spec]], allow_multiple_matches: bool = False ) -> List[spack.spec.Spec]: """Returns a list of specs matching the not necessarily concretized specs given from cli Args: specs: list of specs to be matched against installed packages allow_multiple_matches: if True multiple matches are admitted """ # List of specs that match expressions given via command line specs_from_cli = [] has_errors = False for spec in specs: matching = spack.store.STORE.db.query_local(spec, installed=InstallRecordStatus.INSTALLED) # For each spec provided, make sure it refers to only one package. # Fail and ask user to be unambiguous if it doesn't if not allow_multiple_matches and len(matching) > 1: tty.error("{0} matches multiple packages:".format(spec)) sys.stderr.write("\n") spack.cmd.display_specs(matching, output=sys.stderr, **display_args) sys.stderr.write("\n") sys.stderr.flush() has_errors = True # No installed package matches the query if len(matching) == 0 and spec is not None: tty.die(f"{spec} does not match any installed packages.") specs_from_cli.extend(matching) if has_errors: tty.die(error_message) return specs_from_cli def do_mark(specs, explicit): """Marks all the specs in a list. Args: specs (list): list of specs to be marked explicit (bool): whether to mark specs as explicitly installed """ with spack.store.STORE.db.write_transaction(): for spec in specs: spack.store.STORE.db.mark(spec, "explicit", explicit) def mark_specs(args, specs): mark_list = find_matching_specs(specs, args.all) # Mark everything on the list do_mark(mark_list, args.explicit) def mark(parser, args): if not args.specs and not args.all: tty.die( "mark requires at least one package argument.", " Use `spack mark --all` to mark ALL packages.", ) # [None] here handles the --all case by forcing all specs to be returned specs = spack.cmd.parse_specs(args.specs) if args.specs else [None] mark_specs(args, specs) ================================================ FILE: lib/spack/spack/cmd/mirror.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys from concurrent.futures import as_completed import spack.caches import spack.cmd import spack.concretize import spack.config import spack.environment as ev import spack.llnl.util.lang as lang import spack.llnl.util.tty as tty import spack.llnl.util.tty.colify as colify import spack.mirrors.mirror import spack.mirrors.utils import spack.repo import spack.spec import spack.util.parallel import spack.util.web as web_util from spack.cmd.common import arguments from spack.error import SpackError from spack.llnl.string import comma_or description = "manage mirrors (source and binary)" section = "config" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["no_checksum"]) sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="mirror_command") # Create create_parser = sp.add_parser("create", help=mirror_create.__doc__) create_parser.add_argument( "-d", "--directory", default=None, help="directory in which to create mirror" ) create_parser.add_argument( "-a", "--all", action="store_true", help="mirror all versions of all packages in Spack, or all packages" " in the current environment if there is an active environment" " (this requires significant time and space)", ) create_parser.add_argument( "-j", "--jobs", type=int, default=None, help="Use a given number of workers to make the mirror (used in combination with -a)", ) create_parser.add_argument("--file", help="file with specs of packages to put in mirror") create_parser.add_argument( "--exclude-file", help="specs which Spack should not try to add to a mirror" " (listed in a file, one per line)", ) create_parser.add_argument( "--exclude-specs", help="specs which Spack should not try to add to a mirror (specified on command line)", ) create_parser.add_argument( "--skip-unstable-versions", action="store_true", help="don't cache versions unless they identify a stable (unchanging) source code", ) create_parser.add_argument( "-D", "--dependencies", action="store_true", help="also fetch all dependencies" ) create_parser.add_argument( "-n", "--versions-per-spec", help="the number of versions to fetch for each spec, choose 'all' to" " retrieve all versions of each package", ) create_parser.add_argument( "--private", action="store_true", help="for a private mirror, include non-redistributable packages", ) arguments.add_common_arguments(create_parser, ["specs"]) arguments.add_concretizer_args(create_parser) # Destroy destroy_parser = sp.add_parser("destroy", help=mirror_destroy.__doc__) destroy_target = destroy_parser.add_mutually_exclusive_group(required=True) destroy_target.add_argument( "-m", "--mirror-name", metavar="mirror_name", type=str, help="find mirror to destroy by name", ) destroy_target.add_argument( "--mirror-url", metavar="mirror_url", type=str, help="find mirror to destroy by url" ) # Add add_parser = sp.add_parser("add", help=mirror_add.__doc__) add_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror") add_parser.add_argument("url", help="url of mirror directory from 'spack mirror create'") add_parser.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope(), help="configuration scope to modify", ) add_parser.add_argument( "--type", action="append", choices=("binary", "source"), help=( "specify the mirror type: for both binary " "and source use ``--type binary --type source`` (default)" ), ) add_parser.add_argument( "--autopush", action="store_true", help=("set mirror to push automatically after installation"), ) add_parser_signed = add_parser.add_mutually_exclusive_group(required=False) add_parser_signed.add_argument( "--unsigned", help="do not require signing and signature verification when pushing and installing from " "this build cache", action="store_false", default=None, dest="signed", ) add_parser_signed.add_argument( "--signed", help="require signing and signature verification when pushing and installing from this " "build cache", action="store_true", default=None, dest="signed", ) add_parser.add_argument( "--name", "-n", action="store", dest="view_name", help="Name of the index view for a binary mirror", ) arguments.add_connection_args(add_parser, False) # Remove remove_parser = sp.add_parser("remove", aliases=["rm"], help=mirror_remove.__doc__) remove_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror") remove_parser.add_argument( "--scope", action=arguments.ConfigScope, default=None, help="configuration scope to modify" ) remove_parser.add_argument( "--all-scopes", action="store_true", default=False, help="remove from all config scopes (default: highest scope with matching mirror)", ) # Set-Url set_url_parser = sp.add_parser("set-url", help=mirror_set_url.__doc__) set_url_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror") set_url_parser.add_argument("url", help="url of mirror directory from 'spack mirror create'") set_url_push_or_fetch = set_url_parser.add_mutually_exclusive_group(required=False) set_url_push_or_fetch.add_argument( "--push", action="store_true", help="set only the URL used for uploading" ) set_url_push_or_fetch.add_argument( "--fetch", action="store_true", help="set only the URL used for downloading" ) set_url_parser.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope(), help="configuration scope to modify", ) arguments.add_connection_args(set_url_parser, False) # Set set_parser = sp.add_parser("set", help=mirror_set.__doc__) set_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror") set_parser_push_or_fetch = set_parser.add_mutually_exclusive_group(required=False) set_parser_push_or_fetch.add_argument( "--push", action="store_true", help="modify just the push connection details" ) set_parser_push_or_fetch.add_argument( "--fetch", action="store_true", help="modify just the fetch connection details" ) set_parser.add_argument( "--type", action="append", choices=("binary", "source"), help=( "specify the mirror type: for both binary " "and source use ``--type binary --type source``" ), ) set_parser.add_argument("--url", help="url of mirror directory from 'spack mirror create'") set_parser_autopush = set_parser.add_mutually_exclusive_group(required=False) set_parser_autopush.add_argument( "--autopush", help="set mirror to push automatically after installation", action="store_true", default=None, dest="autopush", ) set_parser_autopush.add_argument( "--no-autopush", help="set mirror to not push automatically after installation", action="store_false", default=None, dest="autopush", ) set_parser_unsigned = set_parser.add_mutually_exclusive_group(required=False) set_parser_unsigned.add_argument( "--unsigned", help="do not require signing and signature verification when pushing and installing from " "this build cache", action="store_false", default=None, dest="signed", ) set_parser_unsigned.add_argument( "--signed", help="require signing and signature verification when pushing and installing from this " "build cache", action="store_true", default=None, dest="signed", ) set_parser.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope(), help="configuration scope to modify", ) arguments.add_connection_args(set_parser, False) # List list_parser = sp.add_parser("list", aliases=["ls"], help=mirror_list.__doc__) list_parser.add_argument( "--scope", action=arguments.ConfigScope, help="configuration scope to read from" ) def _configure_access_pair(args, id_tok, id_variable_tok, secret_variable_tok, default=None): """Configure the access_pair options""" # Check if any of the arguments are set to update this access_pair. # If none are set, then skip computing the new access pair args_id = getattr(args, id_tok) args_id_variable = getattr(args, id_variable_tok) args_secret_variable = getattr(args, secret_variable_tok) if not any([args_id, args_id_variable, args_secret_variable]): return None def _default_value(id_): if isinstance(default, list): return default[0] if id_ == "id" else default[1] elif isinstance(default, dict): return default.get(id_) else: return None def _default_variable(id_): if isinstance(default, dict): return default.get(id_ + "_variable") else: return None id_ = None id_variable = None secret_variable = None # Get the value/default value if the argument of the inverse if not args_id_variable: id_ = getattr(args, id_tok) or _default_value("id") if not args_id: id_variable = getattr(args, id_variable_tok) or _default_variable("id") secret_variable = getattr(args, secret_variable_tok) or _default_variable("secret") if (id_ or id_variable) and secret_variable: return dict( [ (("id", id_) if id_ else ("id_variable", id_variable)), ("secret_variable", secret_variable), ] ) else: if id_ or id_variable or secret_variable is not None: id_arg_tok = id_tok.replace("_", "-") secret_variable_arg_tok = secret_variable_tok.replace("_", "-") tty.warn( "Expected both parts of the access pair to be specified. " f"(i.e. --{id_arg_tok} and --{secret_variable_arg_tok})" ) return None def mirror_add(args): """add a mirror to Spack""" if ( args.s3_access_key_id or args.s3_access_key_id_variable or args.s3_access_key_secret_variable or args.s3_access_token_variable or args.s3_profile or args.s3_endpoint_url or args.view_name or args.type or args.oci_username or args.oci_username_variable or args.oci_password_variable or args.autopush or args.signed is not None ): connection = {"url": args.url} # S3 Connection access_pair = _configure_access_pair( args, "s3_access_key_id", "s3_access_key_id_variable", "s3_access_key_secret_variable" ) if access_pair: connection["access_pair"] = access_pair if args.s3_access_token_variable: connection["access_token_variable"] = args.s3_access_token_variable if args.s3_profile: connection["profile"] = args.s3_profile if args.s3_endpoint_url: connection["endpoint_url"] = args.s3_endpoint_url # OCI Connection access_pair = _configure_access_pair( args, "oci_username", "oci_username_variable", "oci_password_variable" ) if access_pair: connection["access_pair"] = access_pair if args.type: connection["binary"] = "binary" in args.type connection["source"] = "source" in args.type if args.autopush: connection["autopush"] = args.autopush if args.signed is not None: connection["signed"] = args.signed if args.view_name: connection["view"] = args.view_name mirror = spack.mirrors.mirror.Mirror(connection, name=args.name) else: mirror = spack.mirrors.mirror.Mirror(args.url, name=args.name) spack.mirrors.utils.add(mirror, args.scope) def mirror_remove(args): """remove a mirror by name""" name = args.name scopes = [args.scope] if args.scope else reversed(list(spack.config.CONFIG.scopes.keys())) removed = False for scope in scopes: removed_from_this_scope = spack.mirrors.utils.remove(name, scope) if removed_from_this_scope: tty.msg(f"Removed mirror {name} from {scope} scope") removed |= removed_from_this_scope if removed and not args.all_scopes: return if not removed: tty.die(f"No mirror with name {name} in {comma_or(scopes)} scope") def _configure_mirror(args): mirrors = spack.config.get("mirrors", scope=args.scope) if args.name not in mirrors: tty.die(f"No mirror found with name {args.name}.") entry = spack.mirrors.mirror.Mirror(mirrors[args.name], args.name) direction = "fetch" if args.fetch else "push" if args.push else None changes = {} if args.url: changes["url"] = args.url default_access_pair = entry._get_value("access_pair", direction or "fetch") # TODO: Init access_pair args with the fetch/push/base values in the current mirror state access_pair = _configure_access_pair( args, "s3_access_key_id", "s3_access_key_id_variable", "s3_access_key_secret_variable", default=default_access_pair, ) if access_pair: changes["access_pair"] = access_pair if getattr(args, "s3_access_token_variable", None): changes["access_token_variable"] = args.s3_access_token_variable if args.s3_profile: changes["profile"] = args.s3_profile if args.s3_endpoint_url: changes["endpoint_url"] = args.s3_endpoint_url access_pair = _configure_access_pair( args, "oci_username", "oci_username_variable", "oci_password_variable", default=default_access_pair, ) if access_pair: changes["access_pair"] = access_pair if getattr(args, "signed", None) is not None: changes["signed"] = args.signed if getattr(args, "autopush", None) is not None: changes["autopush"] = args.autopush # argparse cannot distinguish between --binary and --no-binary when same dest :( # notice that set-url does not have these args, so getattr if getattr(args, "type", None): changes["binary"] = "binary" in args.type changes["source"] = "source" in args.type changed = entry.update(changes, direction) if changed: mirrors[args.name] = entry.to_dict() spack.config.set("mirrors", mirrors, scope=args.scope) else: tty.msg("No changes made to mirror %s." % args.name) def mirror_set(args): """configure the connection details of a mirror""" _configure_mirror(args) def mirror_set_url(args): """change the URL of a mirror""" _configure_mirror(args) def mirror_list(args): """print out available mirrors to the console""" mirrors = spack.mirrors.mirror.MirrorCollection(scope=args.scope) if not mirrors: tty.msg("No mirrors configured.") return mirrors.display() def specs_from_text_file(filename, concretize=False): """Return a list of specs read from a text file. The file should contain one spec per line. Args: filename (str): name of the file containing the abstract specs. concretize (bool): if True concretize the specs before returning the list. """ with open(filename, "r", encoding="utf-8") as f: specs_in_file = f.readlines() specs_in_file = [s.strip() for s in specs_in_file] return spack.cmd.parse_specs(" ".join(specs_in_file), concretize=concretize) def concrete_specs_from_user(args): """Return the list of concrete specs that the user wants to mirror. The list is passed either from command line or from a text file. """ specs = concrete_specs_from_cli_or_file(args) specs = extend_with_additional_versions(specs, num_versions=versions_per_spec(args)) if args.dependencies: specs = extend_with_dependencies(specs) specs = filter_externals(specs) specs = list(set(specs)) specs.sort(key=lambda s: (s.name, s.version)) return specs def extend_with_additional_versions(specs, num_versions): if num_versions == "all": mirror_specs = spack.mirrors.utils.get_all_versions(specs) else: mirror_specs = spack.mirrors.utils.get_matching_versions(specs, num_versions=num_versions) mirror_specs = [spack.concretize.concretize_one(x) for x in mirror_specs] return mirror_specs def filter_externals(specs): specs, external_specs = lang.stable_partition(specs, predicate_fn=lambda x: not x.external) for spec in external_specs: msg = "Skipping {0} as it is an external spec." tty.msg(msg.format(spec.cshort_spec)) return specs def extend_with_dependencies(specs): """Extend the input list by adding all the dependencies explicitly.""" result = set() for spec in specs: for s in spec.traverse(): result.add(s) return list(result) def concrete_specs_from_cli_or_file(args): if args.specs: specs = spack.cmd.parse_specs(args.specs, concretize=False) if not specs: raise SpackError("unable to parse specs from command line") if args.file: specs = specs_from_text_file(args.file, concretize=False) if not specs: raise SpackError("unable to parse specs from file '{}'".format(args.file)) concrete_specs = spack.cmd.matching_specs_from_env(specs) return concrete_specs class IncludeFilter: def __init__(self, args): self.exclude_specs = [] if args.exclude_file: self.exclude_specs.extend(specs_from_text_file(args.exclude_file, concretize=False)) if args.exclude_specs: self.exclude_specs.extend(spack.cmd.parse_specs(str(args.exclude_specs).split())) self.private = args.private def __call__(self, x): return all([self._not_license_excluded(x), self._not_cmdline_excluded(x)]) def _not_license_excluded(self, x): """True if the spec is for a private mirror, or as long as the package does not explicitly forbid redistributing source.""" if self.private: return True elif spack.repo.PATH.get_pkg_class(x.fullname).redistribute_source(x): return True else: tty.debug( "Skip adding {0} to mirror: the package.py file" " indicates that a public mirror should not contain" " it.".format(x.name) ) return False def _not_cmdline_excluded(self, x): """True if a spec was not explicitly excluded by the user.""" return not any(x.satisfies(y) for y in self.exclude_specs) def concrete_specs_from_environment(): env = ev.active_environment() assert env, "an active environment is required" mirror_specs = env.all_specs() mirror_specs = filter_externals(mirror_specs) return mirror_specs def all_specs_with_all_versions(): specs = [spack.spec.Spec(n) for n in spack.repo.all_package_names()] mirror_specs = spack.mirrors.utils.get_all_versions(specs) mirror_specs.sort(key=lambda s: (s.name, s.version)) return mirror_specs def versions_per_spec(args): """Return how many versions should be mirrored per spec.""" if not args.versions_per_spec: num_versions = 1 elif args.versions_per_spec == "all": num_versions = "all" else: try: num_versions = int(args.versions_per_spec) except ValueError: raise SpackError( "'--versions-per-spec' must be a number or 'all', got '{0}'".format( args.versions_per_spec ) ) return num_versions def process_mirror_stats(present, mirrored, error): p, m, e = len(present), len(mirrored), len(error) tty.msg( "Archive stats:", " %-4d already present" % p, " %-4d added" % m, " %-4d failed to fetch." % e, ) if error: tty.error("Failed downloads:") colify.colify(s.cformat("{name}{@version}") for s in error) sys.exit(1) def mirror_create(args): """create a directory to be used as a spack mirror, and fill it with package archives""" if args.file and args.all: raise SpackError( "cannot specify specs with a file if you chose to mirror all specs with '--all'" ) if args.file and args.specs: raise SpackError("cannot specify specs with a file AND on command line") if not args.specs and not args.file and not args.all: raise SpackError( "no packages were specified.", "To mirror all packages, use the '--all' option " "(this will require significant time and space).", ) if args.versions_per_spec and args.all: raise SpackError( "cannot specify '--versions_per-spec' and '--all' together", "The option '--all' already implies mirroring all versions for each package.", ) # When no directory is provided, the source dir is used path = args.directory or spack.caches.fetch_cache_location() mirror_specs = _specs_to_mirror(args) workers = args.jobs if workers is None: if args.all: workers = min( 16, spack.config.determine_number_of_jobs(parallel=True), len(mirror_specs) ) else: workers = 1 create_mirror_for_all_specs( mirror_specs, path=path, skip_unstable_versions=args.skip_unstable_versions, workers=workers, ) def _specs_to_mirror(args): include_fn = IncludeFilter(args) if args.all and not ev.active_environment(): mirror_specs = all_specs_with_all_versions() elif args.all and ev.active_environment(): mirror_specs = concrete_specs_from_environment() else: mirror_specs = concrete_specs_from_user(args) mirror_specs, _ = lang.stable_partition(mirror_specs, predicate_fn=include_fn) return mirror_specs def create_mirror_for_one_spec(candidate, mirror_cache): pkg_cls = spack.repo.PATH.get_pkg_class(candidate.name) pkg_obj = pkg_cls(spack.spec.Spec(candidate)) mirror_stats = spack.mirrors.utils.MirrorStatsForOneSpec(candidate) spack.mirrors.utils.create_mirror_from_package_object(pkg_obj, mirror_cache, mirror_stats) mirror_stats.finalize() return mirror_stats def create_mirror_for_all_specs(mirror_specs, path, skip_unstable_versions, workers): mirror_cache = spack.mirrors.utils.get_mirror_cache( path, skip_unstable_versions=skip_unstable_versions ) mirror_stats = spack.mirrors.utils.MirrorStatsForAllSpecs() with spack.util.parallel.make_concurrent_executor(jobs=workers) as executor: # Submit tasks to the process pool futures = [ executor.submit(create_mirror_for_one_spec, candidate, mirror_cache) for candidate in mirror_specs ] for mirror_future in as_completed(futures): ext_mirror_stats = mirror_future.result() mirror_stats.merge(ext_mirror_stats) process_mirror_stats(*mirror_stats.stats()) return mirror_stats def create(path, specs, skip_unstable_versions=False): """Create a directory to be used as a spack mirror, and fill it with package archives. Arguments: path: Path to create a mirror directory hierarchy in. specs: Any package versions matching these specs will be added \ to the mirror. skip_unstable_versions: if true, this skips adding resources when they do not have a stable archive checksum (as determined by ``fetch_strategy.stable_target``) Returns: A tuple of lists, each containing specs * present: Package specs that were already present. * mirrored: Package specs that were successfully mirrored. * error: Package specs that failed to mirror due to some error. """ # automatically spec-ify anything in the specs array. specs = [s if isinstance(s, spack.spec.Spec) else spack.spec.Spec(s) for s in specs] mirror_stats = create_mirror_for_all_specs(specs, path, skip_unstable_versions, workers=1) return mirror_stats.stats() def mirror_destroy(args): """given a url, recursively delete everything under it""" mirror_url = None if args.mirror_name: result = spack.mirrors.mirror.MirrorCollection().lookup(args.mirror_name) mirror_url = result.push_url elif args.mirror_url: mirror_url = args.mirror_url web_util.remove_url(mirror_url, recursive=True) def mirror(parser, args): action = { "create": mirror_create, "destroy": mirror_destroy, "add": mirror_add, "remove": mirror_remove, "rm": mirror_remove, "set-url": mirror_set_url, "set": mirror_set, "list": mirror_list, "ls": mirror_list, } if args.no_checksum: spack.config.set("config:checksum", False, scope="command_line") action[args.mirror_command](args) ================================================ FILE: lib/spack/spack/cmd/module.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse from typing import Callable, Dict import spack.cmd.modules.lmod import spack.cmd.modules.tcl description = "generate/manage module files" section = "user environment" level = "short" _subcommands: Dict[str, Callable] = {} def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="module_command") spack.cmd.modules.lmod.add_command(sp, _subcommands) spack.cmd.modules.tcl.add_command(sp, _subcommands) def module(parser, args): _subcommands[args.module_command](parser, args) ================================================ FILE: lib/spack/spack/cmd/modules/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Implementation details of the ``spack module`` command.""" import collections import os import shutil import sys import spack.cmd import spack.config import spack.error import spack.modules import spack.modules.common import spack.repo from spack.cmd import MultipleSpecsMatch, NoSpecMatches from spack.cmd.common import arguments from spack.llnl.util import filesystem, tty from spack.llnl.util.tty import color description = "manipulate module files" section = "environment" level = "short" def setup_parser(subparser): subparser.add_argument( "-n", "--name", action="store", dest="module_set_name", default="default", help="named module set to use from modules configuration", ) sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="subparser_name") refresh_parser = sp.add_parser("refresh", help="regenerate module files") refresh_parser.add_argument( "--delete-tree", help="delete the module file tree before refresh", action="store_true" ) refresh_parser.add_argument( "--upstream-modules", help="generate modules for packages installed upstream", action="store_true", ) arguments.add_common_arguments(refresh_parser, ["constraint", "yes_to_all"]) find_parser = sp.add_parser("find", help="find module files for packages") find_parser.add_argument( "--full-path", help="display full path to module file", action="store_true" ) arguments.add_common_arguments(find_parser, ["constraint", "recurse_dependencies"]) rm_parser = sp.add_parser("rm", help="remove module files") arguments.add_common_arguments(rm_parser, ["constraint", "yes_to_all"]) loads_parser = sp.add_parser( "loads", help="prompt the list of modules associated with a constraint" ) add_loads_arguments(loads_parser) arguments.add_common_arguments(loads_parser, ["constraint"]) return sp def add_loads_arguments(subparser): subparser.add_argument( "--input-only", action="store_false", dest="shell", help="generate input for module command (instead of a shell script)", ) subparser.add_argument( "-p", "--prefix", dest="prefix", default="", help="prepend to module names when issuing module load commands", ) subparser.add_argument( "-x", "--exclude", dest="exclude", action="append", default=[], help="exclude package from output; may be specified multiple times", ) arguments.add_common_arguments(subparser, ["recurse_dependencies"]) def one_spec_or_raise(specs): """Ensures exactly one spec has been selected, or raises the appropriate exception. """ # Ensure a single spec matches the constraint if len(specs) == 0: raise NoSpecMatches() if len(specs) > 1: raise MultipleSpecsMatch() # Get the spec and module type return specs[0] def check_module_set_name(name): modules = spack.config.get("modules") if name != "prefix_inspections" and name in modules: return names = [k for k in modules if k != "prefix_inspections"] if not names: raise spack.error.ConfigError( f"Module set configuration is missing. Cannot use module set '{name}'" ) pretty_names = "', '".join(names) raise spack.error.ConfigError( f"Cannot use invalid module set '{name}'.", f"Valid module set names are: '{pretty_names}'.", ) _missing_modules_warning = ( "Modules have been omitted for one or more specs, either" " because they were excluded or because the spec is" " associated with a package that is installed upstream and" " that installation has not generated a module file. Rerun" " this command with debug output enabled for more details." ) def loads(module_type, specs, args, out=None): """Prompt the list of modules associated with a list of specs""" check_module_set_name(args.module_set_name) out = sys.stdout if out is None else out # Get a comprehensive list of specs if args.recurse_dependencies: specs_from_user_constraint = specs[:] specs = [] # FIXME : during module file creation nodes seem to be visited # FIXME : multiple times even if cover='nodes' is given. This # FIXME : work around permits to get a unique list of spec anyhow. # FIXME : (same problem as in spack/modules.py) seen = set() seen_add = seen.add for spec in specs_from_user_constraint: specs.extend( [ item for item in spec.traverse(order="post", cover="nodes") if not (item in seen or seen_add(item)) ] ) modules = list( ( spec, spack.modules.get_module( module_type, spec, get_full_path=False, module_set_name=args.module_set_name, required=False, ), ) for spec in specs ) module_commands = {"tcl": "module load ", "lmod": "module load "} d = {"command": "" if not args.shell else module_commands[module_type], "prefix": args.prefix} exclude_set = set(args.exclude) load_template = "{comment}{exclude}{command}{prefix}{name}" for spec, mod in modules: if not mod: module_output_for_spec = "## excluded or missing from upstream: {0}".format( spec.format() ) else: d["exclude"] = "## " if spec.name in exclude_set else "" d["comment"] = "" if not args.shell else "# {0}\n".format(spec.format()) d["name"] = mod module_output_for_spec = load_template.format(**d) out.write(module_output_for_spec) out.write("\n") if not all(mod for _, mod in modules): tty.warn(_missing_modules_warning) def find(module_type, specs, args): """Retrieve paths or use names of module files""" check_module_set_name(args.module_set_name) single_spec = one_spec_or_raise(specs) if args.recurse_dependencies: dependency_specs_to_retrieve = list( single_spec.traverse(root=False, order="post", cover="nodes", deptype=("link", "run")) ) else: dependency_specs_to_retrieve = [] try: modules = [ spack.modules.get_module( module_type, spec, args.full_path, module_set_name=args.module_set_name, required=False, ) for spec in dependency_specs_to_retrieve ] modules.append( spack.modules.get_module( module_type, single_spec, args.full_path, module_set_name=args.module_set_name, required=True, ) ) except spack.modules.common.ModuleNotFoundError as e: tty.die(e.message) if not all(modules): tty.warn(_missing_modules_warning) modules = list(x for x in modules if x) print(" ".join(modules)) def rm(module_type, specs, args): """Deletes the module files associated with every spec in specs, for every module type in module types. """ check_module_set_name(args.module_set_name) module_cls = spack.modules.module_types[module_type] module_exist = lambda x: os.path.exists(module_cls(x, args.module_set_name).layout.filename) specs_with_modules = [spec for spec in specs if module_exist(spec)] modules = [module_cls(spec, args.module_set_name) for spec in specs_with_modules] if not modules: tty.die("No module file matches your query") # Ask for confirmation if not args.yes_to_all: msg = "You are about to remove {0} module files for:\n" tty.msg(msg.format(module_type)) spack.cmd.display_specs(specs_with_modules, long=True) print("") answer = tty.get_yes_or_no("Do you want to proceed?") if not answer: tty.die("Will not remove any module files") # Remove the module files for s in modules: s.remove() def refresh(module_type, specs, args): """Regenerates the module files for every spec in specs and every module type in module types. """ check_module_set_name(args.module_set_name) # Prompt a message to the user about what is going to change if not specs: tty.msg("No package matches your query") return if not args.upstream_modules: specs = list(s for s in specs if not s.installed_upstream) if not args.yes_to_all: msg = "You are about to regenerate {types} module files for:\n" tty.msg(msg.format(types=module_type)) spack.cmd.display_specs(specs, long=True) print("") answer = tty.get_yes_or_no("Do you want to proceed?") if not answer: tty.die("Module file regeneration aborted.") # Cycle over the module types and regenerate module files cls = spack.modules.module_types[module_type] # Skip unknown packages. writers = [ cls(spec, args.module_set_name) for spec in specs if spack.repo.PATH.exists(spec.name) ] # Filter excluded packages early writers = [x for x in writers if not x.conf.excluded] # Detect name clashes in module files file2writer = collections.defaultdict(list) for item in writers: file2writer[item.layout.filename].append(item) if len(file2writer) != len(writers): spec_fmt_str = "{name}@={version}%{compiler}/{hash:7} {variants} arch={arch}" message = "Name clashes detected in module files:\n" for filename, writer_list in file2writer.items(): if len(writer_list) > 1: message += "\nfile: {0}\n".format(filename) for x in writer_list: message += "spec: {0}\n".format(x.spec.format(spec_fmt_str)) tty.error(message) tty.error("Operation aborted") raise SystemExit(1) if len(writers) == 0: msg = "Nothing to be done for {0} module files." tty.msg(msg.format(module_type)) return # If we arrived here we have at least one writer module_type_root = writers[0].layout.dirname() # Proceed regenerating module files tty.msg("Regenerating {name} module files".format(name=module_type)) if os.path.isdir(module_type_root) and args.delete_tree: shutil.rmtree(module_type_root, ignore_errors=False) filesystem.mkdirp(module_type_root) # Dump module index after potentially removing module tree spack.modules.common.generate_module_index( module_type_root, writers, overwrite=args.delete_tree ) errors = [] for x in writers: try: x.write(overwrite=True) except spack.error.SpackError as e: msg = f"{x.layout.filename}: {e.message}" errors.append(msg) except Exception as e: msg = f"{x.layout.filename}: {str(e)}" errors.append(msg) if errors: errors.insert(0, color.colorize("@*{some module files could not be written}")) tty.warn("\n".join(errors)) #: Dictionary populated with the list of sub-commands. #: Each sub-command must be callable and accept 3 arguments: #: #: - module_type: the type of module it refers to #: - specs : the list of specs to be processed #: - args : namespace containing the parsed command line arguments callbacks = {"refresh": refresh, "rm": rm, "find": find, "loads": loads} def modules_cmd(parser, args, module_type, callbacks=callbacks): # Qualifiers to be used when querying the db for specs constraint_qualifiers = { "refresh": { "installed": True, "predicate_fn": lambda x: spack.repo.PATH.exists(x.spec.name), } } query_args = constraint_qualifiers.get(args.subparser_name, {}) # Get the specs that match the query from the DB specs = args.specs(**query_args) try: callbacks[args.subparser_name](module_type, specs, args) except MultipleSpecsMatch: query = " ".join(str(s) for s in args.constraint_specs) msg = f"the constraint '{query}' matches multiple packages:\n" for s in specs: spec_fmt = ( "{hash:7} {name}{@version}{compiler_flags}{variants}" "{ platform=architecture.platform}{ os=architecture.os}" "{ target=architecture.target}" "{%compiler}" ) msg += "\t" + s.cformat(spec_fmt) + "\n" tty.die(msg, "In this context exactly *one* match is needed.") except NoSpecMatches: query = " ".join(str(s) for s in args.constraint_specs) msg = f"the constraint '{query}' matches no package." tty.die(msg, "In this context exactly *one* match is needed.") ================================================ FILE: lib/spack/spack/cmd/modules/lmod.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import functools import spack.cmd.common.arguments import spack.cmd.modules import spack.config import spack.modules import spack.modules.lmod def add_command(parser, command_dict): lmod_parser = parser.add_parser("lmod", help="manipulate hierarchical module files") sp = spack.cmd.modules.setup_parser(lmod_parser) # Set default module file for a package setdefault_parser = sp.add_parser( "setdefault", help="set the default module file for a package" ) spack.cmd.common.arguments.add_common_arguments(setdefault_parser, ["constraint"]) callbacks = dict(spack.cmd.modules.callbacks.items()) callbacks["setdefault"] = setdefault command_dict["lmod"] = functools.partial( spack.cmd.modules.modules_cmd, module_type="lmod", callbacks=callbacks ) def setdefault(module_type, specs, args): """set the default module file, when multiple are present""" # For details on the underlying mechanism see: # # https://lmod.readthedocs.io/en/latest/060_locating.html#marking-a-version-as-default # spack.cmd.modules.one_spec_or_raise(specs) spec = specs[0] data = {"modules": {args.module_set_name: {"lmod": {"defaults": [str(spec)]}}}} # Need to clear the cache if a SpackCommand is called during scripting spack.modules.lmod.configuration_registry = {} scope = spack.config.InternalConfigScope("lmod-setdefault", data) with spack.config.override(scope): writer = spack.modules.module_types["lmod"](spec, args.module_set_name) writer.update_module_defaults() ================================================ FILE: lib/spack/spack/cmd/modules/tcl.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import functools import spack.cmd.common.arguments import spack.cmd.modules import spack.config import spack.modules import spack.modules.tcl def add_command(parser, command_dict): tcl_parser = parser.add_parser("tcl", help="manipulate non-hierarchical module files") sp = spack.cmd.modules.setup_parser(tcl_parser) # Set default module file for a package setdefault_parser = sp.add_parser( "setdefault", help="set the default module file for a package" ) spack.cmd.common.arguments.add_common_arguments(setdefault_parser, ["constraint"]) callbacks = dict(spack.cmd.modules.callbacks.items()) callbacks["setdefault"] = setdefault command_dict["tcl"] = functools.partial( spack.cmd.modules.modules_cmd, module_type="tcl", callbacks=callbacks ) def setdefault(module_type, specs, args): """set the default module file, when multiple are present""" # Currently, accepts only a single matching spec spack.cmd.modules.one_spec_or_raise(specs) spec = specs[0] data = {"modules": {args.module_set_name: {"tcl": {"defaults": [str(spec)]}}}} spack.modules.tcl.configuration_registry = {} scope = spack.config.InternalConfigScope("tcl-setdefault", data) with spack.config.override(scope): writer = spack.modules.module_types["tcl"](spec, args.module_set_name) writer.update_module_defaults() ================================================ FILE: lib/spack/spack/cmd/patch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd import spack.config import spack.environment as ev import spack.llnl.util.tty as tty import spack.package_base import spack.traverse from spack.cmd.common import arguments description = "patch expanded sources in preparation for install" section = "build" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["no_checksum", "specs"]) arguments.add_concretizer_args(subparser) def patch(parser, args): if not args.specs: env = ev.active_environment() if not env: tty.die("`spack patch` requires a spec or an active environment") return _patch_env(env) if args.no_checksum: spack.config.set("config:checksum", False, scope="command_line") specs = spack.cmd.parse_specs(args.specs, concretize=False) specs = spack.cmd.matching_specs_from_env(specs) for spec in specs: _patch(spec.package) def _patch_env(env: ev.Environment): tty.msg(f"Patching specs from environment {env.name}") for spec in spack.traverse.traverse_nodes(env.concrete_roots()): _patch(spec.package) def _patch(pkg: spack.package_base.PackageBase): pkg.stage.keep = True with pkg.stage: pkg.do_patch() tty.msg(f"Patched {pkg.name} in {pkg.stage.path}") ================================================ FILE: lib/spack/spack/cmd/pkg.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import sys import spack.cmd import spack.llnl.util.tty as tty import spack.repo import spack.util.executable as exe import spack.util.package_hash as ph from spack.cmd.common import arguments from spack.llnl.util.tty.colify import colify description = "query packages associated with particular git revisions" section = "developer" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="pkg_command") add_parser = sp.add_parser("add", help=pkg_add.__doc__) arguments.add_common_arguments(add_parser, ["packages"]) list_parser = sp.add_parser("list", help=pkg_list.__doc__) list_parser.add_argument( "rev", default="HEAD", nargs="?", help="revision to list packages for" ) diff_parser = sp.add_parser("diff", help=pkg_diff.__doc__) diff_parser.add_argument( "rev1", nargs="?", default="HEAD^", help="revision to compare against" ) diff_parser.add_argument( "rev2", nargs="?", default="HEAD", help="revision to compare to rev1 (default is HEAD)" ) add_parser = sp.add_parser("added", help=pkg_added.__doc__) add_parser.add_argument("rev1", nargs="?", default="HEAD^", help="revision to compare against") add_parser.add_argument( "rev2", nargs="?", default="HEAD", help="revision to compare to rev1 (default is HEAD)" ) add_parser = sp.add_parser("changed", help=pkg_changed.__doc__) add_parser.add_argument("rev1", nargs="?", default="HEAD^", help="revision to compare against") add_parser.add_argument( "rev2", nargs="?", default="HEAD", help="revision to compare to rev1 (default is HEAD)" ) add_parser.add_argument( "-t", "--type", action="store", default="C", help="types of changes to show (A: added, R: removed, C: changed); default is 'C'", ) rm_parser = sp.add_parser("removed", help=pkg_removed.__doc__) rm_parser.add_argument("rev1", nargs="?", default="HEAD^", help="revision to compare against") rm_parser.add_argument( "rev2", nargs="?", default="HEAD", help="revision to compare to rev1 (default is HEAD)" ) # explicitly add help for `spack pkg grep` with just `--help` and NOT `-h`. This is so # that the very commonly used -h (no filename) argument can be passed through to grep grep_parser = sp.add_parser("grep", help=pkg_grep.__doc__, add_help=False) grep_parser.add_argument( "grep_args", nargs=argparse.REMAINDER, default=None, help="arguments for grep" ) grep_parser.add_argument("--help", action="help", help="show this help message and exit") source_parser = sp.add_parser("source", help=pkg_source.__doc__) source_parser.add_argument( "-c", "--canonical", action="store_true", default=False, help="dump canonical source as used by package hash", ) arguments.add_common_arguments(source_parser, ["spec"]) hash_parser = sp.add_parser("hash", help=pkg_hash.__doc__) arguments.add_common_arguments(hash_parser, ["spec"]) def pkg_add(args): """add a package to the git stage with ``git add``""" spack.repo.add_package_to_git_stage(args.packages, spack.repo.builtin_repo()) def pkg_list(args): """list packages associated with a particular spack git revision""" colify(spack.repo.list_packages(args.rev, spack.repo.builtin_repo())) def pkg_diff(args): """compare packages available in two different git revisions""" u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo()) if u1: print("%s:" % args.rev1) colify(sorted(u1), indent=4) if u1: print() if u2: print("%s:" % args.rev2) colify(sorted(u2), indent=4) def pkg_removed(args): """show packages removed since a commit""" u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo()) if u1: colify(sorted(u1)) def pkg_added(args): """show packages added since a commit""" u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo()) if u2: colify(sorted(u2)) def pkg_changed(args): """show packages changed since a commit""" packages = spack.repo.get_all_package_diffs( args.type, spack.repo.builtin_repo(), args.rev1, args.rev2 ) if packages: colify(sorted(packages)) def pkg_source(args): """dump source code for a package""" specs = spack.cmd.parse_specs(args.spec, concretize=False) if len(specs) != 1: tty.die("spack pkg source requires exactly one spec") spec = specs[0] filename = spack.repo.PATH.filename_for_package_name(spec.name) # regular source dump -- just get the package and print its contents if args.canonical: message = "Canonical source for %s:" % filename content = ph.canonical_source(spec) else: message = "Source for %s:" % filename with open(filename, encoding="utf-8") as f: content = f.read() if sys.stdout.isatty(): tty.msg(message) sys.stdout.write(content) def pkg_hash(args): """dump canonical source code hash for a package spec""" specs = spack.cmd.parse_specs(args.spec, concretize=False) for spec in specs: print(ph.package_hash(spec)) def get_grep(required=False): """Get a grep command to use with ``spack pkg grep``.""" grep = exe.which(os.environ.get("SPACK_GREP") or "grep", required=required) if grep: grep.ignore_quotes = True # allow `spack pkg grep '"quoted string"'` without warning return grep def pkg_grep(args, unknown_args): """grep for strings in package.py files from all repositories""" grep = get_grep(required=True) # add a little color to the output if we can if "GNU" in grep("--version", output=str): grep.add_default_arg("--color=auto") all_paths = spack.repo.PATH.all_package_paths() if not all_paths: return 0 # no packages to search # these args start every command invocation (grep arg1 arg2 ...) all_prefix_args = grep.exe + args.grep_args + unknown_args prefix_length = sum(spack.cmd.converted_arg_length(arg) for arg in all_prefix_args) + len( all_prefix_args ) # set up iterator and save the first group to ensure we don't end up with a group of size 1 groups = spack.cmd.group_arguments(all_paths, prefix_length=prefix_length) # You can force GNU grep to show filenames on every line with -H, but not POSIX grep. # POSIX grep only shows filenames when you're grepping 2 or more files. Since we # don't know which one we're running, we ensure there are always >= 2 files by # saving the prior group of paths and adding it to a straggling group of 1 if needed. # This works unless somehow there is only one package in all of Spack. prior_paths = next(groups) # grep returns 1 for nothing found, 0 for something found, and > 1 for error return_code = 1 # assemble args and run grep on a group of paths def grep_group(paths): all_args = args.grep_args + unknown_args + paths grep(*all_args, fail_on_error=False) return grep.returncode for paths in groups: if len(paths) == 1: # Only the very last group can have length 1. If it does, combine # it with the prior group to ensure more than one path is grepped. prior_paths += paths else: # otherwise run grep on the prior group error = grep_group(prior_paths) if error != 1: return_code = error if error > 1: # fail fast on error return error prior_paths = paths # Handle the last remaining group after the loop error = grep_group(prior_paths) if error != 1: return_code = error return return_code def pkg(parser, args, unknown_args): if not spack.cmd.spack_is_git_repo(): tty.die("This spack is not a git clone. Can't use 'spack pkg'") action = { "add": pkg_add, "added": pkg_added, "changed": pkg_changed, "diff": pkg_diff, "hash": pkg_hash, "list": pkg_list, "removed": pkg_removed, "source": pkg_source, } # grep is special as it passes unknown arguments through if args.pkg_command == "grep": return pkg_grep(args, unknown_args) elif unknown_args: tty.die("unrecognized arguments: %s" % " ".join(unknown_args)) else: return action[args.pkg_command](args) ================================================ FILE: lib/spack/spack/cmd/providers.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import io import sys import spack.cmd import spack.llnl.util.tty.colify as colify import spack.repo description = "list packages that provide a particular virtual package" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.epilog = "If called without argument returns the list of all valid virtual packages" subparser.add_argument( "virtual_package", nargs="*", help="find packages that provide this virtual package" ) def providers(parser, args): valid_virtuals = sorted(spack.repo.PATH.provider_index.providers.keys()) buffer = io.StringIO() isatty = sys.stdout.isatty() if isatty: buffer.write("Virtual packages:\n") colify.colify(valid_virtuals, output=buffer, tty=isatty, indent=4) valid_virtuals_str = buffer.getvalue() # If called without arguments, list all the virtual packages if not args.virtual_package: print(valid_virtuals_str) return # Otherwise, parse the specs from command line specs = spack.cmd.parse_specs(args.virtual_package) # Check prerequisites non_virtual = [ str(s) for s in specs if not spack.repo.PATH.is_virtual(s.name) or s.name not in valid_virtuals ] if non_virtual: msg = "non-virtual specs cannot be part of the query " msg += "[{0}]\n".format(", ".join(non_virtual)) msg += valid_virtuals_str raise ValueError(msg) # Display providers for spec in specs: if sys.stdout.isatty(): print("{0}:".format(spec)) spack.cmd.display_specs(sorted(spack.repo.PATH.providers_for(spec))) print("") ================================================ FILE: lib/spack/spack/cmd/pydoc.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse description = "run pydoc from within spack" section = "developer" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument("entity", help="run pydoc help on entity") def pydoc(parser, args): help(args.entity) ================================================ FILE: lib/spack/spack/cmd/python.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import code import os import platform import runpy import sys import spack import spack.llnl.util.tty as tty import spack.repo description = "launch an interpreter as spack would launch a command" section = "developer" level = "long" IS_WINDOWS = sys.platform == "win32" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-V", "--version", action="store_true", dest="python_version", help="print the Python version number and exit", ) subparser.add_argument("-c", dest="python_command", help="command to execute") subparser.add_argument( "-u", dest="unbuffered", action="store_true", help="for compatibility with xdist, do not use without adding -u to the interpreter", ) subparser.add_argument( "-i", dest="python_interpreter", help="python interpreter", choices=["python", "ipython"], default="python", ) subparser.add_argument( "-m", dest="module", action="store", help="run library module as a script" ) subparser.add_argument( "--path", action="store_true", dest="show_path", help="show path to python interpreter that spack uses", ) subparser.add_argument( "python_args", nargs=argparse.REMAINDER, help="file to run plus arguments" ) def python(parser, args, unknown_args): if args.python_version: print("Python", platform.python_version()) return if args.show_path: print(sys.executable) return if args.module: sys.argv = ["spack-python"] + unknown_args + args.python_args runpy.run_module(args.module, run_name="__main__", alter_sys=True) return if unknown_args: tty.die("Unknown arguments:", " ".join(unknown_args)) # Unexpected behavior from supplying both if args.python_command and args.python_args: tty.die("You can only specify a command OR script, but not both.") # Ensure that spack.repo.PATH is initialized spack.repo.PATH.repos # Run user choice of interpreter if args.python_interpreter == "ipython": return ipython_interpreter(args) return python_interpreter(args) def ipython_interpreter(args): """An ipython interpreter is intended to be interactive, so it doesn't support running a script or arguments """ try: import IPython # type: ignore[import] except ImportError: tty.die("ipython is not installed, install and try again.") if "PYTHONSTARTUP" in os.environ: startup_file = os.environ["PYTHONSTARTUP"] if os.path.isfile(startup_file): with open(startup_file, encoding="utf-8") as startup: exec(startup.read()) # IPython can also support running a script OR command, not both if args.python_args: IPython.start_ipython(argv=args.python_args) elif args.python_command: IPython.start_ipython(argv=["-c", args.python_command]) else: header = "Spack version %s\nPython %s, %s %s" % ( spack.spack_version, platform.python_version(), platform.system(), platform.machine(), ) __name__ = "__main__" # noqa: F841 IPython.embed(module="__main__", header=header) def python_interpreter(args): """A python interpreter is the default interpreter""" if args.python_args and not args.python_command: sys.argv = args.python_args runpy.run_path(args.python_args[0], run_name="__main__") else: # Fake a main python shell by setting __name__ to __main__. console = code.InteractiveConsole({"__name__": "__main__", "spack": spack}) if "PYTHONSTARTUP" in os.environ: startup_file = os.environ["PYTHONSTARTUP"] if os.path.isfile(startup_file): with open(startup_file, encoding="utf-8") as startup: console.runsource(startup.read(), startup_file, "exec") if args.python_command: propagate_exceptions_from(console) console.runsource(args.python_command) else: # no readline module on Windows if not IS_WINDOWS: # Provides readline support, allowing user to use arrow keys console.push("import readline") # Provide tabcompletion console.push("from rlcompleter import Completer") console.push("readline.set_completer(Completer(locals()).complete)") console.push('readline.parse_and_bind("tab: complete")') console.interact( "Spack version %s\nPython %s, %s %s" % ( spack.spack_version, platform.python_version(), platform.system(), platform.machine(), ) ) def propagate_exceptions_from(console): """Set sys.excepthook to let uncaught exceptions return 1 to the shell. Args: console (code.InteractiveConsole): the console that needs a change in sys.excepthook """ console.push("import sys") console.push("_wrapped_hook = sys.excepthook") console.push("def _hook(exc_type, exc_value, exc_tb):") console.push(" _wrapped_hook(exc_type, exc_value, exc_tb)") console.push(" sys.exit(1)") console.push("") console.push("sys.excepthook = _hook") ================================================ FILE: lib/spack/spack/cmd/reindex.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import shutil import spack.database import spack.store from spack.llnl.util import tty description = "rebuild Spack's package database" section = "admin" level = "long" def reindex(parser, args): current_index = spack.store.STORE.db._index_path needs_backup = os.path.isfile(current_index) if needs_backup: backup = f"{current_index}.bkp" shutil.copy(current_index, backup) tty.msg("Created a backup copy of the DB at", backup) spack.store.STORE.reindex() extra = ["If you need to restore, replace it with the backup."] if needs_backup else [] tty.msg( f"The DB at {current_index} has been reindexed to v{spack.database._DB_VERSION}", *extra ) ================================================ FILE: lib/spack/spack/cmd/remove.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd import spack.llnl.util.tty as tty from spack.cmd.common import arguments description = "remove specs from an environment" section = "environments" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-a", "--all", action="store_true", help="remove all specs from (clear) the environment" ) subparser.add_argument( "-l", "--list-name", dest="list_name", default="specs", help="name of the list to remove specs from", ) subparser.add_argument( "-f", "--force", action="store_true", help="remove concretized spec (if any) immediately" ) arguments.add_common_arguments(subparser, ["specs"]) def remove(parser, args): env = spack.cmd.require_active_env(cmd_name="remove") with env.write_transaction(): if args.all: env.clear() else: for spec in spack.cmd.parse_specs(args.specs): env.remove(spec, args.list_name, force=args.force) tty.msg(f"{spec} has been removed from {env.manifest}") env.write() ================================================ FILE: lib/spack/spack/cmd/repo.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import shlex import sys import tempfile from typing import Any, Dict, Generator, List, Optional, Tuple, Union import spack import spack.caches import spack.ci import spack.config import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.repo import spack.spec import spack.util.executable import spack.util.git import spack.util.path import spack.util.spack_json as sjson import spack.util.spack_yaml from spack.cmd.common import arguments from spack.error import SpackError from spack.llnl.util.tty import color from spack.version import StandardVersion from . import doc_dedented, doc_first_line description = "manage package source repositories" section = "config" level = "long" def setup_parser(subparser: argparse.ArgumentParser): sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="repo_command") # Create create_parser = sp.add_parser( "create", description=doc_dedented(repo_create), help=doc_first_line(repo_create) ) create_parser.add_argument("directory", help="directory to create the repo in") create_parser.add_argument( "namespace", help="name or namespace to identify packages in the repository" ) create_parser.add_argument( "-d", "--subdirectory", action="store", dest="subdir", default=spack.repo.packages_dir_name, help="subdirectory to store packages in the repository\n\n" "default 'packages'. use an empty string for no subdirectory", ) # List list_parser = sp.add_parser( "list", aliases=["ls"], description=doc_dedented(repo_list), help=doc_first_line(repo_list) ) list_parser.add_argument( "--scope", action=arguments.ConfigScope, type=arguments.config_scope_readable_validator, help="configuration scope to read from", ) output_group = list_parser.add_mutually_exclusive_group() output_group.add_argument("--names", action="store_true", help="show configuration names only") output_group.add_argument( "--namespaces", action="store_true", help="show repository namespaces only" ) output_group.add_argument( "--json", action="store_true", help="output repositories as machine-readable json records" ) # Add add_parser = sp.add_parser( "add", description=doc_dedented(repo_add), help=doc_first_line(repo_add) ) add_parser.add_argument( "path_or_repo", help="path or git repository of a Spack package repository" ) # optional positional argument for destination name in case of git repository add_parser.add_argument( "destination", nargs="?", default=None, help="destination to clone git repository into (defaults to cache directory)", ) add_parser.add_argument( "--name", action="store", help="config name for the package repository, defaults to the namespace of the repository", ) add_parser.add_argument( "--path", help="relative path to the Spack package repository inside a git repository. Can be " "repeated to add multiple package repositories in case of a monorepo", action="append", default=[], ) add_parser.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope(), help="configuration scope to modify", ) # Set (modify existing repository configuration) set_parser = sp.add_parser( "set", description=doc_dedented(repo_set), help=doc_first_line(repo_set) ) set_parser.add_argument("namespace", help="namespace of a Spack package repository") set_parser.add_argument( "--destination", help="destination to clone git repository into", action="store" ) set_parser.add_argument( "--path", help="relative path to the Spack package repository inside a git repository. Can be " "repeated to add multiple package repositories in case of a monorepo", action="append", default=[], ) set_parser.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope(), help="configuration scope to modify", ) # Remove remove_parser = sp.add_parser( "remove", description=doc_dedented(repo_remove), help=doc_first_line(repo_remove), aliases=["rm"], ) remove_parser.add_argument( "namespace_or_path", help="namespace or path of a Spack package repository" ) remove_parser.add_argument( "--scope", action=arguments.ConfigScope, default=None, help="configuration scope to modify" ) remove_parser.add_argument( "--all-scopes", action="store_true", default=False, help="remove from all config scopes (default: highest scope with matching repo)", ) # Migrate migrate_parser = sp.add_parser( "migrate", description=doc_dedented(repo_migrate), help=doc_first_line(repo_migrate) ) migrate_parser.add_argument( "namespace_or_path", help="path to a Spack package repository directory" ) patch_or_fix = migrate_parser.add_mutually_exclusive_group(required=True) patch_or_fix.add_argument( "--dry-run", action="store_true", help="do not modify the repository, but dump a patch file", ) patch_or_fix.add_argument( "--fix", action="store_true", help="automatically migrate the repository to the latest Package API", ) # Update update_parser = sp.add_parser( "update", description=doc_dedented(repo_update), help=doc_first_line(repo_update) ) update_parser.add_argument("names", nargs="*", default=[], help="repositories to update") update_parser.add_argument( "--remote", "-r", default="origin", nargs="?", help="name of remote to check for branches, tags, or commits", ) update_parser.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope(), help="configuration scope to modify", ) update_parser.add_argument( "--branch", "-b", nargs="?", default=None, help="name of a branch to change to" ) refspec = update_parser.add_mutually_exclusive_group(required=False) refspec.add_argument("--tag", "-t", nargs="?", default=None, help="name of a tag to change to") refspec.add_argument( "--commit", "-c", nargs="?", default=None, help="name of a commit to change to" ) # Show updates show_version_updates_parser = sp.add_parser( "show-version-updates", help=repo_show_version_updates.__doc__ ) show_version_updates_parser.add_argument( "--no-manual-packages", action="store_true", help="exclude manual packages" ) show_version_updates_parser.add_argument( "--no-git-versions", action="store_true", help="exclude versions from git" ) show_version_updates_parser.add_argument( "--only-redistributable", action="store_true", help="exclude non-redistributable packages" ) show_version_updates_parser.add_argument( "repository", help="name or path of the repository to analyze" ) show_version_updates_parser.add_argument( "from_ref", help="git ref from which to start looking at changes" ) show_version_updates_parser.add_argument("to_ref", help="git ref to end looking at changes") def repo_create(args): """create a new package repository""" full_path, namespace = spack.repo.create_repo(args.directory, args.namespace, args.subdir) tty.msg("Created repo with namespace '%s'." % namespace) tty.msg("To register it with spack, run this command:", "spack repo add %s" % full_path) def _add_repo( path_or_repo: str, name: Optional[str], scope: Optional[str], paths: List[str], destination: Optional[str], config: Optional[spack.config.Configuration] = None, ) -> str: config = config or spack.config.CONFIG existing: Dict[str, Any] = config.get("repos", default={}, scope=scope) if name and name in existing: raise SpackError(f"A repository with the name '{name}' already exists.") # Interpret as a git URL when it contains a colon at index 2 or more, not preceded by a # forward slash. That allows C:/ windows paths, while following git's convention to distinguish # between local paths on the one hand and URLs and SCP like syntax on the other. entry: Union[str, Dict[str, Any]] colon_idx = path_or_repo.find(":") if colon_idx > 1 and "/" not in path_or_repo[:colon_idx]: # git URL entry = {"git": path_or_repo} if len(paths) >= 1: entry["paths"] = paths if destination: entry["destination"] = destination else: # local path if destination: raise SpackError("The 'destination' argument is only valid for git repositories") elif paths: raise SpackError("The --paths flag is only valid for git repositories") entry = spack.util.path.canonicalize_path(path_or_repo) descriptor = spack.repo.parse_config_descriptor( name or "", entry, lock=spack.repo.package_repository_lock() ) descriptor.initialize(git=spack.util.executable.which("git")) packages_repos = descriptor.construct(cache=spack.caches.MISC_CACHE) usable_repos: Dict[str, spack.repo.Repo] = {} for _path, _repo_or_err in packages_repos.items(): if isinstance(_repo_or_err, Exception): tty.warn(f"Skipping package repository '{_path}' due to: {_repo_or_err}") else: usable_repos[_path] = _repo_or_err if not usable_repos: raise SpackError(f"No package repository could be constructed from {path_or_repo}") # For the config key, default to --name, then to the namespace if there's only one repo. # Otherwise, the name is unclear and we require the user to specify it. if name: key = name elif len(usable_repos) == 1: key = next(iter(usable_repos.values())).namespace else: raise SpackError("Multiple package repositories found, please specify a name with --name.") if key in existing: raise SpackError(f"A repository with the name '{key}' already exists.") # Prepend the new repository config.set("repos", spack.util.spack_yaml.syaml_dict({key: entry, **existing}), scope) return key def repo_add(args): """add package repositories to Spack's configuration""" name = _add_repo( path_or_repo=args.path_or_repo, name=args.name, scope=args.scope, paths=args.path, destination=args.destination, ) tty.msg(f"Added repo to config with name '{name}'.") def repo_remove(args): """remove a repository from Spack's configuration""" scopes = [args.scope] if args.scope else reversed(list(spack.config.CONFIG.scopes.keys())) found_and_removed = False for scope in scopes: found_and_removed |= _remove_repo(args.namespace_or_path, scope) if found_and_removed and not args.all_scopes: return if not found_and_removed: tty.die(f"No repository with path or namespace: {args.namespace_or_path}") def _remove_repo(namespace_or_path, scope): repos: Dict[str, str] = spack.config.get("repos", scope=scope) if namespace_or_path in repos: # delete by name (from config) key = namespace_or_path else: # delete by namespace or path (requires constructing the repo) canon_path = spack.util.path.canonicalize_path(namespace_or_path) descriptors = spack.repo.RepoDescriptors.from_config( spack.repo.package_repository_lock(), spack.config.CONFIG, scope=scope ) for name, descriptor in descriptors.items(): descriptor.initialize(fetch=False) # For now you cannot delete monorepos with multiple package repositories from config, # hence "all" and not "any". We can improve this later if needed. if all( r.namespace == namespace_or_path or r.root == canon_path for r in descriptor.construct(cache=spack.caches.MISC_CACHE).values() if isinstance(r, spack.repo.Repo) ): key = name break else: return False del repos[key] spack.config.set("repos", repos, scope) tty.msg(f"Removed repository '{namespace_or_path}' from scope '{scope}'.") return True def repo_list(args): """show registered repositories and their namespaces List all package repositories known to Spack. Repositories can be local directories or remote git repositories. """ descriptors = spack.repo.RepoDescriptors.from_config( lock=spack.repo.package_repository_lock(), config=spack.config.CONFIG, scope=args.scope ) # --names: just print config names if args.names: for name in descriptors: print(name) return # --namespaces: print all repo namespaces if args.namespaces: for name, path, maybe_repo in _iter_repos_from_descriptors(descriptors): if isinstance(maybe_repo, spack.repo.Repo): print(maybe_repo.namespace) return # Collect all repository information repo_info = [] for name, path, maybe_repo in _iter_repos_from_descriptors(descriptors): if isinstance(maybe_repo, spack.repo.Repo): status = "installed" namespace = maybe_repo.namespace api = maybe_repo.package_api_str repo_path = maybe_repo.root elif maybe_repo is None: # Uninitialized Git-based repo case status = "uninitialized" namespace = name api = "" repo_path = path else: # Exception/error case status = "error" namespace = name api = "" repo_path = path # Add the repo info to our list repo_info.append( { "name": name, "namespace": namespace, "path": repo_path, "api_version": api, "status": status, "error": str(maybe_repo) if isinstance(maybe_repo, Exception) else None, } ) # Output in JSON format if requested if args.json: sjson.dump(repo_info, sys.stdout) return # Default table format with aligned output formatted_repo_info = [] for repo in repo_info: if repo["status"] == "installed": status = "@g{[+]}" elif repo["status"] == "uninitialized": status = "@K{ - }" else: # error status = "@r{[-]}" formatted_repo_info.append((status, repo["namespace"], repo["api_version"], repo["path"])) if formatted_repo_info: max_namespace_width = max(len(namespace) for _, namespace, _, _ in formatted_repo_info) + 3 max_api_width = max(len(api) for _, _, api, _ in formatted_repo_info) + 3 # Print aligned output for status, namespace, api, path in formatted_repo_info: cpath = color.cescape(path) color.cprint( f"{status} {namespace:<{max_namespace_width}} {api:<{max_api_width}} {cpath}" ) def _get_repo(name_or_path: str) -> Optional[spack.repo.Repo]: """get a repo by path or namespace""" try: return spack.repo.from_path(name_or_path) except spack.repo.RepoError: pass descriptors = spack.repo.RepoDescriptors.from_config( spack.repo.package_repository_lock(), spack.config.CONFIG ) repo_path, _ = descriptors.construct(cache=spack.caches.MISC_CACHE, fetch=False) for repo in repo_path.repos: if repo.namespace == name_or_path: return repo return None def repo_migrate(args: Any) -> int: """migrate a package repository to the latest Package API""" from spack.repo_migrate import migrate_v1_to_v2, migrate_v2_imports repo = _get_repo(args.namespace_or_path) if repo is None: tty.die(f"No such repository: {args.namespace_or_path}") if args.dry_run: fd, patch_file_path = tempfile.mkstemp( suffix=".patch", prefix="repo-migrate-", dir=os.getcwd() ) patch_file = os.fdopen(fd, "bw") tty.msg(f"Patch file will be written to {patch_file_path}") else: patch_file_path = None patch_file = None try: if (1, 0) <= repo.package_api < (2, 0): success, repo_v2 = migrate_v1_to_v2(repo, patch_file=patch_file) exit_code = 0 if success else 1 elif (2, 0) <= repo.package_api < (3, 0): repo_v2 = None exit_code = ( 0 if migrate_v2_imports(repo.packages_path, repo.root, patch_file=patch_file) else 1 ) else: repo_v2 = None exit_code = 0 finally: if patch_file is not None: patch_file.flush() patch_file.close() if patch_file_path: tty.warn( f"No changes were made to the '{repo.namespace}' repository with. Review " f"the changes written to {patch_file_path}. Run \n\n" f" spack repo migrate --fix {args.namespace_or_path}\n\n" "to upgrade the repo." ) elif exit_code == 1: tty.error( f"Repository '{repo.namespace}' could not be migrated to the latest Package API. " "Please check the error messages above." ) elif isinstance(repo_v2, spack.repo.Repo): tty.info( f"Repository '{repo_v2.namespace}' was successfully migrated from " f"package API {repo.package_api_str} to {repo_v2.package_api_str}." ) tty.warn( "Remove the old repository from Spack's configuration and add the new one using:\n" f" spack repo remove {shlex.quote(repo.root)}\n" f" spack repo add {shlex.quote(repo_v2.root)}" ) else: tty.info(f"Repository '{repo.namespace}' was successfully migrated") return exit_code def repo_set(args): """modify an existing repository configuration""" namespace = args.namespace # First, check if the repository exists across all scopes for validation all_repos: Dict[str, Any] = spack.config.get("repos", default={}) if namespace not in all_repos: raise SpackError(f"No repository with namespace '{namespace}' found in configuration.") # Validate that it's a git repository if not isinstance(all_repos[namespace], dict): raise SpackError( f"Repository '{namespace}' is not a git repository. " "The 'set' command only works with git repositories." ) # Now get the repos for the specific scope we're modifying scope_repos: Dict[str, Any] = spack.config.get("repos", default={}, scope=args.scope) updated_entry = scope_repos[namespace] if namespace in scope_repos else {} if args.destination: updated_entry["destination"] = args.destination if args.path: updated_entry["paths"] = args.path scope_repos[namespace] = updated_entry spack.config.set("repos", scope_repos, args.scope) tty.msg(f"Updated repo '{namespace}'") def _iter_repos_from_descriptors( descriptors: spack.repo.RepoDescriptors, ) -> Generator[Tuple[str, str, Union[spack.repo.Repo, Exception, None]], None, None]: """Iterate through repository descriptors and yield (name, path, maybe_repo) tuples. Yields: Tuple of (config_name, path, maybe_repo) where maybe_repo is a Repo instance if it could be instantiated, an Exception if it could not be instantiated, or None if it was not initialized yet. """ for name, descriptor in descriptors.items(): descriptor.initialize(fetch=False) repos_for_descriptor = descriptor.construct(cache=spack.caches.MISC_CACHE) for path, maybe_repo in repos_for_descriptor.items(): yield name, path, maybe_repo # If there are no repos, it means it's not yet cloned; yield descriptor info if not repos_for_descriptor and isinstance(descriptor, spack.repo.RemoteRepoDescriptor): yield name, descriptor.repository, None # None indicates remote descriptor def repo_update(args): """update one or more package repositories""" descriptors = spack.repo.RepoDescriptors.from_config( spack.repo.package_repository_lock(), spack.config.CONFIG ) git_flags = ["commit", "tag", "branch"] active_flag = next((attr for attr in git_flags if getattr(args, attr)), None) if active_flag and len(args.names) != 1: raise SpackError( f"Unable to set --{active_flag} because more than one namespace was given." if len(args.names) > 1 else f"Unable to apply --{active_flag} without a namespace" ) for name in args.names: if name not in descriptors: raise SpackError(f"{name} is not a known repository name.") # filter descriptors when namespaces are provided as arguments descriptors = spack.repo.RepoDescriptors( {name: descriptor for name, descriptor in descriptors.items() if name in args.names} ) # Get the repos for the specific scope we're modifying scope_repos: Dict[str, Any] = spack.config.get("repos", default={}, scope=args.scope) for name, descriptor in descriptors.items(): if not isinstance(descriptor, spack.repo.RemoteRepoDescriptor): continue if active_flag: # update the git commit, tag, or branch of the descriptor setattr(descriptor, active_flag, getattr(args, active_flag)) updated_entry = scope_repos[name] if name in scope_repos else {} # prune previous values of git fields for entry in {"commit", "tag"} - {active_flag}: setattr(descriptor, entry, None) updated_entry.pop(entry, None) updated_entry[active_flag] = args.commit or args.tag or args.branch scope_repos[name] = updated_entry git = spack.util.git.git(required=True) previous_commit = descriptor.get_commit(git=git) descriptor.update(git=git, remote=args.remote) new_commit = descriptor.get_commit(git=git) if previous_commit == new_commit: tty.msg(f"{name}: Already up to date.") else: fails = [ r for r in descriptor.construct(cache=spack.caches.MISC_CACHE).values() if type(r) is spack.repo.BadRepoVersionError ] if fails: min_ver = ".".join(str(n) for n in spack.min_package_api_version) max_ver = ".".join(str(n) for n in spack.package_api_version) tty.error( f"{name}: repo is too new for this version of Spack. ", f" Spack supports API v{min_ver} to v{max_ver}, but repo is {fails[0].api}", " Please upgrade Spack or revert with:\n", f" spack repo update --commit {previous_commit}\n", ) else: tty.msg(f"{name}: Updated successfully.") if active_flag: spack.config.set("repos", scope_repos, args.scope) def repo_show_version_updates(args): """show version specs that were added between two commits""" # Get the repository by name or path repo = _get_repo(args.repository) if repo is None: tty.die(f"No such repository: {args.repository}") # Get packages that were changed or added between the refs pkgs = spack.repo.get_all_package_diffs("AC", repo, args.from_ref, args.to_ref) # Filter out manual packages if requested if args.no_manual_packages: pkgs = { pkg_name for pkg_name in pkgs if not spack.repo.PATH.get_pkg_class(pkg_name).manual_download } if not pkgs: tty.info("No packages were added or changed between the specified refs", stream=sys.stderr) return 0 # Collect version specs that were added specs_to_output = [] for pkg_name in pkgs: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) path = spack.repo.PATH.package_path(pkg_name) # Get all versions with checksums or commits version_to_checksum: Dict[StandardVersion, str] = {} for version in pkg_cls.versions: version_dict = pkg_cls.versions[version] if "sha256" in version_dict: version_to_checksum[version] = version_dict["sha256"] elif "commit" in version_dict: version_to_checksum[version] = version_dict["commit"] # Find versions added between the refs with fs.working_dir(os.path.dirname(path)): added_checksums = spack.ci.filter_added_checksums( version_to_checksum.values(), path, from_ref=args.from_ref, to_ref=args.to_ref ) new_versions = [v for v, c in version_to_checksum.items() if c in added_checksums] # Create specs for new versions for version in new_versions: version_spec = spack.spec.Spec(pkg_name) version_spec.constrain(f"@={version}") specs_to_output.append(version_spec) # Filter out git versions if requested if args.no_git_versions: specs_to_output = [ spec for spec in specs_to_output if "commit" not in spack.repo.PATH.get_pkg_class(spec.name).versions[spec.version] ] # Filter out non-redistributable packages if requested if args.only_redistributable: specs_to_output = [ spec for spec in specs_to_output if spack.repo.PATH.get_pkg_class(spec.name).redistribute_source(spec) ] if not specs_to_output: tty.info("No new package versions found between the specified refs", stream=sys.stderr) return 0 # Output specs one per line for spec in specs_to_output: print(spec) def repo(parser, args): return { "create": repo_create, "list": repo_list, "ls": repo_list, "add": repo_add, "set": repo_set, "remove": repo_remove, "rm": repo_remove, "migrate": repo_migrate, "update": repo_update, "show-version-updates": repo_show_version_updates, }[args.repo_command](args) ================================================ FILE: lib/spack/spack/cmd/resource.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.repo description = "list downloadable resources (tarballs, repos, patches)" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="resource_command") list_parser = sp.add_parser("list", help=resource_list.__doc__) list_parser.add_argument( "--only-hashes", action="store_true", help="only print sha256 hashes of resources" ) show_parser = sp.add_parser("show", help=resource_show.__doc__) show_parser.add_argument("hash", action="store") def _show_patch(sha256): """Show a record from the patch index.""" patches = spack.repo.PATH.get_patch_index().index data = patches.get(sha256) if not data: candidates = [k for k in patches if k.startswith(sha256)] if not candidates: tty.die("no such resource: %s" % sha256) elif len(candidates) > 1: tty.die("%s: ambiguous hash prefix. Options are:", *candidates) sha256 = candidates[0] data = patches.get(sha256) color.cprint("@c{%s}" % sha256) for package, rec in data.items(): owner = rec["owner"] if "relative_path" in rec: pkg_dir = spack.repo.PATH.get_pkg_class(owner).package_dir path = os.path.join(pkg_dir, rec["relative_path"]) print(" path: %s" % path) else: print(" url: %s" % rec["url"]) print(" applies to: %s" % package) if owner != package: print(" patched by: %s" % owner) def resource_list(args): """list all resources known to spack (currently just patches)""" patches = spack.repo.PATH.get_patch_index().index for sha256 in patches: if args.only_hashes: print(sha256) else: _show_patch(sha256) def resource_show(args): """show a resource, identified by its checksum""" _show_patch(args.hash) def resource(parser, args): action = {"list": resource_list, "show": resource_show} action[args.resource_command](args) ================================================ FILE: lib/spack/spack/cmd/restage.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd import spack.llnl.util.tty as tty from spack.cmd.common import arguments description = "revert checked out package source code" section = "build" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["specs"]) def restage(parser, args): if not args.specs: tty.die("spack restage requires at least one package spec.") specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: spec.package.do_restage() ================================================ FILE: lib/spack/spack/cmd/solve.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import re import sys import spack import spack.cmd import spack.cmd.spec import spack.config import spack.environment import spack.hash_types as ht import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.package_base import spack.solver.asp as asp import spack.spec description = "concretize a specs using an ASP solver" section = "developer" level = "long" #: output options show_options = ("asp", "opt", "output", "solutions") def setup_parser(subparser: argparse.ArgumentParser) -> None: # Solver arguments subparser.add_argument( "--show", action="store", default="opt,solutions", help="select outputs\n\ncomma-separated list of:\n" " asp asp program text\n" " opt optimization criteria for best model\n" " output raw clingo output\n" " solutions models found by asp program\n" " all all of the above", ) subparser.add_argument( "--timers", action="store_true", default=False, help="print out timers for different solve phases", ) subparser.add_argument( "--stats", action="store_true", default=False, help="print out statistics from clingo" ) spack.cmd.spec.setup_parser(subparser) def _process_result(result, show, required_format, kwargs): opt, _, _ = min(result.answers) if ("opt" in show) and (not required_format): tty.msg("Best of %d considered solutions." % result.nmodels) print() maxlen = max(len(s.name) for s in result.criteria) color.cprint("@*{ Priority Value Criterion}") for i, criterion in enumerate(result.criteria, 1): value = f"@K{{{criterion.value:>5}}}" grey_out = True if criterion.value > 0: value = f"@*{{{criterion.value:>5}}}" grey_out = False if grey_out: lc = "@K" elif criterion.kind == asp.OptimizationKind.CONCRETE: lc = "@b" elif criterion.kind == asp.OptimizationKind.BUILD: lc = "@g" else: lc = "@y" color.cprint(f" @K{{{i:8}}} {value} {lc}{{{criterion.name:<{maxlen}}}}") print() print() color.cprint(" @*{Legend:}") color.cprint(" @g{Specs to be built}") color.cprint(" @b{Reused specs}") color.cprint(" @y{Other criteria}") print() # dump the solutions as concretized specs if "solutions" in show: if required_format: for spec in result.specs: # With -y, just print YAML to output. if required_format == "yaml": # use write because to_yaml already has a newline. sys.stdout.write(spec.to_yaml(hash=ht.dag_hash)) elif required_format == "json": sys.stdout.write(spec.to_json(hash=ht.dag_hash)) else: sys.stdout.write(spack.spec.tree(result.specs, color=sys.stdout.isatty(), **kwargs)) print() if result.unsolved_specs and "solutions" in show: tty.msg(asp.Result.format_unsolved(result.unsolved_specs)) def solve(parser, args): # these are the same options as `spack spec` install_status_fn = spack.spec.Spec.install_status fmt = spack.spec.DISPLAY_FORMAT if args.namespaces: fmt = "{namespace}." + fmt kwargs = { "cover": args.cover, "format": fmt, "hashlen": None if args.very_long else 7, "show_types": args.types, "status_fn": install_status_fn if args.install_status else None, "hashes": args.long or args.very_long, "highlight_version_fn": ( spack.package_base.non_preferred_version if args.non_defaults else None ), "highlight_variant_fn": ( spack.package_base.non_default_variant if args.non_defaults else None ), } # process output options show = re.split(r"\s*,\s*", args.show) if "all" in show: show = show_options for d in show: if d not in show_options: raise ValueError( "Invalid option for '--show': '%s'\nchoose from: (%s)" % (d, ", ".join(show_options + ("all",))) ) # Format required for the output (JSON, YAML or None) required_format = args.format # If we have an active environment, pick the specs from there env = spack.environment.active_environment() if args.specs: specs = spack.cmd.parse_specs(args.specs) elif env: specs = list(env.user_specs) else: tty.die("spack solve requires at least one spec or an active environment") solver = asp.Solver() output = sys.stdout if "asp" in show else None setup_only = set(show) == {"asp"} unify = spack.config.get("concretizer:unify") allow_deprecated = spack.config.get("config:deprecated", False) if unify == "when_possible": for idx, result in enumerate( solver.solve_in_rounds( specs, out=output, timers=args.timers, stats=args.stats, allow_deprecated=allow_deprecated, ) ): if "solutions" in show: tty.msg("ROUND {0}".format(idx)) tty.msg("") else: print("% END ROUND {0}\n".format(idx)) if not setup_only: _process_result(result, show, required_format, kwargs) elif unify: # set up solver parameters # Note: reuse and other concretizer prefs are passed as configuration result = solver.solve( specs, out=output, timers=args.timers, stats=args.stats, setup_only=setup_only, allow_deprecated=allow_deprecated, ) if not setup_only: _process_result(result, show, required_format, kwargs) else: for spec in specs: tty.msg("SOLVING SPEC:", spec) result = solver.solve( [spec], out=output, timers=args.timers, stats=args.stats, setup_only=setup_only, allow_deprecated=allow_deprecated, ) if not setup_only: _process_result(result, show, required_format, kwargs) ================================================ FILE: lib/spack/spack/cmd/spec.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys import spack import spack.cmd import spack.environment as ev import spack.hash_types as ht import spack.llnl.util.lang as lang import spack.llnl.util.tty as tty import spack.package_base import spack.spec import spack.store import spack.traverse from spack.cmd.common import arguments description = "show what would be installed, given a spec" section = "build" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.epilog = """\ when an environment is active and no specs are provided, the environment root \ specs are used instead for further documentation regarding the spec syntax, see: spack help --spec """ arguments.add_common_arguments(subparser, ["long", "very_long", "namespaces"]) install_status_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(install_status_group, ["install_status", "no_install_status"]) format_group = subparser.add_mutually_exclusive_group() format_group.add_argument( "-y", "--yaml", action="store_const", dest="format", default=None, const="yaml", help="print concrete spec as YAML", ) format_group.add_argument( "-j", "--json", action="store_const", dest="format", default=None, const="json", help="print concrete spec as JSON", ) format_group.add_argument( "--format", action="store", default=None, help="print concrete spec with the specified format string", ) arguments.add_common_arguments(format_group, ["show_non_defaults"]) subparser.add_argument( "-c", "--cover", action="store", default="nodes", choices=["nodes", "edges", "paths"], help="how extensively to traverse the DAG (default: nodes)", ) subparser.add_argument( "-t", "--types", action="store_true", default=False, help="show dependency types" ) arguments.add_common_arguments(subparser, ["specs"]) arguments.add_concretizer_args(subparser) def spec(parser, args): install_status_fn = spack.spec.Spec.install_status fmt = spack.spec.DISPLAY_FORMAT if args.namespaces: fmt = "{namespace}." + fmt # use a read transaction if we are getting install status for every # spec in the DAG. This avoids repeatedly querying the DB. tree_context = lang.nullcontext if args.install_status: tree_context = spack.store.STORE.db.read_transaction env = ev.active_environment() if args.specs: concrete_specs = spack.cmd.parse_specs(args.specs, concretize=True) elif env: env.concretize() concrete_specs = env.concrete_roots() else: tty.die("spack spec requires at least one spec or an active environment") # With --yaml, --json, or --format, just print the raw specs to output if args.format: for spec in concrete_specs: if args.format == "yaml": # use write because to_yaml already has a newline. sys.stdout.write(spec.to_yaml(hash=ht.dag_hash)) elif args.format == "json": print(spec.to_json(hash=ht.dag_hash)) else: print(spec.format(args.format)) return with tree_context(): print( spack.spec.tree( concrete_specs, cover=args.cover, format=fmt, hashlen=None if args.very_long else 7, show_types=args.types, status_fn=install_status_fn if args.install_status else None, hashes=args.long or args.very_long, key=spack.traverse.by_dag_hash, highlight_version_fn=( spack.package_base.non_preferred_version if args.non_defaults else None ), highlight_variant_fn=( spack.package_base.non_default_variant if args.non_defaults else None ), ) ) ================================================ FILE: lib/spack/spack/cmd/stage.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import spack.cmd import spack.config import spack.environment as ev import spack.llnl.util.tty as tty import spack.package_base import spack.traverse from spack.cmd.common import arguments description = "expand downloaded archive in preparation for install" section = "build" level = "long" class StageFilter: """ Encapsulation of reasons to skip staging """ def __init__(self, exclusions, skip_installed): """ :param exclusions: A list of specs to skip if satisfied. :param skip_installed: A boolean indicating whether to skip already installed specs. """ self.exclusions = exclusions self.skip_installed = skip_installed def __call__(self, spec): """filter action, true means spec should be filtered""" if spec.external: return True if self.skip_installed and spec.installed: return True if any(spec.satisfies(exclude) for exclude in self.exclusions): return True return False def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["no_checksum", "specs"]) subparser.add_argument( "-p", "--path", dest="path", help="path to stage package, does not add to spack tree" ) subparser.add_argument( "-e", "--exclude", action="append", default=[], help="exclude packages that satisfy the specified specs", ) subparser.add_argument( "-s", "--skip-installed", action="store_true", help="dont restage already installed specs" ) arguments.add_concretizer_args(subparser) def stage(parser, args): if args.no_checksum: spack.config.set("config:checksum", False, scope="command_line") exclusion_specs = spack.cmd.parse_specs(args.exclude, concretize=False) filter = StageFilter(exclusion_specs, args.skip_installed) if not args.specs: env = ev.active_environment() if not env: tty.die("`spack stage` requires a spec or an active environment") return _stage_env(env, filter) specs = spack.cmd.parse_specs(args.specs, concretize=False) # We temporarily modify the working directory when setting up a stage, so we need to # convert this to an absolute path here in order for it to remain valid later. custom_path = os.path.abspath(args.path) if args.path else None # prevent multiple specs from extracting in the same folder if len(specs) > 1 and custom_path: tty.die("`--path` requires a single spec, but multiple were provided") specs = spack.cmd.matching_specs_from_env(specs) for spec in specs: spec = spack.cmd.matching_spec_from_env(spec) if filter(spec): continue pkg = spec.package if custom_path: pkg.path = custom_path _stage(pkg) def _stage_env(env: ev.Environment, filter): tty.msg(f"Staging specs from environment {env.name}") for spec in spack.traverse.traverse_nodes(env.concrete_roots()): if filter(spec): continue _stage(spec.package) def _stage(pkg: spack.package_base.PackageBase): # Use context manager to ensure we don't restage while an installation is in progress # keep = True ensures that the stage is not removed after exiting the context manager pkg.stage.keep = True with pkg.stage: pkg.do_stage() tty.msg(f"Staged {pkg.name} in {pkg.stage.path}") ================================================ FILE: lib/spack/spack/cmd/style.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import ast import os import re import sys from pathlib import Path from typing import Dict, List, Optional, Set, Union import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.paths import spack.repo import spack.util.git from spack.cmd.common.spec_strings import ( _check_spec_strings, _spec_str_default_handler, _spec_str_fix_handler, ) from spack.llnl.util.filesystem import working_dir from spack.util.executable import Executable, which description = "runs source code style checks on spack" section = "developer" level = "long" #: List of paths to exclude from checks -- relative to spack root exclude_paths = [os.path.relpath(spack.paths.vendor_path, spack.paths.prefix)] #: Order in which tools should be run. #: The list maps an executable name to a method to ensure the tool is #: bootstrapped or present in the environment. tool_names = ["import", "ruff-format", "ruff-check", "mypy"] #: warnings to ignore in mypy mypy_ignores = [ # same as `disable_error_code = "annotation-unchecked"` in pyproject.toml, which # doesn't exist in mypy 0.971 for Python 3.6 "[annotation-unchecked]" ] #: decorator for adding tools to the list class tool: def __init__( self, name: str, cmd: Optional[str] = None, required: bool = False, external: bool = True ) -> None: self.name = name self.external = external self.required = required self.cmd = cmd if cmd else name def __call__(self, fun): self.fun = fun tools[self.name] = self return fun @property def installed(self) -> bool: return bool(which(self.cmd)) if self.external else True @property def executable(self) -> Optional[Executable]: return which(self.cmd) if self.external else None #: tools we run in spack style tools: Dict[str, tool] = {} def changed_files(base="develop", untracked=True, all_files=False, root=None) -> List[Path]: """Get list of changed files in the Spack repository. Arguments: base (str): name of base branch to evaluate differences with. untracked (bool): include untracked files in the list. all_files (bool): list all files in the repository. root (str): use this directory instead of the Spack prefix. """ if root is None: root = spack.paths.prefix git = spack.util.git.git(required=True) # ensure base is in the repo base_sha = git( "rev-parse", "--quiet", "--verify", "--revs-only", base, fail_on_error=False, output=str ) if git.returncode != 0: tty.die( "This repository does not have a '%s' revision." % base, "spack style needs this branch to determine which files changed.", "Ensure that '%s' exists, or specify files to check explicitly." % base, ) range = "{0}...".format(base_sha.strip()) git_args = [ # Add changed files committed since branching off of develop ["diff", "--name-only", "--diff-filter=ACMR", range], # Add changed files that have been staged but not yet committed ["diff", "--name-only", "--diff-filter=ACMR", "--cached"], # Add changed files that are unstaged ["diff", "--name-only", "--diff-filter=ACMR"], ] # Add new files that are untracked if untracked: git_args.append(["ls-files", "--exclude-standard", "--other"]) # add everything if the user asked for it if all_files: git_args.append(["ls-files", "--exclude-standard"]) excludes = [os.path.realpath(os.path.join(root, f)) for f in exclude_paths] changed = set() for arg_list in git_args: files = git(*arg_list, output=str).split("\n") for f in files: # Ignore non-Python files if not (f.endswith(".py") or f == "bin/spack"): continue # Ignore files in the exclude locations if any(os.path.realpath(f).startswith(e) for e in excludes): continue changed.add(Path(f)) return sorted(changed) def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-b", "--base", action="store", default="develop", help="branch to compare against to determine changed files (default: develop)", ) subparser.add_argument( "-a", "--all", action="store_true", help="check all files, not just changed files (applies only to Import Check)", ) subparser.add_argument( "-r", "--root-relative", action="store_true", default=False, help="print root-relative paths (default: cwd-relative)", ) subparser.add_argument( "-U", "--no-untracked", dest="untracked", action="store_false", default=True, help="exclude untracked files from checks", ) subparser.add_argument( "-f", "--fix", action="store_true", default=False, help="format automatically if possible (e.g., with isort, black)", ) subparser.add_argument( "--root", action="store", default=None, help="style check a different spack instance" ) tool_group = subparser.add_mutually_exclusive_group() tool_group.add_argument( "-t", "--tool", action="append", help="specify which tools to run (default: %s)" % ", ".join(tool_names), ) tool_group.add_argument( "-s", "--skip", metavar="TOOL", action="append", help="specify tools to skip (choose from %s)" % ", ".join(tool_names), ) subparser.add_argument( "--spec-strings", action="store_true", help="upgrade spec strings in Python, JSON and YAML files for compatibility with Spack " "v1.0 and v0.x. Example: spack style ``--spec-strings $(git ls-files)``. Note: must be " "used only on specs from spack v0.X.", ) subparser.add_argument("files", nargs=argparse.REMAINDER, help="specific files to check") def cwd_relative(path: Path, root: Union[Path, str], initial_working_dir: Path) -> Path: """Translate prefix-relative path to current working directory-relative.""" if path.is_absolute(): return path return Path(os.path.relpath((root / path), initial_working_dir)) def rewrite_and_print_output( output, root, working_dir, root_relative, re_obj=re.compile(r"^(.+):([0-9]+):"), replacement=r"{0}:{1}:", ): """rewrite output with :: format to respect path args""" # print results relative to current working directory def translate(match): return replacement.format( cwd_relative(Path(match.group(1)), root, working_dir), *list(match.groups()[1:]) ) for line in output.split("\n"): if not line: continue if any(ignore in line for ignore in mypy_ignores): # some mypy annotations can't be disabled in older mypys (e.g. .971, which # is the only mypy that supports python 3.6), so we filter them here. continue if not root_relative and re_obj: line = re_obj.sub(translate, line) print(line) def print_tool_result(tool, returncode): if returncode == 0: color.cprint(" @g{%s checks were clean}" % tool) else: color.cprint(" @r{%s found errors}" % tool) @tool("ruff-check", cmd="ruff") def ruff_check(file_list, args): """Run the ruff-check command. Handles config and non generic ruff argument logic""" cmd_args = ["--config", os.path.join(spack.paths.prefix, "pyproject.toml"), "--quiet"] if args.fix: cmd_args += ["--fix", "--no-unsafe-fixes"] else: cmd_args += ["--no-fix"] return run_ruff( file_list, "check", cmd_args, args.root, args.initial_working_dir, args.root_relative ) @tool("ruff-format", cmd="ruff") def ruff_format(file_list, args): """Run the ruff format command""" cmd_args = ["--config", os.path.join(spack.paths.prefix, "pyproject.toml"), "--quiet"] if not args.fix: cmd_args += ["--check", "--diff"] return run_ruff( file_list, "format", cmd_args, args.root, args.initial_working_dir, args.root_relative ) def run_ruff( file_list: List[Path], cmd: str, args: List[str], root: Path, working_dir: Path, root_relative: bool, ): """Run the ruff tool""" ruff_cmd = tools[f"ruff-{cmd}"].executable if not ruff_cmd: tty.warn("Cannot execute requested tool: ruff\nCannot find tool") return -1 files = (str(x) for x in file_list) if color.get_color_when(): args += ("--color", "auto") pat = re.compile("would reformat +(.*)") replacement = "would reformat {0}" packed_args = (cmd,) + (*args,) + tuple(files) output = ruff_cmd(*packed_args, fail_on_error=False, output=str, error=str) returncode = ruff_cmd.returncode rewrite_and_print_output(output, root, working_dir, root_relative, pat, replacement) print_tool_result(f"ruff-{cmd}", returncode) return returncode @tool("mypy") def run_mypy(file_list, args): mypy_cmd = tools["mypy"].executable if not mypy_cmd: tty.warn("Cannot execute requested tool: mypy\nCannot find tool") return -1 # always run with config from running spack prefix common_mypy_args = [ "--config-file", os.path.join(spack.paths.prefix, "pyproject.toml"), "--show-error-codes", ] mypy_arg_sets = [common_mypy_args + ["--package", "spack", "--package", "llnl"]] if "SPACK_MYPY_CHECK_PACKAGES" in os.environ: mypy_arg_sets.append( common_mypy_args + ["--package", "packages", "--disable-error-code", "no-redef"] ) returncode = 0 for mypy_args in mypy_arg_sets: output = mypy_cmd(*mypy_args, fail_on_error=False, output=str) returncode |= mypy_cmd.returncode rewrite_and_print_output(output, args.root, args.initial_working_dir, args.root_relative) print_tool_result("mypy", returncode) return returncode def _module_part(root: Path, expr: str): parts = expr.split(".") # spack.pkg is for repositories, don't try to resolve it here. if expr.startswith(spack.repo.PKG_MODULE_PREFIX_V1) or expr == "spack.pkg": return None while parts: f1 = (root / "lib" / "spack").joinpath(*parts).with_suffix(".py") f2 = (root / "lib" / "spack").joinpath(*parts, "__init__.py") if ( f1.exists() # ensure case sensitive match and any(p.name == f"{parts[-1]}.py" for p in f1.parent.iterdir()) or f2.exists() ): return ".".join(parts) parts.pop() return None def _run_import_check( file_list: List[Path], *, fix: bool, root_relative: bool, root: Path, working_dir: Path, out=sys.stdout, base="develop", all=False, ): if sys.version_info < (3, 9): print("import check requires Python 3.9 or later") return 0 is_use = re.compile(r"(? List[str]: return [t for t in tools_to_run if not tools[t].installed] def _bootstrap_dev_dependencies(): import spack.bootstrap with spack.bootstrap.ensure_bootstrap_configuration(): spack.bootstrap.ensure_environment_dependencies() def style(parser, args): if args.spec_strings: if not args.files: tty.die("No files provided to check spec strings.") handler = _spec_str_fix_handler if args.fix else _spec_str_default_handler return _check_spec_strings(args.files, handler) # save initial working directory for relativizing paths later args.initial_working_dir = Path.cwd() # ensure that the config files we need actually exist in the spack prefix. # assertions b/c users should not ever see these errors -- they're checked in CI. assert (Path(spack.paths.prefix) / "pyproject.toml").is_file() # validate spack root if the user provided one args.root = Path(args.root).resolve() if args.root else Path(spack.paths.prefix) spack_script = args.root / "bin" / "spack" if not spack_script.exists(): tty.die("This does not look like a valid spack root.", "No such file: '%s'" % spack_script) def prefix_relative(path: Union[Path, str]) -> Path: return Path(os.path.relpath(os.path.abspath(os.path.realpath(path)), args.root)) file_list = [prefix_relative(file) for file in args.files] # process --tool and --skip arguments selected = set(tool_names) if args.tool is not None: selected = validate_toolset(args.tool) if args.skip is not None: selected -= validate_toolset(args.skip) if not selected: tty.msg("Nothing to run.") return tools_to_run = [t for t in tool_names if t in selected] if missing_tools(tools_to_run): _bootstrap_dev_dependencies() return_code = 0 with working_dir(str(args.root)): print_style_header(file_list, args, tools_to_run) for tool_name in tools_to_run: tool = tools[tool_name] tty.msg(f"Running {tool.name} checks") return_code |= tool.fun(file_list, args) if return_code == 0: tty.msg(color.colorize("@*{spack style checks were clean}")) else: tty.error(color.colorize("@*{spack style found errors}")) return return_code ================================================ FILE: lib/spack/spack/cmd/tags.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import io import sys from typing import Dict, Iterable, List import spack.environment import spack.llnl.string import spack.llnl.util.tty as tty import spack.llnl.util.tty.colify as colify import spack.repo description = "show package tags and associated packages" section = "query" level = "long" def report_tags(category, tags): buffer = io.StringIO() isatty = sys.stdout.isatty() if isatty: num = len(tags) fmt = "{0} package tag".format(category) buffer.write("{0}:\n".format(spack.llnl.string.plural(num, fmt))) if tags: colify.colify(tags, output=buffer, tty=isatty, indent=4) else: buffer.write(" None\n") print(buffer.getvalue()) def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.epilog = ( "Tags from known packages will be used if no tags are provided on " "the command\nline. If tags are provided, packages with at least one " "will be reported.\n\nYou are not allowed to provide tags and use " "'--all' at the same time." ) subparser.add_argument( "-i", "--installed", action="store_true", default=False, help="show information for installed packages only", ) subparser.add_argument( "-a", "--all", action="store_true", default=False, help="show packages for all available tags", ) subparser.add_argument("tag", nargs="*", help="show packages with the specified tag") def tags(parser, args): # Disallow combining all option with (positional) tags to avoid confusion if args.all and args.tag: tty.die("Use the '--all' option OR provide tag(s) on the command line") # Provide a nice, simple message if database is empty if args.installed and not spack.environment.installed_specs(): tty.msg("No installed packages") return # unique list of available tags available_tags = sorted(spack.repo.PATH.tag_index.tags) if not available_tags: tty.msg("No tagged packages") return show_packages = args.tag or args.all # Only report relevant, available tags if no packages are to be shown if not show_packages: if not args.installed: report_tags("available", available_tags) else: tag_pkgs = packages_with_tags(available_tags, True, True) tags = tag_pkgs.keys() if tag_pkgs else [] report_tags("installed", tags) return # Report packages associated with tags buffer = io.StringIO() isatty = sys.stdout.isatty() tags = args.tag if args.tag else available_tags tag_pkgs = packages_with_tags(tags, args.installed, False) missing = "No installed packages" if args.installed else "None" for tag in sorted(tag_pkgs): # TODO: Remove the sorting once we're sure no one has an old # TODO: tag cache since it can accumulate duplicates. packages = sorted(list(set(tag_pkgs[tag]))) if isatty: buffer.write("{0}:\n".format(tag)) if packages: colify.colify(packages, output=buffer, tty=isatty, indent=4) else: buffer.write(" {0}\n".format(missing)) buffer.write("\n") print(buffer.getvalue()) def packages_with_tags( tags: Iterable[str], installed: bool, skip_empty: bool ) -> Dict[str, List[str]]: """ Returns a dict, indexed by tag, containing lists of names of packages containing the tag or, if no tags, for all available tags. Arguments: tags: list of tags of interest or None for all installed: True if want names of packages that are installed; otherwise, False if want all packages with the tag skip_empty: True if exclude tags with no associated packages; otherwise, False if want entries for all tags even when no such tagged packages """ tag_pkgs: Dict[str, List[str]] = {} name_filter = {x.name for x in spack.environment.installed_specs()} if installed else None for tag in tags: packages = spack.repo.PATH.tag_index.get_packages(tag) if name_filter is not None: packages = [p for p in packages if p in name_filter] if packages or not skip_empty: tag_pkgs[tag] = packages return tag_pkgs ================================================ FILE: lib/spack/spack/cmd/test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import fnmatch import os import re import shutil import sys from collections import Counter import spack.cmd import spack.config import spack.environment as ev import spack.install_test import spack.repo import spack.store from spack.cmd.common import arguments from spack.llnl.util import tty from spack.llnl.util.tty import colify from . import doc_dedented, doc_first_line description = "run spack's tests for an install" section = "build" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="test_command") # Run run_parser = sp.add_parser( "run", description=doc_dedented(test_run), help=doc_first_line(test_run) ) run_parser.add_argument( "--alias", help="provide an alias for this test-suite for subsequent access" ) run_parser.add_argument( "--fail-fast", action="store_true", help="stop tests for each package after the first failure", ) run_parser.add_argument( "--fail-first", action="store_true", help="stop after the first failed package" ) run_parser.add_argument( "--externals", action="store_true", help="test packages that are externally installed" ) run_parser.add_argument( "-x", "--explicit", action="store_true", help="only test packages that are explicitly installed", ) run_parser.add_argument( "--keep-stage", action="store_true", help="keep testing directory for debugging" ) arguments.add_common_arguments(run_parser, ["log_format"]) run_parser.add_argument("--log-file", default=None, help="filename for the log file") arguments.add_cdash_args(run_parser, False) run_parser.add_argument( "--help-cdash", action="store_true", help="show usage instructions for CDash reporting" ) run_parser.add_argument( "--timeout", type=int, default=None, help="maximum time (in seconds) that tests are allowed to run", ) cd_group = run_parser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ["clean", "dirty"]) arguments.add_common_arguments(run_parser, ["installed_specs"]) # List list_parser = sp.add_parser( "list", description=doc_dedented(test_list), help=doc_first_line(test_list) ) list_parser.add_argument( "-a", "--all", action="store_true", dest="list_all", help="list all packages with tests (not just installed)", ) list_parser.add_argument("tag", nargs="*", help="limit packages to those with all listed tags") # Find find_parser = sp.add_parser( "find", description=doc_dedented(test_find), help=doc_first_line(test_find) ) find_parser.add_argument( "filter", nargs=argparse.REMAINDER, help="optional case-insensitive glob patterns to filter results", ) # Status status_parser = sp.add_parser( "status", description=doc_dedented(test_status), help=doc_first_line(test_status) ) status_parser.add_argument( "names", nargs=argparse.REMAINDER, help="test suites for which to print status" ) # Results results_parser = sp.add_parser( "results", description=doc_dedented(test_results), help=doc_first_line(test_results) ) results_parser.add_argument( "-l", "--logs", action="store_true", help="print the test log for each matching package" ) results_parser.add_argument( "-f", "--failed", action="store_true", help="only show results for failed tests of matching packages", ) results_parser.add_argument( "names", nargs=argparse.REMAINDER, metavar="[name(s)] [-- installed_specs]...", help="suite names and installed package constraints", ) results_parser.epilog = ( "Test results will be filtered by space-" "separated suite name(s) and installed\nspecs when provided. " "If names are provided, then only results for those test\nsuites " "will be shown. If installed specs are provided, then only results" "\nmatching those specs will be shown." ) # Remove remove_parser = sp.add_parser( "remove", description=doc_dedented(test_remove), help=doc_first_line(test_remove) ) arguments.add_common_arguments(remove_parser, ["yes_to_all"]) remove_parser.add_argument( "names", nargs=argparse.REMAINDER, help="test suites to remove from test stage" ) def test_run(args): """\ run tests for the specified installed packages if no specs are listed, run tests for all packages in the current environment or all installed packages if there is no active environment """ if args.alias: suites = spack.install_test.get_named_test_suites(args.alias) if suites: tty.die('Test suite "{0}" already exists. Try another alias.'.format(args.alias)) # cdash help option if args.help_cdash: arguments.print_cdash_help() return arguments.sanitize_reporter_options(args) # set config option for fail-fast if args.fail_fast: spack.config.set("config:fail_fast", True, scope="command_line") explicit = args.explicit or None explicit_str = "explicitly " if args.explicit else "" # Get specs to test env = ev.active_environment() hashes = env.all_hashes() if env else None specs = spack.cmd.parse_specs(args.specs) if args.specs else [None] specs_to_test = [] for spec in specs: matching = spack.store.STORE.db.query_local(spec, hashes=hashes, explicit=explicit) if spec and not matching: tty.warn(f"No {explicit_str}installed packages match spec {spec}") # TODO: Need to write out a log message and/or CDASH Testing # output that package not installed IF continue to process # these issues here. # if args.log_format: # # Proceed with the spec assuming the test process # # to ensure report package as skipped (e.g., for CI) # specs_to_test.append(spec) specs_to_test.extend(matching) # test_stage_dir test_suite = spack.install_test.TestSuite(specs_to_test, args.alias) test_suite.ensure_stage() tty.msg(f"Spack test {test_suite.name}") # Set up reporter reporter = args.reporter() if args.log_format else None try: test_suite( remove_directory=not args.keep_stage, dirty=args.dirty, fail_first=args.fail_first, externals=args.externals, timeout=args.timeout, ) finally: if reporter: report_file = report_filename(args, test_suite) reporter.test_report(report_file, test_suite.reports) def report_filename(args, test_suite): return os.path.abspath(args.log_file or "test-{}".format(test_suite.name)) def test_list(args): """list installed packages with available tests""" tagged = spack.repo.PATH.packages_with_tags(*args.tag) if args.tag else set() def has_test_and_tags(pkg_class): tests = spack.install_test.test_functions(pkg_class) return len(tests) and (not args.tag or pkg_class.name in tagged) if args.list_all: report_packages = [ pkg_class.name for pkg_class in spack.repo.PATH.all_package_classes() if has_test_and_tags(pkg_class) ] if sys.stdout.isatty(): filtered = " tagged" if args.tag else "" tty.msg("{0}{1} packages with tests.".format(len(report_packages), filtered)) colify.colify(report_packages) return # TODO: This can be extended to have all of the output formatting options # from `spack find`. env = ev.active_environment() hashes = env.all_hashes() if env else None specs = spack.store.STORE.db.query(hashes=hashes) specs = list( filter(lambda s: has_test_and_tags(spack.repo.PATH.get_pkg_class(s.fullname)), specs) ) spack.cmd.display_specs(specs, long=True) def test_find(args): # TODO: merge with status (noargs) """\ find tests that are running or have available results displays aliases for tests that have them, otherwise test suite content hashes """ test_suites = spack.install_test.get_all_test_suites() # Filter tests by filter argument if args.filter: def create_filter(f): raw = fnmatch.translate("f" if "*" in f or "?" in f else "*" + f + "*") return re.compile(raw, flags=re.IGNORECASE) filters = [create_filter(f) for f in args.filter] def match(t, f): return f.match(t) test_suites = [ t for t in test_suites if any(match(t.alias, f) for f in filters) and os.path.isdir(t.stage) ] names = [t.name for t in test_suites] if names: # TODO: Make these specify results vs active msg = "Spack test results available for the following tests:\n" msg += " %s\n" % " ".join(names) msg += " Run `spack test remove` to remove all tests" tty.msg(msg) else: msg = "No test results match the query\n" msg += " Tests may have been removed using `spack test remove`" tty.msg(msg) def test_status(args): """get the current status for the specified Spack test suite(s)""" if args.names: test_suites = [] for name in args.names: test_suite = spack.install_test.get_test_suite(name) if test_suite: test_suites.append(test_suite) else: tty.msg("No test suite %s found in test stage" % name) else: test_suites = spack.install_test.get_all_test_suites() if not test_suites: tty.msg("No test suites with status to report") for test_suite in test_suites: # TODO: Make this handle capability tests too # TODO: Make this handle tests running in another process tty.msg("Test suite %s completed" % test_suite.name) def _report_suite_results(test_suite, args, constraints): """Report the relevant test suite results.""" # TODO: Make this handle capability tests too # The results file may turn out to be a placeholder for future work if constraints: # TBD: Should I be refactoring or re-using ConstraintAction? qspecs = spack.cmd.parse_specs(constraints) specs = {} for spec in qspecs: for s in spack.store.STORE.db.query(spec, installed=True): specs[s.dag_hash()] = s specs = sorted(specs.values()) test_specs = dict((test_suite.test_pkg_id(s), s) for s in test_suite.specs if s in specs) else: test_specs = dict((test_suite.test_pkg_id(s), s) for s in test_suite.specs) if not test_specs: return if os.path.exists(test_suite.results_file): results_desc = "Failing results" if args.failed else "Results" matching = ", spec matching '{0}'".format(" ".join(constraints)) if constraints else "" tty.msg("{0} for test suite '{1}'{2}:".format(results_desc, test_suite.name, matching)) results = {} with open(test_suite.results_file, "r", encoding="utf-8") as f: for line in f: pkg_id, status = line.split() results[pkg_id] = status tty.msg("test specs:") counts = Counter() for pkg_id in test_specs: if pkg_id in results: status = results[pkg_id] # Backward-compatibility: NO-TESTS => NO_TESTS status = "NO_TESTS" if status == "NO-TESTS" else status status = spack.install_test.TestStatus[status] counts[status] += 1 if args.failed and status != spack.install_test.TestStatus.FAILED: continue msg = " {0} {1}".format(pkg_id, status) if args.logs: spec = test_specs[pkg_id] log_file = test_suite.log_file_for_spec(spec) if os.path.isfile(log_file): with open(log_file, "r", encoding="utf-8") as f: msg += "\n{0}".format("".join(f.readlines())) tty.msg(msg) spack.install_test.write_test_summary(counts) else: msg = "Test %s has no results.\n" % test_suite.name msg += " Check if it is running with " msg += "`spack test status %s`" % test_suite.name tty.msg(msg) def test_results(args): """get the results from Spack test suite(s) (default all)""" if args.names: try: sep_index = args.names.index("--") names = args.names[:sep_index] constraints = args.names[sep_index + 1 :] except ValueError: names = args.names constraints = None else: names, constraints = None, None if names: test_suites = [spack.install_test.get_test_suite(name) for name in names] test_suites = list(filter(lambda ts: ts is not None, test_suites)) if not test_suites: tty.msg("No test suite(s) found in test stage: {0}".format(", ".join(names))) else: test_suites = spack.install_test.get_all_test_suites() if not test_suites: tty.msg("No test suites with results to report") for test_suite in test_suites: _report_suite_results(test_suite, args, constraints) def test_remove(args): """\ remove results from Spack test suite(s) (default all) if no test suite is listed, remove results for all suites. removed tests can no longer be accessed for results or status, and will not appear in ``spack test list`` results """ if args.names: test_suites = [] for name in args.names: test_suite = spack.install_test.get_test_suite(name) if test_suite: test_suites.append(test_suite) else: tty.msg("No test suite %s found in test stage" % name) else: test_suites = spack.install_test.get_all_test_suites() if not test_suites: tty.msg("No test suites to remove") return if not args.yes_to_all: msg = "The following test suites will be removed:\n\n" msg += " " + " ".join(test.name for test in test_suites) + "\n" tty.msg(msg) answer = tty.get_yes_or_no("Do you want to proceed?", default=False) if not answer: tty.msg("Aborting removal of test suites") return for test_suite in test_suites: shutil.rmtree(test_suite.stage) def test(parser, args): globals()["test_%s" % args.test_command](args) ================================================ FILE: lib/spack/spack/cmd/test_env.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.cmd.common.env_utility as env_utility from spack.context import Context description = ( "run a command in a spec's test environment,\nor dump its environment to screen or file" ) section = "developer" level = "long" setup_parser = env_utility.setup_parser def test_env(parser, args): env_utility.emulate_env_utility("test-env", Context.TEST, args) ================================================ FILE: lib/spack/spack/cmd/tutorial.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import shutil import spack import spack.cmd import spack.config import spack.llnl.util.tty as tty import spack.paths import spack.util.git import spack.util.gpg from spack.cmd.common import arguments from spack.llnl.util.filesystem import working_dir from spack.util.spack_yaml import syaml_dict description = "set up spack for our tutorial (WARNING: modifies config!)" section = "config" level = "long" # tutorial configuration parameters tutorial_branch = "releases/v1.1" tutorial_mirror = "file:///mirror" tutorial_key = os.path.join(spack.paths.share_path, "keys", "tutorial.pub") # configs to remove rm_configs = [ "~/.spack/linux/compilers.yaml", "~/.spack/packages.yaml", "~/.spack/mirrors.yaml", "~/.spack/modules.yaml", "~/.spack/config.yaml", ] def setup_parser(subparser: argparse.ArgumentParser) -> None: arguments.add_common_arguments(subparser, ["yes_to_all"]) def tutorial(parser, args): if not spack.cmd.spack_is_git_repo(): tty.die("This command requires a git installation of Spack!") if not args.yes_to_all: tty.msg( "This command will set up Spack for the tutorial at " "https://spack-tutorial.readthedocs.io.", "", ) tty.warn( "This will modify your Spack configuration by:", " - deleting some configuration in ~/.spack", " - adding a mirror and trusting its public key", " - checking out a particular branch of Spack", "", ) if not tty.get_yes_or_no("Are you sure you want to proceed?"): tty.die("Aborted") rm_cmds = [f"rm -f {f}" for f in rm_configs] tty.msg("Reverting compiler and repository configuration", *rm_cmds) for path in rm_configs: if os.path.exists(path): shutil.rmtree(path, ignore_errors=True) tty.msg( "Ensuring that the tutorial binary mirror is configured:", f"spack mirror add tutorial {tutorial_mirror}", ) mirror_config = syaml_dict() mirror_config["tutorial"] = tutorial_mirror spack.config.set("mirrors", mirror_config, scope="user") tty.msg("Ensuring that we trust tutorial binaries", f"spack gpg trust {tutorial_key}") spack.util.gpg.trust(tutorial_key) # Note that checkout MUST be last. It changes Spack under our feet. # If you don't put this last, you'll get import errors for the code # that follows (exacerbated by the various lazy singletons we use) tty.msg(f"Ensuring we're on the {tutorial_branch} branch") git = spack.util.git.git(required=True) with working_dir(spack.paths.prefix): git("checkout", tutorial_branch) # NO CODE BEYOND HERE ================================================ FILE: lib/spack/spack/cmd/undevelop.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import spack.cmd import spack.config import spack.llnl.util.tty as tty import spack.spec from spack.cmd.common import arguments description = "remove specs from an environment" section = "environments" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "--no-modify-concrete-specs", action="store_false", dest="apply_changes", help=( "do not mutate concrete specs to remove dev_path provenance." " This requires running `spack concretize -f` later to apply changes to concrete specs" ), ) subparser.add_argument( "-a", "--all", action="store_true", help="remove all specs from (clear) the environment" ) arguments.add_common_arguments(subparser, ["specs"]) def _update_config(specs_to_remove): def change_fn(dev_config): modified = False for spec in specs_to_remove: if spec.name in dev_config: tty.msg("Undevelop: removing {0}".format(spec.name)) del dev_config[spec.name] modified = True return modified spack.config.update_all("develop", change_fn) def undevelop(parser, args): # TODO: when https://github.com/spack/spack/pull/35307 is merged, # an active env is not required if a scope is specified env = spack.cmd.require_active_env(cmd_name="undevelop") if args.all: remove_specs = [spack.spec.Spec(s) for s in env.dev_specs] else: remove_specs = spack.cmd.parse_specs(args.specs) with env.write_transaction(): _update_config(remove_specs) if args.apply_changes: for spec in remove_specs: env.apply_develop(spec, path=None) updated_all_dev_specs = set(spack.config.get("develop")) remove_spec_names = set(x.name for x in remove_specs) not_fully_removed = updated_all_dev_specs & remove_spec_names if not_fully_removed: tty.msg( "The following specs could not be removed as develop specs" " - see `spack config blame develop` to locate files requiring" f" manual edits: {', '.join(not_fully_removed)}" ) ================================================ FILE: lib/spack/spack/cmd/uninstall.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys from typing import Dict, List, Optional import spack.cmd import spack.cmd.common.confirmation as confirmation import spack.environment as ev import spack.package_base import spack.spec import spack.store import spack.traverse as traverse from spack.cmd.common import arguments from spack.llnl.util import tty from spack.llnl.util.tty.colify import colify from ..enums import InstallRecordStatus description = "remove installed packages" section = "build" level = "short" error_message = """You can either: a) use a more specific spec, or b) specify the spec by its hash (e.g. `spack uninstall /hash`), or c) use `spack uninstall --all` to uninstall ALL matching specs. """ # Arguments for display_specs when we find ambiguity display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4} def setup_parser(subparser: argparse.ArgumentParser) -> None: epilog_msg = ( "Specs to be uninstalled are specified using the spec syntax" " (`spack help --spec`) and can be identified by their " "hashes. To remove packages that are needed only at build " "time and were not explicitly installed see `spack gc -h`." "\n\nWhen using the --all option ALL packages matching the " "supplied specs will be uninstalled. For instance, " "`spack uninstall --all libelf` uninstalls all the versions " "of `libelf` currently present in Spack's store. If no spec " "is supplied, all installed packages will be uninstalled. " "If used in an environment, all packages in the environment " "will be uninstalled." ) subparser.epilog = epilog_msg subparser.add_argument( "-f", "--force", action="store_true", dest="force", help="remove regardless of whether other packages or environments depend on this one", ) subparser.add_argument( "--remove", action="store_true", dest="remove", help="if in an environment, then the spec should also be removed from " "the environment description", ) arguments.add_common_arguments( subparser, ["recurse_dependents", "yes_to_all", "installed_specs"] ) subparser.add_argument( "-a", "--all", action="store_true", dest="all", help="remove ALL installed packages that match each supplied spec", ) subparser.add_argument( "--origin", dest="origin", help="only remove DB records with the specified origin" ) def find_matching_specs( env: Optional[ev.Environment], specs: List[spack.spec.Spec], allow_multiple_matches: bool = False, origin=None, ) -> List[spack.spec.Spec]: """Returns a list of specs matching the not necessarily concretized specs given from cli Args: env: optional active environment specs: list of specs to be matched against installed packages allow_multiple_matches: if True multiple matches are admitted origin: origin of the spec """ # constrain uninstall resolution to current environment if one is active hashes = env.all_hashes() if env else None # List of specs that match expressions given via command line specs_from_cli: List[spack.spec.Spec] = [] has_errors = False for spec in specs: matching = spack.store.STORE.db.query_local( spec, hashes=hashes, installed=(InstallRecordStatus.INSTALLED | InstallRecordStatus.DEPRECATED), origin=origin, ) # For each spec provided, make sure it refers to only one package. # Fail and ask user to be unambiguous if it doesn't if not allow_multiple_matches and len(matching) > 1: tty.error("{0} matches multiple packages:".format(spec)) sys.stderr.write("\n") spack.cmd.display_specs(matching, output=sys.stderr, **display_args) sys.stderr.write("\n") sys.stderr.flush() has_errors = True # No installed package matches the query if len(matching) == 0 and spec is not None: if env: pkg_type = "packages in environment '%s'" % env.name else: pkg_type = "installed packages" tty.die("{0} does not match any {1}.".format(spec, pkg_type)) specs_from_cli.extend(matching) if has_errors: tty.die(error_message) return specs_from_cli def installed_dependents(specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]: # Note: the combination of arguments (in particular order=breadth # and root=False) ensures dependents and matching_specs are non-overlapping; # In the extreme case of "spack uninstall --all" we get the entire database as # input; in that case we return an empty list. def is_installed(spec): record = spack.store.STORE.db.query_local_by_spec_hash(spec.dag_hash()) return record and record.installed all_specs = traverse.traverse_nodes( specs, root=False, order="breadth", cover="nodes", deptype=("link", "run"), direction="parents", key=lambda s: s.dag_hash(), ) with spack.store.STORE.db.read_transaction(): return [spec for spec in all_specs if is_installed(spec)] def dependent_environments( specs: List[spack.spec.Spec], current_env: Optional[ev.Environment] = None ) -> Dict[ev.Environment, List[spack.spec.Spec]]: # For each tracked environment, get the specs we would uninstall from it. # Don't instantiate current environment twice. env_names = ev.all_environment_names() if current_env: env_names = (name for name in env_names if name != current_env.name) # Mapping from Environment -> non-zero list of specs contained in it. other_envs_to_specs: Dict[ev.Environment, List[spack.spec.Spec]] = {} for other_env in (ev.Environment(ev.root(name)) for name in env_names): specs_in_other_env = all_specs_in_env(other_env, specs) if specs_in_other_env: other_envs_to_specs[other_env] = specs_in_other_env return other_envs_to_specs def all_specs_in_env(env: ev.Environment, specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]: """Given a list of specs, return those that are in the env""" hashes = set(env.all_hashes()) return [s for s in specs if s.dag_hash() in hashes] def _remove_from_env(spec, env): """Remove a spec from an environment if it is a root.""" try: # try removing the spec from the current active # environment. this will fail if the spec is not a root env.remove(spec, force=True) except ev.SpackEnvironmentError: pass # ignore non-root specs def do_uninstall(specs: List[spack.spec.Spec], force: bool = False): # TODO: get rid of the call-sites that use this function, # so that we don't have to do a dance of list -> set -> list -> set hashes_to_remove = set(s.dag_hash() for s in specs) for s in traverse.traverse_nodes( specs, order="topo", direction="children", root=True, cover="nodes", deptype="all" ): if s.dag_hash() in hashes_to_remove: spack.package_base.PackageBase.uninstall_by_spec(s, force=force) def get_uninstall_list(args, specs: List[spack.spec.Spec], env: Optional[ev.Environment]): """Returns unordered uninstall_list and remove_list: these may overlap (some things may be both uninstalled and removed from the current environment). It is assumed we are in an environment if ``--remove`` is specified (this method raises an exception otherwise).""" if args.remove and not env: raise ValueError("Can only use --remove when in an environment") # Gets the list of installed specs that match the ones given via cli # args.all takes care of the case where '-a' is given in the cli matching_specs = find_matching_specs(env, specs, args.all, origin=args.origin) dependent_specs = installed_dependents(matching_specs) all_uninstall_specs = matching_specs + dependent_specs if args.dependents else matching_specs other_dependent_envs = dependent_environments(all_uninstall_specs, current_env=env) # There are dependents and we didn't ask to remove dependents dangling_dependents = dependent_specs and not args.dependents # An environment different than the current env depends on # one or more of the list of all specs to be uninstalled. dangling_environments = not args.remove and other_dependent_envs has_error = not args.force and (dangling_dependents or dangling_environments) if has_error: msgs = [] tty.info("Refusing to uninstall the following specs") spack.cmd.display_specs(matching_specs, **display_args) if dangling_dependents: print() tty.info("The following dependents are still installed:") spack.cmd.display_specs(dependent_specs, **display_args) msgs.append("use `spack uninstall --dependents` to remove dependents too") if dangling_environments: print() tty.info("The following environments still reference these specs:") colify([e.name for e in other_dependent_envs.keys()], indent=4) if env: msgs.append("use `spack remove` to remove the spec from the current environment") msgs.append("use `spack env remove` to remove environments") msgs.append("use `spack uninstall --force` to override") print() tty.die("There are still dependents.", *msgs) # If we are in an environment, this will track specs in this environment # which should only be removed from the environment rather than uninstalled remove_only = [] if args.remove and not args.force: for specs_in_other_env in other_dependent_envs.values(): remove_only.extend(specs_in_other_env) if remove_only: tty.info( "The following specs will be removed but not uninstalled because" " they are also used by another environment: {speclist}".format( speclist=", ".join(x.name for x in remove_only) ) ) # Compute the set of specs that should be removed from the current env. # This may overlap (some specs may be uninstalled and also removed from # the current environment). remove_specs = all_specs_in_env(env, all_uninstall_specs) if env and args.remove else [] return list(set(all_uninstall_specs) - set(remove_only)), remove_specs def uninstall_specs(args, specs): env = ev.active_environment() uninstall_list, remove_list = get_uninstall_list(args, specs, env) if not uninstall_list: tty.warn("There are no package to uninstall.") return if not args.yes_to_all: confirmation.confirm_action(uninstall_list, "uninstalled", "uninstall") # Uninstall everything on the list do_uninstall(uninstall_list, args.force) if env: with env.write_transaction(): for spec in remove_list: _remove_from_env(spec, env) env.write() env.regenerate_views() def uninstall(parser, args): if not args.specs and not args.all: tty.die( "uninstall requires at least one package argument.", " Use `spack uninstall --all` to uninstall ALL packages.", ) # [None] here handles the --all case by forcing all specs to be returned specs = spack.cmd.parse_specs(args.specs) if args.specs else [None] uninstall_specs(args, specs) ================================================ FILE: lib/spack/spack/cmd/unit_test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import collections import io import os import re import sys import spack.extensions try: import pytest except ImportError: pytest = None # type: ignore import spack.llnl.util.filesystem import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as color import spack.paths from spack.llnl.util.tty.colify import colify description = "run spack's unit tests (wrapper around pytest)" section = "developer" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: subparser.add_argument( "-H", "--pytest-help", action="store_true", default=False, help="show full pytest help, with advanced options", ) subparser.add_argument( "-n", "--numprocesses", type=int, default=1, help="run tests in parallel up to this wide, default 1 for sequential", ) # extra spack arguments to list tests list_group = subparser.add_argument_group("listing tests") list_mutex = list_group.add_mutually_exclusive_group() list_mutex.add_argument( "-l", "--list", action="store_const", default=None, dest="list", const="list", help="list test filenames", ) list_mutex.add_argument( "-L", "--list-long", action="store_const", default=None, dest="list", const="long", help="list all test functions", ) list_mutex.add_argument( "-N", "--list-names", action="store_const", default=None, dest="list", const="names", help="list full names of all tests", ) # use tests for extension subparser.add_argument( "--extension", default=None, help="run test for a given spack extension" ) # spell out some common pytest arguments, so they'll show up in help pytest_group = subparser.add_argument_group( "common pytest arguments (spack unit-test --pytest-help for more)" ) pytest_group.add_argument( "-s", action="append_const", dest="parsed_args", const="-s", help="print output while tests run (disable capture)", ) pytest_group.add_argument( "-k", action="store", metavar="EXPRESSION", dest="expression", help="filter tests by keyword (can also use w/list options)", ) pytest_group.add_argument( "--showlocals", action="append_const", dest="parsed_args", const="--showlocals", help="show local variable values in tracebacks", ) # remainder is just passed to pytest subparser.add_argument("pytest_args", nargs=argparse.REMAINDER, help="arguments for pytest") def do_list(args, extra_args): """Print a lists of tests than what pytest offers.""" def colorize(c, prefix): if isinstance(prefix, tuple): return "::".join(color.colorize("@%s{%s}" % (c, p)) for p in prefix if p != "()") return color.colorize("@%s{%s}" % (c, prefix)) # To list the files we just need to inspect the filesystem, # which doesn't need to wait for pytest collection and doesn't # require parsing pytest output files = spack.llnl.util.filesystem.find( root=spack.paths.test_path, files="*.py", recursive=True ) files = [ os.path.relpath(f, start=spack.paths.spack_root) for f in files if not f.endswith(("conftest.py", "__init__.py")) ] old_output = sys.stdout try: sys.stdout = output = io.StringIO() pytest.main(["--collect-only"] + extra_args) finally: sys.stdout = old_output lines = output.getvalue().split("\n") tests = collections.defaultdict(set) # collect tests into sections node_regexp = re.compile(r"(\s*)<([^ ]*) ['\"]?([^']*)['\"]?>") key_parts, name_parts = [], [] for line in lines: match = node_regexp.match(line) if not match: continue indent, nodetype, name = match.groups() # strip parametrized tests if "[" in name: name = name[: name.index("[")] len_indent = len(indent) if os.path.isabs(name): name = os.path.relpath(name, start=spack.paths.spack_root) item = (len_indent, name, nodetype) # Reduce the parts to the scopes that are of interest name_parts = [x for x in name_parts if x[0] < len_indent] key_parts = [x for x in key_parts if x[0] < len_indent] # From version 3.X to version 6.X the output format # changed a lot in pytest, and probably will change # in the future - so this manipulation might be fragile if nodetype.lower() == "function": name_parts.append(item) key_end = os.path.join(*key_parts[-1][1].split("/")) key = next(f for f in files if f.endswith(key_end)) tests[key].add(tuple(x[1] for x in name_parts)) elif nodetype.lower() == "class": name_parts.append(item) elif nodetype.lower() in ("package", "module"): key_parts.append(item) if args.list == "list": files = set(tests.keys()) color_files = [colorize("B", file) for file in sorted(files)] colify(color_files) elif args.list == "long": for prefix, functions in sorted(tests.items()): path = colorize("*B", prefix) + "::" functions = [colorize("c", f) for f in sorted(functions)] color.cprint(path) colify(functions, indent=4) print() else: # args.list == "names" all_functions = [ colorize("*B", prefix) + "::" + colorize("c", f) for prefix, functions in sorted(tests.items()) for f in sorted(functions) ] colify(all_functions) def add_back_pytest_args(args, unknown_args): """Add parsed pytest args, unknown args, and remainder together. We add some basic pytest arguments to the Spack parser to ensure that they show up in the short help, so we have to reassemble things here. """ result = args.parsed_args or [] result += unknown_args or [] result += args.pytest_args or [] if args.expression: result += ["-k", args.expression] return result def unit_test(parser, args, unknown_args): global pytest import spack.bootstrap # Ensure clingo is available before switching to the # mock configuration used by unit tests with spack.bootstrap.ensure_bootstrap_configuration(): spack.bootstrap.ensure_clingo_importable_or_raise() if pytest is None: spack.bootstrap.ensure_environment_dependencies() import pytest if args.pytest_help: # make the pytest.main help output more accurate sys.argv[0] = "spack unit-test" return pytest.main(["-h"]) # add back any parsed pytest args we need to pass to pytest pytest_args = add_back_pytest_args(args, unknown_args) # The default is to test the core of Spack. If the option `--extension` # has been used, then test that extension. pytest_root = spack.paths.spack_root if args.extension: pytest_root = spack.extensions.load_extension(args.extension) if args.numprocesses is not None and args.numprocesses > 1: try: import xdist # noqa: F401 except ImportError: tty.error("parallel unit-test requires pytest-xdist module") return 1 pytest_args.extend( [ "--dist", "loadfile", "--tx", f"{args.numprocesses}*popen//python=spack-tmpconfig spack python", ] ) # pytest.ini lives in the root of the spack repository. with spack.llnl.util.filesystem.working_dir(pytest_root): if args.list: do_list(args, pytest_args) return return pytest.main(pytest_args) ================================================ FILE: lib/spack/spack/cmd/unload.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import sys import spack.cmd import spack.cmd.common import spack.error import spack.store import spack.user_environment as uenv from spack.cmd.common import arguments description = "remove package from the user environment" section = "user environment" level = "short" def setup_parser(subparser: argparse.ArgumentParser) -> None: """Parser is only constructed so that this prints a nice help message with -h.""" arguments.add_common_arguments(subparser, ["installed_specs"]) shells = subparser.add_mutually_exclusive_group() shells.add_argument( "--sh", action="store_const", dest="shell", const="sh", help="print sh commands to activate the environment", ) shells.add_argument( "--csh", action="store_const", dest="shell", const="csh", help="print csh commands to activate the environment", ) shells.add_argument( "--fish", action="store_const", dest="shell", const="fish", help="print fish commands to load the package", ) shells.add_argument( "--bat", action="store_const", dest="shell", const="bat", help="print bat commands to load the package", ) shells.add_argument( "--pwsh", action="store_const", dest="shell", const="pwsh", help="print pwsh commands to load the package", ) subparser.add_argument( "-a", "--all", action="store_true", help="unload all loaded Spack packages" ) def unload(parser, args): """unload spack packages from the user environment""" if args.specs and args.all: raise spack.error.SpackError( "Cannot specify specs on command line when unloading all specs with '--all'" ) hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(os.pathsep) if args.specs: specs = [ spack.cmd.disambiguate_spec_from_hashes(spec, hashes) for spec in spack.cmd.parse_specs(args.specs) ] else: specs = spack.store.STORE.db.query(hashes=hashes) if not args.shell: specs_str = " ".join(args.specs) or "SPECS" spack.cmd.common.shell_init_instructions( "spack unload", " eval `spack unload {sh_arg}` %s" % specs_str ) return 1 env_mod = uenv.environment_modifications_for_specs(*specs).reversed() for spec in specs: env_mod.remove_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) cmds = env_mod.shell_modifications(args.shell) sys.stdout.write(cmds) ================================================ FILE: lib/spack/spack/cmd/url.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import urllib.parse from collections import defaultdict import spack.fetch_strategy as fs import spack.llnl.util.tty.color as color import spack.repo import spack.spec import spack.url import spack.util.crypto as crypto from spack.llnl.util import tty from spack.url import ( UndetectableNameError, UndetectableVersionError, UrlParseError, color_url, parse_name, parse_name_offset, parse_version, parse_version_offset, substitute_version, substitution_offsets, ) from spack.util.naming import simplify_name description = "debugging tool for url parsing" section = "developer" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="subcommand") # Parse parse_parser = sp.add_parser("parse", help="attempt to parse a url") parse_parser.add_argument("url", help="url to parse") parse_parser.add_argument( "-s", "--spider", action="store_true", help="spider the source page for versions" ) # List list_parser = sp.add_parser("list", help="list urls in all packages") list_parser.add_argument( "-c", "--color", action="store_true", help="color the parsed version and name in the urls shown " "(versions will be cyan, name red)", ) list_parser.add_argument( "-e", "--extrapolation", action="store_true", help="color the versions used for extrapolation as well " "(additional versions will be green, names magenta)", ) excl_args = list_parser.add_mutually_exclusive_group() excl_args.add_argument( "-n", "--incorrect-name", action="store_true", help="only list urls for which the name was incorrectly parsed", ) excl_args.add_argument( "-N", "--correct-name", action="store_true", help="only list urls for which the name was correctly parsed", ) excl_args.add_argument( "-v", "--incorrect-version", action="store_true", help="only list urls for which the version was incorrectly parsed", ) excl_args.add_argument( "-V", "--correct-version", action="store_true", help="only list urls for which the version was correctly parsed", ) # Summary sp.add_parser("summary", help="print a summary of how well we are parsing package urls") # Stats stats_parser = sp.add_parser( "stats", help="print statistics on versions and checksums for all packages" ) stats_parser.add_argument( "--show-issues", action="store_true", help="show packages with issues (md5 hashes, http urls)", ) def url(parser, args): action = {"parse": url_parse, "list": url_list, "summary": url_summary, "stats": url_stats} action[args.subcommand](args) def url_parse(args): url = args.url tty.msg("Parsing URL: {0}".format(url)) print() ver, vs, vl, vi, vregex = parse_version_offset(url) tty.msg("Matched version regex {0:>2}: r{1!r}".format(vi, vregex)) name, ns, nl, ni, nregex = parse_name_offset(url, ver) tty.msg("Matched name regex {0:>2}: r{1!r}".format(ni, nregex)) print() tty.msg("Detected:") try: print_name_and_version(url) except UrlParseError as e: tty.error(str(e)) print(" name: {0}".format(name)) print(" version: {0}".format(ver)) print() tty.msg("Substituting version 9.9.9b:") newurl = substitute_version(url, "9.9.9b") print_name_and_version(newurl) if args.spider: print() tty.msg("Spidering for versions:") versions = spack.url.find_versions_of_archive(url) if not versions: print(" Found no versions for {0}".format(name)) return max_len = max(len(str(v)) for v in versions) for v in sorted(versions): print("{0:{1}} {2}".format(v, max_len, versions[v])) def url_list(args): urls = set() # Gather set of URLs from all packages for pkg_cls in spack.repo.PATH.all_package_classes(): url = getattr(pkg_cls, "url", None) urls = url_list_parsing(args, urls, url, pkg_cls) for params in pkg_cls.versions.values(): url = params.get("url", None) urls = url_list_parsing(args, urls, url, pkg_cls) # Print URLs for url in sorted(urls): if args.color or args.extrapolation: print(color_url(url, subs=args.extrapolation, errors=True)) else: print(url) # Return the number of URLs that were printed, only for testing purposes return len(urls) def url_summary(args): # Collect statistics on how many URLs were correctly parsed total_urls = 0 correct_names = 0 correct_versions = 0 # Collect statistics on which regexes were matched and how often name_regex_dict = dict() right_name_count = defaultdict(int) wrong_name_count = defaultdict(int) version_regex_dict = dict() right_version_count = defaultdict(int) wrong_version_count = defaultdict(int) tty.msg("Generating a summary of URL parsing in Spack...") # Loop through all packages for pkg_cls in spack.repo.PATH.all_package_classes(): urls = set() pkg = pkg_cls(spack.spec.Spec(pkg_cls.name)) url = getattr(pkg, "url", None) if url: urls.add(url) for params in pkg.versions.values(): url = params.get("url", None) if url: urls.add(url) # Calculate statistics for url in urls: total_urls += 1 # Parse versions version = None try: version, vs, vl, vi, vregex = parse_version_offset(url) version_regex_dict[vi] = vregex if version_parsed_correctly(pkg, version): correct_versions += 1 right_version_count[vi] += 1 else: wrong_version_count[vi] += 1 except UndetectableVersionError: pass # Parse names try: name, ns, nl, ni, nregex = parse_name_offset(url, version) name_regex_dict[ni] = nregex if name_parsed_correctly(pkg, name): correct_names += 1 right_name_count[ni] += 1 else: wrong_name_count[ni] += 1 except UndetectableNameError: pass print() print(" Total URLs found: {0}".format(total_urls)) print( " Names correctly parsed: {0:>4}/{1:>4} ({2:>6.2%})".format( correct_names, total_urls, correct_names / total_urls ) ) print( " Versions correctly parsed: {0:>4}/{1:>4} ({2:>6.2%})".format( correct_versions, total_urls, correct_versions / total_urls ) ) print() tty.msg("Statistics on name regular expressions:") print() print(" Index Right Wrong Total Regular Expression") for ni in sorted(name_regex_dict.keys()): print( " {0:>5} {1:>5} {2:>5} {3:>5} r{4!r}".format( ni, right_name_count[ni], wrong_name_count[ni], right_name_count[ni] + wrong_name_count[ni], name_regex_dict[ni], ) ) print() tty.msg("Statistics on version regular expressions:") print() print(" Index Right Wrong Total Regular Expression") for vi in sorted(version_regex_dict.keys()): print( " {0:>5} {1:>5} {2:>5} {3:>5} r{4!r}".format( vi, right_version_count[vi], wrong_version_count[vi], right_version_count[vi] + wrong_version_count[vi], version_regex_dict[vi], ) ) print() # Return statistics, only for testing purposes return (total_urls, correct_names, correct_versions, right_name_count, right_version_count) def url_stats(args): # dictionary of issue type -> package -> descriptions issues = defaultdict(lambda: defaultdict(lambda: [])) class UrlStats: def __init__(self): self.total = 0 self.schemes = defaultdict(lambda: 0) self.checksums = defaultdict(lambda: 0) self.url_type = defaultdict(lambda: 0) self.git_type = defaultdict(lambda: 0) def add(self, pkg_name, fetcher): self.total += 1 url_type = fetcher.url_attr self.url_type[url_type or "no code"] += 1 if url_type == "url": digest = getattr(fetcher, "digest", None) if digest: algo = crypto.hash_algo_for_digest(digest) else: algo = "no checksum" self.checksums[algo] += 1 if algo == "md5": md5_hashes = issues["md5 hashes"] md5_hashes[pkg_name].append(fetcher.url) # parse out the URL scheme (https/http/ftp/etc.) urlinfo = urllib.parse.urlparse(fetcher.url) self.schemes[urlinfo.scheme] += 1 if urlinfo.scheme == "http": http_urls = issues["http urls"] http_urls[pkg_name].append(fetcher.url) elif url_type == "git": if getattr(fetcher, "commit", None): self.git_type["commit"] += 1 elif getattr(fetcher, "branch", None): self.git_type["branch"] += 1 elif getattr(fetcher, "tag", None): self.git_type["tag"] += 1 else: self.git_type["no ref"] += 1 npkgs = 0 version_stats = UrlStats() resource_stats = UrlStats() for pkg_cls in spack.repo.PATH.all_package_classes(): npkgs += 1 for v in list(pkg_cls.versions): try: pkg = pkg_cls(spack.spec.Spec(pkg_cls.name)) fetcher = fs.for_package_version(pkg, v) except (fs.InvalidArgsError, fs.FetcherConflict): continue version_stats.add(pkg_cls.name, fetcher) for _, resources in pkg_cls.resources.items(): for resource in resources: resource_stats.add(pkg_cls.name, resource.fetcher) # print a nice summary table tty.msg("URL stats for %d packages:" % npkgs) def print_line(): print("-" * 62) def print_stat(indent, name, stat_name=None): width = 20 - indent fmt = " " * indent fmt += "%%-%ds" % width if stat_name is None: print(fmt % name) else: fmt += "%12d%8.1f%%%12d%8.1f%%" v = getattr(version_stats, stat_name).get(name, 0) r = getattr(resource_stats, stat_name).get(name, 0) print( fmt % (name, v, v / version_stats.total * 100, r, r / resource_stats.total * 100) ) print_line() print("%-20s%12s%9s%12s%9s" % ("stat", "versions", "%", "resources", "%")) print_line() print_stat(0, "url", "url_type") print_stat(4, "schemes") schemes = set(version_stats.schemes) | set(resource_stats.schemes) for scheme in schemes: print_stat(8, scheme, "schemes") print_stat(4, "checksums") checksums = set(version_stats.checksums) | set(resource_stats.checksums) for checksum in checksums: print_stat(8, checksum, "checksums") print_line() types = set(version_stats.url_type) | set(resource_stats.url_type) types -= set(["url", "git"]) for url_type in sorted(types): print_stat(0, url_type, "url_type") print_line() print_stat(0, "git", "url_type") git_types = set(version_stats.git_type) | set(resource_stats.git_type) for git_type in sorted(git_types): print_stat(4, git_type, "git_type") print_line() if args.show_issues: total_issues = sum( len(issues) for _, pkg_issues in issues.items() for _, issues in pkg_issues.items() ) print() tty.msg("Found %d issues." % total_issues) for issue_type, pkgs in issues.items(): tty.msg("Package URLs with %s" % issue_type) for pkg_cls, pkg_issues in pkgs.items(): color.cprint(" @*C{%s}" % pkg_cls) for issue in pkg_issues: print(" %s" % issue) def print_name_and_version(url): """Prints a URL. Underlines the detected name with dashes and the detected version with tildes. Args: url (str): The url to parse """ name, ns, nl, ntup, ver, vs, vl, vtup = substitution_offsets(url) underlines = [" "] * max(ns + nl, vs + vl) for i in range(ns, ns + nl): underlines[i] = "-" for i in range(vs, vs + vl): underlines[i] = "~" print(" {0}".format(url)) print(" {0}".format("".join(underlines))) def url_list_parsing(args, urls, url, pkg): """Helper function for :func:`url_list`. Args: args (argparse.Namespace): The arguments given to ``spack url list`` urls (set): List of URLs that have already been added url (str or None): A URL to potentially add to ``urls`` depending on ``args`` pkg (spack.package_base.PackageBase): The Spack package Returns: set: The updated set of ``urls`` """ if url: if args.correct_name or args.incorrect_name: # Attempt to parse the name try: name = parse_name(url) if args.correct_name and name_parsed_correctly(pkg, name): # Add correctly parsed URLs urls.add(url) elif args.incorrect_name and not name_parsed_correctly(pkg, name): # Add incorrectly parsed URLs urls.add(url) except UndetectableNameError: if args.incorrect_name: # Add incorrectly parsed URLs urls.add(url) elif args.correct_version or args.incorrect_version: # Attempt to parse the version try: version = parse_version(url) if args.correct_version and version_parsed_correctly(pkg, version): # Add correctly parsed URLs urls.add(url) elif args.incorrect_version and not version_parsed_correctly(pkg, version): # Add incorrectly parsed URLs urls.add(url) except UndetectableVersionError: if args.incorrect_version: # Add incorrectly parsed URLs urls.add(url) else: urls.add(url) return urls def name_parsed_correctly(pkg, name): """Determine if the name of a package was correctly parsed. Args: pkg (spack.package_base.PackageBase): The Spack package name (str): The name that was extracted from the URL Returns: bool: True if the name was correctly parsed, else False """ pkg_name = remove_prefix(pkg.name) name = simplify_name(name) return name == pkg_name def version_parsed_correctly(pkg, version): """Determine if the version of a package was correctly parsed. Args: pkg (spack.package_base.PackageBase): The Spack package version (str): The version that was extracted from the URL Returns: bool: True if the name was correctly parsed, else False """ version = remove_separators(version) # If the version parsed from the URL is listed in a version() # directive, we assume it was correctly parsed for pkg_version in pkg.versions: pkg_version = remove_separators(pkg_version) if pkg_version == version: return True return False def remove_prefix(pkg_name): """Remove build system prefix (``'py-'``, ``'perl-'``, etc.) from a package name. After determining a name, ``spack create`` determines a build system. Some build systems prepend a special string to the front of the name. Since this can't be guessed from the URL, it would be unfair to say that these names are incorrectly parsed, so we remove them. Args: pkg_name (str): the name of the package Returns: str: the name of the package with any build system prefix removed """ prefixes = [ "r-", "py-", "tcl-", "lua-", "perl-", "ruby-", "llvm-", "intel-", "votca-", "octave-", "gtkorvo-", ] prefix = next((p for p in prefixes if pkg_name.startswith(p)), "") return pkg_name[len(prefix) :] def remove_separators(version): """Remove separator characters (``.``, ``_``, and ``-``) from a version. A version like 1.2.3 may be displayed as 1_2_3 in the URL. Make sure 1.2.3, 1-2-3, 1_2_3, and 123 are considered equal. Unfortunately, this also means that 1.23 and 12.3 are equal. Args: version (str or spack.version.Version): A version Returns: str: The version with all separator characters removed """ version = str(version) version = version.replace(".", "") version = version.replace("_", "") version = version.replace("-", "") return version ================================================ FILE: lib/spack/spack/cmd/verify.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import io from typing import List, Optional import spack.cmd import spack.environment as ev import spack.llnl.util.tty as tty import spack.spec import spack.store import spack.verify import spack.verify_libraries from spack.cmd.common import arguments from spack.llnl.string import plural from spack.llnl.util.filesystem import visit_directory_tree description = "verify spack installations on disk" section = "admin" level = "long" MANIFEST_SUBPARSER: Optional[argparse.ArgumentParser] = None def setup_parser(subparser: argparse.ArgumentParser): global MANIFEST_SUBPARSER sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="verify_command") MANIFEST_SUBPARSER = sp.add_parser( "manifest", help=verify_manifest.__doc__, description=verify_manifest.__doc__ ) MANIFEST_SUBPARSER.add_argument( "-l", "--local", action="store_true", help="verify only locally installed packages" ) MANIFEST_SUBPARSER.add_argument( "-j", "--json", action="store_true", help="output json-formatted errors" ) MANIFEST_SUBPARSER.add_argument("-a", "--all", action="store_true", help="verify all packages") MANIFEST_SUBPARSER.add_argument( "specs_or_files", nargs=argparse.REMAINDER, help="specs or files to verify" ) manifest_sp_type = MANIFEST_SUBPARSER.add_mutually_exclusive_group() manifest_sp_type.add_argument( "-s", "--specs", action="store_const", const="specs", dest="type", default="specs", help="treat entries as specs (default)", ) manifest_sp_type.add_argument( "-f", "--files", action="store_const", const="files", dest="type", default="specs", help="treat entries as absolute filenames\n\ncannot be used with '-a'", ) libraries_subparser = sp.add_parser( "libraries", help=verify_libraries.__doc__, description=verify_libraries.__doc__ ) arguments.add_common_arguments(libraries_subparser, ["constraint"]) versions_subparser = sp.add_parser( "versions", help=verify_versions.__doc__, description=verify_versions.__doc__ ) arguments.add_common_arguments(versions_subparser, ["constraint"]) def verify(parser, args): cmd = args.verify_command if cmd == "libraries": return verify_libraries(args) elif cmd == "manifest": return verify_manifest(args) elif cmd == "versions": return verify_versions(args) parser.error("invalid verify subcommand") def verify_versions(args): """Check that all versions of installed packages are known to Spack and non-deprecated. Reports errors for any of the following: 1. Installed package not loadable from the repo 2. Installed package version not known by the package recipe 3. Installed package version deprecated in the package recipe """ specs = args.specs(installed=True) msg_lines = _verify_version(specs) if msg_lines: tty.die("\n".join(msg_lines)) def _verify_version(specs): """Helper method for verify_versions.""" missing_package = [] unknown_version = [] deprecated_version = [] for spec in specs: try: pkg = spec.package except Exception as e: tty.debug(str(e)) missing_package.append(spec) continue if spec.version not in pkg.versions: unknown_version.append(spec) continue if pkg.versions[spec.version].get("deprecated", False): deprecated_version.append(spec) msg_lines = [] if missing_package or unknown_version or deprecated_version: errors = len(missing_package) + len(unknown_version) + len(deprecated_version) msg_lines = [f"{errors} installed packages have unknown/deprecated versions\n"] msg_lines += [ f" Cannot check version for {spec} at {spec.prefix}. Cannot load package." for spec in missing_package ] msg_lines += [ f" Spec {spec} at {spec.prefix} has version {spec.version} unknown to Spack." for spec in unknown_version ] msg_lines += [ f" Spec {spec} at {spec.prefix} has deprecated version {spec.version}." for spec in deprecated_version ] return msg_lines def verify_libraries(args): """verify that shared libraries of install packages can be located in rpaths (Linux only)""" specs_from_db = [s for s in args.specs(installed=True) if not s.external] tty.info(f"Checking {len(specs_from_db)} packages for shared library resolution") errors = 0 for spec in specs_from_db: try: pkg = spec.package except Exception: tty.warn(f"Skipping {spec.cformat('{name}{@version}{/hash}')} due to missing package") error_msg = _verify_libraries(spec, pkg.unresolved_libraries) if error_msg is not None: errors += 1 tty.error(error_msg) if errors: tty.error(f"Cannot resolve shared libraries in {plural(errors, 'package')}") return 1 def _verify_libraries(spec: spack.spec.Spec, unresolved_libraries: List[str]) -> Optional[str]: """Go over the prefix of the installed spec and verify its shared libraries can be resolved.""" visitor = spack.verify_libraries.ResolveSharedElfLibDepsVisitor( [*spack.verify_libraries.ALLOW_UNRESOLVED, *unresolved_libraries] ) visit_directory_tree(spec.prefix, visitor) if not visitor.problems: return None output = io.StringIO() visitor.write(output, indent=4, brief=True) message = output.getvalue().rstrip() return f"{spec.cformat('{name}{@version}{/hash}')}: {spec.prefix}:\n{message}" def verify_manifest(args): """verify that install directories have not been modified since installation""" local = args.local if args.type == "files": if args.all: MANIFEST_SUBPARSER.error("cannot use --all with --files") for file in args.specs_or_files: results = spack.verify.check_file_manifest(file) if results.has_errors(): if args.json: print(results.json_string()) else: print(results) return 0 else: spec_args = spack.cmd.parse_specs(args.specs_or_files) if args.all: query = spack.store.STORE.db.query_local if local else spack.store.STORE.db.query # construct spec list if spec_args: spec_list = spack.cmd.parse_specs(args.specs_or_files) specs = [] for spec in spec_list: specs += query(spec, installed=True) else: specs = query(installed=True) elif args.specs_or_files: # construct disambiguated spec list env = ev.active_environment() specs = list(map(lambda x: spack.cmd.disambiguate_spec(x, env, local=local), spec_args)) else: MANIFEST_SUBPARSER.error("use --all or specify specs to verify") for spec in specs: tty.debug("Verifying package %s") results = spack.verify.check_spec_manifest(spec) if results.has_errors(): if args.json: print(results.json_string()) else: tty.msg("In package %s" % spec.format("{name}/{hash:7}")) print(results) return 1 else: tty.debug(results) ================================================ FILE: lib/spack/spack/cmd/versions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import sys import spack.llnl.util.tty as tty import spack.repo import spack.spec from spack.cmd.common import arguments from spack.llnl.util.tty.colify import colify from spack.version import infinity_versions, ver description = "list available versions of a package" section = "query" level = "long" def setup_parser(subparser: argparse.ArgumentParser) -> None: output = subparser.add_mutually_exclusive_group() output.add_argument( "-s", "--safe", action="store_true", help="only list safe versions of the package" ) output.add_argument( "-r", "--remote", action="store_true", help="only list remote versions of the package" ) output.add_argument( "-n", "--new", action="store_true", help="only list remote versions newer than the latest checksummed version", ) arguments.add_common_arguments(subparser, ["package", "jobs"]) def versions(parser, args): spec = spack.spec.Spec(args.package) pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) pkg = pkg_cls(spec) safe_versions = pkg.versions if not (args.remote or args.new): if sys.stdout.isatty(): tty.msg("Safe versions (already checksummed):") if not safe_versions: if sys.stdout.isatty(): tty.warn(f"Found no versions for {pkg.name}") tty.debug("Manually add versions to the package.") else: colify(sorted(safe_versions, reverse=True), indent=2) if args.safe: return fetched_versions = pkg.fetch_remote_versions(args.jobs) if args.new: if sys.stdout.isatty(): tty.msg("New remote versions (not yet checksummed):") numeric_safe_versions = list( filter(lambda v: str(v) not in infinity_versions, safe_versions) ) highest_safe_version = max(numeric_safe_versions) remote_versions = set([ver(v) for v in set(fetched_versions) if v > highest_safe_version]) else: if sys.stdout.isatty(): tty.msg("Remote versions (not yet checksummed):") remote_versions = set(fetched_versions).difference(safe_versions) if not remote_versions: if sys.stdout.isatty(): if not fetched_versions: tty.warn(f"Found no versions for {pkg.name}") tty.debug( "Check the list_url and list_depth attributes of " "the package to help Spack find versions." ) else: tty.warn(f"Found no unchecksummed versions for {pkg.name}") else: colify(sorted(remote_versions, reverse=True), indent=2) ================================================ FILE: lib/spack/spack/cmd/view.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Produce a "view" of a Spack DAG. A "view" is file hierarchy representing the union of a number of Spack-installed package file hierarchies. The union is formed from: - specs resolved from the package names given by the user (the seeds) - all dependencies of the seeds unless user specifies ``--no-dependencies`` - less any specs with names matching the regular expressions given by ``--exclude`` The ``view`` can be built and tore down via a number of methods (the "actions"): - symlink :: a file system view which is a directory hierarchy that is the union of the hierarchies of the installed packages in the DAG where installed files are referenced via symlinks. - hardlink :: like the symlink view but hardlinks are used. - statlink :: a view producing a status report of a symlink or hardlink view. The file system view concept is inspired by Nix, implemented by Brett Viren ca 2016. All operations on views are performed via proxy objects such as YamlFilesystemView. """ import argparse import sys import spack.cmd import spack.environment as ev import spack.filesystem_view as fsv import spack.llnl.util.tty as tty import spack.schema.projections import spack.store from spack.config import validate from spack.llnl.util.link_tree import MergeConflictError from spack.util import spack_yaml as s_yaml description = "manipulate view directories in the filesystem" section = "environments" level = "short" actions_link = ["symlink", "add", "soft", "hardlink", "hard", "copy", "relocate"] actions_remove = ["remove", "rm"] actions_status = ["statlink", "status", "check"] def disambiguate_in_view(specs, view): """ When dealing with querying actions (remove/status) we only need to disambiguate among specs in the view """ view_specs = set(view.get_all_specs()) def squash(matching_specs): if not matching_specs: tty.die("Spec matches no installed packages.") matching_in_view = [ms for ms in matching_specs if ms in view_specs] spack.cmd.ensure_single_spec_or_die("Spec", matching_in_view) return matching_in_view[0] if matching_in_view else matching_specs[0] # make function always return a list to keep consistency between py2/3 return list(map(squash, map(spack.store.STORE.db.query, specs))) def setup_parser(sp: argparse.ArgumentParser) -> None: setattr(setup_parser, "parser", sp) sp.add_argument( "-v", "--verbose", action="store_true", default=False, help="if not verbose only warnings/errors will be printed", ) sp.add_argument( "-e", "--exclude", action="append", default=[], help="exclude packages with names matching the given regex pattern", ) sp.add_argument( "-d", "--dependencies", choices=["true", "false", "yes", "no"], default="true", help="link/remove/list dependencies", ) ssp = sp.add_subparsers(metavar="ACTION", dest="action") # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. file_system_view_actions = { "symlink": ssp.add_parser( "symlink", aliases=["add", "soft"], help="add package files to a filesystem view via symbolic links", ), "hardlink": ssp.add_parser( "hardlink", aliases=["hard"], help="add packages files to a filesystem view via hard links", ), "copy": ssp.add_parser( "copy", aliases=["relocate"], help="add package files to a filesystem view via copy/relocate", ), "remove": ssp.add_parser( "remove", aliases=["rm"], help="remove packages from a filesystem view" ), "statlink": ssp.add_parser( "statlink", aliases=["status", "check"], help="check status of packages in a filesystem view", ), } # All these options and arguments are common to every action. for cmd, act in file_system_view_actions.items(): act.add_argument("path", nargs=1, help="path to file system view directory") if cmd in ("symlink", "hardlink", "copy"): # invalid for remove/statlink, for those commands the view needs to # already know its own projections. act.add_argument( "--projection-file", dest="projection_file", type=spack.cmd.extant_file, help="initialize view using projections from file", ) if cmd == "remove": grp = act.add_mutually_exclusive_group(required=True) act.add_argument( "--no-remove-dependents", action="store_true", help="do not remove dependents of specified specs", ) # with all option, spec is an optional argument grp.add_argument( "specs", nargs="*", default=[], metavar="spec", action="store", help="seed specs of the packages to view", ) grp.add_argument("-a", "--all", action="store_true", help="act on all specs in view") elif cmd == "statlink": act.add_argument( "specs", nargs="*", metavar="spec", action="store", help="seed specs of the packages to view", ) else: # without all option, spec is required act.add_argument( "specs", nargs="+", metavar="spec", action="store", help="seed specs of the packages to view", ) for cmd in ("symlink", "hardlink", "copy"): act = file_system_view_actions[cmd] act.add_argument("-i", "--ignore-conflicts", action="store_true") def view(parser, args): """Produce a view of a set of packages.""" if sys.platform == "win32" and args.action in ("hardlink", "hard"): # Hard-linked views are not yet allowed on Windows. # See https://github.com/spack/spack/pull/46335#discussion_r1757411915 tty.die("Hard linking is not supported on Windows. Please use symlinks or copy methods.") specs = spack.cmd.parse_specs(args.specs) path = args.path[0] if args.action in actions_link and args.projection_file: # argparse confirms file exists with open(args.projection_file, "r", encoding="utf-8") as f: projections_data = s_yaml.load(f) validate(projections_data, spack.schema.projections.schema) ordered_projections = projections_data["projections"] else: ordered_projections = {} # What method are we using for this view link_type = args.action if args.action in actions_link else "symlink" view = fsv.YamlFilesystemView( path, spack.store.STORE.layout, projections=ordered_projections, ignore_conflicts=getattr(args, "ignore_conflicts", False), link_type=link_type, verbose=args.verbose, ) # Process common args and specs if getattr(args, "all", False): specs = view.get_all_specs() if len(specs) == 0: tty.warn("Found no specs in %s" % path) elif args.action in actions_link: # only link commands need to disambiguate specs env = ev.active_environment() specs = [spack.cmd.disambiguate_spec(s, env) for s in specs] elif args.action in actions_status: # no specs implies all if len(specs) == 0: specs = view.get_all_specs() else: specs = disambiguate_in_view(specs, view) else: # status and remove can map a partial spec to packages in view specs = disambiguate_in_view(specs, view) with_dependencies = args.dependencies.lower() in ["true", "yes"] # Map action to corresponding functionality if args.action in actions_link: try: view.add_specs(*specs, with_dependencies=with_dependencies, exclude=args.exclude) except MergeConflictError: tty.info( "Some file blocked the merge, adding the '-i' flag will " "ignore this conflict. For more information see e.g. " "https://github.com/spack/spack/issues/9029" ) raise elif args.action in actions_remove: view.remove_specs( *specs, with_dependencies=with_dependencies, exclude=args.exclude, with_dependents=not args.no_remove_dependents, ) elif args.action in actions_status: view.print_status(*specs, with_dependencies=with_dependencies) else: tty.error('Unknown action: "%s"' % args.action) ================================================ FILE: lib/spack/spack/compilers/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/compilers/adaptor.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import enum from typing import Dict, List import spack.spec from spack.llnl.util import lang from .libraries import CompilerPropertyDetector class Languages(enum.Enum): C = "c" CXX = "cxx" FORTRAN = "fortran" class CompilerAdaptor: """Provides access to compiler attributes via ``Package.compiler``. Useful for packages which do not yet access compiler properties via ``self.spec[language]``. """ def __init__( self, compiled_spec: spack.spec.Spec, compilers: Dict[Languages, spack.spec.Spec] ) -> None: if not compilers: raise AttributeError(f"{compiled_spec} has no 'compiler' attribute") self.compilers = compilers self.compiled_spec = compiled_spec def _lang_exists_or_raise(self, name: str, *, lang: Languages) -> None: if lang not in self.compilers: raise AttributeError( f"'{self.compiled_spec}' has no {lang.value} compiler, so the " f"'{name}' property cannot be retrieved" ) def _maybe_return_attribute(self, name: str, *, lang: Languages) -> str: self._lang_exists_or_raise(name, lang=lang) return getattr(self.compilers[lang].package, name) @property def cc_rpath_arg(self) -> str: self._lang_exists_or_raise("cc_rpath_arg", lang=Languages.C) return self.compilers[Languages.C].package.rpath_arg @property def cxx_rpath_arg(self) -> str: self._lang_exists_or_raise("cxx_rpath_arg", lang=Languages.CXX) return self.compilers[Languages.CXX].package.rpath_arg @property def fc_rpath_arg(self) -> str: self._lang_exists_or_raise("fc_rpath_arg", lang=Languages.FORTRAN) return self.compilers[Languages.FORTRAN].package.rpath_arg @property def f77_rpath_arg(self) -> str: self._lang_exists_or_raise("f77_rpath_arg", lang=Languages.FORTRAN) return self.compilers[Languages.FORTRAN].package.rpath_arg @property def linker_arg(self) -> str: return self._maybe_return_attribute("linker_arg", lang=Languages.C) @property def name(self): return next(iter(self.compilers.values())).name @property def version(self): return next(iter(self.compilers.values())).version def implicit_rpaths(self) -> List[str]: result, seen = [], set() for compiler in self.compilers.values(): if compiler in seen: continue seen.add(compiler) result.extend(CompilerPropertyDetector(compiler).implicit_rpaths()) return result @property def opt_flags(self) -> List[str]: return next(iter(self.compilers.values())).package.opt_flags @property def debug_flags(self) -> List[str]: return next(iter(self.compilers.values())).package.debug_flags @property def openmp_flag(self) -> str: return next(iter(self.compilers.values())).package.openmp_flag @property def cxx98_flag(self) -> str: return self.compilers[Languages.CXX].package.standard_flag( language=Languages.CXX.value, standard="98" ) @property def cxx11_flag(self) -> str: return self.compilers[Languages.CXX].package.standard_flag( language=Languages.CXX.value, standard="11" ) @property def cxx14_flag(self) -> str: return self.compilers[Languages.CXX].package.standard_flag( language=Languages.CXX.value, standard="14" ) @property def cxx17_flag(self) -> str: return self.compilers[Languages.CXX].package.standard_flag( language=Languages.CXX.value, standard="17" ) @property def cxx20_flag(self) -> str: return self.compilers[Languages.CXX].package.standard_flag( language=Languages.CXX.value, standard="20" ) @property def cxx23_flag(self) -> str: return self.compilers[Languages.CXX].package.standard_flag( language=Languages.CXX.value, standard="23" ) @property def c99_flag(self) -> str: return self.compilers[Languages.C].package.standard_flag( language=Languages.C.value, standard="99" ) @property def c11_flag(self) -> str: return self.compilers[Languages.C].package.standard_flag( language=Languages.C.value, standard="11" ) @property def c17_flag(self) -> str: return self.compilers[Languages.C].package.standard_flag( language=Languages.C.value, standard="17" ) @property def c23_flag(self) -> str: return self.compilers[Languages.C].package.standard_flag( language=Languages.C.value, standard="23" ) @property def cc_pic_flag(self) -> str: self._lang_exists_or_raise("cc_pic_flag", lang=Languages.C) return self.compilers[Languages.C].package.pic_flag @property def cxx_pic_flag(self) -> str: self._lang_exists_or_raise("cxx_pic_flag", lang=Languages.CXX) return self.compilers[Languages.CXX].package.pic_flag @property def fc_pic_flag(self) -> str: self._lang_exists_or_raise("fc_pic_flag", lang=Languages.FORTRAN) return self.compilers[Languages.FORTRAN].package.pic_flag @property def f77_pic_flag(self) -> str: self._lang_exists_or_raise("f77_pic_flag", lang=Languages.FORTRAN) return self.compilers[Languages.FORTRAN].package.pic_flag @property def prefix(self) -> str: return next(iter(self.compilers.values())).prefix @property def extra_rpaths(self) -> List[str]: compiler = next(iter(self.compilers.values())) return getattr(compiler, "extra_attributes", {}).get("extra_rpaths", []) @property def cc(self): return self._maybe_return_attribute("cc", lang=Languages.C) @property def cxx(self): return self._maybe_return_attribute("cxx", lang=Languages.CXX) @property def fc(self): self._lang_exists_or_raise("fc", lang=Languages.FORTRAN) return self.compilers[Languages.FORTRAN].package.fortran @property def f77(self): self._lang_exists_or_raise("f77", lang=Languages.FORTRAN) return self.compilers[Languages.FORTRAN].package.fortran @property def stdcxx_libs(self): return self._maybe_return_attribute("stdcxx_libs", lang=Languages.CXX) class DeprecatedCompiler(lang.DeprecatedProperty): def __init__(self) -> None: super().__init__(name="compiler") def factory(self, instance, owner) -> CompilerAdaptor: spec = instance.spec if not spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") compilers = {} for language in Languages: deps = spec.dependencies(virtuals=[language.value]) if deps: compilers[language] = deps[0] return CompilerAdaptor(instance, compilers) ================================================ FILE: lib/spack/spack/compilers/config.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module contains functions related to finding compilers on the system, and configuring Spack to use multiple compilers. """ import os import re import sys import warnings from typing import Any, Dict, List, Optional, Tuple import spack.config import spack.detection import spack.detection.path import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.platforms import spack.repo import spack.spec from spack.externals import ExternalSpecsParser, external_spec, extract_dicts_from_configuration from spack.operating_systems import windows_os from spack.util.environment import get_path #: Tag used to identify packages providing a compiler COMPILER_TAG = "compiler" def compiler_config_files(): config_files = [] configuration = spack.config.CONFIG for scope in configuration.writable_scopes: name = scope.name from_packages_yaml = CompilerFactory.from_packages_yaml(configuration, scope=name) if from_packages_yaml: config_files.append(configuration.get_config_filename(name, "packages")) return config_files def add_compiler_to_config(new_compilers, *, scope=None) -> None: """Add a Compiler object to the configuration, at the required scope.""" by_name: Dict[str, List[spack.spec.Spec]] = {} for x in new_compilers: by_name.setdefault(x.name, []).append(x) spack.detection.update_configuration(by_name, buildable=True, scope=scope) def find_compilers( path_hints: Optional[List[str]] = None, *, scope: Optional[str] = None, max_workers: Optional[int] = None, ) -> List[spack.spec.Spec]: """Searches for compiler in the paths given as argument. If any new compiler is found, the configuration is updated, and the list of new compiler objects is returned. Args: path_hints: list of path hints where to look for. A sensible default based on the ``PATH`` environment variable will be used if the value is None scope: configuration scope to modify max_workers: number of processes used to search for compilers """ if path_hints is None: path_hints = get_path("PATH") default_paths = fs.search_paths_for_executables(*path_hints) if sys.platform == "win32": default_paths.extend(windows_os.WindowsOs().compiler_search_paths) compiler_pkgs = spack.repo.PATH.packages_with_tags(COMPILER_TAG, full=True) detected_packages = spack.detection.by_path( compiler_pkgs, path_hints=default_paths, max_workers=max_workers ) new_compilers = spack.detection.update_configuration( detected_packages, buildable=True, scope=scope ) return new_compilers def select_new_compilers( candidates: List[spack.spec.Spec], *, scope: Optional[str] = None ) -> List[spack.spec.Spec]: """Given a list of compilers, remove those that are already defined in the configuration. """ compilers_in_config = all_compilers_from(configuration=spack.config.CONFIG, scope=scope) return [c for c in candidates if c not in compilers_in_config] def supported_compilers() -> List[str]: """Returns all the currently supported compiler packages""" return sorted(spack.repo.PATH.packages_with_tags(COMPILER_TAG)) def all_compilers(scope: Optional[str] = None, init_config: bool = True) -> List[spack.spec.Spec]: """Returns all the compilers from the current global configuration. Args: scope: configuration scope from which to extract the compilers. If None, the merged configuration is used. init_config: if True, search for compilers if none is found in configuration. """ compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope) if not compilers and init_config: _init_packages_yaml(spack.config.CONFIG, scope=scope) compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope) return compilers def _init_packages_yaml( configuration: spack.config.Configuration, *, scope: Optional[str] ) -> None: # Try importing from compilers.yaml legacy_compilers = CompilerFactory.from_compilers_yaml(configuration, scope=scope) if legacy_compilers: by_name: Dict[str, List[spack.spec.Spec]] = {} for legacy in legacy_compilers: by_name.setdefault(legacy.name, []).append(legacy) spack.detection.update_configuration(by_name, buildable=True, scope=scope) tty.info( "Compilers have been converted from 'compilers.yaml' and written to " "'packages.yaml'. Use of 'compilers.yaml' is deprecated, and will be " "ignored in future versions of Spack" ) return # Look for compilers in PATH new_compilers = find_compilers(scope=scope) if not new_compilers: raise NoAvailableCompilerError( "no compiler configured, and Spack cannot find working compilers in PATH" ) tty.info("Compilers have been configured automatically from PATH inspection") def all_compilers_from( configuration: spack.config.Configuration, scope: Optional[str] = None ) -> List[spack.spec.Spec]: """Returns all the compilers from the current global configuration. Args: configuration: configuration to be queried scope: configuration scope from which to extract the compilers. If None, the merged configuration is used. """ compilers = CompilerFactory.from_packages_yaml(configuration, scope=scope) return compilers class CompilerRemover: """Removes compiler from configuration.""" def __init__(self, configuration: spack.config.Configuration) -> None: self.configuration = configuration self.marked_packages_yaml: List[Tuple[str, Any]] = [] def mark_compilers(self, *, match: str, scope: Optional[str] = None) -> List[spack.spec.Spec]: """Marks compilers to be removed in configuration, and returns a corresponding list of specs. Args: match: constraint that the compiler must match to be removed. scope: scope where to remove the compiler. If None, all writeable scopes are checked. """ self.marked_packages_yaml = [] candidate_scopes = [scope] if scope is None: candidate_scopes = [x.name for x in self.configuration.writable_scopes] return self._mark_in_packages_yaml(match, candidate_scopes) def _mark_in_packages_yaml(self, match, candidate_scopes): compiler_package_names = supported_compilers() all_removals = [] for current_scope in candidate_scopes: packages_yaml = self.configuration.get("packages", scope=current_scope) if not packages_yaml: continue removed_from_scope = [] for name, entry in packages_yaml.items(): if name not in compiler_package_names: continue externals_config = entry.get("externals", None) if not externals_config: continue def _partition_match(external_yaml): return not external_spec(external_yaml).satisfies(match) to_keep, to_remove = spack.llnl.util.lang.stable_partition( externals_config, _partition_match ) if not to_remove: continue removed_from_scope.extend(to_remove) entry["externals"] = to_keep if not removed_from_scope: continue self.marked_packages_yaml.append((current_scope, packages_yaml)) all_removals.extend([external_spec(x) for x in removed_from_scope]) return all_removals def flush(self): """Removes from configuration the specs that have been marked by the previous call of ``remove_compilers``. """ for scope, packages_yaml in self.marked_packages_yaml: self.configuration.set("packages", packages_yaml, scope=scope) def compilers_for_arch( arch_spec: spack.spec.ArchSpec, *, scope: Optional[str] = None ) -> List[spack.spec.Spec]: """Returns the compilers that can be used on the input architecture""" compilers = all_compilers_from(spack.config.CONFIG, scope=scope) query = f"platform={arch_spec.platform} target=:{arch_spec.target}" return [x for x in compilers if x.satisfies(query)] _EXTRA_ATTRIBUTES_KEY = "extra_attributes" def name_os_target(spec: spack.spec.Spec) -> Tuple[str, str, str]: if not spec.architecture: host_platform = spack.platforms.host() operating_system = host_platform.operating_system("default_os") target = host_platform.target("default_target") else: target = spec.architecture.target if not target: target = spack.platforms.host().target("default_target") target = target.family operating_system = spec.os if not operating_system: host_platform = spack.platforms.host() operating_system = host_platform.operating_system("default_os") return spec.name, str(operating_system), str(target) class CompilerFactory: """Class aggregating all ways of constructing a list of compiler specs from config entries.""" @staticmethod def from_packages_yaml( configuration: spack.config.Configuration, *, scope: Optional[str] = None ) -> List[spack.spec.Spec]: """Returns the compiler specs defined in the "packages" section of the configuration""" compiler_package_names = supported_compilers() packages_yaml = configuration.deepcopy_as_builtin("packages", scope=scope) init_external_dicts = extract_dicts_from_configuration(packages_yaml) init_external_dicts = list( x for x in init_external_dicts if spack.spec.Spec(x["spec"]).name in compiler_package_names ) externals_dicts = [] for current in init_external_dicts: if _EXTRA_ATTRIBUTES_KEY not in current: header = f"The external spec '{current['spec']}' cannot be used as a compiler" tty.debug(f"[{__file__}] {header}: missing the '{_EXTRA_ATTRIBUTES_KEY}' key") continue externals_dicts.append(current) external_parser = ExternalSpecsParser(externals_dicts) return external_parser.all_specs() @staticmethod def from_legacy_yaml(compiler_dict: Dict[str, Any]) -> List[spack.spec.Spec]: """Returns a list of external specs, corresponding to a compiler entry from compilers.yaml. """ result = [] candidate_paths = [x for x in compiler_dict["paths"].values() if x is not None] finder = spack.detection.path.ExecutablesFinder() for pkg_name in spack.repo.PATH.packages_with_tags("compiler"): pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) pattern = re.compile(r"|".join(finder.search_patterns(pkg=pkg_cls))) filtered_paths = [x for x in candidate_paths if pattern.search(os.path.basename(x))] try: detected = finder.detect_specs( pkg=pkg_cls, paths=filtered_paths, repo_path=spack.repo.PATH ) except Exception: warnings.warn( f"[{__name__}] cannot detect {pkg_name} from the " f"following paths: {', '.join(filtered_paths)}" ) continue for s in detected: for key in ("flags", "environment", "extra_rpaths"): if key in compiler_dict: s.extra_attributes[key] = compiler_dict[key] if "modules" in compiler_dict: s.external_modules = list(compiler_dict["modules"]) result.extend(detected) return result @staticmethod def from_compilers_yaml( configuration: spack.config.Configuration, *, scope: Optional[str] = None ) -> List[spack.spec.Spec]: """Returns the compiler specs defined in the "compilers" section of the configuration""" result: List[spack.spec.Spec] = [] for item in configuration.get("compilers", scope=scope): result.extend(CompilerFactory.from_legacy_yaml(item["compiler"])) return result class UnknownCompilerError(spack.error.SpackError): def __init__(self, compiler_name): super().__init__(f"Spack doesn't support the requested compiler: {compiler_name}") class NoAvailableCompilerError(spack.error.SpackError): pass ================================================ FILE: lib/spack/spack/compilers/error.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import warnings from ..error import SpackAPIWarning, SpackError class CompilerAccessError(SpackError): def __init__(self, compiler, paths): super().__init__( f"Compiler '{compiler.spec}' has executables that are missing" f" or are not executable: {paths}" ) class UnsupportedCompilerFlag(SpackError): """Raised when a compiler does not support a flag type (e.g. a flag to enforce a language standard). """ def __init__(self, message, long_message=None): warnings.warn( "UnsupportedCompilerFlag is deprecated, use CompilerError instead", SpackAPIWarning, stacklevel=2, ) ================================================ FILE: lib/spack/spack/compilers/flags.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from typing import List, Tuple def tokenize_flags(flags_values: str, propagate: bool = False) -> List[Tuple[str, bool]]: """Given a compiler flag specification as a string, this returns a list where the entries are the flags. For compiler options which set values using the syntax ``-flag value``, this function groups flags and their values together. Any token not preceded by a ``-`` is considered the value of a prior flag.""" tokens = flags_values.split() if not tokens: return [] flag = tokens[0] flags_with_propagation = [] for token in tokens[1:]: if not token.startswith("-"): flag += " " + token else: flags_with_propagation.append((flag, propagate)) flag = token flags_with_propagation.append((flag, propagate)) return flags_with_propagation ================================================ FILE: lib/spack/spack/compilers/libraries.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import contextlib import hashlib import json import os import re import shutil import stat import sys import tempfile from typing import Dict, List, Optional, Set, Tuple, cast import spack.caches import spack.llnl.path import spack.llnl.util.lang import spack.schema.environment import spack.spec import spack.util.executable import spack.util.libc import spack.util.module_cmd from spack.llnl.util import tty from spack.llnl.util.filesystem import path_contains_subdirectory, paths_containing_libs from spack.util.environment import filter_system_paths from spack.util.file_cache import FileCache #: regex for parsing linker lines _LINKER_LINE = re.compile(r"^( *|.*[/\\])" r"(link|ld|([^/\\]+-)?ld|collect2)" r"[^/\\]*( |$)") #: components of linker lines to ignore _LINKER_LINE_IGNORE = re.compile(r"(collect2 version|^[A-Za-z0-9_]+=|/ldfe )") #: regex to match linker search paths _LINK_DIR_ARG = re.compile(r"^-L(.:)?(?P[/\\].*)") #: regex to match linker library path arguments _LIBPATH_ARG = re.compile(r"^[-/](LIBPATH|libpath):(?P.*)") @spack.llnl.path.system_path_filter def parse_non_system_link_dirs(compiler_debug_output: str) -> List[str]: """Parses link paths out of compiler debug output. Args: compiler_debug_output: compiler debug output as a string Returns: Implicit link paths parsed from the compiler output """ link_dirs = _parse_link_paths(compiler_debug_output) # Remove directories that do not exist. Some versions of the Cray compiler # report nonexistent directories link_dirs = filter_non_existing_dirs(link_dirs) # Return set of directories containing needed compiler libs, minus # system paths. Note that 'filter_system_paths' only checks for an # exact match, while 'in_system_subdirectory' checks if a path contains # a system directory as a subdirectory link_dirs = filter_system_paths(link_dirs) return list(p for p in link_dirs if not in_system_subdirectory(p)) def filter_non_existing_dirs(dirs): return [d for d in dirs if os.path.isdir(d)] def in_system_subdirectory(path): system_dirs = [ "/lib/", "/lib64/", "/usr/lib/", "/usr/lib64/", "/usr/local/lib/", "/usr/local/lib64/", ] return any(path_contains_subdirectory(path, x) for x in system_dirs) def _parse_link_paths(string): """Parse implicit link paths from compiler debug output. This gives the compiler runtime library paths that we need to add to the RPATH of generated binaries and libraries. It allows us to ensure, e.g., that codes load the right libstdc++ for their compiler. """ lib_search_paths = False raw_link_dirs = [] for line in string.splitlines(): if lib_search_paths: if line.startswith("\t"): raw_link_dirs.append(line[1:]) continue else: lib_search_paths = False elif line.startswith("Library search paths:"): lib_search_paths = True if not _LINKER_LINE.match(line): continue if _LINKER_LINE_IGNORE.match(line): continue tty.debug(f"implicit link dirs: link line: {line}") next_arg = False for arg in line.split(): if arg in ("-L", "-Y"): next_arg = True continue if next_arg: raw_link_dirs.append(arg) next_arg = False continue link_dir_arg = _LINK_DIR_ARG.match(arg) if link_dir_arg: link_dir = link_dir_arg.group("dir") raw_link_dirs.append(link_dir) link_dir_arg = _LIBPATH_ARG.match(arg) if link_dir_arg: link_dir = link_dir_arg.group("dir") raw_link_dirs.append(link_dir) implicit_link_dirs = list() visited = set() for link_dir in raw_link_dirs: normalized_path = os.path.abspath(link_dir) if normalized_path not in visited: implicit_link_dirs.append(normalized_path) visited.add(normalized_path) tty.debug(f"implicit link dirs: result: {', '.join(implicit_link_dirs)}") return implicit_link_dirs class CompilerPropertyDetector: """Detects compiler properties of a given compiler spec. Useful for compiler wrappers.""" def __init__(self, compiler_spec: spack.spec.Spec): assert compiler_spec.concrete, "only concrete compiler specs are allowed" self.spec = compiler_spec self.cache = COMPILER_CACHE @contextlib.contextmanager def compiler_environment(self): """Sets the environment to run this compiler""" # No modifications for Spack managed compilers if not self.spec.external: yield return # Avoid modifying os.environ if possible. environment = self.spec.extra_attributes.get("environment", {}) modules = self.spec.external_modules or [] if not self.spec.external_modules and not environment: yield return # store environment to replace later backup_env = os.environ.copy() try: # load modules and set env variables for module in modules: spack.util.module_cmd.load_module(module) # apply other compiler environment changes spack.schema.environment.parse(environment).apply_modifications() yield finally: # Restore environment regardless of whether inner code succeeded os.environ.clear() os.environ.update(backup_env) def _compile_dummy_c_source(self) -> Optional[str]: compiler_pkg = self.spec.package if getattr(compiler_pkg, "cc"): cc = compiler_pkg.cc ext = "c" else: cc = compiler_pkg.cxx ext = "cc" if not cc or not self.spec.package.verbose_flags: return None try: tmpdir = tempfile.mkdtemp(prefix="spack-implicit-link-info") fout = os.path.join(tmpdir, "output") fin = os.path.join(tmpdir, f"main.{ext}") with open(fin, "w", encoding="utf-8") as csource: csource.write( "int main(int argc, char* argv[]) { (void)argc; (void)argv; return 0; }\n" ) cc_exe = spack.util.executable.Executable(cc) if self.spec.external: compiler_flags = self.spec.extra_attributes.get("flags", {}) for flag_type in [ "cflags" if cc == compiler_pkg.cc else "cxxflags", "cppflags", "ldflags", ]: current_flags = compiler_flags.get(flag_type, "").strip() if current_flags: cc_exe.add_default_arg(*current_flags.split(" ")) with self.compiler_environment(): return cc_exe("-v", fin, "-o", fout, output=str, error=str) except spack.util.executable.ProcessError as pe: tty.debug(f"ProcessError: Command exited with non-zero status: {pe.long_message}") return None finally: shutil.rmtree(tmpdir, ignore_errors=True) def compiler_verbose_output(self) -> Optional[str]: """Get the compiler verbose output from the cache or by compiling a dummy C source.""" return self.cache.get(self.spec).c_compiler_output def default_dynamic_linker(self) -> Optional[str]: """Determine the default dynamic linker path from the compiler verbose output.""" output = self.compiler_verbose_output() if not output: return None return spack.util.libc.parse_dynamic_linker(output) def default_libc(self) -> Optional[spack.spec.Spec]: """Determine libc targeted by the compiler from link line""" # technically this should be testing the target platform of the compiler, but we don't have # that, so stick to host platform for now. if sys.platform in ("darwin", "win32"): return None dynamic_linker = self.default_dynamic_linker() if dynamic_linker is None: return None return spack.util.libc.libc_from_dynamic_linker(dynamic_linker) def implicit_rpaths(self) -> List[str]: """Obtain the implicit rpaths to be added from the default ``-L`` link directories, excluding system directories.""" output = self.compiler_verbose_output() if output is None: return [] link_dirs = parse_non_system_link_dirs(output) all_required_libs = list(self.spec.package.implicit_rpath_libs) + [ "libc", "libc++", "libstdc++", ] dynamic_linker = self.default_dynamic_linker() result = DefaultDynamicLinkerFilter(dynamic_linker)( paths_containing_libs(link_dirs, all_required_libs) ) return list(result) class DefaultDynamicLinkerFilter: """Remove rpaths to directories that are default search paths of the dynamic linker.""" _CACHE: Dict[Optional[str], Set[Tuple[int, int]]] = {} def __init__(self, dynamic_linker: Optional[str]) -> None: if dynamic_linker not in DefaultDynamicLinkerFilter._CACHE: # Identify directories by (inode, device) tuple, which handles symlinks too. default_path_identifiers: Set[Tuple[int, int]] = set() if not dynamic_linker: self.default_path_identifiers = None return for path in spack.util.libc.default_search_paths_from_dynamic_linker(dynamic_linker): try: s = os.stat(path) if stat.S_ISDIR(s.st_mode): default_path_identifiers.add((s.st_ino, s.st_dev)) except OSError: continue DefaultDynamicLinkerFilter._CACHE[dynamic_linker] = default_path_identifiers self.default_path_identifiers = DefaultDynamicLinkerFilter._CACHE[dynamic_linker] def is_dynamic_loader_default_path(self, p: str) -> bool: if self.default_path_identifiers is None: return False try: s = os.stat(p) return (s.st_ino, s.st_dev) in self.default_path_identifiers except OSError: return False def __call__(self, dirs: List[str]) -> List[str]: if not self.default_path_identifiers: return dirs return [p for p in dirs if not self.is_dynamic_loader_default_path(p)] def dynamic_linker_filter_for(node: spack.spec.Spec) -> Optional[DefaultDynamicLinkerFilter]: compiler = compiler_spec(node) if compiler is None: return None detector = CompilerPropertyDetector(compiler) dynamic_linker = detector.default_dynamic_linker() if dynamic_linker is None: return None return DefaultDynamicLinkerFilter(dynamic_linker) def compiler_spec(node: spack.spec.Spec) -> Optional[spack.spec.Spec]: """Returns a compiler :class:`~spack.spec.Spec` associated with the node passed as argument. The function looks for a ``c``, ``cxx``, and ``fortran`` compiler in that order, and returns the first found. If the node does not depend on any of these languages, it returns :obj:`None`. Use of this function is *discouraged*, because a single spec can have multiple compilers associated with it, and this function only returns one of them. It can be better to refer to compilers on a per-language basis, through the language virtuals: ``spec["c"]``, ``spec["cxx"]``, and ``spec["fortran"]``. """ for language in ("c", "cxx", "fortran"): candidates = node.dependencies(virtuals=[language]) if candidates: break else: return None return candidates[0] class CompilerCacheEntry: """Deserialized cache entry for a compiler""" __slots__ = ("c_compiler_output",) def __init__(self, c_compiler_output: Optional[str]): self.c_compiler_output = c_compiler_output @property def empty(self) -> bool: """Sometimes the compiler is temporarily broken, preventing us from getting output. The call site determines if that is a problem.""" return self.c_compiler_output is None @classmethod def from_dict(cls, data: Dict[str, Optional[str]]): if not isinstance(data, dict): raise ValueError(f"Invalid {cls.__name__} data") c_compiler_output = data.get("c_compiler_output") if not isinstance(c_compiler_output, (str, type(None))): raise ValueError(f"Invalid {cls.__name__} data") return cls(c_compiler_output) class CompilerCache: """Base class for compiler output cache. Default implementation does not cache anything.""" def value(self, compiler: spack.spec.Spec) -> Dict[str, Optional[str]]: return {"c_compiler_output": CompilerPropertyDetector(compiler)._compile_dummy_c_source()} def get(self, compiler: spack.spec.Spec) -> CompilerCacheEntry: return CompilerCacheEntry.from_dict(self.value(compiler)) class FileCompilerCache(CompilerCache): """Cache for compiler output, which is used to determine implicit link paths, the default libc version, and the compiler version.""" name = os.path.join("compilers", "compilers.json") def __init__(self, cache: "FileCache") -> None: self.cache = cache self._data: Dict[str, Dict[str, Optional[str]]] = {} def _get_entry(self, key: str, *, allow_empty: bool) -> Optional[CompilerCacheEntry]: try: entry = CompilerCacheEntry.from_dict(self._data[key]) return entry if allow_empty or not entry.empty else None except ValueError: del self._data[key] except KeyError: pass return None def get(self, compiler: spack.spec.Spec) -> CompilerCacheEntry: # Cache hit with self.cache.read_transaction(self.name) as f: if f is not None: try: self._data = json.loads(f.read()) if not isinstance(self._data, dict): self._data = {} except json.JSONDecodeError: self._data = {} else: self._data = {} key = self._key(compiler) value = self._get_entry(key, allow_empty=False) if value is not None: return value # Cache miss with self.cache.write_transaction(self.name) as (old, new): if old is not None: try: self._data = json.loads(old.read()) if not isinstance(self._data, dict): self._data = {} except json.JSONDecodeError: self._data = {} else: self._data = {} # Use cache entry that may have been created by another process in the meantime. entry = self._get_entry(key, allow_empty=True) # Finally compute the cache entry if entry is None: self._data[key] = self.value(compiler) entry = CompilerCacheEntry.from_dict(self._data[key]) new.write(json.dumps(self._data, separators=(",", ":"))) return entry def _key(self, compiler: spack.spec.Spec) -> str: as_bytes = json.dumps(compiler.to_dict(), separators=(",", ":")).encode("utf-8") return hashlib.sha256(as_bytes).hexdigest() def _make_compiler_cache(): return FileCompilerCache(spack.caches.MISC_CACHE) COMPILER_CACHE = cast(CompilerCache, spack.llnl.util.lang.Singleton(_make_compiler_cache)) ================================================ FILE: lib/spack/spack/concretize.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """High-level functions to concretize list of specs""" import importlib import sys import time from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple, Union import spack.compilers import spack.compilers.config import spack.config import spack.error import spack.llnl.util.tty as tty import spack.repo import spack.util.parallel from spack.spec import ArchSpec, CompilerSpec, Spec SpecPairInput = Tuple[Spec, Optional[Spec]] SpecPair = Tuple[Spec, Spec] TestsType = Union[bool, Iterable[str]] if TYPE_CHECKING: from spack.solver.reuse import SpecFiltersFactory def _concretize_specs_together( abstract_specs: Sequence[Spec], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ) -> List[Spec]: """Given a number of specs as input, tries to concretize them together. Args: abstract_specs: abstract specs to be concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. factory: optional factory to produce a list of specs to be reused """ from spack.solver.asp import Solver allow_deprecated = spack.config.get("config:deprecated", False) result = Solver(specs_factory=factory).solve( abstract_specs, tests=tests, allow_deprecated=allow_deprecated ) return [s.copy() for s in result.specs] def concretize_together( spec_list: Sequence[SpecPairInput], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ) -> List[SpecPair]: """Given a number of specs as input, tries to concretize them together. Args: spec_list: list of tuples to concretize. First entry is abstract spec, second entry is already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. factory: optional factory to produce a list of specs to be reused """ to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list] abstract_specs = [abstract for abstract, _ in spec_list] concrete_specs = _concretize_specs_together(to_concretize, tests=tests, factory=factory) return list(zip(abstract_specs, concrete_specs)) def concretize_together_when_possible( spec_list: Sequence[SpecPairInput], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ) -> List[SpecPair]: """Given a number of specs as input, tries to concretize them together to the extent possible. See documentation for ``unify: when_possible`` concretization for the precise definition of "to the extent possible". Args: spec_list: list of tuples to concretize. First entry is abstract spec, second entry is already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. factory: optional factory to produce a list of specs to be reused """ from spack.solver.asp import Solver to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list] old_concrete_to_abstract = { concrete: abstract for (abstract, concrete) in spec_list if concrete } result_by_user_spec: Dict[Spec, Spec] = {} allow_deprecated = spack.config.get("config:deprecated", False) j = 0 start = time.monotonic() for result in Solver(specs_factory=factory).solve_in_rounds( to_concretize, tests=tests, allow_deprecated=allow_deprecated ): now = time.monotonic() duration = now - start percentage = int((j + 1) / len(to_concretize) * 100) for abstract, concrete in result.specs_by_input.items(): tty.verbose( f"{duration:6.1f}s [{percentage:3d}%] {concrete.cformat('{hash:7}')} " f"{abstract.colored_str}" ) j += 1 sys.stdout.flush() result_by_user_spec.update(result.specs_by_input) start = now # If the "abstract" spec is a concrete spec from the previous concretization # translate it back to an abstract spec. Otherwise, keep the abstract spec return [ (old_concrete_to_abstract.get(abstract, abstract), concrete) for abstract, concrete in sorted(result_by_user_spec.items()) ] def concretize_separately( spec_list: Sequence[SpecPairInput], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ) -> List[SpecPair]: """Concretizes the input specs separately from each other. Args: spec_list: list of tuples to concretize. First entry is abstract spec, second entry is already concrete spec or None if not yet concretized tests: list of package names for which to consider tests dependencies. If True, all nodes will have test dependencies. If False, test dependencies will be disregarded. factory: optional factory to produce a list of specs to be reused """ from spack.bootstrap import ( ensure_bootstrap_configuration, ensure_clingo_importable_or_raise, ensure_winsdk_external_or_raise, ) to_concretize = [abstract for abstract, concrete in spec_list if not concrete] args = [ (i, str(abstract), tests, factory) for i, abstract in enumerate(to_concretize) if not abstract.concrete ] ret = [(i, abstract) for i, abstract in enumerate(to_concretize) if abstract.concrete] try: # Ensure we don't try to bootstrap clingo in parallel importlib.import_module("clingo") except ImportError: with ensure_bootstrap_configuration(): ensure_clingo_importable_or_raise() # ensure we don't try to detect winsdk in parallel if sys.platform == "win32": ensure_winsdk_external_or_raise() # Ensure all the indexes have been built or updated, since # otherwise the processes in the pool may timeout on waiting # for a write lock. We do this indirectly by retrieving the # provider index, which should in turn trigger the update of # all the indexes if there's any need for that. _ = spack.repo.PATH.provider_index # Ensure we have compilers in packages.yaml to avoid that # processes try to write the config file in parallel _ = spack.compilers.config.all_compilers() # Early return if there is nothing to do if len(args) == 0: # Still have to combine the things that were passed in as abstract with the things # that were passed in as pairs return [(abstract, concrete) for abstract, (_, concrete) in zip(to_concretize, ret)] + [ (abstract, concrete) for abstract, concrete in spec_list if concrete ] # Solve the environment in parallel on Linux num_procs = min(len(args), spack.config.determine_number_of_jobs(parallel=True)) msg = "Starting concretization" # no parallel conc on Windows if not sys.platform == "win32" and num_procs > 1: msg += f" pool with {num_procs} processes" tty.msg(msg) for j, (i, concrete, duration) in enumerate( spack.util.parallel.imap_unordered( _concretize_task, args, processes=num_procs, debug=tty.is_debug(), maxtaskperchild=1 ) ): ret.append((i, concrete)) percentage = int((j + 1) / len(args) * 100) tty.verbose( f"{duration:6.1f}s [{percentage:3d}%] {concrete.cformat('{hash:7}')} " f"{to_concretize[i].colored_str}" ) sys.stdout.flush() # Add specs in original order ret.sort(key=lambda x: x[0]) return [(abstract, concrete) for abstract, (_, concrete) in zip(to_concretize, ret)] + [ (abstract, concrete) for abstract, concrete in spec_list if concrete ] def _concretize_task( packed_arguments: Tuple[int, str, TestsType, Optional["SpecFiltersFactory"]], ) -> Tuple[int, Spec, float]: index, spec_str, tests, factory = packed_arguments with tty.SuppressOutput(msg_enabled=False): start = time.time() spec = concretize_one(Spec(spec_str), tests=tests, factory=factory) return index, spec, time.time() - start def concretize_one( spec: Union[str, Spec], *, tests: TestsType = False, factory: Optional["SpecFiltersFactory"] = None, ) -> Spec: """Return a concretized copy of the given spec. Args: tests: if False disregard test dependencies, if a list of names activate them for the packages in the list, if True activate test dependencies for all packages. """ from spack.solver.asp import Solver, SpecBuilder if isinstance(spec, str): spec = Spec(spec) spec = spec.lookup_hash() if spec.concrete: return spec.copy() for node in spec.traverse(): if not node.name: raise spack.error.SpecError( f"Spec {node} has no name; cannot concretize an anonymous spec" ) allow_deprecated = spack.config.get("config:deprecated", False) result = Solver(specs_factory=factory).solve( [spec], tests=tests, allow_deprecated=allow_deprecated ) # take the best answer opt, i, answer = min(result.answers) name = spec.name # TODO: Consolidate this code with similar code in solve.py if spack.repo.PATH.is_virtual(spec.name): providers = [s.name for s in answer.values() if s.package.provides(name)] name = providers[0] node = SpecBuilder.make_node(pkg=name) assert node in answer, ( f"cannot find {name} in the list of specs {','.join([n.pkg for n in answer.keys()])}" ) concretized = answer[node] return concretized class UnavailableCompilerVersionError(spack.error.SpackError): """Raised when there is no available compiler that satisfies a compiler spec.""" def __init__(self, compiler_spec: CompilerSpec, arch: Optional[ArchSpec] = None) -> None: err_msg = f"No compilers with spec {compiler_spec} found" if arch: err_msg += f" for operating system {arch.os} and target {arch.target}." super().__init__( err_msg, "Run 'spack compiler find' to add compilers or " "'spack compilers' to see which compilers are already recognized" " by spack.", ) ================================================ FILE: lib/spack/spack/config.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module implements Spack's configuration file handling. This implements Spack's configuration system, which handles merging multiple scopes with different levels of precedence. See the documentation on :ref:`configuration-scopes` for details on how Spack's configuration system behaves. The scopes set up here are: #. ``spack`` in ``$spack/etc/spack`` - controls all built-in spack scopes, except default #. ``defaults`` in ``$spack/etc/spack/defaults`` - defaults that Spack needs to function Important functions in this module are: * :func:`~spack.config.Configuration.get_config` * :func:`~spack.config.Configuration.update_config` ``get_config`` reads in YAML data for a particular scope and returns it. Callers can then modify the data and write it back with ``update_config``. When read in, Spack validates configurations with jsonschemas. The schemas are in submodules of :py:mod:`spack.schema`. """ import contextlib import copy import functools import os import os.path import pathlib import re import sys import tempfile from collections import defaultdict from itertools import chain from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union, cast from spack.vendor import jsonschema import spack.error import spack.paths import spack.schema import spack.schema.bootstrap import spack.schema.cdash import spack.schema.ci import spack.schema.compilers import spack.schema.concretizer import spack.schema.config import spack.schema.definitions import spack.schema.develop import spack.schema.env import spack.schema.env_vars import spack.schema.include import spack.schema.merged import spack.schema.mirrors import spack.schema.modules import spack.schema.packages import spack.schema.repos import spack.schema.toolchains import spack.schema.upstreams import spack.schema.view import spack.util.executable import spack.util.git import spack.util.hash import spack.util.remote_file_cache as rfc_util import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml from spack.llnl.util import filesystem, lang, tty from spack.util.cpus import cpus_available from spack.util.spack_yaml import get_mark_from_yaml_data from .enums import ConfigScopePriority #: Dict from section names -> schema for that section SECTION_SCHEMAS: Dict[str, Any] = { "compilers": spack.schema.compilers.schema, "concretizer": spack.schema.concretizer.schema, "definitions": spack.schema.definitions.schema, "env_vars": spack.schema.env_vars.schema, "include": spack.schema.include.schema, "view": spack.schema.view.schema, "develop": spack.schema.develop.schema, "mirrors": spack.schema.mirrors.schema, "repos": spack.schema.repos.schema, "packages": spack.schema.packages.schema, "modules": spack.schema.modules.schema, "config": spack.schema.config.schema, "upstreams": spack.schema.upstreams.schema, "bootstrap": spack.schema.bootstrap.schema, "ci": spack.schema.ci.schema, "cdash": spack.schema.cdash.schema, "toolchains": spack.schema.toolchains.schema, } # Same as above, but including keys for environments # this allows us to unify config reading between configs and environments _ALL_SCHEMAS: Dict[str, Any] = { **SECTION_SCHEMAS, spack.schema.env.TOP_LEVEL_KEY: spack.schema.env.schema, } #: Path to the main configuration scope CONFIGURATION_DEFAULTS_PATH = ("defaults", os.path.join(spack.paths.etc_path, "defaults")) #: Hard-coded default values for some key configuration options. #: This ensures that Spack will still work even if config.yaml in #: the defaults scope is removed. CONFIG_DEFAULTS = { "config": { "debug": False, "connect_timeout": 10, "verify_ssl": True, "checksum": True, "dirty": False, "build_jobs": min(16, cpus_available()), "build_stage": "$tempdir/spack-stage", "license_dir": spack.paths.default_license_dir, }, "concretizer": {"externals": {"completion": "default_variants"}}, } #: metavar to use for commands that accept scopes #: this is shorter and more readable than listing all choices SCOPES_METAVAR = "{defaults,system,site,user,command_line} or env:ENVIRONMENT" #: Base name for the (internal) overrides scope. _OVERRIDES_BASE_NAME = "overrides-" #: Type used for raw YAML configuration YamlConfigDict = Dict[str, Any] #: safeguard for recursive includes -- maximum include depth MAX_RECURSIVE_INCLUDES = 100 class ConfigScope: def __init__(self, name: str, included: bool = False) -> None: self.name = name self.writable = False self.sections = syaml.syaml_dict() self.prefer_modify = False self.included = included #: included configuration scopes self._included_scopes: Optional[List["ConfigScope"]] = None @property def included_scopes(self) -> List["ConfigScope"]: """Memoized list of included scopes, in the order they appear in this scope.""" if self._included_scopes is None: self._included_scopes = [] includes = self.get_section("include") if includes: include_paths = [included_path(data) for data in includes["include"]] included_scopes = chain(*[include.scopes(self) for include in include_paths]) # Do not include duplicate scopes for included_scope in included_scopes: if any([included_scope.name == scope.name for scope in self._included_scopes]): tty.warn(f"Ignoring duplicate included scope: {included_scope.name}") continue if included_scope not in self._included_scopes: self._included_scopes.append(included_scope) return self._included_scopes @property def exists(self) -> bool: """Whether the config object indicated by the scope can be read""" return True def override_include(self): """Whether the ``include::`` section of this scope should override lower scopes.""" include = self.sections.get("include") if not include: return False # override if this has an include section and there is an override attribute on # the include key in the dict and it is set to True. return getattr(next(iter(include.keys()), None), "override", False) def transitive_includes(self, _names: Optional[Set[str]] = None) -> Set[str]: """Get name of this scope and names of its transitively included scopes.""" if _names is None: _names = _set() _names.add(self.name) for scope in self.included_scopes: _names |= scope.transitive_includes(_names=_names) return _names def get_section_filename(self, section: str) -> str: raise NotImplementedError def get_section(self, section: str) -> Optional[YamlConfigDict]: raise NotImplementedError def _write_section(self, section: str) -> None: raise NotImplementedError def clear(self) -> None: """Empty cached config information.""" self.sections = syaml.syaml_dict() def __repr__(self) -> str: return f"" class DirectoryConfigScope(ConfigScope): """Config scope backed by a directory containing one file per section.""" def __init__( self, name: str, path: str, *, writable: bool = True, prefer_modify: bool = True, included: bool = False, ) -> None: super().__init__(name, included) self.path = path self.writable = writable self.prefer_modify = prefer_modify @property def exists(self) -> bool: return os.path.exists(self.path) def get_section_filename(self, section: str) -> str: """Returns the filename associated with a given section""" _validate_section_name(section) return os.path.join(self.path, f"{section}.yaml") def get_section(self, section: str) -> Optional[YamlConfigDict]: """Returns the data associated with a given section if the scope exists""" if not self.exists: tty.debug(f"Attempting to read from missing scope: {self} at {self.path}") return {} return self._get_section(section) def _get_section(self, section: str) -> Optional[YamlConfigDict]: """get_section but without the existence check""" if section not in self.sections: path = self.get_section_filename(section) schema = SECTION_SCHEMAS[section] data = read_config_file(path, schema) self.sections[section] = data return self.sections[section] def _write_section(self, section: str) -> None: if not self.writable: raise spack.error.ConfigError(f"Cannot write to immutable scope {self}") filename = self.get_section_filename(section) data = self._get_section(section) if data is None: return validate(data, SECTION_SCHEMAS[section]) try: filesystem.mkdirp(self.path) fd, tmp = tempfile.mkstemp(dir=self.path, suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8") as f: syaml.dump_config(data, stream=f, default_flow_style=False) filesystem.rename(tmp, filename) except Exception: os.unlink(tmp) raise except (syaml.SpackYAMLError, OSError) as e: raise ConfigFileError(f"cannot write to '{filename}'") from e class SingleFileScope(ConfigScope): """This class represents a configuration scope in a single YAML file.""" def __init__( self, name: str, path: str, schema: YamlConfigDict, *, yaml_path: Optional[List[str]] = None, writable: bool = True, prefer_modify: bool = True, included: bool = False, ) -> None: """Similar to ``ConfigScope`` but can be embedded in another schema. Arguments: schema (dict): jsonschema for the file to read yaml_path (list): path in the schema where config data can be found. If the schema accepts the following yaml data, the yaml_path would be ['outer', 'inner'] .. code-block:: yaml outer: inner: config: install_tree: $spack/opt/spack """ super().__init__(name, included) self._raw_data: Optional[YamlConfigDict] = None self.schema = schema self.path = path self.writable = writable self.prefer_modify = prefer_modify self.yaml_path = yaml_path or [] @property def exists(self) -> bool: return os.path.exists(self.path) def get_section_filename(self, section) -> str: return self.path def get_section(self, section: str) -> Optional[YamlConfigDict]: # read raw data from the file, which looks like: # { # 'config': { # ... data ... # }, # 'packages': { # ... data ... # }, # } # # To preserve overrides up to the section level (e.g. to override # the "packages" section with the "::" syntax), data in self.sections # looks like this: # { # 'config': { # 'config': { # ... data ... # } # }, # 'packages': { # 'packages': { # ... data ... # } # } # } if not self.exists: tty.debug(f"Attempting to read from missing scope: {self} at {self.path}") return {} # This bit ensures we have read the file and have # the raw data in memory if self._raw_data is None: self._raw_data = read_config_file(self.path, self.schema) if self._raw_data is None: return None # Here we know we have the raw data and ensure we # populate the sections dictionary, which may be # cleared by the clear() method if not self.sections: section_data = self._raw_data for key in self.yaml_path: if section_data is None: return None section_data = section_data[key] for section_key, data in section_data.items(): self.sections[section_key] = {section_key: data} return self.sections.get(section, None) def _write_section(self, section: str) -> None: if not self.writable: raise spack.error.ConfigError(f"Cannot write to immutable scope {self}") data_to_write: Optional[YamlConfigDict] = self._raw_data # If there is no existing data, this section SingleFileScope has never # been written to disk. We need to construct the portion of the data # from the root of self._raw_data to the level at which the config # sections are defined. That requires creating keys for every entry in # self.yaml_path if not data_to_write: data_to_write = {} # reverse because we construct it from the inside out for key in reversed(self.yaml_path): data_to_write = {key: data_to_write} # data_update_pointer is a pointer to the part of data_to_write # that we are currently updating. # We start by traversing into the data to the point at which the # config sections are defined. This means popping the keys from # self.yaml_path data_update_pointer = data_to_write for key in self.yaml_path: data_update_pointer = data_update_pointer[key] # For each section, update the data at the level of our pointer # with the data from the section for key, data in self.sections.items(): data_update_pointer[key] = data[key] validate(data_to_write, self.schema) try: parent = os.path.dirname(self.path) filesystem.mkdirp(parent) fd, tmp = tempfile.mkstemp(dir=parent, suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8") as f: syaml.dump_config(data_to_write, stream=f, default_flow_style=False) filesystem.rename(tmp, self.path) except Exception: os.unlink(tmp) raise except (syaml.SpackYAMLError, OSError) as e: raise ConfigFileError(f"cannot write to config file {str(e)}") from e def __repr__(self) -> str: return f"" class InternalConfigScope(ConfigScope): """An internal configuration scope that is not persisted to a file. This is for spack internal use so that command-line options and config file settings are accessed the same way, and Spack can easily override settings from files. """ def __init__(self, name: str, data: Optional[YamlConfigDict] = None) -> None: super().__init__(name) self.sections = syaml.syaml_dict() if data is not None: data = InternalConfigScope._process_dict_keyname_overrides(data) for section in data: dsec = data[section] validate({section: dsec}, SECTION_SCHEMAS[section]) self.sections[section] = _mark_internal(syaml.syaml_dict({section: dsec}), name) def get_section(self, section: str) -> Optional[YamlConfigDict]: """Just reads from an internal dictionary.""" if section not in self.sections: self.sections[section] = None return self.sections[section] def _write_section(self, section: str) -> None: """This only validates, as the data is already in memory.""" data = self.get_section(section) if data is not None: validate(data, SECTION_SCHEMAS[section]) self.sections[section] = _mark_internal(data, self.name) def __repr__(self) -> str: return f"" def clear(self) -> None: # no cache to clear here. pass @staticmethod def _process_dict_keyname_overrides(data: YamlConfigDict) -> YamlConfigDict: """Turn a trailing `:' in a key name into an override attribute.""" # Below we have a lot of type directives, since we hack on types and monkey-patch them # by adding attributes that otherwise they won't have. result: YamlConfigDict = {} for sk, sv in data.items(): if sk.endswith(":"): key = syaml.syaml_str(sk[:-1]) key.override = True # type: ignore[attr-defined] elif sk.endswith("+"): key = syaml.syaml_str(sk[:-1]) key.prepend = True # type: ignore[attr-defined] elif sk.endswith("-"): key = syaml.syaml_str(sk[:-1]) key.append = True # type: ignore[attr-defined] else: key = sk # type: ignore[assignment] if isinstance(sv, dict): result[key] = InternalConfigScope._process_dict_keyname_overrides(sv) else: result[key] = copy.copy(sv) return result def _config_mutator(method): """Decorator to mark all the methods in the Configuration class that mutate the underlying configuration. Used to clear the memoization cache. """ @functools.wraps(method) def _method(self, *args, **kwargs): self._get_config_memoized.cache_clear() return method(self, *args, **kwargs) return _method ScopeWithOptionalPriority = Union[ConfigScope, Tuple[int, ConfigScope]] ScopeWithPriority = Tuple[int, ConfigScope] class Configuration: """A hierarchical configuration, merging a number of scopes at different priorities.""" # convert to typing.OrderedDict when we drop 3.6, or OrderedDict when we reach 3.9 scopes: lang.PriorityOrderedMapping[str, ConfigScope] def __init__(self) -> None: self.scopes = lang.PriorityOrderedMapping() self.updated_scopes_by_section: Dict[str, List[ConfigScope]] = defaultdict(list) def ensure_unwrapped(self) -> "Configuration": """Ensure we unwrap this object from any dynamic wrapper (like Singleton)""" return self def highest(self) -> ConfigScope: """Scope with the highest precedence""" return next(self.scopes.reversed_values()) # type: ignore @_config_mutator def push_scope_incremental( self, scope: ConfigScope, priority: Optional[int] = None, _depth: int = 0 ) -> Generator["Configuration", None, None]: """Adds a scope to the Configuration, at a given priority. ``push_scope_incremental`` yields included scopes incrementally, so that their data can be used by higher priority scopes during config initialization. If you push a scope that includes other, low-priority scopes, they will be pushed on first, before the scope that included them. If a priority is not given, it is assumed to be the current highest priority. Args: scope: scope to be added priority: priority of the scope """ # TODO: As a follow on to #48784, change this to create a graph of the # TODO: includes AND ensure properly sorted such that the order included # TODO: at the highest level is reflected in the value of an option that # TODO: is set in multiple included files. # before pushing the scope itself, push included scopes recursively, at the same priority for included_scope in reversed(scope.included_scopes): if _depth + 1 > MAX_RECURSIVE_INCLUDES: # make sure we're not recursing endlessly mark = "" if hasattr(included_scope, "path") and syaml.marked(included_scope.path): mark = included_scope.path._start_mark # type: ignore raise RecursiveIncludeError( f"Maximum include recursion exceeded in {included_scope.name}", str(mark) ) # record this inclusion so that remove_scope() can use it self.push_scope(included_scope, priority=priority, _depth=_depth + 1) yield self tty.debug(f"[CONFIGURATION: PUSH SCOPE]: {str(scope)}, priority={priority}", level=2) self.scopes.add(scope.name, value=scope, priority=priority) yield self @_config_mutator def push_scope( self, scope: ConfigScope, priority: Optional[int] = None, _depth: int = 0 ) -> None: """Add a scope to the Configuration, at a given priority. If a priority is not given, it is assumed to be the current highest priority. Args: scope: scope to be added priority: priority of the scope """ # Use push_scope_incremental to do the real work. It returns a generator, which needs # to be consumed to get each of the yielded scopes added to the scope stack. # It will usually yield one scope, but if there are includes it will yield those first, # before the scope we're actually pushing. for _ in self.push_scope_incremental(scope=scope, priority=priority, _depth=_depth): pass @_config_mutator def remove_scope(self, scope_name: str) -> Optional[ConfigScope]: """Removes a scope by name, and returns it. If the scope does not exist, returns None.""" try: scope = self.scopes.remove(scope_name) tty.debug(f"[CONFIGURATION: REMOVE SCOPE]: {str(scope)}", level=2) except KeyError as e: tty.debug(f"[CONFIGURATION: REMOVE SCOPE]: {e}", level=2) return None # transitively remove included scopes for included_scope in scope.included_scopes: assert included_scope.name in self.scopes, ( f"Included scope '{included_scope.name}' was never added to configuration!" ) self.remove_scope(included_scope.name) return scope @property def writable_scopes(self) -> Generator[ConfigScope, None, None]: """Generator of writable scopes with an associated file.""" return (s for s in self.scopes.values() if s.writable) @property def existing_scopes(self) -> Generator[ConfigScope, None, None]: """Generator of existing scopes. These are self.scopes where the scope has a representation on the filesystem or is internal""" return (s for s in self.scopes.values() if s.exists) def highest_precedence_scope(self) -> ConfigScope: """Writable scope with the highest precedence.""" scope = next(s for s in self.scopes.reversed_values() if s.writable) # if a scope prefers that we edit another, respect that. while scope: preferred = scope scope = next( (s for s in scope.included_scopes if s.writable and s.prefer_modify), None ) return preferred def matching_scopes(self, reg_expr) -> List[ConfigScope]: """ List of all scopes whose names match the provided regular expression. For example, ``matching_scopes(r'^command')`` will return all scopes whose names begin with ``command``. """ return [s for s in self.scopes.values() if re.search(reg_expr, s.name)] def _validate_scope(self, scope: Optional[str]) -> ConfigScope: """Ensure that scope is valid in this configuration. This should be used by routines in ``config.py`` to validate scope name arguments, and to determine a default scope where no scope is specified. Raises: ValueError: if ``scope`` is not valid Returns: ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid """ if scope is None: # default to the scope with highest precedence. return self.highest_precedence_scope() elif scope in self.scopes: return self.scopes[scope] else: raise ValueError( f"Invalid config scope: '{scope}'. Must be one of " f"{[k for k in self.scopes.keys()]}" ) def get_config_filename(self, scope: str, section: str) -> str: """For some scope and section, get the name of the configuration file.""" scope = self._validate_scope(scope) return scope.get_section_filename(section) @_config_mutator def clear_caches(self) -> None: """Clears the caches for configuration files, This will cause files to be re-read upon the next request.""" for scope in self.scopes.values(): scope.clear() @_config_mutator def update_config( self, section: str, update_data: Dict, scope: Optional[str] = None, force: bool = False ) -> None: """Update the configuration file for a particular scope. Overwrites contents of a section in a scope with update_data, then writes out the config file. update_data should have the top-level section name stripped off (it will be re-added). Data itself can be a list, dict, or any other yaml-ish structure. Configuration scopes that are still written in an old schema format will fail to update unless ``force`` is True. Args: section: section of the configuration to be updated update_data: data to be used for the update scope: scope to be updated force: force the update """ if self.updated_scopes_by_section.get(section) and not force: msg = ( 'The "{0}" section of the configuration needs to be written' " to disk, but is currently using a deprecated format. " "Please update it using:\n\n" "\tspack config [--scope=] update {0}\n\n" "Note that previous versions of Spack will not be able to " "use the updated configuration." ) msg = msg.format(section) raise RuntimeError(msg) _validate_section_name(section) # validate section name scope = self._validate_scope(scope) # get ConfigScope object # manually preserve comments need_comment_copy = section in scope.sections and scope.sections[section] if need_comment_copy: comments = syaml.extract_comments(scope.sections[section][section]) # read only the requested section's data. scope.sections[section] = syaml.syaml_dict({section: update_data}) if need_comment_copy and comments: syaml.set_comments(scope.sections[section][section], data_comments=comments) scope._write_section(section) def get_config( self, section: str, scope: Optional[str] = None, _merged_scope: Optional[str] = None ) -> YamlConfigDict: """Get configuration settings for a section. If ``scope`` is ``None`` or not provided, return the merged contents of all of Spack's configuration scopes. If ``scope`` is provided, return only the configuration as specified in that scope. This off the top-level name from the YAML section. That is, for a YAML config file that looks like this:: config: install_tree: root: $spack/opt/spack build_stage: - $tmpdir/$user/spack-stage ``get_config('config')`` will return:: { 'install_tree': { 'root': '$spack/opt/spack', } 'build_stage': ['$tmpdir/$user/spack-stage'] } """ return self._get_config_memoized(section, scope=scope, _merged_scope=_merged_scope) def deepcopy_as_builtin( self, section: str, scope: Optional[str] = None, *, line_info: bool = False ) -> Dict[str, Any]: """Get a deep copy of a section with native Python types, excluding YAML metadata.""" return syaml.deepcopy_as_builtin( self.get_config(section, scope=scope), line_info=line_info ) def _filter_overridden(self, scopes: List[ConfigScope], includes: bool = False): """Filter out overridden scopes. NOTE: this does not yet handle diamonds or nested `include::` in lists. It is sufficient for include::[] in an env, which allows isolation. The ``includes`` option controls whether to return all active scopes (``includes=False``) or all scopes whose includes have not been overridden (``includes=True``). """ # find last override in scopes i = next((i for i, s in reversed(list(enumerate(scopes))) if s.override_include()), -1) if i < 0: return scopes # no overrides keep = _set(s.name for s in scopes[i:]) keep |= _set(s.name for s in self.scopes.priority_values(ConfigScopePriority.DEFAULTS)) if not includes: # For all sections except for the include section: # non-included scopes are still active, as are scopes included # from the overriding scope # Transitive scopes from the overriding scope are not included keep |= _set([s.name for s in scopes[i].included_scopes]) keep |= _set([s.name for s in scopes if not s.included]) # return scopes to keep, with order preserved return [s for s in scopes if s.name in keep] @property def active_include_section_scopes(self) -> List[ConfigScope]: """Return a list of all scopes whose includes have not been overridden by include::. This is different from the active scopes because the ``spack`` scope can be active while its includes are overwritten, as can the transitive includes from the overriding scope.""" return self._filter_overridden([s for s in self.scopes.values()], includes=True) @property def active_scopes(self) -> List[ConfigScope]: """Return a list of scopes that have not been overridden by include::.""" return self._filter_overridden([s for s in self.scopes.values()]) @lang.memoized def _get_config_memoized( self, section: str, scope: Optional[str], _merged_scope: Optional[str] ) -> YamlConfigDict: """Memoized helper for ``get_config()``. Note that the memoization cache for this function is cleared whenever any function decorated with ``@_config_mutator`` is called. """ _validate_section_name(section) if scope is not None and _merged_scope is not None: raise ValueError("Cannot specify both scope and _merged_scope") elif scope is not None: scopes = [self._validate_scope(scope)] elif _merged_scope is not None: scope_stack = list(self.scopes.values()) merge_idx = next(i for i, s in enumerate(scope_stack) if s.name == _merged_scope) scopes = scope_stack[: merge_idx + 1] else: scopes = list(self.scopes.values()) # filter any scopes overridden by `include::` scopes = self._filter_overridden(scopes) merged_section: Dict[str, Any] = syaml.syaml_dict() updated_scopes = [] for config_scope in scopes: if section == "include" and config_scope not in self.active_include_section_scopes: continue # read potentially cached data from the scope. data = config_scope.get_section(section) if data and section == "include": # Include overrides are handled by `_filter_overridden` above. Any remaining # includes at this point are *not* actually overridden -- they're scopes with # ConfigScopePriority.DEFAULT, which we currently do *not* remove with # `include::`, because these scopes are needed for Spack to function correctly. # So, we ignore :: here. data = data.copy() data["include"] = data.pop("include") # strip override # Skip empty configs if not isinstance(data, dict) or section not in data: continue # If configuration is in an old format, transform it and keep track of the scope that # may need to be written out to disk. if _update_in_memory(data, section): updated_scopes.append(config_scope) merged_section = spack.schema.merge_yaml(merged_section, data) self.updated_scopes_by_section[section] = updated_scopes # no config files -- empty config. if section not in merged_section: return syaml.syaml_dict() # take the top key off before returning. ret = merged_section[section] if isinstance(ret, dict): ret = syaml.syaml_dict(ret) return ret def get(self, path: str, default: Optional[Any] = None, scope: Optional[str] = None) -> Any: """Get a config section or a single value from one. Accepts a path syntax that allows us to grab nested config map entries. Getting the ``config`` section would look like:: spack.config.get("config") and the ``dirty`` section in the ``config`` scope would be:: spack.config.get("config:dirty") We use ``:`` as the separator, like YAML objects. """ parts = process_config_path(path) section = parts.pop(0) value = self.get_config(section, scope=scope) while parts: key = parts.pop(0) # cannot use value.get(key, default) in case there is another part # and default is not a dict if key not in value: return default value = value[key] return value @_config_mutator def set(self, path: str, value: Any, scope: Optional[str] = None) -> None: """Convenience function for setting single values in config files. Accepts the path syntax described in ``get()``. """ if ":" not in path: # handle bare section name as path self.update_config(path, value, scope=scope) return parts = process_config_path(path) section = parts.pop(0) section_data = self.get_config(section, scope=scope) data = section_data while len(parts) > 1: key = parts.pop(0) if spack.schema.override(key): new = type(data[key])() del data[key] else: new = data[key] if isinstance(new, dict): # Make it an ordered dict new = syaml.syaml_dict(new) # reattach to parent object data[key] = new data = new if spack.schema.override(parts[0]): data.pop(parts[0], None) # update new value data[parts[0]] = value self.update_config(section, section_data, scope=scope) def __iter__(self): """Iterate over scopes in this configuration.""" yield from self.scopes.values() def print_section( self, section: str, yaml: bool = True, blame: bool = False, *, scope: Optional[str] = None ) -> None: """Print a configuration to stdout. Arguments: section: The configuration section to print. yaml: If True, output in YAML format, otherwise JSON (ignored when blame is True). blame: Whether to include source locations for each entry. scope: The configuration scope to use. """ try: data = syaml.syaml_dict() data[section] = self.get_config(section, scope=scope) if yaml or blame: syaml.dump_config(data, stream=sys.stdout, default_flow_style=False, blame=blame) else: sjson.dump(data, sys.stdout) sys.stdout.write("\n") except (syaml.SpackYAMLError, OSError) as e: raise spack.error.ConfigError(f"cannot read '{section}' configuration") from e @contextlib.contextmanager def override( path_or_scope: Union[ConfigScope, str], value: Optional[Any] = None ) -> Generator[Configuration, None, None]: """Simple way to override config settings within a context. Arguments: path_or_scope (ConfigScope or str): scope or single option to override value (object or None): value for the single option Temporarily push a scope on the current configuration, then remove it after the context completes. If a single option is provided, create an internal config scope for it and push/pop that scope. """ if isinstance(path_or_scope, ConfigScope): overrides = path_or_scope CONFIG.push_scope(path_or_scope, priority=None) else: base_name = _OVERRIDES_BASE_NAME # Ensure the new override gets a unique scope name current_overrides = [s.name for s in CONFIG.matching_scopes(rf"^{base_name}")] num_overrides = len(current_overrides) while True: scope_name = f"{base_name}{num_overrides}" if scope_name in current_overrides: num_overrides += 1 else: break overrides = InternalConfigScope(scope_name) CONFIG.push_scope(overrides, priority=None) CONFIG.set(path_or_scope, value, scope=scope_name) try: yield CONFIG finally: scope = CONFIG.remove_scope(overrides.name) assert scope is overrides #: Class for the relevance of an optional path conditioned on a limited #: python code that evaluates to a boolean and or explicit specification #: as optional. class OptionalInclude: """Base properties for all includes.""" name: str when: str optional: bool prefer_modify: bool remote: bool _scopes: List[ConfigScope] def __init__(self, entry: dict): self.name = entry.get("name", "") self.when = entry.get("when", "") self.optional = entry.get("optional", False) self.prefer_modify = entry.get("prefer_modify", False) self.remote = False self._scopes = [] @staticmethod def _parent_scope_directory(parent_scope: Optional[ConfigScope]) -> Optional[str]: """Return the directory of the parent scope, or ``None`` if unavailable. Normalizes ``SingleFileScope`` to its containing directory. """ path = getattr(parent_scope, "path", "") if parent_scope else "" if not path: return None return os.path.dirname(path) if os.path.isfile(path) else path def base_directory( self, path_or_url: str, parent_scope: Optional[ConfigScope] = None ) -> Optional[str]: """Return the local directory to use for this include. For remote includes this is the cache destination directory. For local relative includes this is the working directory from which to resolve the path. Args: path_or_url: path or URL of the include parent_scope: including scope Returns: ``None`` for a local include without an enclosing parent scope; an appropriate subdirectory of the enclosing (parent) scope's writable directory (when available); otherwise a stable temporary directory. """ scope_dir = self._parent_scope_directory(parent_scope) if not self.remote: return scope_dir def _subdir(): # Prefer the provided include name over the git repository name. # If neither, use a hash of the url or path for uniqueness. if self.name: return self.name match = re.search(r"/([^/]+?)(\.git)?$", path_or_url) if match: if not os.path.splitext(match.group(1))[1]: return match.group(1) return spack.util.hash.b32_hash(path_or_url)[-7:] # For remote includes, prefer a writable subdirectory of the parent scope. if scope_dir and filesystem.can_write_to_dir(scope_dir): assert parent_scope is not None subdir = os.path.join("includes", _subdir()) if parent_scope.name.startswith("env:"): subdir = os.path.join(".spack-env", subdir) return os.path.join(scope_dir, subdir) # Fall back to a stable, unique, temporary directory, logging the reason. tmpdir = tempfile.gettempdir() if path_or_url: pre = self.name or getattr(parent_scope, "name", "") subdir = f"{pre}:{path_or_url}" if pre else path_or_url tmpdir = os.path.join(tmpdir, spack.util.hash.b32_hash(subdir)[-7:]) if not scope_dir: tty.debug(f"No parent scope directory for include ({self}). Using {tmpdir}.") else: assert parent_scope is not None tty.debug( f"Parent scope {parent_scope.name}'s directory ({scope_dir}) is not writable. " f"Using {tmpdir}." ) return tmpdir def _scope( self, path: str, config_path: str, parent_scope: ConfigScope ) -> Optional[ConfigScope]: """Instantiate a configuration scope for the configuration path. Args: path: raw include path config_path: configuration path parent_scope: including scope Returns: configuration scopes Raises: ValueError: the required configuration path does not exist """ # circular dependencies import spack.util.path # Ignore included concrete environment files (i.e., ``spack.lock``) # since they are not normal configuration (scope) files and their # processing is handled when the environment is processed. if path and os.path.basename(path) == "spack.lock": tty.debug( f"Ignoring inclusion of '{path}' since environment lock files " "are processed elsewhere" ) return None # Ensure the parent scope is valid self._validate_parent_scope(parent_scope) # Determine the configuration scope name config_name = self.name or parent_scope.name # But ensure that name is unique if there are multiple paths. if not self.name or len(getattr(self, "paths", [])) > 1: parent_path = pathlib.Path(getattr(parent_scope, "path", "")) real_path = pathlib.Path(spack.util.path.substitute_path_variables(path)) try: included_name = real_path.relative_to(parent_path) except ValueError: included_name = real_path if sys.platform == "win32": # Clean windows path for use in config name that looks nicer # ie. The path: C:\\some\\path\\to\\a\\file # becomes C/some/path/to/a/file included_name = included_name.as_posix().replace(":", "") config_name = f"{config_name}:{included_name}" _, ext = os.path.splitext(config_path) ext_is_yaml = ext == ".yaml" or ext == ".yml" is_dir = os.path.isdir(config_path) exists = os.path.exists(config_path) if not exists and not self.optional: dest = f" at ({config_path})" if config_path != os.path.normpath(path) else "" raise ValueError(f"Required path ({path}) does not exist{dest}") if (exists and not is_dir) or ext_is_yaml: tty.debug(f"Creating SingleFileScope {config_name} for '{config_path}'") return SingleFileScope( config_name, config_path, spack.schema.merged.schema, prefer_modify=self.prefer_modify, included=True, ) if ext and not is_dir: raise ValueError( f"File-based scope does not exist yet: should have a .yaml/.yml extension \ for file scopes, or no extension for directory scopes (currently {ext})" ) # directories are treated as regular ConfigScopes # assign by "default" tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'") return DirectoryConfigScope( config_name, config_path, prefer_modify=self.prefer_modify, included=True ) def _validate_parent_scope(self, parent_scope: ConfigScope): """Validates that a parent scope is a valid configuration object""" # enforced by type checking but those can always be # type: ignore'd assert isinstance(parent_scope, ConfigScope), ( f"Includes must be within a configuration scope (ConfigScope), not {type(parent_scope)}" # noqa: E501 ) assert parent_scope.name.strip(), "Parent scope of an include must have a name" def evaluate_condition(self) -> bool: """Evaluate the include condition: Returns: ``True`` if the include condition is satisfied; else ``False``. """ # circular dependencies import spack.spec return (not self.when) or spack.spec.eval_conditional(self.when) def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: """Instantiate configuration scopes. Args: parent_scope: including scope Returns: configuration scopes for configuration files IF the when condition is satisfied; otherwise, an empty list. Raises: ValueError: the required configuration path does not exist """ raise NotImplementedError("must be implemented in derived classes") @property def paths(self) -> List[str]: """Path(s) associated with the include.""" raise NotImplementedError("must be implemented in derived classes") class IncludePath(OptionalInclude): path: str sha256: str destination: Optional[str] def __init__(self, entry: dict): # circular dependencies import spack.util.path super().__init__(entry) path_override_env_var = entry.get("path_override_env_var", "") if path_override_env_var and path_override_env_var in os.environ: path = os.environ[path_override_env_var] else: path = entry.get("path", "") self.path = spack.util.path.substitute_path_variables(path) self.sha256 = entry.get("sha256", "") self.remote = "sha256" in entry self.destination = None def __repr__(self): return ( f"IncludePath({self.path}, sha256={self.sha256}, " f"when='{self.when}', optional={self.optional})" ) def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: """Instantiate a configuration scope for the included path. Args: parent_scope: including scope Returns: configuration scopes IF the when condition is satisfied; otherwise, an empty list. Raises: ConfigFileError: unable to access remote configuration file ValueError: included path has an unsupported URL scheme, is required but does not exist; configuration stage directory argument is missing """ if not self.evaluate_condition(): tty.debug(f"Include condition is not satisfied in {self}") return [] if self._scopes: tty.debug(f"Using existing scopes: {[s.name for s in self._scopes]}") return self._scopes # An absolute path does not need a local base directory. if os.path.isabs(self.path): tty.debug(f"The included path ({self}) is absolute so needs no base directory") base = None else: base = self.base_directory(self.path, parent_scope) # Make sure to use a proper working directory when obtaining the local # path for a local (or remote) file. tty.debug(f"Local base directory for {self.path} is {base}") config_path = rfc_util.local_path(self.path, self.sha256, base) assert config_path self.destination = config_path scope = self._scope(self.path, self.destination, parent_scope) if scope is not None: self._scopes = [scope] return self._scopes @property def paths(self) -> List[str]: """Path(s) associated with the include.""" return [self.path] class GitIncludePaths(OptionalInclude): git: str branch: str commit: str tag: str _paths: List[str] destination: Optional[str] def __init__(self, entry: dict): # circular dependencies import spack.util.path super().__init__(entry) self.git = spack.util.path.substitute_path_variables(entry.get("git", "")) self.branch = entry.get("branch", "") self.commit = entry.get("commit", "") self.tag = entry.get("tag", "") self._paths = [ spack.util.path.substitute_path_variables(path) for path in entry.get("paths", []) ] self.destination = None self.remote = True if not self.branch and not self.commit and not self.tag: raise spack.error.ConfigError( "Git include paths ({self}) must specify one or more of: branch, commit, tag" ) if not self._paths: raise spack.error.ConfigError( "Git include paths ({self}) must include one or more relative paths" ) def __repr__(self): if self.branch: identifier = f"branch={self.branch}" else: identifier = f"commit={self.commit}, tag={self.tag}" return ( f"GitIncludePaths('{self.name}', {self.git}, paths={self._paths}, " f"{identifier}, when='{self.when}', optional={self.optional})" ) def _clone(self, parent_scope: ConfigScope) -> Optional[str]: """Clone the repository. Args: parent_scope: enclosing scope Returns: destination path if cloned or ``None`` """ if self.fetched(): tty.debug(f"Repository ({self.git}) already cloned to {self.destination}") return self.destination # environment includes should be located under the environment destination = self.base_directory(self.git, parent_scope) assert destination, f"{self} requires a local cache directory" tty.debug(f"Cloning {self.git} into {destination}") with filesystem.working_dir(destination, create=True): if not os.path.exists(".git"): try: tty.debug("Initializing the git repository") spack.util.git.init_git_repo(self.git) except spack.util.executable.ProcessError as e: raise spack.error.ConfigError( f"Unable to initialize repository ({self.git}) under {destination}: {e}" ) try: if self.commit: tty.debug(f"Pulling commit {self.commit}") spack.util.git.pull_checkout_commit(self.commit) elif self.tag: tty.debug(f"Pulling tag {self.tag}") spack.util.git.pull_checkout_tag(self.tag) elif self.branch: # if the branch already exists we should use the # previously configured remote tty.debug(f"Pulling branch {self.branch}") try: git = spack.util.git.git(required=True) output = git("config", f"branch.{self.branch}.remote", output=str) remote = output.strip() except spack.util.executable.ProcessError: remote = "origin" spack.util.git.pull_checkout_branch(self.branch, remote=remote) else: raise spack.error.ConfigError(f"Missing or unsupported options in {self}") except spack.util.executable.ProcessError as e: raise spack.error.ConfigError( f"Unable to check out repository ({self}) in {destination}: {e}" ) # only set the destination on successful clone/checkout self.destination = destination return self.destination def fetched(self) -> bool: return bool(self.destination) and os.path.exists( os.path.join(self.destination, ".git") # type: ignore[arg-type] ) def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]: """Instantiate configuration scopes for the included paths. Args: parent_scope: including scope Returns: configuration scopes IF the when condition is satisfied; otherwise, an empty list. Raises: ConfigFileError: unable to access remote configuration file(s) ValueError: included path has an unsupported URL scheme, is required but does not exist; configuration stage directory argument is missing """ if not self.evaluate_condition(): tty.debug(f"Include condition is not satisfied in {self}") return [] if self._scopes: tty.debug(f"Using existing scopes: {[s.name for s in self._scopes]}") return self._scopes destination = self._clone(parent_scope) if not destination: raise spack.error.ConfigError(f"Unable to cache the include: {self}") scopes: List[ConfigScope] = [] for path in self.paths: config_path = str(pathlib.Path(destination) / path) scope = self._scope(path, config_path, parent_scope) if scope is not None: scopes.append(scope) # cache the scopes if successfully able to process all of them if scopes: self._scopes = scopes return self._scopes @property def paths(self) -> List[str]: """Path(s) associated with the include.""" return self._paths def included_path(entry: Union[str, pathlib.Path, dict]) -> Union[IncludePath, GitIncludePaths]: """Convert the included paths entry into the appropriate optional include. Args: entry: include configuration entry Returns: converted entry, where an empty ``when`` means the path is not conditionally included """ if isinstance(entry, (str, pathlib.Path)): return IncludePath({"path": str(entry)}) if entry.get("path", ""): return IncludePath(entry) return GitIncludePaths(entry) def paths_from_includes(includes: List[Union[str, dict]]) -> List[str]: """The path(s) from the configured includes. Args: includes: include configuration information Returns: list of path or an empty list if there are none """ paths = [] for entry in includes: include = included_path(entry) paths.extend(include.paths) return paths def config_paths_from_entry_points() -> List[Tuple[str, str]]: """Load configuration paths from entry points A python package can register entry point metadata so that Spack can find its configuration by adding the following to the project's pyproject.toml: .. code-block:: toml [project.entry-points."spack.config"] baz = "baz:get_spack_config_path" The function ``get_spack_config_path`` returns the path to the package's spack configuration scope """ config_paths: List[Tuple[str, str]] = [] for entry_point in lang.get_entry_points(group="spack.config"): hook = entry_point.load() if callable(hook): config_path = hook() if config_path and os.path.exists(config_path): config_paths.append(("plugin-%s" % entry_point.name, str(config_path))) return config_paths def create_incremental() -> Generator[Configuration, None, None]: """Singleton Configuration instance. This constructs one instance associated with this module and returns it. It is bundled inside a function so that configuration can be initialized lazily. """ # Default scopes are builtins and the default scope within the Spack instance. # These are versioned with Spack and can be overridden by systems, sites or user scopes. cfg = create_from( (ConfigScopePriority.DEFAULTS, InternalConfigScope("_builtin", CONFIG_DEFAULTS)), (ConfigScopePriority.DEFAULTS, DirectoryConfigScope(*CONFIGURATION_DEFAULTS_PATH)), ) yield cfg # Initial topmost scope is spack (the config scope in the spack instance). # It includes the user, site, and system scopes. Environments and command # line scopes go above this. configuration_paths = [("spack", os.path.join(spack.paths.etc_path))] # Python packages can register configuration scopes via entry_points configuration_paths.extend(config_paths_from_entry_points()) # add each scope for name, path in configuration_paths: # yield the config incrementally so that each config level's init code can get # data from the one below. This can be tricky, but it enables us to have a # single unified config system. # # TODO: think about whether we want to restrict what types of config can be used # at each level. e.g., we may want to just more forcibly disallow remote # config (which uses ssl and other config options) for some of the scopes, # to make the bootstrap issues more explicit, even if allowing config scope # init to reference lower scopes is more flexible. yield from cfg.push_scope_incremental( DirectoryConfigScope(name, path), priority=ConfigScopePriority.CONFIG_FILES ) def create() -> Configuration: """Create a configuration using create_incremental(), return the last yielded result.""" return list(create_incremental())[-1] #: This is the singleton configuration instance for Spack. CONFIG = cast(Configuration, lang.Singleton(create_incremental)) def add_from_file(filename: str, scope: Optional[str] = None) -> None: """Add updates to a config from a filename""" # Extract internal attributes, if we are dealing with an environment data = read_config_file(filename) if data is None: return if spack.schema.env.TOP_LEVEL_KEY in data: data = data[spack.schema.env.TOP_LEVEL_KEY] msg = ( "unexpected 'None' value when retrieving configuration. " "Please submit a bug-report at https://github.com/spack/spack/issues" ) assert data is not None, msg # update all sections from config dict # We have to iterate on keys to keep overrides from the file for section in data.keys(): if section in SECTION_SCHEMAS.keys(): # Special handling for compiler scope difference # Has to be handled after we choose a section if scope is None: scope = default_modify_scope(section) value = data[section] existing = get(section, scope=scope) new = spack.schema.merge_yaml(existing, value) # We cannot call config.set directly (set is a type) CONFIG.set(section, new, scope) def add(fullpath: str, scope: Optional[str] = None) -> None: """Add the given configuration to the specified config scope. Add accepts a path. If you want to add from a filename, use add_from_file""" components = process_config_path(fullpath) has_existing_value = True path = "" override = False value = components[-1] if not isinstance(value, syaml.syaml_str): value = syaml.load_config(value) for idx, name in enumerate(components[:-1]): # First handle double colons in constructing path colon = "::" if override else ":" if path else "" path += colon + name if getattr(name, "override", False): override = True else: override = False # Test whether there is an existing value at this level existing = get(path, scope=scope) if existing is None: has_existing_value = False # We've nested further than existing config, so we need the # type information for validation to know how to handle bare # values appended to lists. existing = get_valid_type(path) # construct value from this point down for component in reversed(components[idx + 1 : -1]): value: Dict[str, str] = {component: value} # type: ignore[no-redef] break if override: path += "::" if has_existing_value: existing = get(path, scope=scope) # append values to lists if isinstance(existing, list) and not isinstance(value, list): value: List[str] = [value] # type: ignore[no-redef] # merge value into existing new = spack.schema.merge_yaml(existing, value) CONFIG.set(path, new, scope) def get(path: str, default: Optional[Any] = None, scope: Optional[str] = None) -> Any: """Module-level wrapper for ``Configuration.get()``.""" return CONFIG.get(path, default, scope) _set = set #: save this before defining set -- maybe config.set was ill-advised :) def set(path: str, value: Any, scope: Optional[str] = None) -> None: """Convenience function for setting single values in config files. Accepts the path syntax described in ``get()``. """ result = CONFIG.set(path, value, scope) return result def scopes() -> lang.PriorityOrderedMapping[str, ConfigScope]: """Convenience function to get list of configuration scopes.""" return CONFIG.scopes def writable_scopes() -> List[ConfigScope]: """Return list of writable scopes. Higher-priority scopes come first in the list.""" scopes = [x for x in CONFIG.scopes.values() if x.writable] scopes.reverse() return scopes def existing_scopes() -> List[ConfigScope]: """Return list of existing scopes. Scopes where Spack is aware of said scope, and the scope has a representation on the filesystem or are internal scopes. Higher-priority scopes come first in the list.""" scopes = [x for x in CONFIG.scopes.values() if x.exists] scopes.reverse() return scopes def writable_scope_names() -> List[str]: return list(x.name for x in writable_scopes()) def existing_scope_names() -> List[str]: return list(x.name for x in existing_scopes()) def matched_config(cfg_path: str) -> List[Tuple[str, Any]]: return [(scope, get(cfg_path, scope=scope)) for scope in writable_scope_names()] def change_or_add( section_name: str, find_fn: Callable[[str], bool], update_fn: Callable[[str], None] ) -> None: """Change or add a subsection of config, with additional logic to select a reasonable scope where the change is applied. Search through config scopes starting with the highest priority: the first matching a criteria (determined by ``find_fn``) is updated; if no such config exists, find the first config scope that defines any config for the named section; if no scopes define any related config, then update the highest-priority config scope. """ configs_by_section = matched_config(section_name) found = False for scope, section in configs_by_section: found = find_fn(section) if found: break if found: update_fn(section) CONFIG.set(section_name, section, scope=scope) return # If no scope meets the criteria specified by ``find_fn``, # then look for a scope that has any content (for the specified # section name) for scope, section in configs_by_section: if section: update_fn(section) found = True break if found: CONFIG.set(section_name, section, scope=scope) return # If no scopes define any config for the named section, then # modify the highest-priority scope. scope, section = configs_by_section[0] update_fn(section) CONFIG.set(section_name, section, scope=scope) def update_all(section_name: str, change_fn: Callable[[str], bool]) -> None: """Change a config section, which may have details duplicated across multiple scopes. """ configs_by_section = matched_config("develop") for scope, section in configs_by_section: modified = change_fn(section) if modified: CONFIG.set(section_name, section, scope=scope) def _validate_section_name(section: str) -> None: """Exit if the section is not a valid section.""" if section not in SECTION_SCHEMAS: raise ConfigSectionError( f"Invalid config section: '{section}'. Options are: {' '.join(SECTION_SCHEMAS.keys())}" ) def validate( data: YamlConfigDict, schema: YamlConfigDict, filename: Optional[str] = None ) -> YamlConfigDict: """Validate data read in from a Spack YAML file. Arguments: data: data read from a Spack YAML file schema: jsonschema to validate data This leverages the line information (start_mark, end_mark) stored on Spack YAML structures. """ try: spack.schema.Validator(schema).validate(data) except jsonschema.ValidationError as e: if hasattr(e.instance, "lc"): line_number = e.instance.lc.line + 1 else: line_number = None raise ConfigFormatError(e, data, filename, line_number) from e # return the validated data so that we can access the raw data # mostly relevant for environments return data def read_config_file( path: str, schema: Optional[YamlConfigDict] = None ) -> Optional[YamlConfigDict]: """Read a YAML configuration file. User can provide a schema for validation. If no schema is provided, we will infer the schema from the top-level key.""" # Dev: Inferring schema and allowing it to be provided directly allows us # to preserve flexibility in calling convention (don't need to provide # schema when it's not necessary) while allowing us to validate against a # known schema when the top-level key could be incorrect. try: with open(path, encoding="utf-8") as f: tty.debug(f"Reading config from file {path}") data = syaml.load_config(f) if data: if schema is None: key = next(iter(data)) schema = _ALL_SCHEMAS[key] validate(data, schema) return data except FileNotFoundError: # Ignore nonexistent files. tty.debug(f"Skipping nonexistent config path {path}", level=3) return None except OSError as e: raise ConfigFileError(f"Path is not a file or is not readable: {path}: {str(e)}") from e except StopIteration as e: raise ConfigFileError(f"Config file is empty or is not a valid YAML dict: {path}") from e except syaml.SpackYAMLError as e: raise ConfigFileError(str(e)) from e def _mark_internal(data, name): """Add a simple name mark to raw YAML/JSON data. This is used by `spack config blame` to show where config lines came from. """ if isinstance(data, dict): d = syaml.syaml_dict( (_mark_internal(k, name), _mark_internal(v, name)) for k, v in data.items() ) elif isinstance(data, list): d = syaml.syaml_list(_mark_internal(e, name) for e in data) else: d = syaml.syaml_type(data) if syaml.markable(d): d._start_mark = syaml.name_mark(name) d._end_mark = syaml.name_mark(name) return d def get_valid_type(path): """Returns an instance of a type that will pass validation for path. The instance is created by calling the constructor with no arguments. If multiple types will satisfy validation for data at the configuration path given, the priority order is ``list``, ``dict``, ``str``, ``bool``, ``int``, ``float``. """ types = { "array": list, "object": syaml.syaml_dict, "string": str, "boolean": bool, "integer": int, "number": float, } components = process_config_path(path) section = components[0] # Use None to construct the test data test_data = None for component in reversed(components): test_data = {component: test_data} try: validate(test_data, SECTION_SCHEMAS[section]) except (ConfigFormatError, AttributeError) as e: jsonschema_error = e.validation_error if jsonschema_error.validator == "type": return types[jsonschema_error.validator_value]() elif jsonschema_error.validator in ("anyOf", "oneOf"): for subschema in jsonschema_error.validator_value: schema_type = subschema.get("type") if schema_type is not None: return types[schema_type]() else: return type(None) raise spack.error.ConfigError(f"Cannot determine valid type for path '{path}'.") def remove_yaml(dest, source): """UnMerges source from dest; entries in source take precedence over dest. This routine may modify dest and should be assigned to dest, in case dest was None to begin with, e.g.:: dest = remove_yaml(dest, source) In the result, elements from lists from ``source`` will not appear as elements of lists from ``dest``. Likewise, when iterating over keys or items in merged ``OrderedDict`` objects, keys from ``source`` will not appear as keys in ``dest``. Config file authors can optionally end any attribute in a dict with ``::`` instead of ``:``, and the key will remove the entire section from ``dest`` """ def they_are(t): return isinstance(dest, t) and isinstance(source, t) # If source is None, overwrite with source. if source is None: return dest # Source list is prepended (for precedence) if they_are(list): # Make sure to copy ruamel comments dest[:] = [x for x in dest if x not in source] return dest # Source dict is merged into dest. elif they_are(dict): for sk, sv in source.items(): # always remove the dest items. Python dicts do not overwrite # keys on insert, so this ensures that source keys are copied # into dest along with mark provenance (i.e., file/line info). unmerge = sk in dest old_dest_value = dest.pop(sk, None) if unmerge and not spack.schema.override(sk): dest[sk] = remove_yaml(old_dest_value, sv) return dest # If we reach here source and dest are either different types or are # not both lists or dicts: replace with source. return dest class ConfigPath: quoted_string = "(?:\"[^\"]+\")|(?:'[^']+')" unquoted_string = "[^:'\"]+" element = rf"(?:(?:{quoted_string})|(?:{unquoted_string}))" next_key_pattern = rf"({element}[+-]?)(?:\:|$)" @staticmethod def _split_front(string, extract): m = re.match(extract, string) if not m: return None, None token = m.group(1) return token, string[len(token) :] @staticmethod def _validate(path): """Example valid config paths: x:y:z x:"y":z x:y+:z x:y::z x:y+::z x:y: x:y:: """ first_key, path = ConfigPath._split_front(path, ConfigPath.next_key_pattern) if not first_key: raise ValueError(f"Config path does not start with a parse-able key: {path}") path_elements = [first_key] path_index = 1 while path: separator, path = ConfigPath._split_front(path, r"(\:+)") if not separator: raise ValueError(f"Expected separator for {path}") path_elements[path_index - 1] += separator if not path: break element, remainder = ConfigPath._split_front(path, ConfigPath.next_key_pattern) if not element: # If we can't parse something as a key, then it must be a # value (if it's valid). try: syaml.load_config(path) except syaml.SpackYAMLError as e: raise ValueError( "Remainder of path is not a valid key" f" and does not parse as a value {path}" ) from e element = path path = None # The rest of the path was consumed into the value else: path = remainder path_elements.append(element) path_index += 1 return path_elements @staticmethod def process(path): result = [] quote = "['\"]" seen_override_in_path = False path_elements = ConfigPath._validate(path) last_element_idx = len(path_elements) - 1 for i, element in enumerate(path_elements): override = False append = False prepend = False quoted = False if element.endswith("::") or (element.endswith(":") and i == last_element_idx): if seen_override_in_path: raise syaml.SpackYAMLError( "Meaningless second override indicator `::' in path `{0}'".format(path), "" ) override = True seen_override_in_path = True element = element.rstrip(":") if element.endswith("+"): prepend = True elif element.endswith("-"): append = True element = element.rstrip("+-") if re.match(f"^{quote}", element): quoted = True element = element.strip("'\"") if append or prepend or override or quoted: element = syaml.syaml_str(element) if append: element.append = True if prepend: element.prepend = True if override: element.override = True result.append(element) return result def process_config_path(path: str) -> List[str]: """Process a path argument to config.set() that may contain overrides (``::`` or trailing ``:``) Colons will be treated as static strings if inside of quotes, e.g. ``this:is:a:path:'value:with:colon'`` will yield: .. code-block:: text [this, is, a, path, value:with:colon] The path may consist only of keys (e.g. for a ``get``) or may end in a value. Keys are always strings: if a user encloses a key in quotes, the quotes should be removed. Values with quotes should be treated as strings, but without quotes, may be parsed as a different yaml object (e.g. ``'{}'`` is a dict, but ``'"{}"'`` is a string). This function does not know whether the final element of the path is a key or value, so: * It must strip the quotes, in case it is a key (so we look for ``key`` and not ``"key"``) * It must indicate somehow that the quotes were stripped, in case it is a value (so that we don't process ``"{}"`` as a YAML dict) Therefore, all elements with quotes are stripped, and then also converted to ``syaml_str`` (if treating the final element as a value, the caller should not parse it in this case). """ return ConfigPath.process(path) # # Settings for commands that modify configuration # def default_modify_scope(section: str = "config") -> str: """Return the config scope that commands should modify by default. Commands that modify configuration by default modify the *highest* priority scope. Arguments: section (bool): Section for which to get the default scope. """ return CONFIG.highest_precedence_scope().name def _update_in_memory(data: YamlConfigDict, section: str) -> bool: """Update the format of the configuration data in memory. This function assumes the section is valid (i.e. validation is responsibility of the caller) Args: data: configuration data section: section of the configuration to update Returns: True if the data was changed, False otherwise """ return ensure_latest_format_fn(section)(data) def ensure_latest_format_fn(section: str) -> Callable[[YamlConfigDict], bool]: """Return a function that takes a config dictionary and update it to the latest format. The function returns True iff there was any update. Args: section: section of the configuration e.g. "packages", "config", etc. """ # Every module we need is already imported at the top level, so getattr should not raise return getattr(getattr(spack.schema, section), "update", lambda _: False) @contextlib.contextmanager def use_configuration( *scopes_or_paths: Union[ScopeWithOptionalPriority, str], ) -> Generator[Configuration, None, None]: """Use the configuration scopes passed as arguments within the context manager. This function invalidates caches, and is therefore very slow. Args: *scopes_or_paths: scope objects or paths to be used Returns: Configuration object associated with the scopes passed as arguments """ global CONFIG # Normalize input and construct a Configuration object configuration = create_from(*scopes_or_paths) CONFIG.clear_caches(), configuration.clear_caches() saved_config, CONFIG = CONFIG, configuration try: yield configuration finally: CONFIG = saved_config def _normalize_input(entry: Union[ScopeWithOptionalPriority, str]) -> ScopeWithPriority: if isinstance(entry, tuple): return entry default_priority = ConfigScopePriority.CONFIG_FILES if isinstance(entry, ConfigScope): return default_priority, entry # Otherwise we need to construct it path = os.path.normpath(entry) assert os.path.isdir(path), f'"{path}" must be a directory' name = os.path.basename(path) return default_priority, DirectoryConfigScope(name, path) @lang.memoized def create_from(*scopes_or_paths: Union[ScopeWithOptionalPriority, str]) -> Configuration: """Creates a configuration object from the scopes passed in input. Args: *scopes_or_paths: either a tuple of (priority, ConfigScope), or a ConfigScope, or a string If priority is not given, it is assumed to be ConfigScopePriority.CONFIG_FILES. If a string is given, a DirectoryConfigScope is created from it. Examples: >>> builtin_scope = InternalConfigScope("_builtin", {"config": {"build_jobs": 1}}) >>> cl_scope = InternalConfigScope("command_line", {"config": {"build_jobs": 10}}) >>> cfg = create_from( ... (ConfigScopePriority.COMMAND_LINE, cl_scope), ... (ConfigScopePriority.BUILTIN, builtin_scope) ... ) """ scopes_with_priority = [_normalize_input(x) for x in scopes_or_paths] result = Configuration() for priority, scope in scopes_with_priority: result.push_scope(scope, priority=priority) return result def determine_number_of_jobs( *, parallel: bool = False, max_cpus: int = cpus_available(), config: Optional[Configuration] = None, ) -> int: """ Packages that require sequential builds need 1 job. Otherwise we use the number of jobs set on the command line. If not set, then we use the config defaults (which is usually set through the builtin config scope), but we cap to the number of CPUs available to avoid oversubscription. Parameters: parallel: true when package supports parallel builds max_cpus: maximum number of CPUs to use (defaults to cpus_available()) config: configuration object (defaults to global config) """ if not parallel: return 1 cfg = config or CONFIG # Command line overrides all try: command_line = cfg.get("config:build_jobs", default=None, scope="command_line") if command_line is not None: return command_line except ValueError: pass return min(max_cpus, cfg.get("config:build_jobs", 16)) class ConfigSectionError(spack.error.ConfigError): """Error for referring to a bad config section name in a configuration.""" class ConfigFileError(spack.error.ConfigError): """Issue reading or accessing a configuration file.""" class ConfigFormatError(spack.error.ConfigError): """Raised when a configuration format does not match its schema.""" def __init__( self, validation_error, data: YamlConfigDict, filename: Optional[str] = None, line: Optional[int] = None, ) -> None: # spack yaml has its own file/line marks -- try to find them # we prioritize these over the inputs self.validation_error = validation_error mark = self._get_mark(validation_error, data) if mark: filename = mark.name line = mark.line + 1 self.filename = filename # record this for ruamel.yaml # construct location location = "" if filename: location = f"{filename}" if line is not None: location += f":{line:d}" message = f"{location}: {validation_error.message}" super().__init__(message) def _get_mark(self, validation_error, data): """Get the file/line mark for a validation error from a Spack YAML file.""" # Try various places, starting with instance and parent for obj in (validation_error.instance, validation_error.parent): mark = get_mark_from_yaml_data(obj) if mark: return mark def get_path(path, data): if path: return get_path(path[1:], data[path[0]]) else: return data # Try really hard to get the parent (which sometimes is not # set) This digs it out of the validated structure if it's not # on the validation_error. path = validation_error.path if path: parent = get_path(list(path)[:-1], data) if path[-1] in parent: if isinstance(parent, dict): keylist = list(parent.keys()) elif isinstance(parent, list): keylist = parent idx = keylist.index(path[-1]) mark = getattr(keylist[idx], "_start_mark", None) if mark: return mark # give up and return None if nothing worked return None class RecursiveIncludeError(spack.error.SpackError): """Too many levels of recursive includes.""" ================================================ FILE: lib/spack/spack/container/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Package that provides functions and classes to generate container recipes from a Spack environment """ import warnings import spack.vendor.jsonschema import spack.environment as ev import spack.schema.env as env import spack.util.spack_yaml as syaml from .writers import recipe __all__ = ["validate", "recipe"] def validate(configuration_file): """Validate a Spack environment YAML file that is being used to generate a recipe for a container. Since a few attributes of the configuration must have specific values for the container recipe, this function returns a sanitized copy of the configuration in the input file. If any modification is needed, a warning will be issued. Args: configuration_file (str): path to the Spack environment YAML file Returns: A sanitized copy of the configuration stored in the input file """ with open(configuration_file, encoding="utf-8") as f: config = syaml.load(f) # Ensure we have a "container" attribute with sensible defaults set env_dict = config[ev.TOP_LEVEL_KEY] env_dict.setdefault( "container", {"format": "docker", "images": {"os": "ubuntu:22.04", "spack": "develop"}} ) env_dict["container"].setdefault("format", "docker") env_dict["container"].setdefault("images", {"os": "ubuntu:22.04", "spack": "develop"}) # Remove attributes that are not needed / allowed in the # container recipe for subsection in ("cdash", "gitlab_ci", "modules"): if subsection in env_dict: msg = ( 'the subsection "{0}" in "{1}" is not used when generating' " container recipes and will be discarded" ) warnings.warn(msg.format(subsection, configuration_file)) env_dict.pop(subsection) # Set the default value of the concretization strategy to unify and # warn if the user explicitly set another value env_dict.setdefault("concretizer", {"unify": True}) if env_dict["concretizer"]["unify"] is not True: warnings.warn( '"concretizer:unify" is not set to "true", which means the ' "generated image may contain different variants of the same " 'packages. Set to "true" to get a consistent set of packages.' ) # Check if the install tree was explicitly set to a custom value and warn # that it will be overridden environment_config = env_dict.get("config", {}) if environment_config.get("install_tree", None): msg = ( 'the "config:install_tree" attribute has been set explicitly ' "and will be overridden in the container image" ) warnings.warn(msg) # Likewise for the view environment_view = env_dict.get("view", None) if environment_view: msg = ( 'the "view" attribute has been set explicitly ' "and will be overridden in the container image" ) warnings.warn(msg) spack.vendor.jsonschema.validate(config, schema=env.schema) return config ================================================ FILE: lib/spack/spack/container/images.json ================================================ { "images": { "alpine:3": { "bootstrap": { "template": "container/alpine_3.dockerfile" }, "os_package_manager": "apk" }, "amazonlinux:2": { "bootstrap": { "template": "container/amazonlinux_2.dockerfile" }, "os_package_manager": "yum_amazon" }, "fedora:40": { "bootstrap": { "template": "container/fedora.dockerfile", "image": "docker.io/fedora:40" }, "os_package_manager": "dnf", "build": "spack/fedora40", "final": { "image": "docker.io/fedora:40" } }, "fedora:39": { "bootstrap": { "template": "container/fedora.dockerfile", "image": "docker.io/fedora:39" }, "os_package_manager": "dnf", "build": "spack/fedora39", "final": { "image": "docker.io/fedora:39" } }, "rockylinux:9": { "bootstrap": { "template": "container/rockylinux_9.dockerfile", "image": "docker.io/rockylinux:9" }, "os_package_manager": "dnf_epel", "build": "spack/rockylinux9", "final": { "image": "docker.io/rockylinux:9" } }, "rockylinux:8": { "bootstrap": { "template": "container/rockylinux_8.dockerfile", "image": "docker.io/rockylinux:8" }, "os_package_manager": "dnf_epel", "build": "spack/rockylinux8", "final": { "image": "docker.io/rockylinux:8" } }, "almalinux:9": { "bootstrap": { "template": "container/almalinux_9.dockerfile", "image": "quay.io/almalinuxorg/almalinux:9" }, "os_package_manager": "dnf_epel", "build": "spack/almalinux9", "final": { "image": "quay.io/almalinuxorg/almalinux:9" } }, "almalinux:8": { "bootstrap": { "template": "container/almalinux_8.dockerfile", "image": "quay.io/almalinuxorg/almalinux:8" }, "os_package_manager": "dnf_epel", "build": "spack/almalinux8", "final": { "image": "quay.io/almalinuxorg/almalinux:8" } }, "centos:stream9": { "bootstrap": { "template": "container/centos_stream9.dockerfile", "image": "quay.io/centos/centos:stream9" }, "os_package_manager": "dnf_epel", "build": "spack/centos-stream9", "final": { "image": "quay.io/centos/centos:stream9" } }, "opensuse/leap:15": { "bootstrap": { "template": "container/leap-15.dockerfile" }, "os_package_manager": "zypper", "build": "spack/leap15", "final": { "image": "opensuse/leap:latest" } }, "nvidia/cuda:11.2.1": { "bootstrap": { "template": "container/cuda_11_2_1.dockerfile", "image": "nvidia/cuda:11.2.1-devel" }, "final": { "image": "nvidia/cuda:11.2.1-base" }, "os_package_manager": "apt" }, "ubuntu:24.04": { "bootstrap": { "template": "container/ubuntu_2404.dockerfile" }, "os_package_manager": "apt", "build": "spack/ubuntu-noble" }, "ubuntu:22.04": { "bootstrap": { "template": "container/ubuntu_2204.dockerfile" }, "os_package_manager": "apt", "build": "spack/ubuntu-jammy" }, "ubuntu:20.04": { "bootstrap": { "template": "container/ubuntu_2004.dockerfile" }, "build": "spack/ubuntu-focal", "os_package_manager": "apt" } }, "os_package_managers": { "apk": { "update": "apk update", "install": "apk add --no-cache", "clean": "true" }, "apt": { "update": "apt-get -yqq update && apt-get -yqq upgrade", "install": "apt-get -yqq install", "clean": "rm -rf /var/lib/apt/lists/*" }, "dnf": { "update": "dnf update -y", "install": "dnf install -y", "clean": "rm -rf /var/cache/dnf && dnf clean all" }, "dnf_epel": { "update": "dnf update -y && dnf install -y epel-release && dnf update -y", "install": "dnf install -y", "clean": "rm -rf /var/cache/dnf && dnf clean all" }, "yum": { "update": "yum update -y && yum install -y epel-release && yum update -y", "install": "yum install -y", "clean": "rm -rf /var/cache/yum && yum clean all" }, "yum_amazon": { "update": "yum update -y && amazon-linux-extras install epel -y", "install": "yum install -y", "clean": "rm -rf /var/cache/yum && yum clean all" }, "zypper": { "update": "zypper update -y", "install": "zypper install -y", "clean": "rm -rf /var/cache/zypp && zypper clean -a" } } } ================================================ FILE: lib/spack/spack/container/images.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Manages the details on the images used in the various stages.""" import json import os import shlex import sys import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.util.git #: Global variable used to cache in memory the content of images.json _data = None def data(): """Returns a dictionary with the static data on the images. The dictionary is read from a JSON file lazily the first time this function is called. """ global _data if not _data: json_dir = os.path.abspath(os.path.dirname(__file__)) json_file = os.path.join(json_dir, "images.json") with open(json_file, encoding="utf-8") as f: _data = json.load(f) return _data def build_info(image, spack_version): """Returns the name of the build image and its tag. Args: image (str): image to be used at run-time. Should be of the form : e.g. ``"ubuntu:18.04"`` spack_version (str): version of Spack that we want to use to build Returns: A tuple with (image_name, image_tag) for the build image """ # Don't handle error here, as a wrong image should have been # caught by the JSON schema image_data = data()["images"][image] build_image = image_data.get("build", None) if not build_image: return None, None return build_image, spack_version def os_package_manager_for(image): """Returns the name of the OS package manager for the image passed as argument. Args: image (str): image to be used at run-time. Should be of the form : e.g. ``"ubuntu:18.04"`` Returns: Name of the package manager, e.g. ``"apt"`` or ``"yum"`` """ name = data()["images"][image]["os_package_manager"] return name def all_bootstrap_os(): """Return a list of all the OS that can be used to bootstrap Spack""" return list(data()["images"]) def commands_for(package_manager): """Returns the commands used to update system repositories, install system packages and clean afterwards. Args: package_manager (str): package manager to be used Returns: A tuple of (update, install, clean) commands. """ info = data()["os_package_managers"][package_manager] return info["update"], info["install"], info["clean"] def bootstrap_template_for(image): return data()["images"][image]["bootstrap"]["template"] def _verify_ref(url, ref, enforce_sha): # Do a checkout in a temporary directory msg = 'Cloning "{0}" to verify ref "{1}"'.format(url, ref) tty.info(msg, stream=sys.stderr) git = spack.util.git.git(required=True) with fs.temporary_dir(): git("clone", "-q", url, ".") sha = git( "rev-parse", "-q", ref + "^{commit}", output=str, error=os.devnull, fail_on_error=False ) if git.returncode: msg = '"{0}" is not a valid reference for "{1}"' raise RuntimeError(msg.format(sha, url)) if enforce_sha: ref = sha.strip() return ref def checkout_command(url, ref, enforce_sha, verify): """Return the checkout command to be used in the bootstrap phase. Args: url (str): url of the Spack repository ref (str): either a branch name, a tag or a commit sha enforce_sha (bool): if true turns every verify (bool): """ url = url or "https://github.com/spack/spack.git" ref = ref or "develop" enforce_sha, verify = bool(enforce_sha), bool(verify) # If we want to enforce a sha or verify the ref we need # to checkout the repository locally if enforce_sha or verify: ref = _verify_ref(url, ref, enforce_sha) return " && ".join( [ "git init --quiet", f"git remote add origin {shlex.quote(url)}", f"git fetch --depth=1 origin {shlex.quote(ref)}", "git checkout --detach FETCH_HEAD", ] ) ================================================ FILE: lib/spack/spack/container/writers.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Writers for different kind of recipes and related convenience functions. """ import copy import shlex from collections import namedtuple from typing import Optional import spack.vendor.jsonschema import spack.environment as ev import spack.error import spack.schema.env import spack.tengine as tengine import spack.util.spack_yaml as syaml from .images import ( bootstrap_template_for, build_info, checkout_command, commands_for, data, os_package_manager_for, ) #: Caches all the writers that are currently supported _writer_factory = {} def writer(name): """Decorator to register a factory for a recipe writer. Each factory should take a configuration dictionary and return a properly configured writer that, when called, prints the corresponding recipe. """ def _decorator(factory): _writer_factory[name] = factory return factory return _decorator def create(configuration, last_phase=None): """Returns a writer that conforms to the configuration passed as input. Args: configuration (dict): how to generate the current recipe last_phase (str): last phase to be printed or None to print them all """ name = configuration[ev.TOP_LEVEL_KEY]["container"]["format"] return _writer_factory[name](configuration, last_phase) def recipe(configuration, last_phase=None): """Returns a recipe that conforms to the configuration passed as input. Args: configuration (dict): how to generate the current recipe last_phase (str): last phase to be printed or None to print them all """ return create(configuration, last_phase)() def _stage_base_images(images_config): """Return a tuple with the base images to be used at the various stages. Args: images_config (dict): configuration under container:images """ # If we have custom base images, just return them verbatim. build_stage = images_config.get("build", None) if build_stage: final_stage = images_config["final"] return None, build_stage, final_stage # Check the operating system: this will be the base of the bootstrap # stage, if there, and of the final stage. operating_system = images_config.get("os", None) # Check the OS is mentioned in the internal data stored in a JSON file images_json = data()["images"] if not any(os_name == operating_system for os_name in images_json): msg = 'invalid operating system name "{0}". [Allowed values are {1}]' msg = msg.format(operating_system, ", ".join(data()["images"])) raise ValueError(msg) # Retrieve the build stage spack_info = images_config["spack"] if isinstance(spack_info, dict): build_stage = "bootstrap" else: spack_version = images_config["spack"] image_name, tag = build_info(operating_system, spack_version) build_stage = "bootstrap" if image_name: build_stage = ":".join([image_name, tag]) # Retrieve the bootstrap stage bootstrap_stage = None if build_stage == "bootstrap": bootstrap_stage = images_json[operating_system]["bootstrap"].get("image", operating_system) # Retrieve the final stage final_stage = images_json[operating_system].get("final", {"image": operating_system})["image"] return bootstrap_stage, build_stage, final_stage def _spack_checkout_config(images_config): spack_info = images_config["spack"] url = "https://github.com/spack/spack.git" ref = "develop" resolve_sha, verify = False, False # Config specific values may override defaults if isinstance(spack_info, dict): url = spack_info.get("url", url) ref = spack_info.get("ref", ref) resolve_sha = spack_info.get("resolve_sha", resolve_sha) verify = spack_info.get("verify", verify) else: ref = spack_info return url, ref, resolve_sha, verify class PathContext(tengine.Context): """Generic context used to instantiate templates of recipes that install software in a common location and make it available directly via PATH. """ # Must be set by derived classes template_name: Optional[str] = None def __init__(self, config, last_phase): self.config = config[ev.TOP_LEVEL_KEY] self.container_config = self.config["container"] # Operating system tag as written in the configuration file self.operating_system_key = self.container_config["images"].get("os") # Get base images and verify the OS bootstrap, build, final = _stage_base_images(self.container_config["images"]) self.bootstrap_image = bootstrap self.build_image = build self.final_image = final # Record the last phase self.last_phase = last_phase @tengine.context_property def depfile(self): return self.container_config.get("depfile", False) @tengine.context_property def run(self): """Information related to the run image.""" Run = namedtuple("Run", ["image"]) return Run(image=self.final_image) @tengine.context_property def build(self): """Information related to the build image.""" Build = namedtuple("Build", ["image"]) return Build(image=self.build_image) @tengine.context_property def strip(self): """Whether or not to strip binaries in the image""" return self.container_config.get("strip", True) @tengine.context_property def paths(self): """Important paths in the image""" Paths = namedtuple("Paths", ["environment", "store", "view_parent", "view", "former_view"]) return Paths( environment="/opt/spack-environment", store="/opt/software", view_parent="/opt/views", view="/opt/views/view", former_view="/opt/view", # /opt/view -> /opt/views/view for backward compatibility ) @tengine.context_property def manifest(self): """The spack.yaml file that should be used in the image""" # Copy in the part of spack.yaml prescribed in the configuration file manifest = copy.deepcopy(self.config) manifest.pop("container") # Ensure that a few paths are where they need to be manifest.setdefault("config", syaml.syaml_dict()) manifest["config"]["install_tree"] = {"root": self.paths.store} manifest["view"] = self.paths.view manifest = {"spack": manifest} # Validate the manifest file spack.vendor.jsonschema.validate(manifest, schema=spack.schema.env.schema) return syaml.dump(manifest, default_flow_style=False).strip() @tengine.context_property def os_packages_final(self): """Additional system packages that are needed at run-time.""" try: return self._os_packages_for_stage("final") except Exception as e: msg = f"an error occurred while rendering the 'final' stage of the image: {e}" raise spack.error.SpackError(msg) from e @tengine.context_property def os_packages_build(self): """Additional system packages that are needed at build-time.""" try: return self._os_packages_for_stage("build") except Exception as e: msg = f"an error occurred while rendering the 'build' stage of the image: {e}" raise spack.error.SpackError(msg) from e @tengine.context_property def os_package_update(self): """Whether or not to update the OS package manager cache.""" os_packages = self.container_config.get("os_packages", {}) return os_packages.get("update", True) def _os_packages_for_stage(self, stage): os_packages = self.container_config.get("os_packages", {}) package_list = os_packages.get(stage, None) return self._package_info_from(package_list) def _package_info_from(self, package_list): """Helper method to pack a list of packages with the additional information required by the template. Args: package_list: list of packages Returns: Enough information to know how to update the cache, install a list of packages, and clean in the end. """ if not package_list: return package_list image_config = self.container_config["images"] image = image_config.get("build", None) if image is None: os_pkg_manager = os_package_manager_for(image_config["os"]) else: os_pkg_manager = self._os_pkg_manager() update, install, clean = commands_for(os_pkg_manager) Packages = namedtuple("Packages", ["update", "install", "list", "clean"]) return Packages(update=update, install=install, list=package_list, clean=clean) def _os_pkg_manager(self): try: os_pkg_manager = self.container_config["os_packages"]["command"] except KeyError: msg = ( "cannot determine the OS package manager to use.\n\n\tPlease add an " "appropriate 'os_packages:command' entry to the spack.yaml manifest file\n" ) raise spack.error.SpackError(msg) return os_pkg_manager @tengine.context_property def labels(self): return self.container_config.get("labels", {}) @tengine.context_property def bootstrap(self): """Information related to the build image.""" images_config = self.container_config["images"] bootstrap_recipe = None if self.bootstrap_image: config_args = _spack_checkout_config(images_config) command = checkout_command(*config_args) template_path = bootstrap_template_for(self.operating_system_key) env = tengine.make_environment() context = {"bootstrap": {"image": self.bootstrap_image, "spack_checkout": command}} bootstrap_recipe = env.get_template(template_path).render(**context) Bootstrap = namedtuple("Bootstrap", ["image", "recipe"]) return Bootstrap(image=self.bootstrap_image, recipe=bootstrap_recipe) @tengine.context_property def render_phase(self): render_bootstrap = bool(self.bootstrap_image) render_build = not (self.last_phase == "bootstrap") render_final = self.last_phase in (None, "final") Render = namedtuple("Render", ["bootstrap", "build", "final"]) return Render(bootstrap=render_bootstrap, build=render_build, final=render_final) def __call__(self): """Returns the recipe as a string""" env = tengine.make_environment() template_name = self.container_config.get("template", self.template_name) t = env.get_template(template_name) return t.render(**self.to_dict()) @writer("docker") class DockerContext(PathContext): """Context used to instantiate a Dockerfile""" #: Name of the template used for Dockerfiles template_name = "container/Dockerfile" @tengine.context_property def manifest(self): manifest_str = super().manifest # Docker doesn't support HEREDOC, so we need to resort to # a horrible echo trick to have the manifest in the Dockerfile echoed_lines = [] for idx, line in enumerate(manifest_str.split("\n")): quoted_line = shlex.quote(line) if idx == 0: echoed_lines.append("&& (echo " + quoted_line + " \\") continue echoed_lines.append("&& echo " + quoted_line + " \\") echoed_lines[-1] = echoed_lines[-1].replace(" \\", ")") return "\n".join(echoed_lines) @writer("singularity") class SingularityContext(PathContext): """Context used to instantiate a Singularity definition file""" #: Name of the template used for Singularity definition files template_name = "container/singularity.def" @property def singularity_config(self): return self.container_config.get("singularity", {}) @tengine.context_property def runscript(self): return self.singularity_config.get("runscript", "") @tengine.context_property def startscript(self): return self.singularity_config.get("startscript", "") @tengine.context_property def test(self): return self.singularity_config.get("test", "") @tengine.context_property def help(self): return self.singularity_config.get("help", "") ================================================ FILE: lib/spack/spack/context.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module provides classes used in user and build environment""" from enum import Enum class Context(Enum): """Enum used to indicate the context in which an environment has to be setup: build, run or test.""" BUILD = 1 RUN = 2 TEST = 3 def __str__(self): return ("build", "run", "test")[self.value - 1] @classmethod def from_string(cls, s: str): if s == "build": return Context.BUILD elif s == "run": return Context.RUN elif s == "test": return Context.TEST raise ValueError(f"context should be one of 'build', 'run', 'test', got {s}") ================================================ FILE: lib/spack/spack/cray_manifest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import json import os import traceback import warnings from typing import Any, Dict, Iterable, List, Optional from spack.vendor import jsonschema from spack.vendor.jsonschema import exceptions import spack.cmd import spack.compilers.config import spack.deptypes as dt import spack.error import spack.hash_types as hash_types import spack.llnl.util.tty as tty import spack.platforms import spack.repo import spack.spec import spack.store from spack.detection.path import ExecutablesFinder from spack.schema.cray_manifest import schema as manifest_schema #: Cray systems can store a Spack-compatible description of system #: packages here. default_path = "/opt/cray/pe/cpe-descriptive-manifest/" COMPILER_NAME_TRANSLATION = {"nvidia": "nvhpc", "rocm": "llvm-amdgpu", "clang": "llvm"} def translated_compiler_name(manifest_compiler_name): """ When creating a Compiler object, Spack expects a name matching one of the classes in :mod:`spack.compilers.config`. Names in the Cray manifest may differ; for cases where we know the name refers to a compiler in Spack, this function translates it automatically. This function will raise an error if there is no recorded translation and the name doesn't match a known compiler name. """ if manifest_compiler_name in COMPILER_NAME_TRANSLATION: return COMPILER_NAME_TRANSLATION[manifest_compiler_name] elif manifest_compiler_name in spack.compilers.config.supported_compilers(): return manifest_compiler_name else: raise spack.compilers.config.UnknownCompilerError( f"[CRAY MANIFEST] unknown compiler: {manifest_compiler_name}" ) def compiler_from_entry(entry: dict, *, manifest_path: str) -> Optional[spack.spec.Spec]: # Note that manifest_path is only passed here to compose a # useful warning message when paths appear to be missing. compiler_name = translated_compiler_name(entry["name"]) paths = extract_compiler_paths(entry) # Do a check for missing paths. Note that this isn't possible for # all compiler entries, since their "paths" might actually be # exe names like "cc" that depend on modules being loaded. Cray # manifest entries are always paths though. missing_paths = [x for x in paths if not os.path.exists(x)] if missing_paths: warnings.warn( "Manifest entry refers to nonexistent paths:\n\t" + "\n\t".join(missing_paths) + f"\nfor {entry['name']}@{entry['version']}" + f"\nin {manifest_path}" + "\nPlease report this issue" ) try: compiler_spec = compiler_spec_from_paths(pkg_name=compiler_name, compiler_paths=paths) except spack.error.SpackError as e: tty.debug(f"[CRAY MANIFEST] {e}") return None compiler_spec.constrain( f"platform=linux os={entry['arch']['os']} target={entry['arch']['target']}" ) return compiler_spec def compiler_spec_from_paths(*, pkg_name: str, compiler_paths: Iterable[str]) -> spack.spec.Spec: """Returns the external spec associated with a series of compilers, if any.""" pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) finder = ExecutablesFinder() specs = finder.detect_specs(pkg=pkg_cls, paths=compiler_paths, repo_path=spack.repo.PATH) if not specs or len(specs) > 1: raise CrayCompilerDetectionError( message=f"cannot detect a single {pkg_name} compiler for Cray manifest entry", long_message=f"Analyzed paths are: {', '.join(compiler_paths)}", ) return specs[0] def extract_compiler_paths(entry: Dict[str, Any]) -> List[str]: """Returns the paths to compiler executables, from a dictionary entry in the Cray manifest.""" paths = list(entry["executables"].values()) if "prefix" in entry: paths = [os.path.join(entry["prefix"], relpath) for relpath in paths] return paths def spec_from_entry(entry): arch_str = "" if "arch" in entry: local_platform = spack.platforms.host() spec_platform = entry["arch"]["platform"] # Note that Cray systems are now treated as Linux. Specs # in the manifest which specify "cray" as the platform # should be registered in the DB as "linux" if local_platform.name == "linux" and spec_platform.lower() == "cray": spec_platform = "linux" arch_format = "arch={platform}-{os}-{target}" arch_str = arch_format.format( platform=spec_platform, os=entry["arch"]["platform_os"], target=entry["arch"]["target"]["name"], ) compiler_str = "" if "compiler" in entry: compiler_format = "%{name}@={version}" compiler_str = compiler_format.format( name=translated_compiler_name(entry["compiler"]["name"]), version=entry["compiler"]["version"], ) spec_format = "{name}@={version} {arch}" spec_str = spec_format.format( name=entry["name"], version=entry["version"], compiler=compiler_str, arch=arch_str ) pkg_cls = spack.repo.PATH.get_pkg_class(entry["name"]) if "parameters" in entry: variant_strs = list() for name, value in entry["parameters"].items(): # TODO: also ensure that the variant value is valid? if not pkg_cls.has_variant(name): tty.debug( "Omitting variant {0} for entry {1}/{2}".format( name, entry["name"], entry["hash"][:7] ) ) continue # Value could be a list (of strings), boolean, or string if isinstance(value, str): variant_strs.append("{0}={1}".format(name, value)) else: try: iter(value) variant_strs.append("{0}={1}".format(name, ",".join(value))) continue except TypeError: # Not an iterable pass # At this point not a string or collection, check for boolean if value in [True, False]: bool_symbol = "+" if value else "~" variant_strs.append("{0}{1}".format(bool_symbol, name)) else: raise ValueError( "Unexpected value for {0} ({1}): {2}".format( name, str(type(value)), str(value) ) ) spec_str += " " + " ".join(variant_strs) (spec,) = spack.cmd.parse_specs(spec_str.split()) for ht in [hash_types.dag_hash, hash_types.build_hash, hash_types.full_hash]: setattr(spec, ht.attr, entry["hash"]) spec._concrete = True spec._hashes_final = True spec.external_path = entry["prefix"] spec.origin = "external-db" spec.namespace = pkg_cls.namespace spack.spec.Spec.ensure_valid_variants(spec) return spec def entries_to_specs(entries): spec_dict = {} for entry in entries: try: spec = spec_from_entry(entry) assert spec.concrete, f"{spec} is not concrete" spec_dict[spec._hash] = spec except spack.repo.UnknownPackageError: tty.debug("Omitting package {0}: no corresponding repo package".format(entry["name"])) except spack.error.SpackError: raise except Exception: tty.warn("Could not parse entry: " + str(entry)) for entry in filter(lambda x: "dependencies" in x, entries): dependencies = entry["dependencies"] for name, properties in dependencies.items(): dep_hash = properties["hash"] depflag = dt.canonicalize(properties["type"]) if dep_hash in spec_dict: if entry["hash"] not in spec_dict: continue parent_spec = spec_dict[entry["hash"]] dep_spec = spec_dict[dep_hash] parent_spec._add_dependency(dep_spec, depflag=depflag, virtuals=()) for spec in spec_dict.values(): spack.spec.reconstruct_virtuals_on_edges(spec) return spec_dict def read(path, apply_updates): decode_exception_type = json.decoder.JSONDecodeError try: with open(path, "r", encoding="utf-8") as json_file: json_data = json.load(json_file) jsonschema.validate(json_data, manifest_schema) except (exceptions.ValidationError, decode_exception_type) as e: raise ManifestValidationError("error parsing manifest JSON:", str(e)) from e specs = entries_to_specs(json_data["specs"]) tty.debug("{0}: {1} specs read from manifest".format(path, str(len(specs)))) compilers = [] if "compilers" in json_data: for x in json_data["compilers"]: # We don't want to fail reading the manifest, if a single compiler fails try: candidate = compiler_from_entry(x, manifest_path=path) except Exception: candidate = None if candidate is None: continue compilers.append(candidate) tty.debug(f"{path}: {str(len(compilers))} compilers read from manifest") # Filter out the compilers that already appear in the configuration compilers = spack.compilers.config.select_new_compilers(compilers) if apply_updates and compilers: try: spack.compilers.config.add_compiler_to_config(compilers) except Exception: warnings.warn( f"Could not add compilers from manifest: {path}" "\nPlease reexecute with 'spack -d' and include the stack trace" ) tty.debug(f"Include this\n{traceback.format_exc()}") if apply_updates: for spec in specs.values(): assert spec.concrete, f"{spec} is not concrete" spack.store.STORE.db.add(spec) class ManifestValidationError(spack.error.SpackError): def __init__(self, msg, long_msg=None): super().__init__(msg, long_msg) class CrayCompilerDetectionError(spack.error.SpackError): """Raised if a compiler, listed in the Cray manifest, cannot be detected correctly based on the paths provided. """ ================================================ FILE: lib/spack/spack/database.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Spack's installation tracking database. The database serves two purposes: 1. It implements a cache on top of a potentially very large Spack directory hierarchy, speeding up many operations that would otherwise require filesystem access. 2. It will allow us to track external installations as well as lost packages and their dependencies. Prior to the implementation of this store, a directory layout served as the authoritative database of packages in Spack. This module provides a cache and a sanity checking mechanism for what is in the filesystem. """ import contextlib import datetime import os import pathlib import sys import time from json import JSONDecoder from typing import ( IO, Any, Callable, Container, Dict, Generator, Iterable, List, NamedTuple, Optional, Set, Tuple, Type, Union, ) import spack import spack.repo try: import uuid _use_uuid = True except ImportError: _use_uuid = False pass import spack.deptypes as dt import spack.hash_types as ht import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.spec import spack.traverse as tr import spack.util.lock as lk import spack.util.spack_json as sjson import spack.version as vn from spack.directory_layout import ( DirectoryLayout, DirectoryLayoutError, InconsistentInstallDirectoryError, ) from spack.error import SpackError from spack.util.crypto import bit_length from spack.util.socket import _gethostname from .enums import InstallRecordStatus # TODO: Provide an API automatically retrying a build after detecting and # TODO: clearing a failure. #: DB goes in this directory underneath the root _DB_DIRNAME = ".spack-db" #: DB version. This is stuck in the DB file to track changes in format. #: Increment by one when the database format changes. #: Versions before 5 were not integers. _DB_VERSION = vn.Version("8") #: For any version combinations here, skip reindex when upgrading. #: Reindexing can take considerable time and is not always necessary. _REINDEX_NOT_NEEDED_ON_READ = [ # reindexing takes a significant amount of time, and there's # no reason to do it from DB version 0.9.3 to version 5. The # only difference is that v5 can contain "deprecated_for" # fields. So, skip the reindex for this transition. The new # version is saved to disk the first time the DB is written. (vn.Version("0.9.3"), vn.Version("5")), (vn.Version("5"), vn.Version("6")), (vn.Version("6"), vn.Version("7")), (vn.Version("6"), vn.Version("8")), (vn.Version("7"), vn.Version("8")), ] #: Default timeout for spack database locks in seconds or None (no timeout). #: A balance needs to be struck between quick turnaround for parallel installs #: (to avoid excess delays) and waiting long enough when the system is busy #: (to ensure the database is updated). _DEFAULT_DB_LOCK_TIMEOUT = 120 #: Default timeout for spack package locks in seconds or None (no timeout). #: A balance needs to be struck between quick turnaround for parallel installs #: (to avoid excess delays when performing a parallel installation) and waiting #: long enough for the next possible spec to install (to avoid excessive #: checking of the last high priority package) or holding on to a lock (to #: ensure a failed install is properly tracked). _DEFAULT_PKG_LOCK_TIMEOUT = None #: Types of dependencies tracked by the database #: We store by DAG hash, so we track the dependencies that the DAG hash includes. _TRACKED_DEPENDENCIES = ht.dag_hash.depflag #: Default list of fields written for each install record DEFAULT_INSTALL_RECORD_FIELDS = ( "spec", "ref_count", "path", "installed", "explicit", "installation_time", "deprecated_for", ) #: File where the database is written INDEX_JSON_FILE = "index.json" # Verifier file to check last modification of the DB _INDEX_VERIFIER_FILE = "index_verifier" # Lockfile for the database _LOCK_FILE = "lock" def reader(version: vn.StandardVersion) -> Type["spack.spec.SpecfileReaderBase"]: reader_cls = { vn.StandardVersion.from_string("5"): spack.spec.SpecfileV1, vn.StandardVersion.from_string("6"): spack.spec.SpecfileV3, vn.StandardVersion.from_string("7"): spack.spec.SpecfileV4, vn.StandardVersion.from_string("8"): spack.spec.SpecfileV5, } return reader_cls[version] def _now() -> float: """Returns the time since the epoch""" return time.time() def _autospec(function): """Decorator that automatically converts the argument of a single-arg function to a Spec.""" def converter(self, spec_like, *args, **kwargs): if not isinstance(spec_like, spack.spec.Spec): spec_like = spack.spec.Spec(spec_like) return function(self, spec_like, *args, **kwargs) return converter def normalize_query(installed: Union[bool, InstallRecordStatus]) -> InstallRecordStatus: if installed is True: installed = InstallRecordStatus.INSTALLED elif installed is False: installed = InstallRecordStatus.MISSING return installed class InstallRecord: """A record represents one installation in the DB. The record keeps track of the spec for the installation, its install path, AND whether or not it is installed. We need the installed flag in case a user either: 1. blew away a directory, or 2. used spack uninstall -f to get rid of it If, in either case, the package was removed but others still depend on it, we still need to track its spec, so we don't actually remove from the database until a spec has no installed dependents left. Args: spec: spec tracked by the install record path: path where the spec has been installed installed: whether or not the spec is currently installed ref_count (int): number of specs that depend on this one explicit (bool or None): whether or not this spec was explicitly installed, or pulled-in as a dependency of something else installation_time (datetime.datetime or None): time of the installation """ def __init__( self, spec: "spack.spec.Spec", path: Optional[str], installed: bool, ref_count: int = 0, explicit: bool = False, installation_time: Optional[float] = None, deprecated_for: Optional[str] = None, in_buildcache: bool = False, origin: Optional[str] = None, ) -> None: self.spec = spec self.path = str(path) if path else None self.installed = bool(installed) self.ref_count = ref_count self.explicit = explicit self.installation_time = installation_time or _now() self.deprecated_for = deprecated_for self.in_buildcache = in_buildcache self.origin = origin def install_type_matches(self, installed: InstallRecordStatus) -> bool: if self.installed: return InstallRecordStatus.INSTALLED in installed elif self.deprecated_for: return InstallRecordStatus.DEPRECATED in installed return InstallRecordStatus.MISSING in installed def to_dict(self, include_fields=DEFAULT_INSTALL_RECORD_FIELDS): rec_dict = {} for field_name in include_fields: if field_name == "spec": rec_dict.update({"spec": self.spec.node_dict_with_hashes()}) elif field_name == "deprecated_for" and self.deprecated_for: rec_dict.update({"deprecated_for": self.deprecated_for}) else: rec_dict.update({field_name: getattr(self, field_name)}) if self.origin: rec_dict["origin"] = self.origin return rec_dict @classmethod def from_dict(cls, spec, dictionary): d = dict(dictionary.items()) d.pop("spec", None) # Old databases may have "None" for path for externals if "path" not in d or d["path"] == "None": d["path"] = None if "installed" not in d: d["installed"] = False return InstallRecord(spec, **d) class ForbiddenLockError(SpackError): """Raised when an upstream DB attempts to acquire a lock""" class ForbiddenLock: def __getattr__(self, name): raise ForbiddenLockError(f"Cannot access attribute '{name}' of lock") def __reduce__(self): return ForbiddenLock, tuple() class LockConfiguration(NamedTuple): """Data class to configure locks in Database objects Args: enable: whether to enable locks or not. database_timeout: timeout for the database lock package_timeout: timeout for the package lock """ enable: bool database_timeout: Optional[int] package_timeout: Optional[int] #: Configure a database to avoid using locks NO_LOCK: LockConfiguration = LockConfiguration( enable=False, database_timeout=None, package_timeout=None ) #: Configure the database to use locks without a timeout NO_TIMEOUT: LockConfiguration = LockConfiguration( enable=True, database_timeout=None, package_timeout=None ) #: Default configuration for database locks DEFAULT_LOCK_CFG: LockConfiguration = LockConfiguration( enable=True, database_timeout=_DEFAULT_DB_LOCK_TIMEOUT, package_timeout=_DEFAULT_PKG_LOCK_TIMEOUT, ) def lock_configuration(configuration): """Return a LockConfiguration from a spack.config.Configuration object.""" return LockConfiguration( enable=configuration.get("config:locks", True), database_timeout=configuration.get("config:db_lock_timeout"), package_timeout=configuration.get("config:package_lock_timeout"), ) def prefix_lock_path(root_dir: Union[str, pathlib.Path]) -> pathlib.Path: """Returns the path of the prefix lock file, given the root directory. Args: root_dir: root directory containing the database directory """ return pathlib.Path(root_dir) / _DB_DIRNAME / "prefix_lock" def failures_lock_path(root_dir: Union[str, pathlib.Path]) -> pathlib.Path: """Returns the path of the failures lock file, given the root directory. Args: root_dir: root directory containing the database directory """ return pathlib.Path(root_dir) / _DB_DIRNAME / "prefix_failures" class SpecLocker: """Manages acquiring and releasing read or write locks on concrete specs.""" def __init__(self, lock_path: Union[str, pathlib.Path], default_timeout: Optional[float]): self.lock_path = pathlib.Path(lock_path) self.default_timeout = default_timeout # Maps (spec.dag_hash(), spec.name) to the corresponding lock object self.locks: Dict[Tuple[str, str], lk.Lock] = {} def lock(self, spec: "spack.spec.Spec", timeout: Optional[float] = None) -> lk.Lock: """Returns a lock on a concrete spec. The lock is a byte range lock on the nth byte of a file. The lock file is ``self.lock_path``. n is the sys.maxsize-bit prefix of the DAG hash. This makes likelihood of collision is very low AND it gives us readers-writer lock semantics with just a single lockfile, so no cleanup required. """ assert spec.concrete, "cannot lock a non-concrete spec" timeout = timeout or self.default_timeout key = self._lock_key(spec) if key not in self.locks: self.locks[key] = self.raw_lock(spec, timeout=timeout) else: self.locks[key].default_timeout = timeout return self.locks[key] def raw_lock(self, spec: "spack.spec.Spec", timeout: Optional[float] = None) -> lk.Lock: """Returns a raw lock for a Spec, but doesn't keep track of it.""" return lk.Lock( str(self.lock_path), start=spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), length=1, default_timeout=timeout, desc=spec.name, ) def has_lock(self, spec: "spack.spec.Spec") -> bool: """Returns True if the spec is already managed by this spec locker""" return self._lock_key(spec) in self.locks def _lock_key(self, spec: "spack.spec.Spec") -> Tuple[str, str]: return (spec.dag_hash(), spec.name) @contextlib.contextmanager def write_lock(self, spec: "spack.spec.Spec") -> Generator["SpecLocker", None, None]: lock = self.lock(spec) lock.acquire_write() try: yield self except lk.LockError: # This addresses the case where a nested lock attempt fails inside # of this context manager raise except (Exception, KeyboardInterrupt): lock.release_write() raise else: lock.release_write() def clear(self, spec: "spack.spec.Spec") -> Tuple[bool, Optional[lk.Lock]]: key = self._lock_key(spec) lock = self.locks.pop(key, None) return bool(lock), lock def clear_all(self, clear_fn: Optional[Callable[[lk.Lock], Any]] = None) -> None: if clear_fn is not None: for lock in self.locks.values(): clear_fn(lock) self.locks.clear() class FailureTracker: """Tracks installation failures. Prefix failure marking takes the form of a byte range lock on the nth byte of a file for coordinating between concurrent parallel build processes and a persistent file, named with the full hash and containing the spec, in a subdirectory of the database to enable persistence across overlapping but separate related build processes. The failure lock file lives alongside the install DB. ``n`` is the sys.maxsize-bit prefix of the associated DAG hash to make the likelihood of collision very low with no cleanup required. """ #: root directory of the failure tracker dir: pathlib.Path #: File for locking particular concrete spec hashes locker: SpecLocker def __init__(self, root_dir: Union[str, pathlib.Path], default_timeout: Optional[float]): #: Ensure a persistent location for dealing with parallel installation #: failures (e.g., across near-concurrent processes). self.dir = pathlib.Path(root_dir) / _DB_DIRNAME / "failures" self.locker = SpecLocker(failures_lock_path(root_dir), default_timeout=default_timeout) def _ensure_parent_directories(self) -> None: """Ensure that parent directories of the FailureTracker exist. Accesses the filesystem only once, the first time it's called on a given FailureTracker. """ self.dir.mkdir(parents=True, exist_ok=True) def clear(self, spec: "spack.spec.Spec", force: bool = False) -> None: """Removes any persistent and cached failure tracking for the spec. see :meth:`mark`. Args: spec: the spec whose failure indicators are being removed force: True if the failure information should be cleared when a failure lock exists for the file, or False if the failure should not be cleared (e.g., it may be associated with a concurrent build) """ locked = self.lock_taken(spec) if locked and not force: tty.msg(f"Retaining failure marking for {spec.name} due to lock") return if locked: tty.warn(f"Removing failure marking despite lock for {spec.name}") succeeded, lock = self.locker.clear(spec) if succeeded and lock is not None: lock.release_write() if self.persistent_mark(spec): path = self._path(spec) tty.debug(f"Removing failure marking for {spec.name}") try: path.unlink() except OSError as err: tty.warn( f"Unable to remove failure marking for {spec.name} ({str(path)}): {str(err)}" ) def clear_all(self) -> None: """Force remove install failure tracking files.""" tty.debug("Releasing prefix failure locks") self.locker.clear_all( clear_fn=lambda x: x.release_write() if x.is_write_locked() else True ) tty.debug("Removing prefix failure tracking files") try: marks = os.listdir(str(self.dir)) except FileNotFoundError: return # directory doesn't exist yet except OSError as exc: tty.warn(f"Unable to remove failure marking files: {str(exc)}") return for fail_mark in marks: try: (self.dir / fail_mark).unlink() except OSError as exc: tty.warn(f"Unable to remove failure marking file {fail_mark}: {str(exc)}") def mark(self, spec: "spack.spec.Spec") -> lk.Lock: """Marks a spec as failing to install. Args: spec: spec that failed to install """ self._ensure_parent_directories() # Dump the spec to the failure file for (manual) debugging purposes path = self._path(spec) path.write_text(spec.to_json()) # Also ensure a failure lock is taken to prevent cleanup removal # of failure status information during a concurrent parallel build. if not self.locker.has_lock(spec): try: mark = self.locker.lock(spec) mark.acquire_write() except lk.LockTimeoutError: # Unlikely that another process failed to install at the same # time but log it anyway. tty.debug(f"PID {os.getpid()} failed to mark install failure for {spec.name}") tty.warn(f"Unable to mark {spec.name} as failed.") return self.locker.lock(spec) def has_failed(self, spec: "spack.spec.Spec") -> bool: """Return True if the spec is marked as failed.""" # The failure was detected in this process. if self.locker.has_lock(spec): return True # The failure was detected by a concurrent process (e.g., an srun), # which is expected to be holding a write lock if that is the case. if self.lock_taken(spec): return True # Determine if the spec may have been marked as failed by a separate # spack build process running concurrently. return self.persistent_mark(spec) def lock_taken(self, spec: "spack.spec.Spec") -> bool: """Return True if another process has a failure lock on the spec.""" check = self.locker.raw_lock(spec) return check.is_write_locked() def persistent_mark(self, spec: "spack.spec.Spec") -> bool: """Determine if the spec has a persistent failure marking.""" return self._path(spec).exists() def _path(self, spec: "spack.spec.Spec") -> pathlib.Path: """Return the path to the spec's failure file, which may not exist.""" assert spec.concrete, "concrete spec required for failure path" return self.dir / f"{spec.name}-{spec.dag_hash()}" SelectType = Callable[[InstallRecord], bool] class Database: #: Fields written for each install record record_fields: Tuple[str, ...] = DEFAULT_INSTALL_RECORD_FIELDS def __init__( self, root: str, *, upstream_dbs: Optional[List["Database"]] = None, is_upstream: bool = False, lock_cfg: LockConfiguration = DEFAULT_LOCK_CFG, layout: Optional[DirectoryLayout] = None, ) -> None: """Database for Spack installations. A Database is a cache of Specs data from ``$prefix/spec.yaml`` files in Spack installation directories. Database files (data and lock files) are stored under ``root/.spack-db``, which is created if it does not exist. This is the "database directory". The database will attempt to read an ``index.json`` file in the database directory. If that does not exist, it will create a database when needed by scanning the entire store root for ``spec.json`` files according to Spack's directory layout. Args: root: root directory where to create the database directory. upstream_dbs: upstream databases for this repository. is_upstream: whether this repository is an upstream. lock_cfg: configuration for the locks to be used by this repository. Relevant only if the repository is not an upstream. """ self.root = root self.database_directory = pathlib.Path(self.root) / _DB_DIRNAME self.layout = layout # Set up layout of database files within the db dir self._index_path = self.database_directory / INDEX_JSON_FILE self._verifier_path = self.database_directory / _INDEX_VERIFIER_FILE self._lock_path = self.database_directory / _LOCK_FILE self.is_upstream = is_upstream self.last_seen_verifier = "" # Failed write transactions (interrupted by exceptions) will alert # _write. When that happens, we set this flag to indicate that # future read/write transactions should re-read the DB. Normally it # would make more sense to resolve this at the end of the transaction # but typically a failed transaction will terminate the running # instance of Spack and we don't want to incur an extra read in that # case, so we defer the cleanup to when we begin the next transaction self._state_is_inconsistent = False # initialize rest of state. self.db_lock_timeout = lock_cfg.database_timeout tty.debug(f"DATABASE LOCK TIMEOUT: {str(self.db_lock_timeout)}s") self.lock: Union[ForbiddenLock, lk.Lock] if self.is_upstream: self.lock = ForbiddenLock() else: self.lock = lk.Lock( str(self._lock_path), default_timeout=self.db_lock_timeout, desc="database", enable=lock_cfg.enable, ) self._data: Dict[str, InstallRecord] = {} # For every installed spec we keep track of its install prefix, so that # we can answer the simple query whether a given path is already taken # before installing a different spec. self._installed_prefixes: Set[str] = set() self.upstream_dbs = list(upstream_dbs) if upstream_dbs else [] self._write_transaction_impl = lk.WriteTransaction self._read_transaction_impl = lk.ReadTransaction self._db_version: Optional[vn.ConcreteVersion] = None @property def db_version(self) -> vn.ConcreteVersion: if self._db_version is None: raise AttributeError("version not set -- DB has not been read yet") return self._db_version @db_version.setter def db_version(self, value: vn.ConcreteVersion): self._db_version = value def _ensure_parent_directories(self): """Create the parent directory for the DB, if necessary.""" if not self.is_upstream: self.database_directory.mkdir(parents=True, exist_ok=True) def write_transaction(self): """Get a write lock context manager for use in a ``with`` block.""" return self._write_transaction_impl(self.lock, acquire=self._read, release=self._write) def read_transaction(self): """Get a read lock context manager for use in a ``with`` block.""" return self._read_transaction_impl(self.lock, acquire=self._read) def _write_to_file(self, stream): """Write out the database in JSON format to the stream passed as argument. This function does not do any locking or transactions. """ self._ensure_parent_directories() # map from per-spec hash code to installation record. installs = dict( (k, v.to_dict(include_fields=self.record_fields)) for k, v in self._data.items() ) # database includes installation list and version. # NOTE: this DB version does not handle multiple installs of # the same spec well. If there are 2 identical specs with # different paths, it can't differentiate. # TODO: fix this before we support multiple install locations. database = { "database": { # TODO: move this to a top-level _meta section if we ever # TODO: bump the DB version to 7 "version": str(_DB_VERSION), # dictionary of installation records, keyed by DAG hash "installs": installs, } } try: sjson.dump(database, stream) except (TypeError, ValueError) as e: raise sjson.SpackJSONError("error writing JSON database:", str(e)) def _read_spec_from_dict(self, spec_reader, hash_key, installs, hash=ht.dag_hash): """Recursively construct a spec from a hash in a YAML database. Does not do any locking. """ spec_dict = installs[hash_key]["spec"] # Install records don't include hash with spec, so we add it in here # to ensure it is read properly. if "name" not in spec_dict.keys(): # old format, can't update format here for name in spec_dict: spec_dict[name]["hash"] = hash_key else: # new format, already a singleton spec_dict[hash.name] = hash_key # Build spec from dict first. return spec_reader.from_node_dict(spec_dict) def db_for_spec_hash(self, hash_key): with self.read_transaction(): if hash_key in self._data: return self for db in self.upstream_dbs: if hash_key in db._data: return db def query_by_spec_hash( self, hash_key: str, data: Optional[Dict[str, InstallRecord]] = None ) -> Tuple[bool, Optional[InstallRecord]]: """Get a spec for hash, and whether it's installed upstream. Return: Tuple of bool and optional InstallRecord. The bool tells us whether the record is from an upstream. Its InstallRecord is also returned if available (the record must be checked to know whether the hash is installed). If the record is available locally, this function will always have a preference for returning that, even if it is not installed locally and is installed upstream. """ if data and hash_key in data: return False, data[hash_key] if not data: with self.read_transaction(): if hash_key in self._data: return False, self._data[hash_key] for db in self.upstream_dbs: if hash_key in db._data: return True, db._data[hash_key] return False, None def query_local_by_spec_hash(self, hash_key: str) -> Optional[InstallRecord]: """Get a spec by hash in the local database Return: InstallRecord when installed locally, otherwise None.""" with self.read_transaction(): return self._data.get(hash_key, None) def _assign_build_spec( self, spec_reader: Type["spack.spec.SpecfileReaderBase"], hash_key: str, installs: dict, data: Dict[str, InstallRecord], ): # Add dependencies from other records in the install DB to # form a full spec. spec = data[hash_key].spec spec_node_dict = installs[hash_key]["spec"] if "name" not in spec_node_dict: # old format spec_node_dict = spec_node_dict[spec.name] if "build_spec" in spec_node_dict: assert spec_reader.SPEC_VERSION >= 2, "SpecfileV1 spec cannot have build_spec" _, bhash, _ = spec_reader.extract_build_spec_info_from_node_dict(spec_node_dict) _, build_spec = self.query_by_spec_hash(bhash, data=data) assert build_spec is not None, f"build_spec with hash {bhash} not found in database" spec._build_spec = build_spec.spec def _assign_dependencies( self, spec_reader: Type["spack.spec.SpecfileReaderBase"], hash_key: str, installs: dict, data: Dict[str, InstallRecord], ): # Add dependencies from other records in the install DB to # form a full spec. spec = data[hash_key].spec spec_node_dict = installs[hash_key]["spec"] if "name" not in spec_node_dict: # old format spec_node_dict = spec_node_dict[spec.name] if "dependencies" in spec_node_dict: yaml_deps = spec_node_dict["dependencies"] for dname, dhash, dtypes, _, virtuals, direct in spec_reader.read_specfile_dep_specs( yaml_deps ): # It is important that we always check upstream installations in the same order, # and that we always check the local installation first: if a downstream Spack # installs a package then dependents in that installation could be using it. If a # hash is installed locally and upstream, there isn't enough information to # determine which one a local package depends on, so the convention ensures that # this isn't an issue. _, record = self.query_by_spec_hash(dhash, data=data) child = record.spec if record else None if not child: tty.warn( f"Missing dependency not in database: " f"{spec.cformat('{name}{/hash:7}')} needs {dname}-{dhash[:7]}" ) continue spec._add_dependency( child, depflag=dt.canonicalize(dtypes), virtuals=virtuals, direct=direct ) def _read_from_file(self, filename: pathlib.Path, *, reindex: bool = False) -> None: """Fill database from file, do not maintain old data. Does not do any locking. """ with filename.open("r", encoding="utf-8") as f: self._read_from_stream(f, reindex=reindex) def _read_from_stream(self, stream: IO[str], *, reindex: bool = False) -> None: """Fill database from a text stream, do not maintain old data. Translate the spec portions from node-dict form to spec form. Does not do any locking. """ source = getattr(stream, "name", None) or self._index_path try: # In the future we may use a stream of JSON objects, hence `raw_decode` for compat. fdata, _ = JSONDecoder().raw_decode(stream.read()) except Exception as e: raise CorruptDatabaseError(f"error parsing database at {source}:", str(e)) from e if fdata is None: return def check(cond, msg): if not cond: raise CorruptDatabaseError(f"Spack database is corrupt: {msg}", str(source)) check("database" in fdata, "no 'database' attribute in JSON DB.") # High-level file checks db = fdata["database"] check("version" in db, "no 'version' in JSON DB.") self.db_version = vn.StandardVersion.from_string(db["version"]) if self.db_version > _DB_VERSION: raise InvalidDatabaseVersionError(self, _DB_VERSION, self.db_version) elif self.db_version < _DB_VERSION: installs = self._handle_old_db_versions_read(check, db, reindex=reindex) else: installs = self._handle_current_version_read(check, db) spec_reader = reader(self.db_version) def invalid_record(hash_key, error): return CorruptDatabaseError( f"Invalid record in Spack database: hash: {hash_key}, cause: " f"{type(error).__name__}: {error}", str(source), ) # Build up the database in three passes: # # 1. Read in all specs without dependencies. # 2. Hook dependencies up among specs. # 3. Mark all specs concrete. # # The database is built up so that ALL specs in it share nodes # (i.e., its specs are a true Merkle DAG, unlike most specs.) # Pass 1: Iterate through database and build specs w/o dependencies data: Dict[str, InstallRecord] = {} installed_prefixes: Set[str] = set() for hash_key, rec in installs.items(): try: # This constructs a spec DAG from the list of all installs spec = self._read_spec_from_dict(spec_reader, hash_key, installs) # Insert the brand new spec in the database. Each # spec has its own copies of its dependency specs. # TODO: would a more immmutable spec implementation simplify # this? data[hash_key] = InstallRecord.from_dict(spec, rec) if not spec.external and "installed" in rec and rec["installed"]: installed_prefixes.add(rec["path"]) except Exception as e: raise invalid_record(hash_key, e) from e # Pass 2: Assign dependencies once all specs are created. for hash_key in data: try: self._assign_build_spec(spec_reader, hash_key, installs, data) self._assign_dependencies(spec_reader, hash_key, installs, data) except MissingDependenciesError: raise except Exception as e: raise invalid_record(hash_key, e) from e # Pass 3: Mark all specs concrete. Specs representing real # installations must be explicitly marked. # We do this *after* all dependencies are connected because if we # do it *while* we're constructing specs,it causes hashes to be # cached prematurely. for hash_key, rec in data.items(): rec.spec._mark_root_concrete() self._data = data self._installed_prefixes = installed_prefixes def _handle_current_version_read(self, check, db): check("installs" in db, "no 'installs' in JSON DB.") installs = db["installs"] return installs def _handle_old_db_versions_read(self, check, db, *, reindex: bool): if reindex is False and not self.is_upstream: self.raise_explicit_database_upgrade_error() if not self.is_readable(): raise DatabaseNotReadableError( f"cannot read database v{self.db_version} at {self.root}" ) return self._handle_current_version_read(check, db) def is_readable(self) -> bool: """Returns true if this DB can be read without reindexing""" return (self.db_version, _DB_VERSION) in _REINDEX_NOT_NEEDED_ON_READ def raise_explicit_database_upgrade_error(self): """Raises an ExplicitDatabaseUpgradeError with an appropriate message""" raise ExplicitDatabaseUpgradeError( f"database is v{self.db_version}, but Spack v{spack.__version__} needs v{_DB_VERSION}", long_message=( f"You will need to either:" f"\n" f"\n 1. Migrate the database to v{_DB_VERSION}, or" f"\n 2. Use a new database by changing config:install_tree:root." f"\n" f"\nTo migrate the database at {self.root} " f"\nto version {_DB_VERSION}, run:" f"\n" f"\n spack reindex" f"\n" f"\nNOTE that if you do this, older Spack versions will no longer" f"\nbe able to read the database. However, `spack reindex` will create a backup," f"\nin case you want to revert." f"\n" f"\nIf you still need your old database, you can instead run" f"\n`spack config edit config` and set install_tree:root to a new location." ), ) def reindex(self): """Build database index from scratch based on a directory layout. Locks the DB if it isn't locked already. """ if self.is_upstream: raise UpstreamDatabaseLockingError("Cannot reindex an upstream database") self._ensure_parent_directories() # Special transaction to avoid recursive reindex calls and to # ignore errors if we need to rebuild a corrupt database. def _read_suppress_error(): try: with self._index_path.open("r", encoding="utf-8") as f: self._read_from_stream(f, reindex=True) except FileNotFoundError: pass except (CorruptDatabaseError, DatabaseNotReadableError): self._data = {} self._installed_prefixes = set() with lk.WriteTransaction(self.lock, acquire=_read_suppress_error, release=self._write): old_installed_prefixes, self._installed_prefixes = self._installed_prefixes, set() old_data, self._data = self._data, {} try: self._reindex(old_data) except BaseException: # If anything explodes, restore old data, skip write. self._data = old_data self._installed_prefixes = old_installed_prefixes raise def _reindex(self, old_data: Dict[str, InstallRecord]): # Specs on the file system are the source of truth for record.spec. The old database values # if available are the source of truth for the rest of the record. assert self.layout, "Database layout must be set to reindex" specs_from_fs = self.layout.all_specs() deprecated_for = self.layout.deprecated_for(specs_from_fs) known_specs: List[spack.spec.Spec] = [ *specs_from_fs, *(deprecated for _, deprecated in deprecated_for), *(rec.spec for rec in old_data.values()), ] upstream_hashes = { dag_hash for upstream in self.upstream_dbs for dag_hash in upstream._data } upstream_hashes.difference_update(spec.dag_hash() for spec in known_specs) def create_node(edge: spack.spec.DependencySpec, is_upstream: bool): if is_upstream: return self._data[edge.spec.dag_hash()] = InstallRecord( spec=edge.spec.copy(deps=False), path=edge.spec.external_path if edge.spec.external else None, installed=edge.spec.external, ) # Store all nodes of known specs, excluding ones found in upstreams tr.traverse_breadth_first_with_visitor( known_specs, tr.CoverNodesVisitor( NoUpstreamVisitor(upstream_hashes, create_node), key=tr.by_dag_hash ), ) # Store the prefix and other information for specs were found on the file system for s in specs_from_fs: record = self._data[s.dag_hash()] record.path = s.prefix record.installed = True record.explicit = True # conservative assumption record.installation_time = os.stat(s.prefix).st_ctime # Deprecate specs for new, old in deprecated_for: self._data[old.dag_hash()].deprecated_for = new.dag_hash() # Copy data we have from the old database for old_record in old_data.values(): record = self._data[old_record.spec.dag_hash()] record.explicit = old_record.explicit record.installation_time = old_record.installation_time record.origin = old_record.origin record.deprecated_for = old_record.deprecated_for # Warn when the spec has been removed from the file system (i.e. it was not detected) if not record.installed and old_record.installed: tty.warn( f"Spec {old_record.spec.short_spec} was marked installed in the database " "but was not found on the file system. It is now marked as missing." ) def create_edge(edge: spack.spec.DependencySpec, is_upstream: bool): if not edge.parent: return parent_record = self._data[edge.parent.dag_hash()] if is_upstream: upstream, child_record = self.query_by_spec_hash(edge.spec.dag_hash()) assert upstream and child_record, "Internal error: upstream spec not found" else: child_record = self._data[edge.spec.dag_hash()] parent_record.spec._add_dependency( child_record.spec, depflag=edge.depflag, virtuals=edge.virtuals ) # Then store edges tr.traverse_breadth_first_with_visitor( known_specs, tr.CoverEdgesVisitor( NoUpstreamVisitor(upstream_hashes, create_edge), key=tr.by_dag_hash ), ) # Finally update the ref counts for record in self._data.values(): for dep in record.spec.dependencies(deptype=_TRACKED_DEPENDENCIES): dep_record = self._data.get(dep.dag_hash()) if dep_record: # dep might be upstream dep_record.ref_count += 1 if record.deprecated_for: self._data[record.deprecated_for].ref_count += 1 self._check_ref_counts() def _check_ref_counts(self): """Ensure consistency of reference counts in the DB. Raise an AssertionError if something is amiss. Does no locking. """ counts: Dict[str, int] = {} for key, rec in self._data.items(): counts.setdefault(key, 0) for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES): dep_key = dep.dag_hash() counts.setdefault(dep_key, 0) counts[dep_key] += 1 if rec.deprecated_for: counts.setdefault(rec.deprecated_for, 0) counts[rec.deprecated_for] += 1 for rec in self._data.values(): key = rec.spec.dag_hash() expected = counts[key] found = rec.ref_count if not expected == found: raise AssertionError( "Invalid ref_count: %s: %d (expected %d), in DB %s" % (key, found, expected, self._index_path) ) def _write(self, type=None, value=None, traceback=None): """Write the in-memory database index to its file path. This is a helper function called by the WriteTransaction context manager. If there is an exception while the write lock is active, nothing will be written to the database file, but the in-memory database *may* be left in an inconsistent state. It will be consistent after the start of the next transaction, when it read from disk again. This routine does no locking. """ self._ensure_parent_directories() # Do not write if exceptions were raised if type is not None: # A failure interrupted a transaction, so we should record that # the Database is now in an inconsistent state: we should # restore it in the next transaction self._state_is_inconsistent = True return temp_file = str(self._index_path) + (".%s.%s.temp" % (_gethostname(), os.getpid())) # Write a temporary database file them move it into place try: with open(temp_file, "w", encoding="utf-8") as f: self._write_to_file(f) fs.rename(temp_file, str(self._index_path)) if _use_uuid: with self._verifier_path.open("w", encoding="utf-8") as f: new_verifier = str(uuid.uuid4()) f.write(new_verifier) self.last_seen_verifier = new_verifier except BaseException as e: tty.debug(e) # Clean up temp file if something goes wrong. if os.path.exists(temp_file): os.remove(temp_file) raise def _read(self): """Re-read Database from the data in the set location. This does no locking.""" try: index_file = self._index_path.open("r", encoding="utf-8") except FileNotFoundError: if self.is_upstream: tty.warn(f"upstream not found: {self._index_path}") return with index_file as f: current_verifier = "" if _use_uuid: try: with self._verifier_path.open("r", encoding="utf-8") as vf: current_verifier = vf.read() except BaseException: pass if (current_verifier != self.last_seen_verifier) or (current_verifier == ""): self.last_seen_verifier = current_verifier # Read from file if a database exists self._read_from_stream(f) elif self._state_is_inconsistent: self._read_from_stream(f) self._state_is_inconsistent = False def _add( self, spec: "spack.spec.Spec", explicit: bool = False, installation_time: Optional[float] = None, allow_missing: bool = False, ): """Add an install record for this spec to the database. Also ensures dependencies are present and updated in the DB as either installed or missing. Args: spec: spec to be added explicit: Possible values: True, False, any A spec that was installed following a specific user request is marked as explicit. If instead it was pulled-in as a dependency of a user requested spec it's considered implicit. installation_time: Date and time of installation allow_missing: if True, don't warn when installation is not found on on disk This is useful when installing specs without build/test deps. """ if not spec.concrete: raise NonConcreteSpecAddError("Specs added to DB must be concrete.") key = spec.dag_hash() spec_pkg_hash = spec._package_hash # type: ignore[attr-defined] upstream, record = self.query_by_spec_hash(key) if upstream and record and record.installed: return installation_time = installation_time or _now() for edge in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES): if edge.spec.dag_hash() in self._data: continue self._add( edge.spec, explicit=False, installation_time=installation_time, # allow missing build / test only deps allow_missing=allow_missing or edge.depflag & (dt.BUILD | dt.TEST) == edge.depflag, ) if spec.spliced: self._add(spec.build_spec, explicit=False, allow_missing=True) # Make sure the directory layout agrees whether the spec is installed if not spec.external and self.layout: path = self.layout.path_for_spec(spec) installed = False try: self.layout.ensure_installed(spec) installed = True self._installed_prefixes.add(path) except DirectoryLayoutError as e: if not (allow_missing and isinstance(e, InconsistentInstallDirectoryError)): action = "updated" if key in self._data else "registered" tty.warn( f"{spec.short_spec} is being {action} in the database with prefix {path}, " "but this directory does not contain an installation of " f"the spec, due to: {e}" ) elif spec.external_path: path = spec.external_path installed = True else: path = None installed = True if key not in self._data: # Create a new install record with no deps initially. new_spec = spec.copy(deps=False) self._data[key] = InstallRecord( new_spec, path=path, installed=installed, ref_count=0, explicit=explicit, installation_time=installation_time, origin=None if not hasattr(spec, "origin") else spec.origin, ) # Connect dependencies from the DB to the new copy. for dep in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES): dkey = dep.spec.dag_hash() upstream, record = self.query_by_spec_hash(dkey) assert record, f"Missing dependency {dep.spec.short_spec} in DB" new_spec._add_dependency(record.spec, depflag=dep.depflag, virtuals=dep.virtuals) if not upstream: record.ref_count += 1 # Mark concrete once everything is built, and preserve the original hashes of concrete # specs. new_spec._mark_concrete() new_spec._hash = key new_spec._package_hash = spec_pkg_hash else: # It is already in the database self._data[key].installed = installed self._data[key].installation_time = _now() self._data[key].explicit = explicit @_autospec def add(self, spec: "spack.spec.Spec", *, explicit: bool = False, allow_missing=False) -> None: """Add spec at path to database, locking and reading DB to sync. ``add()`` will lock and read from the DB on disk. """ # TODO: ensure that spec is concrete? # Entire add is transactional. with self.write_transaction(): self._add(spec, explicit=explicit, allow_missing=allow_missing) def _get_matching_spec_key(self, spec: "spack.spec.Spec", **kwargs) -> str: """Get the exact spec OR get a single spec that matches.""" key = spec.dag_hash() _, record = self.query_by_spec_hash(key) if not record: match = self.query_one(spec, **kwargs) if match: return match.dag_hash() raise NoSuchSpecError(spec) return key @_autospec def get_record(self, spec: "spack.spec.Spec", **kwargs) -> Optional[InstallRecord]: key = self._get_matching_spec_key(spec, **kwargs) _, record = self.query_by_spec_hash(key) return record def _decrement_ref_count(self, spec: "spack.spec.Spec") -> None: key = spec.dag_hash() if key not in self._data: # TODO: print something here? DB is corrupt, but # not much we can do. return rec = self._data[key] rec.ref_count -= 1 if rec.ref_count == 0 and not rec.installed: del self._data[key] for dep in spec.dependencies(deptype=_TRACKED_DEPENDENCIES): self._decrement_ref_count(dep) def _increment_ref_count(self, spec: "spack.spec.Spec") -> None: key = spec.dag_hash() if key not in self._data: return rec = self._data[key] rec.ref_count += 1 def _remove(self, spec: "spack.spec.Spec") -> "spack.spec.Spec": """Non-locking version of remove(); does real work.""" key = self._get_matching_spec_key(spec) rec = self._data[key] # This install prefix is now free for other specs to use, even if the # spec is only marked uninstalled. if not rec.spec.external and rec.installed and rec.path: self._installed_prefixes.remove(rec.path) if rec.ref_count > 0: rec.installed = False return rec.spec del self._data[key] # Remove any reference to this node from dependencies and # decrement the reference count rec.spec.detach(deptype=_TRACKED_DEPENDENCIES) for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES): self._decrement_ref_count(dep) if rec.deprecated_for: new_spec = self._data[rec.deprecated_for].spec self._decrement_ref_count(new_spec) # Returns the concrete spec so we know it in the case where a # query spec was passed in. return rec.spec @_autospec def remove(self, spec: "spack.spec.Spec") -> "spack.spec.Spec": """Removes a spec from the database. To be called on uninstall. Reads the database, then: 1. Marks the spec as not installed. 2. Removes the spec if it has no more dependents. 3. If removed, recursively updates dependencies' ref counts and removes them if they are no longer needed. """ # Take a lock around the entire removal. with self.write_transaction(): return self._remove(spec) def deprecator(self, spec: "spack.spec.Spec") -> Optional["spack.spec.Spec"]: """Return the spec that the given spec is deprecated for, or None""" with self.read_transaction(): spec_key = self._get_matching_spec_key(spec) spec_rec = self._data[spec_key] if spec_rec.deprecated_for: return self._data[spec_rec.deprecated_for].spec else: return None def specs_deprecated_by(self, spec: "spack.spec.Spec") -> List["spack.spec.Spec"]: """Return all specs deprecated in favor of the given spec""" with self.read_transaction(): return [ rec.spec for rec in self._data.values() if rec.deprecated_for == spec.dag_hash() ] def _deprecate(self, spec: "spack.spec.Spec", deprecator: "spack.spec.Spec") -> None: spec_key = self._get_matching_spec_key(spec) spec_rec = self._data[spec_key] deprecator_key = self._get_matching_spec_key(deprecator) self._increment_ref_count(deprecator) # If spec was already deprecated, update old deprecator's ref count if spec_rec.deprecated_for: old_repl_rec = self._data[spec_rec.deprecated_for] self._decrement_ref_count(old_repl_rec.spec) spec_rec.deprecated_for = deprecator_key spec_rec.installed = False self._data[spec_key] = spec_rec @_autospec def mark(self, spec: "spack.spec.Spec", key: str, value: Any) -> None: """Mark an arbitrary record on a spec.""" with self.write_transaction(): return self._mark(spec, key, value) def _mark(self, spec: "spack.spec.Spec", key, value) -> None: record = self._data[self._get_matching_spec_key(spec)] setattr(record, key, value) @_autospec def deprecate(self, spec: "spack.spec.Spec", deprecator: "spack.spec.Spec") -> None: """Marks a spec as deprecated in favor of its deprecator""" with self.write_transaction(): return self._deprecate(spec, deprecator) @_autospec def installed_relatives( self, spec: "spack.spec.Spec", direction: tr.DirectionType = "children", transitive: bool = True, deptype: Union[dt.DepFlag, dt.DepTypes] = dt.ALL, ) -> Set["spack.spec.Spec"]: """Return installed specs related to this one.""" if direction not in ("parents", "children"): raise ValueError("Invalid direction: %s" % direction) relatives: Set[spack.spec.Spec] = set() for spec in self.query(spec): if transitive: to_add = spec.traverse(direction=direction, root=False, deptype=deptype) elif direction == "parents": to_add = spec.dependents(deptype=deptype) else: # direction == 'children' to_add = spec.dependencies(deptype=deptype) for relative in to_add: hash_key = relative.dag_hash() _, record = self.query_by_spec_hash(hash_key) if not record: tty.warn( f"Inconsistent state: " f"{'dependent' if direction == 'parents' else 'dependency'} {hash_key} of " f"{spec.dag_hash()} not in DB" ) continue if not record.installed: continue relatives.add(relative) return relatives @_autospec def installed_extensions_for(self, extendee_spec: "spack.spec.Spec"): """Returns the specs of all packages that extend the given spec""" for spec in self.query(): if spec.package.extends(extendee_spec): yield spec.package def _get_by_hash_local( self, dag_hash: str, default: Optional[List["spack.spec.Spec"]] = None, installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY, ) -> Optional[List["spack.spec.Spec"]]: installed = normalize_query(installed) # hash is a full hash and is in the data somewhere if dag_hash in self._data: rec = self._data[dag_hash] if rec.install_type_matches(installed): return [rec.spec] else: return default # check if hash is a prefix of some installed (or previously installed) spec. matches = [ record.spec for h, record in self._data.items() if h.startswith(dag_hash) and record.install_type_matches(installed) ] if matches: return matches # nothing found return default def get_by_hash_local( self, dag_hash: str, default: Optional[List["spack.spec.Spec"]] = None, installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY, ) -> Optional[List["spack.spec.Spec"]]: """Look up a spec in *this DB* by DAG hash, or by a DAG hash prefix. Args: dag_hash: hash (or hash prefix) to look up default: default value to return if dag_hash is not in the DB installed: if ``True``, includes only installed specs in the search; if ``False`` only missing specs. Otherwise, a InstallRecordStatus flag. ``installed`` defaults to ``InstallRecordStatus.ANY`` so we can refer to any known hash. ``query()`` and ``query_one()`` differ in that they only return installed specs by default. """ with self.read_transaction(): return self._get_by_hash_local(dag_hash, default=default, installed=installed) def get_by_hash( self, dag_hash: str, default: Optional[List["spack.spec.Spec"]] = None, installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY, ) -> Optional[List["spack.spec.Spec"]]: """Look up a spec by DAG hash, or by a DAG hash prefix. Args: dag_hash: hash (or hash prefix) to look up default: default value to return if dag_hash is not in the DB installed: if ``True``, includes only installed specs in the search; if ``False`` only missing specs. Otherwise, a InstallRecordStatus flag. ``installed`` defaults to ``InstallRecordStatus.ANY`` so we can refer to any known hash. ``query()`` and ``query_one()`` differ in that they only return installed specs by default. """ spec = self.get_by_hash_local(dag_hash, default=default, installed=installed) if spec is not None: return spec for upstream_db in self.upstream_dbs: spec = upstream_db._get_by_hash_local(dag_hash, default=default, installed=installed) if spec is not None: return spec return default def _query( self, query_spec: Optional[Union[str, "spack.spec.Spec"]] = None, *, predicate_fn: Optional[SelectType] = None, installed: Union[bool, InstallRecordStatus] = True, explicit: Optional[bool] = None, start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None, hashes: Optional[Iterable[str]] = None, in_buildcache: Optional[bool] = None, origin: Optional[str] = None, ) -> List["spack.spec.Spec"]: installed = normalize_query(installed) # Restrict the set of records over which we iterate first matching_hashes = self._data if hashes is not None: matching_hashes = {h: self._data[h] for h in hashes if h in self._data} if isinstance(query_spec, str): query_spec = spack.spec.Spec(query_spec) if query_spec is not None and query_spec.concrete: hash_key = query_spec.dag_hash() if hash_key not in matching_hashes: return [] matching_hashes = {hash_key: matching_hashes[hash_key]} results = [] start_date = start_date or datetime.datetime.min end_date = end_date or datetime.datetime.max deferred = [] for rec in matching_hashes.values(): if origin and not (origin == rec.origin): continue if not rec.install_type_matches(installed): continue if in_buildcache is not None and rec.in_buildcache != in_buildcache: continue if explicit is not None and rec.explicit != explicit: continue if predicate_fn is not None and not predicate_fn(rec): continue if start_date or end_date: inst_date = datetime.datetime.fromtimestamp(rec.installation_time) if not (start_date < inst_date < end_date): continue if query_spec is None or query_spec.concrete: results.append(rec.spec) continue # check anon specs and exact name matches first if not query_spec.name or rec.spec.name == query_spec.name: if rec.spec.satisfies(query_spec): results.append(rec.spec) # save potential virtual matches for later, but not if we already found a match elif not results: deferred.append(rec.spec) # Checking for virtuals is expensive, so we save it for last and only if needed. # If we get here, we didn't find anything in the DB that matched by name. # If we did fine something, the query spec can't be virtual b/c we matched an actual # package installation, so skip the virtual check entirely. If we *didn't* find anything, # check all the deferred specs *if* the query is virtual. if ( not results and query_spec is not None and deferred and spack.repo.PATH.is_virtual(query_spec.name) ): results = [spec for spec in deferred if spec.satisfies(query_spec)] return results def query_local( self, query_spec: Optional[Union[str, "spack.spec.Spec"]] = None, *, predicate_fn: Optional[SelectType] = None, installed: Union[bool, InstallRecordStatus] = True, explicit: Optional[bool] = None, start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None, hashes: Optional[List[str]] = None, in_buildcache: Optional[bool] = None, origin: Optional[str] = None, ) -> List["spack.spec.Spec"]: """Queries the local Spack database. This function doesn't guarantee any sorting of the returned data for performance reason, since comparing specs for __lt__ may be an expensive operation. Args: query_spec: if query_spec is ``None``, match all specs in the database. If it is a spec, return all specs matching ``spec.satisfies(query_spec)``. predicate_fn: optional predicate taking an InstallRecord as argument, and returning whether that record is selected for the query. It can be used to craft criteria that need some data for selection not provided by the Database itself. installed: if ``True``, includes only installed specs in the search. If ``False`` only missing specs, and if ``any``, all specs in database. If an InstallStatus or iterable of InstallStatus, returns specs whose install status matches at least one of the InstallStatus. explicit: a spec that was installed following a specific user request is marked as explicit. If instead it was pulled-in as a dependency of a user requested spec it's considered implicit. start_date: if set considers only specs installed from the starting date. end_date: if set considers only specs installed until the ending date. in_buildcache: specs that are marked in this database as part of an associated binary cache are ``in_buildcache``. All other specs are not. This field is used for querying mirror indices. By default, it does not check this status. hashes: list of hashes used to restrict the search origin: origin of the spec """ with self.read_transaction(): return self._query( query_spec, predicate_fn=predicate_fn, installed=installed, explicit=explicit, start_date=start_date, end_date=end_date, hashes=hashes, in_buildcache=in_buildcache, origin=origin, ) def query( self, query_spec: Optional[Union[str, "spack.spec.Spec"]] = None, *, predicate_fn: Optional[SelectType] = None, installed: Union[bool, InstallRecordStatus] = True, explicit: Optional[bool] = None, start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None, in_buildcache: Optional[bool] = None, hashes: Optional[List[str]] = None, origin: Optional[str] = None, install_tree: str = "all", ) -> List["spack.spec.Spec"]: """Queries the Spack database including all upstream databases. Args: query_spec: if query_spec is ``None``, match all specs in the database. If it is a spec, return all specs matching ``spec.satisfies(query_spec)``. predicate_fn: optional predicate taking an InstallRecord as argument, and returning whether that record is selected for the query. It can be used to craft criteria that need some data for selection not provided by the Database itself. installed: if ``True``, includes only installed specs in the search. If ``False`` only missing specs, and if ``any``, all specs in database. If an InstallStatus or iterable of InstallStatus, returns specs whose install status matches at least one of the InstallStatus. explicit: a spec that was installed following a specific user request is marked as explicit. If instead it was pulled-in as a dependency of a user requested spec it's considered implicit. start_date: if set considers only specs installed from the starting date. end_date: if set considers only specs installed until the ending date. in_buildcache: specs that are marked in this database as part of an associated binary cache are ``in_buildcache``. All other specs are not. This field is used for querying mirror indices. By default, it does not check this status. hashes: list of hashes used to restrict the search install_tree: query ``"all"`` (default), ``"local"``, ``"upstream"``, or upstream path origin: origin of the spec """ valid_trees = ["all", "upstream", "local", self.root] + [u.root for u in self.upstream_dbs] if install_tree not in valid_trees: msg = "Invalid install_tree argument to Database.query()\n" msg += f"Try one of {', '.join(valid_trees)}" tty.error(msg) return [] upstream_results = [] upstreams = self.upstream_dbs if install_tree not in ("all", "upstream"): upstreams = [u for u in self.upstream_dbs if u.root == install_tree] for upstream_db in upstreams: # queries for upstream DBs need to *not* lock - we may not # have permissions to do this and the upstream DBs won't know about # us anyway (so e.g. they should never uninstall specs) upstream_results.extend( upstream_db._query( query_spec, predicate_fn=predicate_fn, installed=installed, explicit=explicit, start_date=start_date, end_date=end_date, hashes=hashes, in_buildcache=in_buildcache, origin=origin, ) or [] ) local_results: Set["spack.spec.Spec"] = set() if install_tree in ("all", "local") or self.root == install_tree: local_results = set( self.query_local( query_spec, predicate_fn=predicate_fn, installed=installed, explicit=explicit, start_date=start_date, end_date=end_date, hashes=hashes, in_buildcache=in_buildcache, origin=origin, ) ) results = list(local_results) + list(x for x in upstream_results if x not in local_results) results.sort() # type: ignore[call-arg,call-overload] return results def query_one( self, query_spec: Optional[Union[str, "spack.spec.Spec"]], predicate_fn: Optional[SelectType] = None, installed: Union[bool, InstallRecordStatus] = True, ) -> Optional["spack.spec.Spec"]: """Query for exactly one spec that matches the query spec. Returns None if no installed package matches. Raises: AssertionError: if more than one spec matches the query. """ concrete_specs = self.query(query_spec, predicate_fn=predicate_fn, installed=installed) assert len(concrete_specs) <= 1 return concrete_specs[0] if concrete_specs else None def missing(self, spec): key = spec.dag_hash() _, record = self.query_by_spec_hash(key) return record and not record.installed def is_occupied_install_prefix(self, path): with self.read_transaction(): return path in self._installed_prefixes def all_hashes(self): """Return dag hash of every spec in the database.""" with self.read_transaction(): return list(self._data.keys()) def unused_specs( self, root_hashes: Optional[Container[str]] = None, deptype: Union[dt.DepFlag, dt.DepTypes] = dt.LINK | dt.RUN, ) -> List["spack.spec.Spec"]: """Return all specs that are currently installed but not needed by root specs. By default, roots are all explicit specs in the database. If a set of root hashes are passed in, they are instead used as the roots. Arguments: root_hashes: optional list of roots to consider when evaluating needed installations. deptype: if a spec is reachable from a root via these dependency types, it is considered needed. By default only link and run dependency types are considered. """ def root(key, record): """Whether a DB record is a root for garbage collection.""" return key in root_hashes if root_hashes is not None else record.explicit with self.read_transaction(): roots = [rec.spec for key, rec in self._data.items() if root(key, rec)] needed = set(id(spec) for spec in tr.traverse_nodes(roots, deptype=deptype)) return [ rec.spec for rec in self._data.values() if id(rec.spec) not in needed and rec.installed ] class NoUpstreamVisitor: """Gives edges to upstream specs, but does follow edges from upstream specs.""" def __init__( self, upstream_hashes: Set[str], on_visit: Callable[["spack.spec.DependencySpec", bool], None], ): self.upstream_hashes = upstream_hashes self.on_visit = on_visit def accept(self, item: tr.EdgeAndDepth) -> bool: self.on_visit(item.edge, self.is_upstream(item)) return True def is_upstream(self, item: tr.EdgeAndDepth) -> bool: return item.edge.spec.dag_hash() in self.upstream_hashes def neighbors(self, item: tr.EdgeAndDepth): # Prune edges from upstream nodes, only follow database tracked dependencies return ( [] if self.is_upstream(item) else item.edge.spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES) ) class UpstreamDatabaseLockingError(SpackError): """Raised when an operation would need to lock an upstream database""" class CorruptDatabaseError(SpackError): """Raised when errors are found while reading the database.""" class NonConcreteSpecAddError(SpackError): """Raised when attempting to add non-concrete spec to DB.""" class MissingDependenciesError(SpackError): """Raised when DB cannot find records for dependencies""" class InvalidDatabaseVersionError(SpackError): """Exception raised when the database metadata is newer than current Spack.""" def __init__(self, database, expected, found): self.expected = expected self.found = found msg = ( f"you need a newer Spack version to read the DB in '{database.root}'. " f"{self.database_version_message}" ) super().__init__(msg) @property def database_version_message(self): return f"The expected DB version is '{self.expected}', but '{self.found}' was found." class ExplicitDatabaseUpgradeError(SpackError): """Raised to request an explicit DB upgrade to the user""" class DatabaseNotReadableError(SpackError): """Raised to signal Database.reindex that the reindex should happen via spec.json""" class NoSuchSpecError(KeyError): """Raised when a spec is not found in the database.""" def __init__(self, spec): self.spec = spec super().__init__(spec) def __str__(self): # This exception is raised frequently, and almost always # caught, so ensure we don't pay the cost of Spec.__str__ # unless the exception is actually printed. return f"No such spec in database: {self.spec}" ================================================ FILE: lib/spack/spack/dependency.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Data structures that represent Spack's dependency relationships.""" from typing import TYPE_CHECKING, Dict, List, Type import spack.deptypes as dt import spack.spec if TYPE_CHECKING: import spack.package_base import spack.patch class Dependency: """Class representing metadata for a dependency on a package. This class differs from ``spack.spec.DependencySpec`` because it represents metadata at the ``Package`` level. ``spack.spec.DependencySpec`` is a descriptor for an actual package configuration, while ``Dependency`` is a descriptor for a package's dependency *requirements*. A dependency is a requirement for a configuration of another package that satisfies a particular spec. The dependency can have *types*, which determine *how* that package configuration is required, e.g. whether it is required for building the package, whether it needs to be linked to, or whether it is needed at runtime so that Spack can call commands from it. A package can also depend on another package with *patches*. This is for cases where the maintainers of one package also maintain special patches for their dependencies. If one package depends on another with patches, a special version of that dependency with patches applied will be built for use by the dependent package. The patches are included in the new version's spec hash to differentiate it from unpatched versions of the same package, so that unpatched versions of the dependency package can coexist with the patched version. """ __slots__ = "pkg", "spec", "patches", "depflag" def __init__( self, pkg: Type["spack.package_base.PackageBase"], spec: spack.spec.Spec, depflag: dt.DepFlag = dt.DEFAULT, ): """Create a new Dependency. Args: pkg: Package that has this dependency spec: Spec indicating dependency requirements type: strings describing dependency relationship """ self.pkg = pkg self.spec = spec # This dict maps condition specs to lists of Patch objects, just # as the patches dict on packages does. self.patches: Dict[spack.spec.Spec, List["spack.patch.Patch"]] = {} self.depflag = depflag @property def name(self) -> str: """Get the name of the dependency package.""" return self.spec.name def __repr__(self) -> str: types = dt.flag_to_chars(self.depflag) if self.patches: return f" {self.spec} [{types}, {self.patches}]>" else: return f" {self.spec} [{types}]>" ================================================ FILE: lib/spack/spack/deptypes.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Data structures that represent Spack's edge types.""" from typing import Iterable, List, Tuple, Union from spack.vendor.typing_extensions import Literal #: Type hint for the low-level dependency input (enum.Flag is too slow) DepFlag = int #: Type hint for the high-level dependency input DepTypes = Union[str, List[str], Tuple[str, ...]] #: Individual dependency types DepType = Literal["build", "link", "run", "test"] # Flag values. NOTE: these values are not arbitrary, since hash computation imposes # the order (link, run, build, test) when depending on the same package multiple times, # and we rely on default integer comparison to sort dependency types. # New dependency types should be appended. LINK = 0b0001 RUN = 0b0010 BUILD = 0b0100 TEST = 0b1000 #: The types of dependency relationships that Spack understands. ALL_TYPES: Tuple[DepType, ...] = ("build", "link", "run", "test") #: Default dependency type if none is specified DEFAULT_TYPES: Tuple[DepType, ...] = ("build", "link") #: A flag with all dependency types set ALL: DepFlag = BUILD | LINK | RUN | TEST #: Default dependency type if none is specified DEFAULT: DepFlag = BUILD | LINK #: A flag with no dependency types set NONE: DepFlag = 0 #: An iterator of all flag components ALL_FLAGS: Tuple[DepFlag, DepFlag, DepFlag, DepFlag] = (BUILD, LINK, RUN, TEST) def compatible(flag1: DepFlag, flag2: DepFlag) -> bool: """Returns True if two depflags can be dependencies from a Spec to deps of the same name. The only allowable separated dependencies are a build-only dependency, combined with a non-build dependency. This separates our two process spaces, build time and run time. These dependency combinations are allowed: * single dep on name: ``[b]``, ``[l]``, ``[r]``, ``[bl]``, ``[br]``, ``[blr]`` * two deps on name: ``[b, l]``, ``[b, r]``, ``[b, lr]`` but none of these make any sense: * two build deps: ``[b, b]``, ``[b, br]``, ``[b, bl]``, ``[b, blr]`` * any two deps that both have an ``l`` or an ``r``, i.e. ``[l, l]``, ``[r, r]``, ``[l, r]``, ``[bl, l]``, ``[bl, r]``""" # Cannot have overlapping build types to two different dependencies if flag1 & flag2: return False # Cannot have two different link/run dependencies for the same name link_run = LINK | RUN if flag1 & link_run and flag2 & link_run: return False return True def flag_from_string(s: str) -> DepFlag: if s == "build": return BUILD elif s == "link": return LINK elif s == "run": return RUN elif s == "test": return TEST else: raise ValueError(f"Invalid dependency type: {s}") def flag_from_strings(deptype: Iterable[str]) -> DepFlag: """Transform an iterable of deptype strings into a flag.""" flag = 0 for deptype_str in deptype: flag |= flag_from_string(deptype_str) return flag def canonicalize(deptype: DepTypes) -> DepFlag: """Convert deptype user input to a DepFlag, or raise ValueError. Args: deptype: string representing dependency type, or a list/tuple of such strings. Can also be the builtin function ``all`` or the string 'all', which result in a tuple of all dependency types known to Spack. """ if deptype in ("all", all): return ALL if isinstance(deptype, str): return flag_from_string(deptype) if isinstance(deptype, (tuple, list, set)): return flag_from_strings(deptype) raise ValueError(f"Invalid dependency type: {deptype!r}") def flag_to_tuple(x: DepFlag) -> Tuple[DepType, ...]: deptype: List[DepType] = [] if x & BUILD: deptype.append("build") if x & LINK: deptype.append("link") if x & RUN: deptype.append("run") if x & TEST: deptype.append("test") return tuple(deptype) def flag_to_string(x: DepFlag) -> DepType: if x == BUILD: return "build" elif x == LINK: return "link" elif x == RUN: return "run" elif x == TEST: return "test" else: raise ValueError(f"Invalid dependency type flag: {x}") def flag_to_chars(depflag: DepFlag) -> str: """Create a string representing deptypes for many dependencies. The string will be some subset of ``blrt``, like ``bl ``, ``b t``, or `` lr `` where each letter in ``blrt`` stands for ``build``, ``link``, ``run``, and ``test`` (the dependency types). For a single dependency, this just indicates that the dependency has the indicated deptypes. For a list of dependnecies, this shows whether ANY dependency in the list has the deptypes (so the deptypes are merged).""" return "".join( t_str[0] if t_flag & depflag else " " for t_str, t_flag in zip(ALL_TYPES, ALL_FLAGS) ) ================================================ FILE: lib/spack/spack/detection/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from .common import executable_prefix, set_virtuals_nonbuildable, update_configuration from .path import by_path, executables_in_path from .test import detection_tests __all__ = [ "by_path", "executables_in_path", "executable_prefix", "update_configuration", "set_virtuals_nonbuildable", "detection_tests", ] ================================================ FILE: lib/spack/spack/detection/common.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Define a common data structure to represent external packages and a function to update packages.yaml given a list of detected packages. Ideally, each detection method should be placed in a specific subpackage and implement at least a function that returns a list of specs. The update in packages.yaml can then be done using the function provided here. The module also contains other functions that might be useful across different detection mechanisms. """ import glob import itertools import os import pathlib import re import sys from typing import Dict, List, Optional, Set, Tuple, Union import spack.config import spack.error import spack.operating_systems.windows_os as winOs import spack.schema import spack.spec import spack.util.environment import spack.util.spack_yaml import spack.util.windows_registry from spack.llnl.util import tty def _externals_in_packages_yaml() -> Set[spack.spec.Spec]: """Returns all the specs mentioned as externals in packages.yaml""" packages_yaml = spack.config.get("packages") already_defined_specs = set() for pkg_name, package_configuration in packages_yaml.items(): for item in package_configuration.get("externals", []): already_defined_specs.add(spack.spec.Spec(item["spec"])) return already_defined_specs ExternalEntryType = Union[str, Dict[str, str]] def _pkg_config_dict( external_pkg_entries: List["spack.spec.Spec"], ) -> Dict[str, Union[bool, List[Dict[str, ExternalEntryType]]]]: """Generate a package specific config dict according to the packages.yaml schema. This does not generate the entire packages.yaml. For example, given some external entries for the CMake package, this could return:: { 'externals': [{ 'spec': 'cmake@3.17.1', 'prefix': '/opt/cmake-3.17.1/' }, { 'spec': 'cmake@3.16.5', 'prefix': '/opt/cmake-3.16.5/' }] } """ pkg_dict = spack.util.spack_yaml.syaml_dict() pkg_dict["externals"] = [] for e in external_pkg_entries: if not _spec_is_valid(e): continue external_items: List[Tuple[str, ExternalEntryType]] = [ ("spec", str(e)), ("prefix", pathlib.Path(e.external_path).as_posix()), ] if e.external_modules: external_items.append(("modules", e.external_modules)) if e.extra_attributes: external_items.append( ("extra_attributes", spack.util.spack_yaml.syaml_dict(e.extra_attributes.items())) ) # external_items.extend(e.spec.extra_attributes.items()) pkg_dict["externals"].append(spack.util.spack_yaml.syaml_dict(external_items)) return pkg_dict def _spec_is_valid(spec: spack.spec.Spec) -> bool: try: str(spec) except spack.error.SpackError: # It is assumed here that we can at least extract the package name from the spec so we # can look up the implementation of determine_spec_details tty.warn(f"Constructed spec for {spec.name} does not have a string representation") return False try: spack.spec.Spec(str(spec)) except spack.error.SpackError: tty.warn( "Constructed spec has a string representation but the string" " representation does not evaluate to a valid spec: {0}".format(str(spec)) ) return False return True def path_to_dict(search_paths: List[str]) -> Dict[str, str]: """Return dictionary[fullpath]: basename from list of paths""" path_to_lib: Dict[str, str] = {} # Reverse order of search directories so that a lib in the first # entry overrides later entries for search_path in reversed(search_paths): try: dir_iter = os.scandir(search_path) except OSError as e: tty.debug(f"cannot scan '{search_path}' for external software: {e}") continue with dir_iter as entries: for entry in entries: try: if entry.is_file(): path_to_lib[entry.path] = entry.name except OSError as e: tty.debug(f"cannot scan '{search_path}' for external software: {e}") return path_to_lib def is_executable(file_path: str) -> bool: """Return True if the path passed as argument is that of an executable""" return os.path.isfile(file_path) and os.access(file_path, os.X_OK) def _convert_to_iterable(single_val_or_multiple): x = single_val_or_multiple if x is None: return [] elif isinstance(x, str): return [x] elif isinstance(x, spack.spec.Spec): # Specs are iterable, but a single spec should be converted to a list return [x] try: iter(x) return x except TypeError: return [x] def executable_prefix(executable_dir: str) -> str: """Given a directory where an executable is found, guess the prefix (i.e. the "root" directory of that installation) and return it. Args: executable_dir: directory where an executable is found """ # Given a prefix where an executable is found, assuming that prefix # contains /bin/, strip off the 'bin' directory to get a Spack-compatible # prefix assert os.path.isdir(executable_dir) components = executable_dir.split(os.sep) # convert to lower to match Bin, BIN, bin lowered_components = executable_dir.lower().split(os.sep) if "bin" not in lowered_components: return executable_dir idx = lowered_components.index("bin") return os.sep.join(components[:idx]) def library_prefix(library_dir: str) -> str: """Given a directory where a library is found, guess the prefix (i.e. the "root" directory of that installation) and return it. Args: library_dir: directory where a library is found """ # Given a prefix where an library is found, assuming that prefix # contains /lib/ or /lib64/, strip off the 'lib' or 'lib64' directory # to get a Spack-compatible prefix assert os.path.isdir(library_dir) components = library_dir.split(os.sep) # convert to lowercase to match lib, LIB, Lib, etc. lowered_components = library_dir.lower().split(os.sep) if "lib64" in lowered_components: idx = lowered_components.index("lib64") return os.sep.join(components[:idx]) elif "lib" in lowered_components: idx = lowered_components.index("lib") return os.sep.join(components[:idx]) elif sys.platform == "win32" and "bin" in lowered_components: idx = lowered_components.index("bin") return os.sep.join(components[:idx]) else: return library_dir def update_configuration( detected_packages: Dict[str, List["spack.spec.Spec"]], scope: Optional[str] = None, buildable: bool = True, ) -> List[spack.spec.Spec]: """Add the packages passed as arguments to packages.yaml Args: detected_packages: list of specs to be added scope: configuration scope where to add the detected packages buildable: whether the detected packages are buildable or not """ predefined_external_specs = _externals_in_packages_yaml() pkg_to_cfg, all_new_specs = {}, [] for package_name, entries in detected_packages.items(): new_entries = [s for s in entries if s not in predefined_external_specs] pkg_config = _pkg_config_dict(new_entries) external_entries = pkg_config.get("externals", []) assert not isinstance(external_entries, bool), "unexpected value for external entry" all_new_specs.extend(new_entries) if buildable is False: pkg_config["buildable"] = False pkg_to_cfg[package_name] = pkg_config scope = scope or spack.config.default_modify_scope() pkgs_cfg = spack.config.get("packages", scope=scope) pkgs_cfg = spack.schema.merge_yaml(pkgs_cfg, pkg_to_cfg) spack.config.set("packages", pkgs_cfg, scope=scope) return all_new_specs def set_virtuals_nonbuildable(virtuals: Set[str], scope: Optional[str] = None) -> List[str]: """Update packages:virtual:buildable:False for the provided virtual packages, if the property is not set by the user. Returns the list of virtual packages that have been updated.""" packages = spack.config.get("packages") new_config = {} for virtual in virtuals: # If the user has set the buildable prop do not override it if virtual in packages and "buildable" in packages[virtual]: continue new_config[virtual] = {"buildable": False} # Update the provided scope spack.config.set( "packages", spack.schema.merge_yaml(spack.config.get("packages", scope=scope), new_config), scope=scope, ) return list(new_config.keys()) def _windows_drive() -> str: """Return Windows drive string extracted from the PROGRAMFILES environment variable, which is guaranteed to be defined for all logins. """ match = re.match(r"([a-zA-Z]:)", os.environ["PROGRAMFILES"]) if match is None: raise RuntimeError("cannot read the PROGRAMFILES environment variable") return match.group(1) class WindowsCompilerExternalPaths: @staticmethod def find_windows_compiler_root_paths() -> List[str]: """Helper for Windows compiler installation root discovery At the moment simply returns location of VS install paths from VSWhere But should be extended to include more information as relevant""" return list(winOs.WindowsOs().vs_install_paths) @staticmethod def find_windows_compiler_cmake_paths() -> List[str]: """Semi hard-coded search path for cmake bundled with MSVC""" return [ os.path.join( path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "CMake", "bin" ) for path in WindowsCompilerExternalPaths.find_windows_compiler_root_paths() ] @staticmethod def find_windows_compiler_ninja_paths() -> List[str]: """Semi hard-coded search heuristic for locating ninja bundled with MSVC""" return [ os.path.join(path, "Common7", "IDE", "CommonExtensions", "Microsoft", "CMake", "Ninja") for path in WindowsCompilerExternalPaths.find_windows_compiler_root_paths() ] @staticmethod def find_windows_compiler_bundled_packages() -> List[str]: """Return all MSVC compiler bundled packages""" return ( WindowsCompilerExternalPaths.find_windows_compiler_cmake_paths() + WindowsCompilerExternalPaths.find_windows_compiler_ninja_paths() ) class WindowsKitExternalPaths: @staticmethod def find_windows_kit_roots() -> List[str]: """Return Windows kit root, typically %programfiles%\\Windows Kits\\10|11\\""" if sys.platform != "win32": return [] program_files = os.environ["PROGRAMFILES(x86)"] kit_base = os.path.join(program_files, "Windows Kits", "**") return glob.glob(kit_base) @staticmethod def find_windows_kit_bin_paths( kit_base: Union[Optional[str], Optional[list]] = None, ) -> List[str]: """Returns Windows kit bin directory per version""" kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base assert kit_base, "Unexpectedly empty value for Windows kit base path" if isinstance(kit_base, str): kit_base = kit_base.split(";") kit_paths = [] for kit in kit_base: kit_bin = os.path.join(kit, "bin") kit_paths.extend(glob.glob(os.path.join(kit_bin, "[0-9]*", "*\\"))) return kit_paths @staticmethod def find_windows_kit_lib_paths( kit_base: Union[Optional[str], Optional[list]] = None, ) -> List[str]: """Returns Windows kit lib directory per version""" kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base assert kit_base, "Unexpectedly empty value for Windows kit base path" if isinstance(kit_base, str): kit_base = kit_base.split(";") kit_paths = [] for kit in kit_base: kit_lib = os.path.join(kit, "Lib") kit_paths.extend(glob.glob(os.path.join(kit_lib, "[0-9]*", "*", "*\\"))) return kit_paths @staticmethod def find_windows_driver_development_kit_paths() -> List[str]: """Provides a list of all installation paths for the WDK by version and architecture """ wdk_content_root = os.getenv("WDKContentRoot") return WindowsKitExternalPaths.find_windows_kit_lib_paths(wdk_content_root) @staticmethod def find_windows_kit_reg_installed_roots_paths() -> List[str]: reg = spack.util.windows_registry.WindowsRegistryView( "SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots", root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE, ) if not reg: # couldn't find key, return empty list return [] kit_root_reg = re.compile(r"KitsRoot[0-9]+") root_paths = [] for kit_root in filter(kit_root_reg.match, reg.get_values().keys()): root_paths.extend( WindowsKitExternalPaths.find_windows_kit_lib_paths(reg.get_value(kit_root).value) ) return root_paths @staticmethod def find_windows_kit_reg_sdk_paths() -> List[str]: sdk_paths = [] sdk_regex = re.compile(r"v[0-9]+.[0-9]+") windows_reg = spack.util.windows_registry.WindowsRegistryView( "SOFTWARE\\WOW6432Node\\Microsoft\\Microsoft SDKs\\Windows", root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE, ) for key in filter(sdk_regex.match, [x.name for x in windows_reg.get_subkeys()]): reg = windows_reg.get_subkey(key) sdk_paths.extend( WindowsKitExternalPaths.find_windows_kit_lib_paths( reg.get_value("InstallationFolder").value ) ) return sdk_paths def find_win32_additional_install_paths() -> List[str]: """Not all programs on Windows live on the PATH Return a list of other potential install locations. """ drive_letter = _windows_drive() windows_search_ext = [] cuda_re = r"CUDA_PATH[a-zA-Z1-9_]*" # The list below should be expanded with other # common Windows install locations as necessary path_ext_keys = ["I_MPI_ONEAPI_ROOT", "MSMPI_BIN", "MLAB_ROOT", "NUGET_PACKAGES"] user = os.environ["USERPROFILE"] add_path = lambda key: re.search(cuda_re, key) or key in path_ext_keys windows_search_ext.extend([os.environ[key] for key in os.environ.keys() if add_path(key)]) # note windows paths are fine here as this method should only ever be invoked # to interact with Windows # Add search path for default Chocolatey (https://github.com/chocolatey/choco) # install directory windows_search_ext.append("%s\\ProgramData\\chocolatey\\bin" % drive_letter) # Add search path for NuGet package manager default install location windows_search_ext.append(os.path.join(user, ".nuget", "packages")) windows_search_ext.extend( spack.config.get("config:additional_external_search_paths", default=[]) ) windows_search_ext.extend(spack.util.environment.get_path("PATH")) return windows_search_ext def compute_windows_program_path_for_package(pkg: "spack.package_base.PackageBase") -> List[str]: """Given a package, attempts to compute its Windows program files location, and returns the list of best guesses. Args: pkg: package for which Program Files location is to be computed """ if sys.platform != "win32": return [] # note windows paths are fine here as this method should only ever be invoked # to interact with Windows program_files = "{}\\Program Files{}\\{}" drive_letter = _windows_drive() return [ program_files.format(drive_letter, arch, name) for arch, name in itertools.product(("", " (x86)"), (pkg.name, pkg.name.capitalize())) ] def compute_windows_user_path_for_package(pkg: "spack.package_base.PackageBase") -> List[str]: """Given a package attempt to compute its user scoped install location, return list of potential locations based on common heuristics. For more info on Windows user specific installs see: https://learn.microsoft.com/en-us/dotnet/api/system.environment.specialfolder?view=netframework-4.8 """ if sys.platform != "win32": return [] # Current user directory user = os.environ["USERPROFILE"] app_data = "AppData" app_data_locations = ["Local", "Roaming"] user_appdata_install_stubs = [os.path.join(app_data, x) for x in app_data_locations] return [ os.path.join(user, app_data, name) for app_data, name in list( itertools.product(user_appdata_install_stubs, (pkg.name, pkg.name.capitalize())) ) ] + [os.path.join(user, name) for name in (pkg.name, pkg.name.capitalize())] ================================================ FILE: lib/spack/spack/detection/path.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Detection of software installed in the system, based on paths inspections and running executables. """ import collections import concurrent.futures import os import pathlib import re import sys import traceback import warnings from typing import Dict, Iterable, List, Optional, Set, Tuple, Type import spack.error import spack.llnl.util.filesystem import spack.llnl.util.lang import spack.llnl.util.tty import spack.spec import spack.util.elf as elf_utils import spack.util.environment import spack.util.environment as environment import spack.util.ld_so_conf import spack.util.parallel from .common import ( WindowsCompilerExternalPaths, WindowsKitExternalPaths, _convert_to_iterable, compute_windows_program_path_for_package, compute_windows_user_path_for_package, executable_prefix, find_win32_additional_install_paths, library_prefix, path_to_dict, ) #: Timeout used for package detection (seconds) DETECTION_TIMEOUT = 60 if sys.platform == "win32": DETECTION_TIMEOUT = 120 def common_windows_package_paths(pkg_cls=None) -> List[str]: """Get the paths for common package installation location on Windows that are outside the PATH Returns [] on unix """ if sys.platform != "win32": return [] paths = WindowsCompilerExternalPaths.find_windows_compiler_bundled_packages() paths.extend(find_win32_additional_install_paths()) paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths()) paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths()) paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths()) if pkg_cls: paths.extend(compute_windows_user_path_for_package(pkg_cls)) paths.extend(compute_windows_program_path_for_package(pkg_cls)) return paths def file_identifier(path): s = os.stat(path) return s.st_dev, s.st_ino def dedupe_paths(paths: List[str]) -> List[str]: """Deduplicate paths based on inode and device number. In case the list contains first a symlink and then the directory it points to, the symlink is replaced with the directory path. This ensures that we pick for example ``/usr/bin`` over ``/bin`` if the latter is a symlink to the former.""" seen: Dict[Tuple[int, int], str] = {} linked_parent_check = lambda x: any( [spack.llnl.util.filesystem.islink(str(y)) for y in pathlib.Path(x).parents] ) for path in paths: identifier = file_identifier(path) if identifier not in seen: seen[identifier] = path # we also want to deprioritize paths if they contain a symlink in any parent # (not just the basedir): e.g. oneapi has "latest/bin", # where "latest" is a symlink to 2025.0" elif not (spack.llnl.util.filesystem.islink(path) or linked_parent_check(path)): seen[identifier] = path return list(seen.values()) def executables_in_path(path_hints: List[str]) -> Dict[str, str]: """Get the paths of all executables available from the current PATH. For convenience, this is constructed as a dictionary where the keys are the executable paths and the values are the names of the executables (i.e. the basename of the executable path). There may be multiple paths with the same basename. In this case it is assumed there are two different instances of the executable. Args: path_hints: list of paths to be searched. If None the list will be constructed based on the PATH environment variable. """ search_paths = spack.llnl.util.filesystem.search_paths_for_executables(*path_hints) # Make use we don't doubly list /usr/lib and /lib etc return path_to_dict(dedupe_paths(search_paths)) def accept_elf(path, host_compat): """Accept an ELF file if the header matches the given compat triplet. In case it's not an ELF (e.g. static library, or some arbitrary file, fall back to is_readable_file).""" # Fast path: assume libraries at least have .so in their basename. # Note: don't replace with splitext, because of libsmth.so.1.2.3 file names. if ".so" not in os.path.basename(path): return spack.llnl.util.filesystem.is_readable_file(path) try: return host_compat == elf_utils.get_elf_compat(path) except (OSError, elf_utils.ElfParsingError): return spack.llnl.util.filesystem.is_readable_file(path) def libraries_in_ld_and_system_library_path( path_hints: Optional[List[str]] = None, ) -> Dict[str, str]: """Get the paths of all libraries available from ``path_hints`` or the following defaults: - Environment variables (Linux: ``LD_LIBRARY_PATH``, Darwin: ``DYLD_LIBRARY_PATH``, and ``DYLD_FALLBACK_LIBRARY_PATH``) - Dynamic linker default paths (glibc: ld.so.conf, musl: ld-musl-.path) - Default system library paths. For convenience, this is constructed as a dictionary where the keys are the library paths and the values are the names of the libraries (i.e. the basename of the library path). There may be multiple paths with the same basename. In this case it is assumed there are two different instances of the library. Args: path_hints: list of paths to be searched. If None the list will be constructed based on the set of LD_LIBRARY_PATH, LIBRARY_PATH, DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH environment variables as well as the standard system library paths. path_hints (list): list of paths to be searched. If ``None``, the default system paths are used. """ if path_hints: search_paths = spack.llnl.util.filesystem.search_paths_for_libraries(*path_hints) else: search_paths = [] # Environment variables if sys.platform == "darwin": search_paths.extend(environment.get_path("DYLD_LIBRARY_PATH")) search_paths.extend(environment.get_path("DYLD_FALLBACK_LIBRARY_PATH")) elif sys.platform.startswith("linux"): search_paths.extend(environment.get_path("LD_LIBRARY_PATH")) # Dynamic linker paths search_paths.extend(spack.util.ld_so_conf.host_dynamic_linker_search_paths()) # Drop redundant paths search_paths = list(filter(os.path.isdir, search_paths)) # Make use we don't doubly list /usr/lib and /lib etc search_paths = dedupe_paths(search_paths) try: host_compat = elf_utils.get_elf_compat(sys.executable) accept = lambda path: accept_elf(path, host_compat) except (OSError, elf_utils.ElfParsingError): accept = spack.llnl.util.filesystem.is_readable_file path_to_lib = {} # Reverse order of search directories so that a lib in the first # search path entry overrides later entries for search_path in reversed(search_paths): for lib in os.listdir(search_path): lib_path = os.path.join(search_path, lib) if accept(lib_path): path_to_lib[lib_path] = lib return path_to_lib def libraries_in_windows_paths(path_hints: Optional[List[str]] = None) -> Dict[str, str]: """Get the paths of all libraries available from the system PATH paths. For more details, see ``libraries_in_ld_and_system_library_path`` regarding return type and contents. Args: path_hints: list of paths to be searched. If None the list will be constructed based on the set of PATH environment variables as well as the standard system library paths. """ search_hints = ( path_hints if path_hints is not None else spack.util.environment.get_path("PATH") ) search_paths = spack.llnl.util.filesystem.search_paths_for_libraries(*search_hints) # on Windows, some libraries (.dlls) are found in the bin directory or sometimes # at the search root. Add both of those options to the search scheme search_paths.extend(spack.llnl.util.filesystem.search_paths_for_executables(*search_hints)) if path_hints is None: # if no user provided path was given, add defaults to the search search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths()) # SDK and WGL should be handled by above, however on occasion the WDK is in an atypical # location, so we handle that case specifically. search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths()) return path_to_dict(search_paths) def _group_by_prefix(paths: List[str]) -> Dict[str, Set[str]]: groups = collections.defaultdict(set) for p in paths: groups[os.path.dirname(p)].add(p) return groups class Finder: """Inspects the file-system looking for packages. Guesses places where to look using PATH.""" def default_path_hints(self) -> List[str]: return [] def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]: """Returns the list of patterns used to match candidate files. Args: pkg: package being detected """ raise NotImplementedError("must be implemented by derived classes") def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]: """Returns a list of candidate files found on the system. Args: patterns: search patterns to be used for matching files paths: paths where to search for files """ raise NotImplementedError("must be implemented by derived classes") def prefix_from_path(self, *, path: str) -> str: """Given a path where a file was found, returns the corresponding prefix. Args: path: path of a detected file """ raise NotImplementedError("must be implemented by derived classes") def detect_specs( self, *, pkg: Type["spack.package_base.PackageBase"], paths: Iterable[str], repo_path ) -> List["spack.spec.Spec"]: """Given a list of files matching the search patterns, returns a list of detected specs. Args: pkg: package being detected paths: files matching the package search patterns """ if not hasattr(pkg, "determine_spec_details"): warnings.warn( f"{pkg.name} must define 'determine_spec_details' in order" f" for Spack to detect externally-provided instances" f" of the package." ) return [] result = [] resolved_specs: Dict[spack.spec.Spec, str] = {} # spec -> prefix of first detection for candidate_path, items_in_prefix in _group_by_prefix( spack.llnl.util.lang.dedupe(paths) ).items(): # TODO: multiple instances of a package can live in the same # prefix, and a package implementation can return multiple specs # for one prefix, but without additional details (e.g. about the # naming scheme which differentiates them), the spec won't be # usable. try: specs = _convert_to_iterable( pkg.determine_spec_details(candidate_path, items_in_prefix) ) except Exception as e: specs = [] if spack.error.SHOW_BACKTRACE: details = traceback.format_exc() else: details = f"[{e.__class__.__name__}: {e}]" warnings.warn( f'error detecting "{pkg.name}" from prefix {candidate_path}: {details}' ) if not specs: files = ", ".join(_convert_to_iterable(items_in_prefix)) spack.llnl.util.tty.debug( f"The following files in {candidate_path} were decidedly not " f"part of the package {pkg.name}: {files}" ) for spec in specs: prefix = self.prefix_from_path(path=candidate_path) if not prefix: continue if spec in resolved_specs: prior_prefix = resolved_specs[spec] warnings.warn( f'"{spec}" detected in "{prefix}" was already detected in "{prior_prefix}"' ) continue resolved_specs[spec] = prefix try: # Validate the spec calling a package specific method pkg_cls = repo_path.get_pkg_class(spec.name) validate_fn = getattr(pkg_cls, "validate_detected_spec", lambda x, y: None) validate_fn(spec, spec.extra_attributes) except Exception as e: msg = ( f'"{spec}" has been detected on the system but will ' f"not be added to packages.yaml [reason={str(e)}]" ) warnings.warn(msg) continue if not spec.external_path: spec.external_path = prefix result.append(spec) return result def find( self, *, pkg_name: str, repository, initial_guess: Optional[List[str]] = None ) -> List["spack.spec.Spec"]: """For a given package, returns a list of detected specs. Args: pkg_name: package being detected repository: repository to retrieve the package initial_guess: initial list of paths to search from the caller if None, default paths are searched. If this is an empty list, nothing will be searched. """ pkg_cls = repository.get_pkg_class(pkg_name) patterns = self.search_patterns(pkg=pkg_cls) if not patterns: return [] if initial_guess is None: initial_guess = self.default_path_hints() initial_guess.extend(common_windows_package_paths(pkg_cls)) candidates = self.candidate_files(patterns=patterns, paths=initial_guess) return self.detect_specs(pkg=pkg_cls, paths=candidates, repo_path=repository) class ExecutablesFinder(Finder): def default_path_hints(self) -> List[str]: return spack.util.environment.get_path("PATH") def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]: result = [] if hasattr(pkg, "executables") and hasattr(pkg, "platform_executables"): result = pkg.platform_executables() return result def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]: executables_by_path = executables_in_path(path_hints=paths) joined_pattern = re.compile(r"|".join(patterns)) result = [path for path, exe in executables_by_path.items() if joined_pattern.search(exe)] result.sort() return result def prefix_from_path(self, *, path: str) -> str: result = executable_prefix(path) if not result: msg = f"no bin/ dir found in {path}. Cannot add it as a Spack package" spack.llnl.util.tty.debug(msg) return result class LibrariesFinder(Finder): """Finds libraries on the system, searching by LD_LIBRARY_PATH, LIBRARY_PATH, DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH, and standard system library paths """ def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]: result = [] if hasattr(pkg, "libraries"): result = pkg.libraries return result def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]: libraries_by_path = ( libraries_in_ld_and_system_library_path(path_hints=paths) if sys.platform != "win32" else libraries_in_windows_paths(path_hints=paths) ) patterns = [re.compile(x) for x in patterns] result = [] for compiled_re in patterns: for path, exe in libraries_by_path.items(): if compiled_re.search(exe): result.append(path) return result def prefix_from_path(self, *, path: str) -> str: result = library_prefix(path) if not result: msg = f"no lib/ or lib64/ dir found in {path}. Cannot add it as a Spack package" spack.llnl.util.tty.debug(msg) return result def by_path( packages_to_search: Iterable[str], *, path_hints: Optional[List[str]] = None, max_workers: Optional[int] = None, ) -> Dict[str, List["spack.spec.Spec"]]: """Return the list of packages that have been detected on the system, keyed by unqualified package name. Args: packages_to_search: list of packages to be detected. Each package can be either unqualified of fully qualified path_hints: initial list of paths to be searched max_workers: maximum number of workers to search for packages in parallel """ from spack.repo import PATH, partition_package_name # TODO: Packages should be able to define both .libraries and .executables in the future # TODO: determine_spec_details should get all relevant libraries and executables in one call executables_finder, libraries_finder = ExecutablesFinder(), LibrariesFinder() detected_specs_by_package: Dict[str, Tuple[concurrent.futures.Future, ...]] = {} result = collections.defaultdict(list) repository = PATH.ensure_unwrapped() executor: concurrent.futures.Executor if max_workers == 1: executor = spack.util.parallel.SequentialExecutor() else: executor = spack.util.parallel.make_concurrent_executor(max_workers, require_fork=False) with executor: for pkg in packages_to_search: executable_future = executor.submit( executables_finder.find, pkg_name=pkg, initial_guess=path_hints, repository=repository, ) library_future = executor.submit( libraries_finder.find, pkg_name=pkg, initial_guess=path_hints, repository=repository, ) detected_specs_by_package[pkg] = executable_future, library_future for pkg_name, futures in detected_specs_by_package.items(): for future in futures: try: detected = future.result(timeout=DETECTION_TIMEOUT) if detected: _, unqualified_name = partition_package_name(pkg_name) result[unqualified_name].extend(detected) except concurrent.futures.TimeoutError: spack.llnl.util.tty.debug( f"[EXTERNAL DETECTION] Skipping {pkg_name}: timeout reached" ) except Exception: spack.llnl.util.tty.debug( f"[EXTERNAL DETECTION] Skipping {pkg_name}: {traceback.format_exc()}" ) return result ================================================ FILE: lib/spack/spack/detection/test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Create and run mock e2e tests for package detection.""" import collections import contextlib import pathlib import tempfile from typing import Any, Deque, Dict, Generator, List, NamedTuple, Tuple import spack.platforms import spack.repo import spack.spec from spack.llnl.util import filesystem from spack.util import spack_yaml from .path import by_path class MockExecutables(NamedTuple): """Mock executables to be used in detection tests""" #: Relative paths for mock executables to be created executables: List[str] #: Shell script for the mock executable script: str class ExpectedTestResult(NamedTuple): """Data structure to model assertions on detection tests""" #: Spec to be detected spec: str #: Attributes expected in the external spec extra_attributes: Dict[str, str] class DetectionTest(NamedTuple): """Data structure to construct detection tests by PATH inspection. Packages may have a YAML file containing the description of one or more detection tests to be performed. Each test creates a few mock executable scripts in a temporary folder, and checks that detection by PATH gives the expected results. """ pkg_name: str layout: List[MockExecutables] results: List[ExpectedTestResult] class Runner: """Runs an external detection test""" def __init__(self, *, test: DetectionTest, repository: spack.repo.RepoPath) -> None: self.test = test self.repository = repository self.tmpdir = tempfile.TemporaryDirectory() def execute(self) -> List[spack.spec.Spec]: """Executes a test and returns the specs that have been detected. This function sets-up a test in a temporary directory, according to the prescriptions in the test layout, then performs a detection by executables and returns the specs that have been detected. """ with self._mock_layout() as path_hints: entries = by_path([self.test.pkg_name], path_hints=path_hints) _, unqualified_name = spack.repo.partition_package_name(self.test.pkg_name) specs = set(entries[unqualified_name]) return list(specs) @contextlib.contextmanager def _mock_layout(self) -> Generator[List[str], None, None]: hints = set() try: for entry in self.test.layout: exes = self._create_executable_scripts(entry) for mock_executable in exes: hints.add(str(mock_executable.parent)) yield list(hints) finally: self.tmpdir.cleanup() def _create_executable_scripts(self, mock_executables: MockExecutables) -> List[pathlib.Path]: import spack.vendor.jinja2 relative_paths = mock_executables.executables script = mock_executables.script script_template = spack.vendor.jinja2.Template("#!/bin/bash\n{{ script }}\n") result = [] for mock_exe_path in relative_paths: rel_path = pathlib.Path(mock_exe_path) abs_path = pathlib.Path(self.tmpdir.name) / rel_path abs_path.parent.mkdir(parents=True, exist_ok=True) abs_path.write_text(script_template.render(script=script)) filesystem.set_executable(abs_path) result.append(abs_path) return result @property def expected_specs(self) -> List[spack.spec.Spec]: return [ spack.spec.Spec.from_detection( item.spec, external_path=self.tmpdir.name, extra_attributes=item.extra_attributes ) for item in self.test.results ] def detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> List[Runner]: """Returns a list of test runners for a given package. Currently, detection tests are specified in a YAML file, called ``detection_test.yaml``, alongside the ``package.py`` file. This function reads that file to create a bunch of ``Runner`` objects. Args: pkg_name: name of the package to test repository: repository where the package lives """ result = [] detection_tests_content = read_detection_tests(pkg_name, repository) current_platform = str(spack.platforms.host()) tests_by_path = detection_tests_content.get("paths", []) for single_test_data in tests_by_path: if current_platform not in single_test_data.get("platforms", [current_platform]): continue mock_executables = [] for layout in single_test_data["layout"]: mock_executables.append( MockExecutables(executables=layout["executables"], script=layout["script"]) ) expected_results = [] for assertion in single_test_data["results"]: expected_results.append( ExpectedTestResult( spec=assertion["spec"], extra_attributes=assertion.get("extra_attributes", {}) ) ) current_test = DetectionTest( pkg_name=pkg_name, layout=mock_executables, results=expected_results ) result.append(Runner(test=current_test, repository=repository)) return result def read_detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> Dict[str, Any]: """Returns the normalized content of the detection_tests.yaml associated with the package passed in input. The content is merged with that of any package that is transitively included using the "includes" attribute. Args: pkg_name: name of the package to test repository: repository in which to search for packages """ content_stack, seen = [], set() included_packages: Deque[str] = collections.deque() root_detection_yaml, result = _detection_tests_yaml(pkg_name, repository) included_packages.extend(result.get("includes", [])) seen |= set(result.get("includes", [])) while included_packages: current_package = included_packages.popleft() try: current_detection_yaml, content = _detection_tests_yaml(current_package, repository) except FileNotFoundError as e: msg = ( f"cannot read the detection tests from the '{current_package}' package, " f"included by {root_detection_yaml}" ) raise FileNotFoundError(msg + f"\n\n\t{e}\n") content_stack.append((current_package, content)) included_packages.extend(x for x in content.get("includes", []) if x not in seen) seen |= set(content.get("includes", [])) result.setdefault("paths", []) for pkg_name, content in content_stack: result["paths"].extend(content.get("paths", [])) return result def _detection_tests_yaml( pkg_name: str, repository: spack.repo.RepoPath ) -> Tuple[pathlib.Path, Dict[str, Any]]: pkg_dir = pathlib.Path(repository.filename_for_package_name(pkg_name)).parent detection_tests_yaml = pkg_dir / "detection_test.yaml" with open(str(detection_tests_yaml), encoding="utf-8") as f: content = spack_yaml.load(f) return detection_tests_yaml, content ================================================ FILE: lib/spack/spack/directives.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This package contains directives that can be used within a package. Directives are functions that can be called inside a package definition to modify the package, for example:: class OpenMpi(Package): depends_on("hwloc") provides("mpi") ... ``provides`` and ``depends_on`` are spack directives. The available directives are: * ``build_system`` * ``conflicts`` * ``depends_on`` * ``extends`` * ``license`` * ``patch`` * ``provides`` * ``resource`` * ``variant`` * ``version`` * ``requires`` * ``redistribute`` They're implemented as functions that return partial functions that are later executed with a package class as first argument:: @directive("example") def example_directive(arg1, arg2): return partial(_execute_example_directive, arg1=arg1, arg2=arg2) def _execute_example_directive(pkg, arg1, arg2): # modify pkg.example based on arg1 and arg2 """ import collections import collections.abc import os import re import warnings from functools import partial from typing import Any, Callable, List, Optional, Tuple, Type, Union import spack.deptypes as dt import spack.error import spack.fetch_strategy import spack.llnl.util.tty.color import spack.package_base import spack.patch import spack.spec import spack.util.crypto import spack.variant from spack.dependency import Dependency from spack.directives_meta import DirectiveError, directive, get_spec from spack.resource import Resource from spack.spec import EMPTY_SPEC from spack.version import StandardVersion, VersionChecksumError, VersionError __all__ = [ "DirectiveError", "version", "conditional", "conflicts", "depends_on", "extends", "maintainers", "license", "provides", "patch", "variant", "resource", "build_system", "requires", "redistribute", "can_splice", ] _patch_order_index = 0 SpecType = str DepType = Union[Tuple[str, ...], str] WhenType = Optional[Union[spack.spec.Spec, str, bool]] PackageType = Type[spack.package_base.PackageBase] Patcher = Callable[[Union[PackageType, Dependency]], None] PatchesType = Union[Patcher, str, List[Union[Patcher, str]]] def _make_when_spec(value: Union[WhenType, Tuple[str, ...]]) -> Optional[spack.spec.Spec]: """Create a ``Spec`` that indicates when a directive should be applied. Directives with ``when`` specs, e.g.: patch('foo.patch', when='@4.5.1:') depends_on('mpi', when='+mpi') depends_on('readline', when=sys.platform() != 'darwin') are applied conditionally depending on the value of the ``when`` keyword argument. Specifically: 1. If the ``when`` argument is ``True``, the directive is always applied 2. If it is ``False``, the directive is never applied 3. If it is a ``Spec`` string, it is applied when the package's concrete spec satisfies the ``when`` spec. The first two conditions are useful for the third example case above. It allows package authors to include directives that are conditional at package definition time, in additional to ones that are evaluated as part of concretization. Arguments: value: a conditional Spec, constant ``bool``, or None if not supplied value indicating when a directive should be applied. It can also be a tuple of when conditions (as strings) to be combined together. """ # This branch is never taken, but our WhenType type annotation allows it, so handle it too. if isinstance(value, spack.spec.Spec): return value if isinstance(value, tuple): assert value, "when stack cannot be empty" # avoid a copy when there's only one condition if len(value) == 1: return get_spec(value[0]) # reduce the when-stack to a single spec by combining all constraints. combined_spec = spack.spec.Spec(value[0]) for cond in value[1:]: combined_spec._constrain_symbolically(get_spec(cond)) return combined_spec # Unsatisfiable conditions are discarded by the caller, and never # added to the package class if value is False: return None # If there is no constraint, the directive should always apply; # represent this by returning the unconstrained `Spec()`, which is # always satisfied. if value is None or value is True: return EMPTY_SPEC # This is conditional on the spec return get_spec(value) SubmoduleCallback = Callable[[spack.package_base.PackageBase], Union[str, List[str], bool]] @directive("versions", supports_when=False) def version( ver: Union[str, int], # this positional argument is deprecated, use sha256=... instead checksum: Optional[str] = None, *, # generic version options preferred: Optional[bool] = None, deprecated: Optional[bool] = None, no_cache: Optional[bool] = None, # url fetch options url: Optional[str] = None, extension: Optional[str] = None, expand: Optional[bool] = None, fetch_options: Optional[dict] = None, # url archive verification options md5: Optional[str] = None, sha1: Optional[str] = None, sha224: Optional[str] = None, sha256: Optional[str] = None, sha384: Optional[str] = None, sha512: Optional[str] = None, # git fetch options git: Optional[str] = None, commit: Optional[str] = None, tag: Optional[str] = None, branch: Optional[str] = None, get_full_repo: Optional[bool] = None, git_sparse_paths: Optional[ Union[List[str], Callable[[spack.package_base.PackageBase], List[str]]] ] = None, submodules: Union[SubmoduleCallback, Optional[bool]] = None, submodules_delete: Optional[bool] = None, # other version control svn: Optional[str] = None, hg: Optional[str] = None, cvs: Optional[str] = None, revision: Optional[str] = None, date: Optional[str] = None, ): """Declare a version for a package with optional metadata for fetching its code. Example:: version("2.1", sha256="...") version("2.0", sha256="...", preferred=True) .. versionchanged:: v2.3 The ``git_sparse_paths`` parameter was added. """ kwargs: dict = { key: value for key, value in ( ("sha256", sha256), ("sha384", sha384), ("sha512", sha512), ("preferred", preferred), ("deprecated", deprecated), ("expand", expand), ("url", url), ("extension", extension), ("no_cache", no_cache), ("fetch_options", fetch_options), ("git", git), ("svn", svn), ("hg", hg), ("cvs", cvs), ("get_full_repo", get_full_repo), ("git_sparse_paths", git_sparse_paths), ("branch", branch), ("submodules", submodules), ("submodules_delete", submodules_delete), ("commit", commit), ("tag", tag), ("revision", revision), ("date", date), ("md5", md5), ("sha1", sha1), ("sha224", sha224), ("checksum", checksum), ) if value is not None } return partial(_execute_version, ver=ver, kwargs=kwargs) def _execute_version(pkg: PackageType, ver: Union[str, int], kwargs: dict): if ( (any(s in kwargs for s in spack.util.crypto.hashes) or "checksum" in kwargs) and hasattr(pkg, "has_code") and not pkg.has_code ): raise VersionChecksumError( f"{pkg.name}: Checksums not allowed in no-code packages (see '{ver}' version)." ) if not isinstance(ver, (int, str)): raise VersionError( f"{pkg.name}: declared version '{ver!r}' in package should be a string or int." ) version = StandardVersion.from_string(str(ver)) # Store kwargs for the package to later with a fetch_strategy. pkg.versions[version] = kwargs @directive("conflicts") def conflicts(conflict_spec: SpecType, when: WhenType = None, msg: Optional[str] = None): """Declare a conflict for a package. A conflict is a spec that is known to be invalid. For example, a package that cannot build with GCC 14 and above can declare:: conflicts("%gcc@14:") To express the same constraint only when the ``foo`` variant is activated:: conflicts("%gcc@14:", when="+foo") Args: conflict_spec: constraint defining the known conflict when: optional condition that triggers the conflict msg: optional user defined message """ return partial(_execute_conflicts, conflict_spec=conflict_spec, when=when, msg=msg) def _execute_conflicts(pkg: PackageType, conflict_spec, when, msg): # If when is not specified the conflict always holds when_spec = _make_when_spec(when) if not when_spec: return # Save in a list the conflicts and the associated custom messages conflict_spec_list = pkg.conflicts.setdefault(when_spec, []) msg_with_name = f"{pkg.name}: {msg}" if msg is not None else msg conflict_spec_list.append((get_spec(conflict_spec), msg_with_name)) @directive("dependencies", can_patch_dependencies=True) def depends_on( spec: SpecType, when: WhenType = None, type: DepType = dt.DEFAULT_TYPES, *, patches: Optional[PatchesType] = None, ): """Declare a dependency on another package. Example:: depends_on("hwloc@2:", when="@1:", type="link") Args: spec: dependency spec when: condition when this dependency applies type: One or more of ``"build"``, ``"run"``, ``"test"``, or ``"link"`` (either a string or tuple). Defaults to ``("build", "link")``. patches: single result of :py:func:`patch` directive, a ``str`` to be passed to ``patch``, or a list of these """ return partial(_execute_depends_on, spec=spec, when=when, type=type, patches=patches) def _execute_depends_on( pkg: PackageType, spec: Union[str, spack.spec.Spec], *, when: WhenType = None, type: DepType = dt.DEFAULT_TYPES, patches: Optional[PatchesType] = None, ): spec = get_spec(spec) if isinstance(spec, str) else spec when_spec = _make_when_spec(when) if not when_spec: return if not spec.name: raise DependencyError( f"Invalid dependency specification in package '{pkg.name}':", str(spec) ) if pkg.name == spec.name: raise CircularReferenceError(f"Package '{pkg.name}' cannot depend on itself.") depflag = dt.canonicalize(type) # call this patches here for clarity -- we want patch to be a list, # but the caller doesn't have to make it one. # Note: we cannot check whether a package is virtual in a directive # because directives are run as part of class instantiation, and specs # instantiate the package class as part of the `virtual` check. # To be technical, specs only instantiate the package class as part of the # virtual check if the provider index hasn't been created yet. # TODO: There could be a cache warming strategy that would allow us to # ensure `Spec.virtual` is a valid thing to call in a directive. # For now, we comment out the following check to allow for virtual packages # with package files. # if patches and spec.virtual: # raise DependencyPatchError("Cannot patch a virtual dependency.") # ensure patches is a list if patches is None: patches = [] elif not isinstance(patches, (list, tuple)): patches = [patches] # this is where we actually add the dependency to this package deps_by_name = pkg.dependencies.setdefault(when_spec, {}) dependency = deps_by_name.get(spec.name) edges = spec.edges_to_dependencies() if edges and not all(x.direct for x in edges): raise DirectiveError( f"the '^' sigil cannot be used in 'depends_on' directives. Please reformulate " f"the directive below as multiple directives:\n\n" f'\tdepends_on("{spec}", when="{when_spec}")\n' ) if not dependency: dependency = Dependency(pkg, spec, depflag=depflag) deps_by_name[spec.name] = dependency else: copy = dependency.spec.copy() copy.constrain(spec, deps=False) dependency.spec = copy dependency.depflag |= depflag # apply patches to the dependency for patch in patches: if isinstance(patch, str): _execute_patch(dependency, url_or_filename=patch) else: assert callable(patch), f"Invalid patch argument: {patch!r}" patch(dependency) @directive("disable_redistribute") def redistribute( source: Optional[bool] = None, binary: Optional[bool] = None, when: WhenType = None ): """Declare that the package source and/or compiled binaries should not be redistributed. By default, packages allow source/binary distribution (in mirrors/build caches resp.). This directive allows users to explicitly disable redistribution for specs. """ return partial(_execute_redistribute, source=source, binary=binary, when=when) def _execute_redistribute( pkg: PackageType, source: Optional[bool], binary: Optional[bool], when: WhenType ): if source is None and binary is None: return elif (source is True) or (binary is True): raise DirectiveError( "Source/binary distribution are true by default, they can only be explicitly disabled." ) if source is None: source = True if binary is None: binary = True when_spec = _make_when_spec(when) if not when_spec: return if source is False: max_constraint = get_spec(f"{pkg.name}@{when_spec.versions}") if not max_constraint.satisfies(when_spec): raise DirectiveError("Source distribution can only be disabled for versions") if when_spec in pkg.disable_redistribute: disable = pkg.disable_redistribute[when_spec] if not source: disable.source = True if not binary: disable.binary = True else: pkg.disable_redistribute[when_spec] = spack.package_base.DisableRedistribute( source=not source, binary=not binary ) @directive(("extendees", "dependencies"), can_patch_dependencies=True) def extends( spec: str, when: WhenType = None, type: DepType = ("build", "run"), *, patches: Optional[PatchesType] = None, ): """Same as :func:`depends_on`, but also adds this package to the extendee list. In case of Python, also adds a dependency on ``python-venv``. .. note:: Notice that the default ``type`` is ``("build", "run")``, which is different from :func:`depends_on` where the default is ``("build", "link")``.""" return partial(_execute_extends, spec=spec, when=when, type=type, patches=patches) def _execute_extends( pkg: PackageType, spec: str, when: WhenType, type: DepType, patches: Optional[PatchesType] ): when_spec = _make_when_spec(when) if not when_spec: return dep_spec = get_spec(spec) _execute_depends_on(pkg, dep_spec, when=when, type=type, patches=patches) # When extending python, also add a dependency on python-venv. This is done so that # Spack environment views are Python virtual environments. if dep_spec.name == "python" and not pkg.name == "python-venv": _execute_depends_on(pkg, "python-venv", when=when, type=("build", "run")) pkg.extendees[dep_spec.name] = (dep_spec, when_spec) @directive(("provided", "provided_together")) def provides(*specs: SpecType, when: WhenType = None): """Declare that this package provides a virtual dependency. If a package provides ``mpi``, other packages can declare that they depend on ``mpi``, and spack can use the providing package to satisfy the dependency. Args: *specs: virtual specs provided by this package when: condition when this provides clause needs to be considered """ return partial(_execute_provides, specs=specs, when=when) def _execute_provides(pkg: PackageType, specs: Tuple[SpecType, ...], when: WhenType): when_spec = _make_when_spec(when) if not when_spec: return spec_objs = [get_spec(x) for x in specs] spec_names = [x.name for x in spec_objs] if len(spec_names) > 1: pkg.provided_together.setdefault(when_spec, []).append(set(spec_names)) for provided_spec in spec_objs: if pkg.name == provided_spec.name: raise CircularReferenceError(f"Package '{pkg.name}' cannot provide itself.") pkg.provided.setdefault(when_spec, set()).add(provided_spec) @directive("splice_specs") def can_splice( target: SpecType, *, when: SpecType, match_variants: Union[None, str, List[str]] = None ): """Declare whether the package is ABI-compatible with another package and thus can be spliced into concrete versions of that package. Args: target: The spec that the current package is ABI-compatible with. when: An anonymous spec constraining current package for when it is ABI-compatible with target. match_variants: A list of variants that must match between target spec and current package, with special value ``*`` which matches all variants. Example: a ``json`` variant is defined on two packages, and they are ABI-compatible whenever they agree on the json variant (regardless of whether it is turned on or off). Note that this cannot be applied to multi-valued variants and multi-valued variants will be skipped by ``*``. """ return partial(_execute_can_splice, target=target, when=when, match_variants=match_variants) def _execute_can_splice( pkg: PackageType, target: SpecType, when: SpecType, match_variants: Union[None, str, List[str]] ): when_spec = _make_when_spec(when) if isinstance(match_variants, str) and match_variants != "*": raise ValueError( "* is the only valid string for match_variants " "if looking to provide a single variant, use " f"[{match_variants}] instead" ) if when_spec is None: return pkg.splice_specs[when_spec] = (get_spec(target), match_variants) @directive("patches") def patch( url_or_filename: str, level: int = 1, when: WhenType = None, working_dir: str = ".", reverse: bool = False, sha256: Optional[str] = None, archive_sha256: Optional[str] = None, ) -> Patcher: """Declare a patch to apply to package sources. A when spec can be provided to indicate that a particular patch should only be applied when the package's spec meets certain conditions. Example:: patch("foo.patch", when="@1.0.0:") patch("https://example.com/foo.patch", sha256="...") Args: url_or_filename: url or relative filename of the patch level: patch level (as in the patch shell command) when: optional anonymous spec that specifies when to apply the patch working_dir: dir to change to before applying reverse: reverse the patch sha256: sha256 sum of the patch, used to verify the patch (only required for URL patches) archive_sha256: sha256 sum of the *archive*, if the patch is compressed (only required for compressed URL patches) """ return partial( _execute_patch, when=when, url_or_filename=url_or_filename, level=level, working_dir=working_dir, reverse=reverse, sha256=sha256, archive_sha256=archive_sha256, ) def _execute_patch( pkg_or_dep: Union[PackageType, Dependency], url_or_filename: str, level: int = 1, when: WhenType = None, working_dir: str = ".", reverse: bool = False, sha256: Optional[str] = None, archive_sha256: Optional[str] = None, ) -> None: pkg = pkg_or_dep.pkg if isinstance(pkg_or_dep, Dependency) else pkg_or_dep if hasattr(pkg, "has_code") and not pkg.has_code: raise UnsupportedPackageDirective( "Patches are not allowed in {0}: package has no code.".format(pkg.name) ) when_spec = _make_when_spec(when) if not when_spec: return # If this spec is identical to some other, then append this # patch to the existing list. cur_patches = pkg_or_dep.patches.setdefault(when_spec, []) global _patch_order_index ordering_key = (pkg.name, _patch_order_index) _patch_order_index += 1 patch: spack.patch.Patch if "://" in url_or_filename: if sha256 is None: raise ValueError("patch() with a url requires a sha256") patch = spack.patch.UrlPatch( pkg, url_or_filename, level, working_dir=working_dir, reverse=reverse, ordering_key=ordering_key, sha256=sha256, archive_sha256=archive_sha256, ) else: patch = spack.patch.FilePatch( pkg, url_or_filename, level, working_dir, reverse, ordering_key=ordering_key ) cur_patches.append(patch) def conditional(*values: Union[str, bool], when: Optional[WhenType] = None): """Conditional values that can be used in variant declarations.""" # _make_when_spec returns None when the condition is statically false. when = _make_when_spec(when) return spack.variant.ConditionalVariantValues( spack.variant.ConditionalValue(x, when=when) for x in values ) @directive("variants") def variant( name: str, default: Optional[Union[bool, str, Tuple[str, ...]]] = None, description: str = "", values: Optional[Union[collections.abc.Sequence, Callable[[Any], bool]]] = None, multi: Optional[bool] = None, validator: Optional[Callable[[str, str, Tuple[Any, ...]], None]] = None, when: Optional[Union[str, bool]] = None, sticky: bool = False, ): """Declare a variant for a package. Packager can specify a default value as well as a text description. Args: name: Name of the variant default: Default value for the variant, if not specified otherwise the default will be False for a boolean variant and 'nothing' for a multi-valued variant description: Description of the purpose of the variant values: Either a tuple of strings containing the allowed values, or a callable accepting one value and returning True if it is valid multi: If False only one value per spec is allowed for this variant validator: Optional group validator to enforce additional logic. It receives the package name, the variant name and a tuple of values and should raise an instance of SpackError if the group doesn't meet the additional constraints when: Optional condition on which the variant applies sticky: The variant should not be changed by the concretizer to find a valid concrete spec Raises: spack.directives_meta.DirectiveError: If arguments passed to the directive are invalid """ return partial( _execute_variant, name=name, default=default, description=description, values=values, multi=multi, validator=validator, when=when, sticky=sticky, ) def _format_error(msg, pkg, name): msg += " @*r{{[{0}, variant '{1}']}}" return spack.llnl.util.tty.color.colorize(msg.format(pkg.name, name)) def _execute_variant( pkg: PackageType, name: str, default: Optional[Union[bool, str, Tuple[str, ...]]], description: str, values: Optional[Union[collections.abc.Sequence, Callable[[Any], bool]]], multi: Optional[bool], validator: Optional[Callable[[str, str, Tuple[Any, ...]], None]], when: Optional[Union[str, bool]], sticky: bool, ): # This validation can be removed at runtime and enforced with an audit in Spack v1.0. # For now it's a warning to let people migrate faster. if not ( default is None or type(default) in (bool, str) or (type(default) is tuple and all(type(x) is str for x in default)) ): if isinstance(default, (list, tuple)): did_you_mean = f"default={','.join(str(x) for x in default)!r}" else: did_you_mean = f"default={str(default)!r}" warnings.warn( f"default value for variant '{name}' is not a boolean or string: default={default!r}. " f"Did you mean {did_you_mean}?", stacklevel=3, category=spack.error.SpackAPIWarning, ) if name in spack.variant.RESERVED_NAMES: raise DirectiveError(_format_error(f"The name '{name}' is reserved by Spack", pkg, name)) # Ensure we have a sequence of allowed variant values, or a # predicate for it. if values is None: if ( default in (True, False) or type(default) is str and default.upper() in ("TRUE", "FALSE") ): values = (True, False) else: values = lambda x: True # The object defining variant values might supply its own defaults for # all the other arguments. Ensure we have no conflicting definitions # in place. for argument in ("default", "multi", "validator"): # TODO: we can consider treating 'default' differently from other # TODO: attributes and let a packager decide whether to use the fluent # TODO: interface or the directive argument if hasattr(values, argument) and locals()[argument] is not None: raise DirectiveError( _format_error( f"Remove specification of {argument} argument: it is handled " "by an attribute of the 'values' argument", pkg, name, ) ) # Allow for the object defining the allowed values to supply its own # default value and group validator, say if it supports multiple values. default = getattr(values, "default", default) validator = getattr(values, "validator", validator) multi = getattr(values, "multi", bool(multi)) # Here we sanitize against a default value being either None # or the empty string, as the former indicates that a default # was not set while the latter will make the variant unparsable # from the command line if isinstance(default, tuple): default = ",".join(default) if default is None or default == "": if default is None: msg = "either a default was not explicitly set, or 'None' was used" else: msg = "the default cannot be an empty string" raise DirectiveError(_format_error(msg, pkg, name)) description = str(description).strip() when_spec = _make_when_spec(when) if not re.match(spack.spec.IDENTIFIER_RE, name): raise DirectiveError("variant", f"Invalid variant name in {pkg.name}: '{name}'") # variants are stored by condition then by name (so only the last variant of a # given name takes precedence *per condition*). # NOTE: variant defaults and values can conflict if when conditions overlap. variants_by_name = pkg.variants.setdefault(when_spec, {}) # type: ignore[arg-type] variants_by_name[name] = spack.variant.Variant( name=name, default=default, description=description, values=values, multi=multi, validator=validator, sticky=sticky, precedence=pkg.num_variant_definitions(), ) @directive("resources") def resource( *, name: Optional[str] = None, destination: str = "", placement: Optional[str] = None, when: WhenType = None, # additional kwargs are as for `version()` **kwargs, ): """Declare an external resource to be fetched and staged when building the package. Based on the keywords present in the dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their own folder inside spack stage area, and then moved into the stage area of the package that needs them. Keyword Arguments: name: name for the resource when: condition defining when the resource is needed destination: path, relative to the package stage area, to which resource should be moved placement: optionally rename the expanded resource inside the destination directory """ return partial( _execute_resource, name=name, destination=destination, placement=placement, when=when, kwargs=kwargs, ) def _execute_resource( pkg: PackageType, name: Optional[str], destination: str, placement: Optional[str], when: WhenType, # additional kwargs are as for `version()` kwargs: dict, ): when_spec = _make_when_spec(when) if not when_spec: return # Check if the path is relative if os.path.isabs(destination): msg = "The destination keyword of a resource directive can't be an absolute path.\n" msg += f"\tdestination : '{destination}\n'" raise RuntimeError(msg) # Check if the path falls within the main package stage area test_path = "stage_folder_root" # Normalized absolute path normalized_destination = os.path.normpath(os.path.join(test_path, destination)) if test_path not in normalized_destination: msg = "Destination of a resource must be within the package stage directory.\n" msg += f"\tdestination : '{destination}'\n" raise RuntimeError(msg) resources = pkg.resources.setdefault(when_spec, []) resources.append( Resource(name, spack.fetch_strategy.from_kwargs(**kwargs), destination, placement) ) def build_system(*values, **kwargs): """Define the build system used by the package. This defines the ``build_system`` variant. Example:: build_system("cmake", "autotools", "meson", default="cmake") """ default = kwargs.get("default", None) or values[0] return variant( "build_system", values=tuple(values), description="Build systems supported by the package", default=default, multi=False, ) @directive(dicts=()) def maintainers(*names: str): """Declare the maintainers of a package. Args: names: GitHub username for the maintainer """ return partial(_execute_maintainer, names=names) def _execute_maintainer(pkg: PackageType, names: Tuple[str, ...]): maintainers = set(pkg.maintainers) maintainers.update(names) pkg.maintainers = sorted(maintainers) @directive("licenses") def license( license_identifier: str, checked_by: Optional[Union[str, List[str]]] = None, when: Optional[Union[str, bool]] = None, ): """Declare the license(s) the software is distributed under. Args: license_identifiers: SPDX identifier specifying the license(s) the software is distributed under. checked_by: string or list of strings indicating which github user checked the license (if any). when: A spec specifying when the license applies. """ return partial(_execute_license, license_identifier=license_identifier, when=when) def _execute_license(pkg: PackageType, license_identifier: str, when: Optional[Union[str, bool]]): # If when is not specified the license always holds when_spec = _make_when_spec(when) if not when_spec: return for other_when_spec in pkg.licenses: if when_spec.intersects(other_when_spec): when_message = "" if when_spec != EMPTY_SPEC: when_message = f"when {when_spec}" other_when_message = "" if other_when_spec != EMPTY_SPEC: other_when_message = f"when {other_when_spec}" err_msg = ( f"{pkg.name} is specified as being licensed as {license_identifier} " f"{when_message}, but it is also specified as being licensed under " f"{pkg.licenses[other_when_spec]} {other_when_message}, which conflict." ) raise OverlappingLicenseError(err_msg) pkg.licenses[when_spec] = license_identifier @directive("requirements") def requires( *requirement_specs: str, policy: str = "one_of", when: Optional[str] = None, msg: Optional[str] = None, ): """Declare that a spec must be satisfied for a package. For instance, a package whose Fortran code can only be compiled with GCC can declare:: requires("%fortran=gcc") A package that requires Apple-Clang on Darwin can declare instead:: requires("%apple-clang", when="platform=darwin", msg="Apple Clang is required on Darwin") Args: requirement_specs: spec expressing the requirement policy: either ``"one_of"`` or ``"any_of"``. If ``"one_of"``, exactly one of the requirements must be satisfied. If ``"any_of"``, at least one of the requirements must be satisfied. Defaults to ``"one_of"``. when: optional constraint that triggers the requirement. If None the requirement is applied unconditionally. msg: optional user defined message """ return partial( _execute_requires, requirement_specs=requirement_specs, policy=policy, when=when, msg=msg ) def _execute_requires( pkg: PackageType, requirement_specs: Tuple[str, ...], policy: str, when: Optional[str], msg: Optional[str], ): if policy not in ("one_of", "any_of"): err_msg = ( f"the 'policy' argument of the 'requires' directive in {pkg.name} is set " f"to a wrong value (only 'one_of' or 'any_of' are allowed)" ) raise DirectiveError(err_msg) when_spec = _make_when_spec(when) if not when_spec: return # Save in a list the requirements and the associated custom messages requirement_list = pkg.requirements.setdefault(when_spec, []) msg_with_name = f"{pkg.name}: {msg}" if msg is not None else msg requirements = tuple(get_spec(s) for s in requirement_specs) requirement_list.append((requirements, policy, msg_with_name)) class DependencyError(DirectiveError): """This is raised when a dependency specification is invalid.""" class CircularReferenceError(DependencyError): """This is raised when something depends on itself.""" class DependencyPatchError(DirectiveError): """Raised for errors with patching dependencies.""" class UnsupportedPackageDirective(DirectiveError): """Raised when an invalid or unsupported package directive is specified.""" class OverlappingLicenseError(DirectiveError): """Raised when two licenses are declared that apply on overlapping specs.""" ================================================ FILE: lib/spack/spack/directives_meta.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import functools from typing import Any, Callable, Dict, List, Set, Tuple, Type, TypeVar, Union from spack.vendor.typing_extensions import ParamSpec import spack.error import spack.repo import spack.spec from spack.llnl.util.lang import dedupe P = ParamSpec("P") R = TypeVar("R") #: Names of possible directives. This list is mostly populated using the @directive decorator. #: Some directives leverage others and in that case are not automatically added. directive_names = ["build_system"] SPEC_CACHE: Dict[str, spack.spec.Spec] = {} def get_spec(spec_str: str) -> spack.spec.Spec: """Get a spec from the cache, or create it if not present.""" if spec_str not in SPEC_CACHE: SPEC_CACHE[spec_str] = spack.spec._ImmutableSpec(spec_str) return SPEC_CACHE[spec_str] class DirectiveMeta(type): """Flushes the directives that were temporarily stored in the staging area into the package. """ #: Registry of {directive_name: [list_of_dicts_it_modifies]} populated by @directive _directive_to_dicts: Dict[str, Tuple[str, ...]] = {} #: Inverted index of {dict_name: [list_of_directives_modifying_it]} _dict_to_directives: Dict[str, List[str]] = collections.defaultdict(list) #: Maps dictionary name to its descriptor instance _descriptor_cache: Dict[str, "DirectiveDictDescriptor"] = {} #: Set of all known directive dictionary names from `@directive(dicts=...)` _directive_dict_names: Set[str] = set() #: Lists of directives to be executed for the class being defined, grouped by directive #: function name (e.g. "depends_on", "version", etc.) _directives_to_be_executed: Dict[str, List[Callable]] = collections.defaultdict(list) #: Stack of when constraints from `with when(...)` context managers _when_constraints_stack: List[str] = [] #: Stack of default args from `with default_args(...)` context managers _default_args_stack: List[dict] = [] #: This property is set *automatically* during class definition as directives are invoked, #: if any ``depends_on`` or ``extends`` calls include patches for dependencies. This flag can #: be used as an optimization to detect whether a package provides patches for dependencies, #: without triggering the expensive deferred execution of those directives (without populating #: the ``dependencies`` dictionary). _patches_dependencies: bool = False def __new__( cls: Type["DirectiveMeta"], name: str, bases: tuple, attr_dict: dict ) -> "DirectiveMeta": attr_dict["_patches_dependencies"] = DirectiveMeta._patches_dependencies # Initialize the attribute containing the list of directives to be executed. Here we go # reversed because we want to execute commands in the order they were defined, following # the MRO. merged: Dict[str, List[Callable]] = {} sources = [getattr(b, "_directives_to_be_executed", None) or {} for b in reversed(bases)] for source in sources: for key, directive_list in source.items(): merged.setdefault(key, []).extend(directive_list) merged = {key: list(dedupe(directive_list)) for key, directive_list in merged.items()} # Add current class's directives (no deduplication needed here) for key, directive_list in DirectiveMeta._directives_to_be_executed.items(): merged.setdefault(key, []).extend(directive_list) attr_dict["_directives_to_be_executed"] = merged DirectiveMeta._directives_to_be_executed.clear() DirectiveMeta._patches_dependencies = False # Add descriptors for all known directive dictionaries for dict_name in DirectiveMeta._directive_dict_names: # Where the actual data will be stored attr_dict[f"_{dict_name}"] = None # Descriptor to lazily initialize and populate the dictionary attr_dict[dict_name] = DirectiveMeta._get_descriptor(dict_name) return super(DirectiveMeta, cls).__new__(cls, name, bases, attr_dict) def __init__(cls: "DirectiveMeta", name: str, bases: tuple, attr_dict: dict): if spack.repo.is_package_module(cls.__module__): # Historically, maintainers was not a directive. They were simply set as class # attributes `maintainers = ["alice", "bob"]`. Therefore, we execute these directives # eagerly. for directive in cls._directives_to_be_executed.get("maintainers", ()): directive(cls) super(DirectiveMeta, cls).__init__(name, bases, attr_dict) @staticmethod def register_directive(name: str, dicts: Tuple[str, ...]) -> None: """Called by @directive to register relationships.""" DirectiveMeta._directive_to_dicts[name] = dicts for d in dicts: DirectiveMeta._dict_to_directives[d].append(name) @staticmethod def _get_descriptor(name: str) -> "DirectiveDictDescriptor": """Returns a singleton descriptor for the given dictionary name.""" if name not in DirectiveMeta._descriptor_cache: DirectiveMeta._descriptor_cache[name] = DirectiveDictDescriptor(name) return DirectiveMeta._descriptor_cache[name] @staticmethod def push_when_constraint(when_spec: str) -> None: """Add a spec to the context constraints.""" DirectiveMeta._when_constraints_stack.append(when_spec) @staticmethod def pop_when_constraint() -> str: """Pop the last constraint from the context""" return DirectiveMeta._when_constraints_stack.pop() @staticmethod def push_default_args(default_args: Dict[str, Any]) -> None: """Push default arguments""" DirectiveMeta._default_args_stack.append(default_args) @staticmethod def pop_default_args() -> dict: """Pop default arguments""" return DirectiveMeta._default_args_stack.pop() @staticmethod def _remove_kwarg_value_directives_from_queue(value) -> None: """Remove directives found in a kwarg value from the execution queue.""" # Certain keyword argument values of directives may themselves be (lists of) directives. An # example of this is ``depends_on(..., patches=[patch(...), ...])``. In that case, we # should not execute those directives as part of the current package, but let the called # directive handle them. This function removes such directives from the execution queue. if isinstance(value, (list, tuple)): for item in value: DirectiveMeta._remove_kwarg_value_directives_from_queue(item) elif callable(value): # directives are always callable # Remove directives args from the exec queue for lst in DirectiveMeta._directives_to_be_executed.values(): for directive in lst: if value is directive: lst.remove(directive) # iterations ends, so mutation is fine break @staticmethod def _get_execution_plan(target_dict: str) -> Tuple[List[str], List[str]]: """Calculates the closure of dicts and directives needed to populate target_dict.""" dicts_involved = {target_dict} directives_involved = set() stack = [target_dict] while stack: current_dict = stack.pop() for directive_name in DirectiveMeta._dict_to_directives.get(current_dict, ()): if directive_name in directives_involved: continue directives_involved.add(directive_name) for other_dict in DirectiveMeta._directive_to_dicts[directive_name]: if other_dict not in dicts_involved: dicts_involved.add(other_dict) stack.append(other_dict) return sorted(dicts_involved), sorted(directives_involved) class DirectiveDictDescriptor: """A descriptor that lazily executes directives on first access.""" def __init__(self, name: str): self.name = name self.private_name = f"_{name}" self.dicts_to_init, self.directives_to_run = DirectiveMeta._get_execution_plan(name) def __get__(self, obj, objtype=None): val = getattr(objtype, self.private_name) if val is not None: return val # The None value is a sentinel for "not yet initialized". for dictionary in self.dicts_to_init: if getattr(objtype, f"_{dictionary}") is None: setattr(objtype, f"_{dictionary}", {}) # Populate these dictionaries by running all directives that modify them for directive_name in self.directives_to_run: directives = objtype._directives_to_be_executed.get(directive_name) if directives: for directive in directives: directive(objtype) return getattr(objtype, self.private_name) class directive: def __init__( self, dicts: Union[Tuple[str, ...], str] = (), supports_when: bool = True, can_patch_dependencies: bool = False, ) -> None: """Decorator for Spack directives. Spack directives allow you to modify a package while it is being defined, e.g. to add version or dependency information. Directives are one of the key pieces of Spack's package "language", which is embedded in python. Here's an example directive:: @directive(dicts="versions") def version(pkg, ...): ... This directive allows you write:: class Foo(Package): version(...) The ``@directive`` decorator handles a couple things for you: 1. Adds the class scope (pkg) as an initial parameter when called, like a class method would. This allows you to modify a package from within a directive, while the package is still being defined. 2. It automatically adds a dictionary called ``versions`` to the package so that you can refer to pkg.versions. Arguments: dicts: A tuple of names of dictionaries to add to the package class if they don't already exist. supports_when: If True, the directive can be used within a ``with when(...)`` context manager. (To be removed when all directives support ``when=`` arguments.) can_patch_dependencies: If True, the directive can patch dependencies. This is used to identify nested directives so they can be removed from the execution queue, and to mark the package as patching dependencies. """ if isinstance(dicts, str): dicts = (dicts,) # Add the dictionary names if not already there DirectiveMeta._directive_dict_names.update(dicts) self.supports_when = supports_when self.can_patch_dependencies = can_patch_dependencies self.dicts = tuple(dicts) def __call__(self, decorated_function: Callable[P, R]) -> Callable[P, R]: directive_names.append(decorated_function.__name__) DirectiveMeta.register_directive(decorated_function.__name__, self.dicts) @functools.wraps(decorated_function) def _wrapper(*args, **_kwargs): # First merge default args with kwargs if DirectiveMeta._default_args_stack: kwargs = {} for default_args in DirectiveMeta._default_args_stack: kwargs.update(default_args) kwargs.update(_kwargs) else: kwargs = _kwargs # Inject when arguments from the `with when(...)` stack. if DirectiveMeta._when_constraints_stack: if not self.supports_when: raise DirectiveError( f'directive "{decorated_function.__name__}" cannot be used within a ' '"when" context since it does not support a "when=" argument' ) if "when" in kwargs: kwargs["when"] = (*DirectiveMeta._when_constraints_stack, kwargs["when"]) else: kwargs["when"] = tuple(DirectiveMeta._when_constraints_stack) # Remove directives passed as arguments, so they are not executed as part of this # class's directive execution, but handled by the called directive instead if self.can_patch_dependencies and "patches" in kwargs: DirectiveMeta._remove_kwarg_value_directives_from_queue(kwargs["patches"]) DirectiveMeta._patches_dependencies = True result = decorated_function(*args, **kwargs) DirectiveMeta._directives_to_be_executed[decorated_function.__name__].append(result) # wrapped function returns same result as original so that we can nest directives return result return _wrapper class DirectiveError(spack.error.SpackError): """This is raised when something is wrong with a package directive.""" ================================================ FILE: lib/spack/spack/directory_layout.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import errno import os import re import shutil import sys from pathlib import Path from typing import Dict, List, Optional, Tuple import spack.config import spack.hash_types as ht import spack.llnl.util.filesystem as fs import spack.projections import spack.spec import spack.util.spack_json as sjson from spack.error import SpackError from spack.llnl.util.filesystem import readlink default_projections = { "all": "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}" } def _check_concrete(spec: "spack.spec.Spec") -> None: """If the spec is not concrete, raise a ValueError""" if not spec.concrete: raise ValueError("Specs passed to a DirectoryLayout must be concrete!") def _get_spec(prefix: str) -> Optional["spack.spec.Spec"]: """Returns a spec if the prefix contains a spec file in the .spack subdir""" for f in ("spec.json", "spec.yaml"): try: return spack.spec.Spec.from_specfile(os.path.join(prefix, ".spack", f)) except Exception: continue return None def specs_from_metadata_dirs(root: str) -> List["spack.spec.Spec"]: stack = [root] specs = [] while stack: prefix = stack.pop() spec = _get_spec(prefix) if spec: spec.set_prefix(prefix) specs.append(spec) continue try: scandir = os.scandir(prefix) except OSError: continue with scandir as entries: for entry in entries: if entry.is_dir(follow_symlinks=False): stack.append(entry.path) return specs class DirectoryLayout: """A directory layout is used to associate unique paths with specs. Different installations are going to want different layouts for their install, and they can use this to customize the nesting structure of spack installs. The default layout is: * / * / * -/ * -- The installation directory projections can be modified with the projections argument.""" def __init__( self, root: str, *, projections: Optional[Dict[str, str]] = None, hash_length: Optional[int] = None, ) -> None: self.root = root projections = projections or default_projections self.projections = {key: projection.lower() for key, projection in projections.items()} # apply hash length as appropriate self.hash_length = hash_length if self.hash_length is not None: for when_spec, projection in self.projections.items(): if "{hash}" not in projection: raise InvalidDirectoryLayoutParametersError( "Conflicting options for installation layout hash length" if "{hash" in projection else "Cannot specify hash length when the hash is not part of all " "install_tree projections" ) self.projections[when_spec] = projection.replace( "{hash}", "{hash:%d}" % self.hash_length ) # If any of these paths change, downstream databases may not be able to # locate files in older upstream databases self.metadata_dir = ".spack" self.deprecated_dir = "deprecated" self.spec_file_name = "spec.json" # Use for checking yaml and deprecated types self._spec_file_name_yaml = "spec.yaml" self.extension_file_name = "extensions.yaml" self.packages_dir = "repos" # archive of package.py files self.manifest_file_name = "install_manifest.json" @property def hidden_file_regexes(self) -> Tuple[str]: return ("^{0}$".format(re.escape(self.metadata_dir)),) def relative_path_for_spec(self, spec: "spack.spec.Spec") -> str: _check_concrete(spec) projection = spack.projections.get_projection(self.projections, spec) path = spec.format_path(projection) return str(Path(path)) def write_spec(self, spec: "spack.spec.Spec", path: str) -> None: """Write a spec out to a file.""" _check_concrete(spec) with open(path, "w", encoding="utf-8") as f: # The hash of the projection is the DAG hash which contains # the full provenance, so it's available if we want it later spec.to_json(f, hash=ht.dag_hash) def write_host_environment(self, spec: "spack.spec.Spec") -> None: """The host environment is a json file with os, kernel, and spack versioning. We use it in the case that an analysis later needs to easily access this information. """ env_file = self.env_metadata_path(spec) environ = spack.spec.get_host_environment_metadata() with open(env_file, "w", encoding="utf-8") as fd: sjson.dump(environ, fd) def read_spec(self, path: str) -> "spack.spec.Spec": """Read the contents of a file and parse them as a spec""" try: with open(path, encoding="utf-8") as f: extension = os.path.splitext(path)[-1].lower() if extension == ".json": spec = spack.spec.Spec.from_json(f) elif extension == ".yaml": # Too late for conversion; spec_file_path() already called. spec = spack.spec.Spec.from_yaml(f) else: raise SpecReadError(f"Did not recognize spec file extension: {extension}") except Exception as e: if spack.config.get("config:debug"): raise raise SpecReadError(f"Unable to read file: {path}", f"Cause: {e}") # Specs read from actual installations are always concrete spec._mark_concrete() return spec def spec_file_path(self, spec: "spack.spec.Spec") -> str: """Gets full path to spec file""" _check_concrete(spec) yaml_path = os.path.join(self.metadata_path(spec), self._spec_file_name_yaml) json_path = os.path.join(self.metadata_path(spec), self.spec_file_name) return yaml_path if os.path.exists(yaml_path) else json_path def deprecated_file_path( self, deprecated_spec: "spack.spec.Spec", deprecator_spec: Optional["spack.spec.Spec"] = None, ) -> str: """Gets full path to spec file for deprecated spec If the deprecator_spec is provided, use that. Otherwise, assume deprecated_spec is already deprecated and its prefix links to the prefix of its deprecator.""" _check_concrete(deprecated_spec) if deprecator_spec: _check_concrete(deprecator_spec) # If deprecator spec is None, assume deprecated_spec already deprecated # and use its link to find the file. base_dir = ( self.path_for_spec(deprecator_spec) if deprecator_spec else readlink(deprecated_spec.prefix) ) yaml_path = os.path.join( base_dir, self.metadata_dir, self.deprecated_dir, deprecated_spec.dag_hash() + "_" + self._spec_file_name_yaml, ) json_path = os.path.join( base_dir, self.metadata_dir, self.deprecated_dir, deprecated_spec.dag_hash() + "_" + self.spec_file_name, ) return yaml_path if os.path.exists(yaml_path) else json_path def metadata_path(self, spec: "spack.spec.Spec") -> str: return os.path.join(spec.prefix, self.metadata_dir) def env_metadata_path(self, spec: "spack.spec.Spec") -> str: return os.path.join(self.metadata_path(spec), "install_environment.json") def build_packages_path(self, spec: "spack.spec.Spec") -> str: return os.path.join(self.metadata_path(spec), self.packages_dir) def create_install_directory(self, spec: "spack.spec.Spec") -> None: _check_concrete(spec) # Create install directory with properly configured permissions # Cannot import at top of file from spack.package_prefs import get_package_dir_permissions, get_package_group # Each package folder can have its own specific permissions, while # intermediate folders (arch/compiler) are set with access permissions # equivalent to the root permissions of the layout. group = get_package_group(spec) perms = get_package_dir_permissions(spec) fs.mkdirp(spec.prefix, mode=perms, group=group, default_perms="parents") fs.mkdirp(self.metadata_path(spec), mode=perms, group=group) # in prefix self.write_spec(spec, self.spec_file_path(spec)) def ensure_installed(self, spec: "spack.spec.Spec") -> None: """ Throws InconsistentInstallDirectoryError if: 1. spec prefix does not exist 2. spec prefix does not contain a spec file, or 3. We read a spec with the wrong DAG hash out of an existing install directory. """ _check_concrete(spec) path = self.path_for_spec(spec) spec_file_path = self.spec_file_path(spec) if not os.path.isdir(path): raise InconsistentInstallDirectoryError( "Install prefix {0} does not exist.".format(path) ) if not os.path.isfile(spec_file_path): raise InconsistentInstallDirectoryError( "Install prefix exists but contains no spec.json:", " " + path ) installed_spec = self.read_spec(spec_file_path) if installed_spec.dag_hash() != spec.dag_hash(): raise InconsistentInstallDirectoryError( "Spec file in %s does not match hash!" % spec_file_path ) def path_for_spec(self, spec: "spack.spec.Spec") -> str: """Return absolute path from the root to a directory for the spec.""" _check_concrete(spec) if spec.external: return spec.external_path path = self.relative_path_for_spec(spec) assert not path.startswith(self.root) return os.path.join(self.root, path) def remove_install_directory(self, spec: "spack.spec.Spec", deprecated: bool = False) -> None: """Removes a prefix and any empty parent directories from the root. Raised RemoveFailedError if something goes wrong. """ path = self.path_for_spec(spec) assert path.startswith(self.root), ( "Attempted to remove dir outside Spack's install tree. PATH: {path}, ROOT: {self.root}" ) if deprecated: if os.path.exists(path): try: metapath = self.deprecated_file_path(spec) os.unlink(path) os.remove(metapath) except OSError as e: raise RemoveFailedError(spec, path, e) from e elif os.path.exists(path): try: if sys.platform == "win32": # Windows readonly files cannot be removed by Python # directly, change permissions before attempting to remove shutil.rmtree( path, ignore_errors=False, onerror=fs.readonly_file_handler(ignore_errors=False), ) else: shutil.rmtree(path) except OSError as e: raise RemoveFailedError(spec, path, e) from e path = os.path.dirname(path) while path != self.root: if os.path.isdir(path): try: os.rmdir(path) except OSError as e: if e.errno == errno.ENOENT: # already deleted, continue with parent pass elif e.errno == errno.ENOTEMPTY: # directory wasn't empty, done return else: raise e path = os.path.dirname(path) def all_specs(self) -> List["spack.spec.Spec"]: """Returns a list of all specs detected in self.root, detected by ``.spack`` directories. Their prefix is set to the directory containing the ``.spack`` directory. Note that these specs may follow a different layout than the current layout if it was changed after installation.""" return specs_from_metadata_dirs(self.root) def deprecated_for( self, specs: List["spack.spec.Spec"] ) -> List[Tuple["spack.spec.Spec", "spack.spec.Spec"]]: """Returns a list of tuples of specs (new, old) where new is deprecated for old""" spec_with_deprecated = [] for spec in specs: try: deprecated = os.scandir( os.path.join(str(spec.prefix), self.metadata_dir, self.deprecated_dir) ) except OSError: continue with deprecated as entries: for entry in entries: try: deprecated_spec = spack.spec.Spec.from_specfile(entry.path) spec_with_deprecated.append((spec, deprecated_spec)) except Exception: continue return spec_with_deprecated class DirectoryLayoutError(SpackError): """Superclass for directory layout errors.""" def __init__(self, message, long_msg=None): super().__init__(message, long_msg) class RemoveFailedError(DirectoryLayoutError): """Raised when a DirectoryLayout cannot remove an install prefix.""" def __init__(self, installed_spec, prefix, error): super().__init__( "Could not remove prefix %s for %s : %s" % (prefix, installed_spec.short_spec, error) ) self.cause = error class InconsistentInstallDirectoryError(DirectoryLayoutError): """Raised when a package seems to be installed to the wrong place.""" def __init__(self, message, long_msg=None): super().__init__(message, long_msg) class SpecReadError(DirectoryLayoutError): """Raised when directory layout can't read a spec.""" class InvalidDirectoryLayoutParametersError(DirectoryLayoutError): """Raised when a invalid directory layout parameters are supplied""" def __init__(self, message, long_msg=None): super().__init__(message, long_msg) class InvalidExtensionSpecError(DirectoryLayoutError): """Raised when an extension file has a bad spec in it.""" class ExtensionAlreadyInstalledError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, ext_spec): super().__init__("%s is already installed in %s" % (ext_spec.short_spec, spec.short_spec)) class ExtensionConflictError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, ext_spec, conflict): super().__init__( "%s cannot be installed in %s because it conflicts with %s" % (ext_spec.short_spec, spec.short_spec, conflict.short_spec) ) ================================================ FILE: lib/spack/spack/enums.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Enumerations used throughout Spack""" import enum class InstallRecordStatus(enum.Flag): """Enum flag to facilitate querying status from the DB""" INSTALLED = enum.auto() DEPRECATED = enum.auto() MISSING = enum.auto() ANY = INSTALLED | DEPRECATED | MISSING class ConfigScopePriority(enum.IntEnum): """Priorities of the different kind of config scopes used by Spack""" DEFAULTS = 0 CONFIG_FILES = 1 ENVIRONMENT = 2 CUSTOM = 3 COMMAND_LINE = 4 # Topmost scope reserved for internal use ENVIRONMENT_SPEC_GROUPS = 5 class PropagationPolicy(enum.Enum): """Enum to specify the behavior of a propagated dependency""" NONE = enum.auto() PREFERENCE = enum.auto() ================================================ FILE: lib/spack/spack/environment/__init__.py ================================================ # -*- coding: utf-8 -*- # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This package implements Spack environments. .. _lockfile-format: ``spack.lock`` format ===================== Spack environments have existed since Spack ``v0.12.0``, and there have been different ``spack.lock`` formats since then. The formats are documented here. The high-level format of a Spack lockfile hasn't changed much between versions, but the contents have. Lockfiles are JSON-formatted and their top-level sections are: 1. ``_meta`` (object): this contains details about the file format, including: * ``file-type``: always ``"spack-lockfile"`` * ``lockfile-version``: an integer representing the lockfile format version * ``specfile-version``: an integer representing the spec format version (since ``v0.17``) 2. ``spack`` (object): optional, this identifies information about Spack used to concretize the environment: * ``type``: required, identifies form Spack version took (e.g., ``git``, ``release``) * ``commit``: the commit if the version is from git * ``version``: the Spack version 3. ``roots`` (list): an ordered list of records representing the roots of the Spack environment. Each has two fields: * ``hash``: a Spack spec hash uniquely identifying the concrete root spec * ``spec``: a string representation of the abstract spec that was concretized 4. ``concrete_specs``: a dictionary containing the specs in the environment. 5. ``include_concrete`` (dictionary): an optional dictionary that includes the roots and concrete specs from the included environments, keyed by the path to that environment Compatibility ------------- New versions of Spack can (so far) read all old lockfile formats -- they are backward-compatible. Old versions cannot read new lockfile formats, and you'll need to upgrade Spack to use them. .. list-table:: Lockfile version compatibility across Spack versions :header-rows: 1 * - Spack version - ``v1`` - ``v2`` - ``v3`` - ``v4`` - ``v5`` - ``v6`` - ``v7`` * - ``v0.12:0.14`` - ✅ - - - - - - * - ``v0.15:0.16`` - ✅ - ✅ - - - - - * - ``v0.17`` - ✅ - ✅ - ✅ - - - - * - ``v0.18:`` - ✅ - ✅ - ✅ - ✅ - - - * - ``v0.22:v0.23`` - ✅ - ✅ - ✅ - ✅ - ✅ - - * - ``v1.0:1.1`` - ✅ - ✅ - ✅ - ✅ - ✅ - ✅ - * - ``v1.2:`` - ✅ - ✅ - ✅ - ✅ - ✅ - ✅ - ✅ Version 1 --------- When lockfiles were first created, there was only one hash in Spack: the DAG hash. This DAG hash (we'll call it the old DAG hash) did *not* include build dependencies -- it only included transitive link and run dependencies. The spec format at this time was keyed by name. Each spec started with a key for its name, whose value was a dictionary of other spec attributes. The lockfile put these name-keyed specs into dictionaries keyed by their DAG hash, and the spec records did not actually have a "hash" field in the lockfile -- you have to associate the hash from the key with the spec record after the fact. Dependencies in original lockfiles were keyed by ``"hash"``, i.e. the old DAG hash. .. code-block:: json { "_meta": { "file-type": "spack-lockfile", "lockfile-version": 1 }, "roots": [ { "hash": "", "spec": "" }, { "hash": "", "spec": "" } ], "concrete_specs": { "": { "... ...": { }, "dependencies": { "depname_1": { "hash": "", "type": ["build", "link"] }, "depname_2": { "hash": "", "type": ["build", "link"] } }, "hash": "" }, "": { "... ...": { }, "dependencies": { "depname_3": { "hash": "", "type": ["build", "link"] }, "depname_4": { "hash": "", "type": ["build", "link"] }, }, "hash": "" }, } } Version 2 --------- Version 2 changes one thing: specs in the lockfile are now keyed by ``build_hash`` instead of the old ``dag_hash``. Specs have a ``hash`` attribute with their real DAG hash, so you can't go by the dictionary key anymore to identify a spec -- you have to read it in and look at ``"hash"``. Dependencies are still keyed by old DAG hash. Even though we key lockfiles by ``build_hash``, specs in Spack were still deployed with the old, coarser DAG hash. This means that in v2 and v3 lockfiles (which are keyed by build hash), there may be multiple versions of the same spec with different build dependencies, which means they will have different build hashes but the same DAG hash. Spack would only have been able to actually install one of these. .. code-block:: json { "_meta": { "file-type": "spack-lockfile", "lockfile-version": 2 }, "roots": [ { "hash": "", "spec": "" }, { "hash": "", "spec": "" } ], "concrete_specs": { "": { "... ...": { }, "dependencies": { "depname_1": { "hash": "", "type": ["build", "link"] }, "depname_2": { "hash": "", "type": ["build", "link"] } }, "hash": "", }, "": { "... ...": { }, "dependencies": { "depname_3": { "hash": "", "type": ["build", "link"] }, "depname_4": { "hash": "", "type": ["build", "link"] } }, "hash": "" } } } Version 3 --------- Version 3 doesn't change the top-level lockfile format, but this was when we changed the specfile format. Specs in ``concrete_specs`` are now keyed by the build hash, with no inner dictionary keyed by their package name. The package name is in a ``name`` field inside each spec dictionary. The ``dependencies`` field in the specs is a list instead of a dictionary, and each element of the list is a record with the name, dependency types, and hash of the dependency. Instead of a key called ``hash``, dependencies are keyed by ``build_hash``. Each spec still has a ``hash`` attribute. Version 3 adds the ``specfile_version`` field to ``_meta`` and uses the new JSON spec format. .. code-block:: json { "_meta": { "file-type": "spack-lockfile", "lockfile-version": 3, "specfile-version": 2 }, "roots": [ { "hash": "", "spec": "" }, { "hash": "", "spec": "" }, ], "concrete_specs": { "": { "... ...": { }, "dependencies": [ { "name": "depname_1", "build_hash": "", "type": ["build", "link"] }, { "name": "depname_2", "build_hash": "", "type": ["build", "link"] }, ], "hash": "", }, "": { "... ...": { }, "dependencies": [ { "name": "depname_3", "build_hash": "", "type": ["build", "link"] }, { "name": "depname_4", "build_hash": "", "type": ["build", "link"] }, ], "hash": "" } } } Version 4 --------- Version 4 removes build hashes and is keyed by the new DAG hash (``hash``). The ``hash`` now includes build dependencies and a canonical hash of the ``package.py`` file. Dependencies are keyed by ``hash`` (DAG hash) as well. There are no more ``build_hash`` fields in the specs, and there are no more issues with lockfiles being able to store multiple specs with the same DAG hash (because the DAG hash is now finer-grained). An optional ``spack`` property may be included to track version information, such as the commit or version. .. code-block:: json { "_meta": { "file-type": "spack-lockfile", "lockfile-version": 4, "specfile-version": 3 }, "roots": [ { "hash": "", "spec": "" }, { "hash": "", "spec": "" } ], "concrete_specs": { "": { "... ...": { }, "dependencies": [ { "name": "depname_1", "hash": "", "type": ["build", "link"] }, { "name": "depname_2", "hash": "", "type": ["build", "link"] } ], "hash": "", }, "": { "... ...": { }, "dependencies": [ { "name": "depname_3", "hash": "", "type": ["build", "link"] }, { "name": "depname_4", "hash": "", "type": ["build", "link"] } ], "hash": "" } } } Version 5 --------- Version 5 doesn't change the top-level lockfile format, but an optional dictionary is added. The dictionary has the ``root`` and ``concrete_specs`` of the included environments, which are keyed by the path to that environment. Since this is optional if the environment does not have any included environments ``include_concrete`` will not be a part of the lockfile. .. code-block:: json { "_meta": { "file-type": "spack-lockfile", "lockfile-version": 5, "specfile-version": 3 }, "roots": [ { "hash": "", "spec": "" }, { "hash": "", "spec": "" } ], "concrete_specs": { "": { "... ...": { }, "dependencies": [ { "name": "depname_1", "hash": "", "type": ["build", "link"] }, { "name": "depname_2", "hash": "", "type": ["build", "link"] } ], "hash": "", }, "": { "... ...": { }, "dependencies": [ { "name": "depname_3", "hash": "", "type": ["build", "link"] }, { "name": "depname_4", "hash": "", "type": ["build", "link"] } ], "hash": "" } } "include_concrete": { "": { "roots": [ { "hash": "", "spec": "" }, { "hash": "", "spec": "" } ], "concrete_specs": { "": { "... ...": { }, "dependencies": [ { "name": "depname_1", "hash": "", "type": ["build", "link"] }, { "name": "depname_2", "hash": "", "type": ["build", "link"] } ], "hash": "", }, "": { "... ...": { }, "dependencies": [ { "name": "depname_3", "hash": "", "type": ["build", "link"] }, { "name": "depname_4", "hash": "", "type": ["build", "link"] } ], "hash": "" } } } } } Version 6 --------- Version 6 uses specs where compilers are modeled as real dependencies, and not as a node attribute. It doesn't change the top-level lockfile format. As part of Spack v1.0, compilers stopped being a node attribute, and became a build-only dependency. Packages may declare a dependency on the c, cxx, or fortran languages, which are now treated as virtuals, and compilers would be providers for one or more of those languages. Compilers can also inject runtime dependency, on the node being compiled. The compiler-wrapper is explicitly represented as a node in the DAG, and enters the hash. .. code-block:: json { "_meta": { "file-type": "spack-lockfile", "lockfile-version": 6, "specfile-version": 5 }, "spack": { "version": "1.0.0.dev0", "type": "git", "commit": "395b34f17417132389a6a8ee4dbf831c4a04f642" }, "roots": [ { "hash": "tivmbe3xjw7oqv4c3jv3v4jw42a7cajq", "spec": "zlib-ng" } ], "concrete_specs": { "tivmbe3xjw7oqv4c3jv3v4jw42a7cajq": { "name": "zlib-ng", "version": "2.2.3", "": {} } "dependencies": [ { "name": "compiler-wrapper", "hash": "n5lamxu36f4cx4sm7m7gocalctve4mcx", "parameters": { "deptypes": [ "build" ], "virtuals": [] } }, { "name": "gcc", "hash": "b375mbpprxze4vxy4ho7aixhuchsime2", "parameters": { "deptypes": [ "build" ], "virtuals": [ "c", "cxx" ] } }, { "": {} } ], "annotations": { "original_specfile_version": 5 }, } } Version 7 --------- Version 7 adds the additional attribute ``group`` to ``roots``. As part of Spack v1.2 each environment can define multiple groups of specs, and fine-tune their concretization separately. This attribute is needed to associate each root spec with the corresponding group. .. code-block:: json { "_meta": { "file-type": "spack-lockfile", "lockfile-version": 7, "specfile-version": 5 }, "spack": { "version": "1.2.0.dev0", "type": "git", "commit": "94b055476f874f424f20e3c0f33b0f22de29220a" }, "roots": [ { "hash": "o72mlpqvb5xijyqg4iyubpnvd5bfcomb", "spec": "hdf5", "group": "default" } ], "concrete_specs": { } } """ from .environment import ( TOP_LEVEL_KEY, Environment, SpackEnvironmentConfigError, SpackEnvironmentDevelopError, SpackEnvironmentError, SpackEnvironmentViewError, activate, active, active_environment, all_environment_names, all_environments, as_env_dir, create, create_in_dir, deactivate, default_manifest_yaml, default_view_name, display_specs, environment_dir_from_name, environment_from_name_or_dir, environment_path_scope, exists, initialize_environment_dir, installed_specs, is_env_dir, is_latest_format, lockfile_include_key, lockfile_name, manifest_file, manifest_include_name, manifest_name, no_active_environment, read, root, spack_env_var, spack_env_view_var, update_yaml, ) __all__ = [ "TOP_LEVEL_KEY", "Environment", "SpackEnvironmentConfigError", "SpackEnvironmentDevelopError", "SpackEnvironmentError", "SpackEnvironmentViewError", "activate", "active", "active_environment", "all_environment_names", "all_environments", "as_env_dir", "create", "create_in_dir", "deactivate", "default_manifest_yaml", "default_view_name", "display_specs", "environment_dir_from_name", "environment_from_name_or_dir", "environment_path_scope", "exists", "initialize_environment_dir", "installed_specs", "is_env_dir", "is_latest_format", "lockfile_include_key", "lockfile_name", "manifest_file", "manifest_include_name", "manifest_name", "no_active_environment", "read", "root", "spack_env_var", "spack_env_view_var", "update_yaml", ] ================================================ FILE: lib/spack/spack/environment/depfile.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This module contains the traversal logic and models that can be used to generate depfiles from an environment. """ import os import re import shlex from enum import Enum from typing import List, Optional import spack.deptypes as dt import spack.environment.environment as ev import spack.paths import spack.spec import spack.traverse as traverse class UseBuildCache(Enum): ONLY = 1 NEVER = 2 AUTO = 3 @staticmethod def from_string(s: str) -> "UseBuildCache": if s == "only": return UseBuildCache.ONLY elif s == "never": return UseBuildCache.NEVER elif s == "auto": return UseBuildCache.AUTO raise ValueError(f"invalid value for UseBuildCache: {s}") def _deptypes(use_buildcache: UseBuildCache): """What edges should we follow for a given node? If it's a cache-only node, then we can drop build type deps.""" return ( dt.LINK | dt.RUN if use_buildcache == UseBuildCache.ONLY else dt.BUILD | dt.LINK | dt.RUN ) class DepfileNode: """Contains a spec, a subset of its dependencies, and a flag whether it should be buildcache only/never/auto.""" def __init__( self, target: spack.spec.Spec, prereqs: List[spack.spec.Spec], buildcache: UseBuildCache ): self.target = MakefileSpec(target) self.prereqs = list(MakefileSpec(x) for x in prereqs) if buildcache == UseBuildCache.ONLY: self.buildcache_flag = "--use-buildcache=only" elif buildcache == UseBuildCache.NEVER: self.buildcache_flag = "--use-buildcache=never" else: self.buildcache_flag = "" class DepfileSpecVisitor: """This visitor produces an adjacency list of a (reduced) DAG, which is used to generate depfile targets with their prerequisites. Currently it only drops build deps when using buildcache only mode. Note that the DAG could be reduced even more by dropping build edges of specs installed at the moment the depfile is generated, but that would produce stateful depfiles that would not fail when the database is wiped later.""" def __init__(self, pkg_buildcache: UseBuildCache, deps_buildcache: UseBuildCache): self.adjacency_list: List[DepfileNode] = [] self.pkg_buildcache = pkg_buildcache self.deps_buildcache = deps_buildcache self.depflag_root = _deptypes(pkg_buildcache) self.depflag_deps = _deptypes(deps_buildcache) def neighbors(self, node): """Produce a list of spec to follow from node""" depflag = self.depflag_root if node.depth == 0 else self.depflag_deps return traverse.sort_edges(node.edge.spec.edges_to_dependencies(depflag=depflag)) def accept(self, node): self.adjacency_list.append( DepfileNode( target=node.edge.spec, prereqs=[edge.spec for edge in self.neighbors(node)], buildcache=self.pkg_buildcache if node.depth == 0 else self.deps_buildcache, ) ) # We already accepted this return True class MakefileSpec(object): """Limited interface to spec to help generate targets etc. without introducing unwanted special characters. """ _pattern = None def __init__(self, spec): self.spec = spec def safe_name(self): return self.safe_format("{name}-{version}-{hash}") def spec_hash(self): return self.spec.dag_hash() def safe_format(self, format_str): unsafe_result = self.spec.format(format_str) if not MakefileSpec._pattern: MakefileSpec._pattern = re.compile(r"[^A-Za-z0-9_.-]") return MakefileSpec._pattern.sub("_", unsafe_result) def unsafe_format(self, format_str): return self.spec.format(format_str) class MakefileModel: """This class produces all data to render a makefile for specs of an environment.""" def __init__( self, env: ev.Environment, roots: List[spack.spec.Spec], adjacency_list: List[DepfileNode], make_prefix: Optional[str], jobserver: bool, ): """ Args: env: environment to generate the makefile for roots: specs that get built in the default target adjacency_list: list of DepfileNode, mapping specs to their dependencies make_prefix: prefix for makefile targets jobserver: when enabled, make will invoke Spack with jobserver support. For dry-run this should be disabled. """ # Currently we can only use depfile with an environment since Spack needs to # find the concrete specs somewhere. self.env_path = env.path # These specs are built in the default target. self.roots = list(MakefileSpec(x) for x in roots) # The SPACK_PACKAGE_IDS variable is "exported", which can be used when including # generated makefiles to add post-install hooks, like pushing to a buildcache, # running tests, etc. if make_prefix is None: self.make_prefix = os.path.join(env.env_subdir_path, "makedeps") self.pkg_identifier_variable = "SPACK_PACKAGE_IDS" else: # NOTE: GNU Make allows directory separators in variable names, so for consistency # we can namespace this variable with the same prefix as targets. self.make_prefix = make_prefix self.pkg_identifier_variable = os.path.join(make_prefix, "SPACK_PACKAGE_IDS") # And here we collect a tuple of (target, prereqs, dag_hash, nice_name, buildcache_flag) self.make_adjacency_list = [ ( item.target.safe_name(), " ".join(self._install_target(s.safe_name()) for s in item.prereqs), item.target.spec_hash(), item.target.unsafe_format( "{name}{@version}{variants}" "{ platform=architecture.platform}{ os=architecture.os}" "{ target=architecture.target}" ), item.buildcache_flag, ) for item in adjacency_list ] # Root specs without deps are the prereqs for the environment target self.root_install_targets = [self._install_target(s.safe_name()) for s in self.roots] self.jobserver_support = "+" if jobserver else "" # All package identifiers, used to generate the SPACK_PACKAGE_IDS variable self.all_pkg_identifiers: List[str] = [] # All install and install-deps targets self.all_install_related_targets: List[str] = [] # Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers # /.spack-env/makedeps/install/pkg-version-hash in case # we don't have a custom make target prefix. self.phony_convenience_targets: List[str] = [] for node in adjacency_list: tgt = node.target.safe_name() self.all_pkg_identifiers.append(tgt) self.all_install_related_targets.append(self._install_target(tgt)) self.all_install_related_targets.append(self._install_deps_target(tgt)) if make_prefix is None: self.phony_convenience_targets.append(os.path.join("install", tgt)) self.phony_convenience_targets.append(os.path.join("install-deps", tgt)) def _target(self, name: str) -> str: # The `all` and `clean` targets are phony. It doesn't make sense to # have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make # sense to have a prefix like `env/all`, `env/clean` when they are # supposed to be included if name in ("all", "clean") and os.path.isabs(self.make_prefix): return name else: return os.path.join(self.make_prefix, name) def _install_target(self, name: str) -> str: return os.path.join(self.make_prefix, "install", name) def _install_deps_target(self, name: str) -> str: return os.path.join(self.make_prefix, "install-deps", name) def to_dict(self): return { "all_target": self._target("all"), "env_target": self._target("env"), "clean_target": self._target("clean"), "all_install_related_targets": " ".join(self.all_install_related_targets), "root_install_targets": " ".join(self.root_install_targets), "dirs_target": self._target("dirs"), "environment": self.env_path, "install_target": self._target("install"), "install_deps_target": self._target("install-deps"), "any_hash_target": self._target("%"), "jobserver_support": self.jobserver_support, "spack_script": shlex.quote(spack.paths.spack_script), "adjacency_list": self.make_adjacency_list, "phony_convenience_targets": " ".join(self.phony_convenience_targets), "pkg_ids_variable": self.pkg_identifier_variable, "pkg_ids": " ".join(self.all_pkg_identifiers), } @property def empty(self): return len(self.roots) == 0 @staticmethod def from_env( env: ev.Environment, *, filter_specs: Optional[List[spack.spec.Spec]] = None, pkg_buildcache: UseBuildCache = UseBuildCache.AUTO, dep_buildcache: UseBuildCache = UseBuildCache.AUTO, make_prefix: Optional[str] = None, jobserver: bool = True, ) -> "MakefileModel": """Produces a MakefileModel from an environment and a list of specs. Args: env: the environment to use filter_specs: if provided, only these specs will be built from the environment, otherwise the environment roots are used. pkg_buildcache: whether to only use the buildcache for top-level specs. dep_buildcache: whether to only use the buildcache for non-top-level specs. make_prefix: the prefix for the makefile targets jobserver: when enabled, make will invoke Spack with jobserver support. For dry-run this should be disabled. """ roots = env.all_matching_specs(*filter_specs) if filter_specs else env.concrete_roots() visitor = DepfileSpecVisitor(pkg_buildcache, dep_buildcache) traverse.traverse_breadth_first_with_visitor( roots, traverse.CoverNodesVisitor(visitor, key=lambda s: s.dag_hash()) ) return MakefileModel(env, roots, visitor.adjacency_list, make_prefix, jobserver) ================================================ FILE: lib/spack/spack/environment/environment.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import collections.abc import contextlib import errno import glob import os import pathlib import re import shutil import stat import warnings from collections.abc import KeysView from typing import ( Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, Union, ) import spack import spack.config import spack.deptypes as dt import spack.error import spack.filesystem_view as fsv import spack.hash_types as ht import spack.installer_dispatch import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr import spack.package_base import spack.paths import spack.repo import spack.schema.env import spack.spec import spack.store import spack.user_environment as uenv import spack.util.environment import spack.util.hash import spack.util.lock as lk import spack.util.path import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.variant as vt from spack import traverse from spack.enums import ConfigScopePriority from spack.llnl.util.filesystem import copy_tree, islink, readlink, symlink from spack.llnl.util.lang import stable_partition from spack.llnl.util.link_tree import ConflictingSpecsError from spack.schema.env import TOP_LEVEL_KEY from spack.spec import Spec from spack.spec_filter import SpecFilter from spack.util.path import substitute_path_variables from .list import SpecList, SpecListError, SpecListParser SpecPair = Tuple[Spec, Spec] DEFAULT_USER_SPEC_GROUP = "default" #: environment variable used to indicate the active environment spack_env_var = "SPACK_ENV" #: environment variable used to indicate the active environment view spack_env_view_var = "SPACK_ENV_VIEW" #: currently activated environment _active_environment: Optional["Environment"] = None # This is used in spack.main to bypass env failures if the command is `spack config edit` # It is used in spack.cmd.config to get the path to a failed env for `spack config edit` #: Validation error for a currently activate environment that failed to parse _active_environment_error: Optional[spack.config.ConfigFormatError] = None #: default path where environments are stored in the spack tree default_env_path = os.path.join(spack.paths.var_path, "environments") #: Name of the input yaml file for an environment manifest_name = "spack.yaml" #: Name of the input yaml file for an environment lockfile_name = "spack.lock" #: Name of the directory where environments store repos, logs, views, configs env_subdir_name = ".spack-env" def env_root_path() -> str: """Override default root path if the user specified it""" return spack.util.path.canonicalize_path( spack.config.get("config:environments_root", default=default_env_path) ) def environment_name(path: Union[str, pathlib.Path]) -> str: """Human-readable representation of the environment. This is the path for independent environments, and just the name for managed environments. """ env_root = pathlib.Path(env_root_path()).resolve() path_path = pathlib.Path(path) # For a managed environment created in Spack, env.path is ENV_ROOT/NAME # For a tracked environment from `spack env track`, the path is symlinked to ENV_ROOT/NAME # So if ENV_ROOT/NAME resolves to env.path we know the environment is tracked/managed. # Otherwise, it is an independent environment and we return the path. # # We resolve both paths fully because the env_root itself could also be a symlink, # and any directory in env.path could be a symlink. if (env_root / path_path.name).resolve() == path_path.resolve(): return path_path.name else: return str(path) def ensure_no_disallowed_env_config_mods(scope: spack.config.ConfigScope) -> None: config = scope.get_section("config") if config and "environments_root" in config["config"]: raise SpackEnvironmentError( "Spack environments are prohibited from modifying 'config:environments_root' " "because it can make the definition of the environment ill-posed. Please " "remove from your environment and place it in a permanent scope such as " "defaults, system, site, etc." ) def default_manifest_yaml(): """default spack.yaml file to put in new environments""" return """\ # This is a Spack Environment file. # # It describes a set of packages to be installed, along with # configuration settings. spack: # add package specs to the `specs` list specs: [] view: true concretizer: unify: {} """.format("true" if spack.config.get("concretizer:unify") else "false") sep_re = re.escape(os.sep) #: regex for validating environment names valid_environment_name_re = rf"^\w[{sep_re}\w-]*$" #: version of the lockfile format. Must increase monotonically. CURRENT_LOCKFILE_VERSION = 7 READER_CLS = { 1: spack.spec.SpecfileV1, 2: spack.spec.SpecfileV1, 3: spack.spec.SpecfileV2, 4: spack.spec.SpecfileV3, 5: spack.spec.SpecfileV4, 6: spack.spec.SpecfileV5, 7: spack.spec.SpecfileV5, } # Magic names # The name of the standalone spec list in the manifest yaml USER_SPECS_KEY = "specs" # The name of the default view (the view loaded on env.activate) default_view_name = "default" # Default behavior to link all packages into views (vs. only root packages) default_view_link = "all" # (DEPRECATED) Use as the heading/name in the manifest is deprecated. # The key for any concrete specs included in a lockfile. lockfile_include_key = "include_concrete" # The name/heading for include paths in the manifest file. manifest_include_name = "include" def installed_specs(): """ Returns the specs of packages installed in the active environment or None if no packages are installed. """ env = active_environment() hashes = env.all_hashes() if env else None return spack.store.STORE.db.query(hashes=hashes) def valid_env_name(name): return re.match(valid_environment_name_re, name) def validate_env_name(name): if not valid_env_name(name): raise ValueError( f"{name}: names may only contain letters, numbers, _, and -, and may not start with -." ) return name def activate(env, use_env_repo=False): """Activate an environment. To activate an environment, we add its manifest's configuration scope to the existing Spack configuration, and we set active to the current environment. Arguments: env (Environment): the environment to activate use_env_repo (bool): use the packages exactly as they appear in the environment's repository """ global _active_environment try: _active_environment = env # Fail early to avoid ending in an invalid state if not isinstance(env, Environment): raise TypeError("`env` should be of type {0}".format(Environment.__name__)) # Check if we need to reinitialize spack.store.STORE and spack.repo.REPO due to # config changes. install_tree_before = spack.config.get("config:install_tree") upstreams_before = spack.config.get("upstreams") repos_before = spack.config.get("repos") env.manifest.prepare_config_scope() install_tree_after = spack.config.get("config:install_tree") upstreams_after = spack.config.get("upstreams") repos_after = spack.config.get("repos") if install_tree_before != install_tree_after or upstreams_before != upstreams_after: setattr(env, "store_token", spack.store.reinitialize()) if repos_before != repos_after: setattr(env, "repo_token", spack.repo.PATH) spack.repo.PATH.disable() new_repo = spack.repo.RepoPath.from_config(spack.config.CONFIG) if use_env_repo: new_repo.put_first(env.repo) spack.repo.enable_repo(new_repo) tty.debug(f"Using environment '{env.name}'") except Exception: _active_environment = None raise def deactivate(): """Undo any configuration or repo settings modified by ``activate()``.""" global _active_environment if not _active_environment: return # If any config changes affected spack.store.STORE or spack.repo.PATH, undo them. store = getattr(_active_environment, "store_token", None) if store is not None: spack.store.restore(store) delattr(_active_environment, "store_token") repo = getattr(_active_environment, "repo_token", None) if repo is not None: spack.repo.PATH.disable() spack.repo.enable_repo(repo) _active_environment.manifest.deactivate_config_scope() tty.debug(f"Deactivated environment '{_active_environment.name}'") _active_environment = None def active_environment() -> Optional["Environment"]: """Returns the active environment when there is any""" return _active_environment def _root(name): """Non-validating version of root(), to be used internally.""" return os.path.join(env_root_path(), name) def root(name): """Get the root directory for an environment by name.""" validate_env_name(name) return _root(name) def exists(name): """Whether an environment with this name exists or not.""" return valid_env_name(name) and os.path.lexists(os.path.join(_root(name), manifest_name)) def active(name): """True if the named environment is active.""" return _active_environment and name == _active_environment.name def is_env_dir(path): """Whether a directory contains a spack environment.""" return os.path.isdir(path) and os.path.exists(os.path.join(path, manifest_name)) def as_env_dir(name_or_dir): """Translate an environment name or directory to the environment directory""" if is_env_dir(name_or_dir): return name_or_dir else: validate_env_name(name_or_dir) if not exists(name_or_dir): raise SpackEnvironmentError("no such environment '%s'" % name_or_dir) return _root(name_or_dir) def environment_from_name_or_dir(name_or_dir): """Get an environment with the supplied name.""" return Environment(as_env_dir(name_or_dir)) def read(name): """Get an environment with the supplied name.""" validate_env_name(name) if not exists(name): raise SpackEnvironmentError("no such environment '%s'" % name) return Environment(root(name)) def create( name: str, init_file: Optional[Union[str, pathlib.Path]] = None, with_view: Optional[Union[str, pathlib.Path, bool]] = None, keep_relative: bool = False, include_concrete: Optional[List[str]] = None, ) -> "Environment": """Create a managed environment in Spack and returns it. A managed environment is created in a root directory managed by this Spack instance, so that Spack can keep track of them. Files with suffix ``.json`` or ``.lock`` are considered lockfiles. Files with any other name are considered manifest files. Args: name: name of the managed environment init_file: either a lockfile, a manifest file, or None with_view: whether a view should be maintained for the environment. If the value is a string, it specifies the path to the view keep_relative: if True, develop paths are copied verbatim into the new environment file, otherwise they are made absolute include_concrete: concrete environment names/paths to be included """ environment_dir = environment_dir_from_name(name, exists_ok=False) return create_in_dir( environment_dir, init_file=init_file, with_view=with_view, keep_relative=keep_relative, include_concrete=include_concrete, ) def create_in_dir( root: Union[str, pathlib.Path], init_file: Optional[Union[str, pathlib.Path]] = None, with_view: Optional[Union[str, pathlib.Path, bool]] = None, keep_relative: bool = False, include_concrete: Optional[List[str]] = None, ) -> "Environment": """Create an environment in the directory passed as input and returns it. Files with suffix ``.json`` or ``.lock`` are considered lockfiles. Files with any other name are considered manifest files. Args: root: directory where to create the environment. init_file: either a lockfile, a manifest file, an env directory, or None with_view: whether a view should be maintained for the environment. If the value is a string, it specifies the path to the view keep_relative: if True, develop paths are copied verbatim into the new environment file, otherwise they are made absolute include_concrete: concrete environment names/paths to be included """ # If the initfile is a named environment, get its path if init_file and exists(str(init_file)): init_file = read(str(init_file)).path initialize_environment_dir(root, envfile=init_file) if with_view is None and keep_relative: return Environment(root) try: manifest = EnvironmentManifestFile(root) if with_view is not None: manifest.set_default_view(with_view) if include_concrete is not None: set_included_envs_to_env_paths(include_concrete) validate_included_envs_exists(include_concrete) validate_included_envs_concrete(include_concrete) manifest.set_include_concrete(include_concrete) manifest.flush() except (spack.config.ConfigFormatError, SpackEnvironmentConfigError) as e: shutil.rmtree(root) raise e env = Environment(root) if init_file: if os.path.isdir(init_file): init_file_dir = init_file copied = True else: init_file_dir = os.path.abspath(os.path.dirname(init_file)) copied = False if not keep_relative: if env.path != init_file_dir: # If we are here, we are creating an environment based on an # spack.yaml file in another directory, and moreover we want # dev paths in this environment to refer to their original # locations. # If the full env was copied including internal files, only rewrite # relative paths outside of env _rewrite_relative_dev_paths_on_relocation(env, init_file_dir, copied_env=copied) _rewrite_relative_repos_paths_on_relocation(env, init_file_dir, copied_env=copied) return env def _rewrite_relative_dev_paths_on_relocation(env, init_file_dir, copied_env=False): """When initializing the environment from a manifest file and we plan to store the environment in a different directory, we have to rewrite relative paths to absolute ones.""" with env: dev_specs = spack.config.get("develop", default={}, scope=env.scope_name) if not dev_specs: return for name, entry in dev_specs.items(): dev_path = substitute_path_variables(entry["path"]) expanded_path = spack.util.path.canonicalize_path(dev_path, default_wd=init_file_dir) # Skip if the substituted and expanded path is the same (e.g. when absolute) if entry["path"] == expanded_path: continue # If copied and it's inside the env, we copied it and don't need to relativize if copied_env and expanded_path.startswith(init_file_dir): continue tty.debug("Expanding develop path for {0} to {1}".format(name, expanded_path)) dev_specs[name]["path"] = expanded_path spack.config.set("develop", dev_specs, scope=env.scope_name) env._dev_specs = None # If we changed the environment's spack.yaml scope, that will not be reflected # in the manifest that we read env._re_read() def _rewrite_relative_repos_paths_on_relocation(env, init_file_dir, copied_env=False): """When initializing the environment from a manifest file and we plan to store the environment in a different directory, we have to rewrite relative repo paths to absolute ones and expand environment variables.""" with env: repos_specs = spack.config.get("repos", default={}, scope=env.scope_name) if not repos_specs: return for name, entry in list(repos_specs.items()): # only rewrite when we have a path-based repository if not isinstance(entry, str): continue repo_path = substitute_path_variables(entry) expanded_path = spack.util.path.canonicalize_path(repo_path, default_wd=init_file_dir) # Skip if the substituted and expanded path is the same (e.g. when absolute) if entry == expanded_path: continue # If copied and it's inside the env, we copied it and don't need to relativize if copied_env and expanded_path.startswith(init_file_dir): continue tty.debug("Expanding repo path for {0} to {1}".format(entry, expanded_path)) repos_specs[name] = expanded_path spack.config.set("repos", repos_specs, scope=env.scope_name) env.repos_specs = None # If we changed the environment's spack.yaml scope, that will not be reflected # in the manifest that we read env._re_read() def environment_dir_from_name(name: str, exists_ok: bool = True) -> str: """Returns the directory associated with a named environment. Args: name: name of the environment exists_ok: if False, raise an error if the environment exists already Raises: SpackEnvironmentError: if exists_ok is False and the environment exists already """ if not exists_ok and exists(name): raise SpackEnvironmentError(f"'{name}': environment already exists at {root(name)}") ensure_env_root_path_exists() validate_env_name(name) return root(name) def ensure_env_root_path_exists(): if not os.path.isdir(env_root_path()): fs.mkdirp(env_root_path()) def set_included_envs_to_env_paths(include_concrete: List[str]) -> None: """If the included environment(s) is the environment name it is replaced by the path to the environment Args: include_concrete: list of env name or path to env""" for i, env_name in enumerate(include_concrete): if is_env_dir(env_name): include_concrete[i] = env_name elif exists(env_name): include_concrete[i] = root(env_name) def validate_included_envs_exists(include_concrete: List[str]) -> None: """Checks that all of the included environments exist Args: include_concrete: list of already existing concrete environments to include Raises: SpackEnvironmentError: if any of the included environments do not exist """ missing_envs = set() for i, env_name in enumerate(include_concrete): if not is_env_dir(env_name): missing_envs.add(env_name) if missing_envs: msg = "The following environment(s) are missing: {0}".format(", ".join(missing_envs)) raise SpackEnvironmentError(msg) def validate_included_envs_concrete(include_concrete: List[str]) -> None: """Checks that all of the included environments are concrete Args: include_concrete: list of already existing concrete environments to include Raises: SpackEnvironmentError: if any of the included environments are not concrete """ non_concrete_envs = set() for env_path in include_concrete: if not os.path.exists(os.path.join(env_path, lockfile_name)): non_concrete_envs.add(environment_name(env_path)) if non_concrete_envs: msg = "The following environment(s) are not concrete: {0}\nPlease run:".format( ", ".join(non_concrete_envs) ) for env in non_concrete_envs: msg += f"\n\t`spack -e {env} concretize`" raise SpackEnvironmentError(msg) def all_environment_names(): """List the names of environments that currently exist.""" # just return empty if the env path does not exist. A read-only # operation like list should not try to create a directory. if not os.path.exists(env_root_path()): return [] env_root = pathlib.Path(env_root_path()).resolve() def yaml_paths(): for root, dirs, files in os.walk(env_root, topdown=True, followlinks=True): dirs[:] = [ d for d in dirs if not d.startswith(".") and not env_root.samefile(os.path.join(root, d)) ] if manifest_name in files: yield os.path.join(root, manifest_name) names = [] for yaml_path in yaml_paths(): candidate = str(pathlib.Path(yaml_path).relative_to(env_root).parent) if valid_env_name(candidate): names.append(candidate) return names def all_environments(): """Generator for all managed Environments.""" for name in all_environment_names(): yield read(name) def _read_yaml(str_or_file): """Read YAML from a file for round-trip parsing.""" try: data = syaml.load_config(str_or_file) except syaml.SpackYAMLError as e: raise SpackEnvironmentConfigError( f"Invalid environment configuration detected: {e.message}", e.filename ) filename = getattr(str_or_file, "name", None) spack.config.validate(data, spack.schema.env.schema, filename) return data def _write_yaml(data, str_or_file): """Write YAML to a file preserving comments and dict order.""" filename = getattr(str_or_file, "name", None) spack.config.validate(data, spack.schema.env.schema, filename) syaml.dump_config(data, str_or_file, default_flow_style=False) def _is_dev_spec_and_has_changed(spec): """Check if the passed spec is a dev build and whether it has changed since the last installation""" # First check if this is a dev build and in the process already try to get # the dev_path if not spec.variants.get("dev_path", None): return False # Now we can check whether the code changed since the last installation if not spec.installed: # Not installed -> nothing to compare against return False # hook so packages can use to write their own method for checking the dev_path # use package so attributes about concretization such as variant state can be # utilized return spec.package.detect_dev_src_change() def _error_on_nonempty_view_dir(new_root): """Defensively error when the target view path already exists and is not an empty directory. This usually happens when the view symlink was removed, but not the directory it points to. In those cases, it's better to just error when the new view dir is non-empty, since it indicates the user removed part but not all of the view, and it likely in an inconsistent state.""" # Check if the target path lexists try: st = os.lstat(new_root) except OSError: return # Empty directories are fine if stat.S_ISDIR(st.st_mode) and len(os.listdir(new_root)) == 0: return # Anything else is an error raise SpackEnvironmentViewError( "Failed to generate environment view, because the target {} already " "exists or is not empty. To update the view, remove this path, and run " "`spack env view regenerate`".format(new_root) ) class ViewDescriptor: def __init__( self, base_path: str, root: str, *, projections: Optional[Dict[str, str]] = None, select: Optional[List[str]] = None, exclude: Optional[List[str]] = None, link: str = default_view_link, link_type: fsv.LinkType = "symlink", groups: Optional[Union[str, List[str]]] = None, ) -> None: self.base = base_path self.raw_root = root self.root = spack.util.path.canonicalize_path(root, default_wd=base_path) self.projections = projections or {} self.select = select or [] self.exclude = exclude or [] self.link_type: fsv.LinkType = fsv.canonicalize_link_type(link_type) self.link = link if isinstance(groups, str): groups = [groups] self.groups: Optional[List[str]] = groups def select_fn(self, spec: Spec) -> bool: return any(spec.satisfies(s) for s in self.select) def exclude_fn(self, spec: Spec) -> bool: return not any(spec.satisfies(e) for e in self.exclude) def update_root(self, new_path: str) -> None: self.raw_root = new_path self.root = spack.util.path.canonicalize_path(new_path, default_wd=self.base) def __eq__(self, other: object) -> bool: return isinstance(other, ViewDescriptor) and all( [ self.root == other.root, self.projections == other.projections, self.select == other.select, self.exclude == other.exclude, self.link == other.link, self.link_type == other.link_type, ] ) def to_dict(self): ret = syaml.syaml_dict([("root", self.raw_root)]) if self.projections: ret["projections"] = self.projections if self.select: ret["select"] = self.select if self.exclude: ret["exclude"] = self.exclude if self.link_type: ret["link_type"] = self.link_type if self.link != default_view_link: ret["link"] = self.link return ret @staticmethod def from_dict(base_path: str, d) -> "ViewDescriptor": return ViewDescriptor( base_path, d["root"], projections=d.get("projections", {}), select=d.get("select", []), exclude=d.get("exclude", []), link=d.get("link", default_view_link), link_type=d.get("link_type", "symlink"), groups=d.get("group", None), ) @property def _current_root(self) -> Optional[str]: if not islink(self.root): return None root = readlink(self.root) if os.path.isabs(root): return root root_dir = os.path.dirname(self.root) return os.path.join(root_dir, root) def _next_root(self, specs): content_hash = self.content_hash(specs) root_dir = os.path.dirname(self.root) root_name = os.path.basename(self.root) return os.path.join(root_dir, "._%s" % root_name, content_hash) def content_hash(self, specs): d = syaml.syaml_dict( [ ("descriptor", self.to_dict()), ("specs", [(spec.dag_hash(), spec.prefix) for spec in sorted(specs)]), ] ) contents = sjson.dump(d) return spack.util.hash.b32_hash(contents) def get_projection_for_spec(self, spec): """Get projection for spec. This function does not require the view to exist on the filesystem.""" return self._view(self.root).get_projection_for_spec(spec) def view(self, new: Optional[str] = None) -> fsv.SimpleFilesystemView: """ Returns a view object for the *underlying* view directory. This means that the self.root symlink is followed, and that the view has to exist on the filesystem (unless ``new``). This function is useful when writing to the view. Raise if new is None and there is no current view Arguments: new: If a string, create a FilesystemView rooted at that path. Default None. This should only be used to regenerate the view, and cannot be used to access specs. """ path = new if new else self._current_root if not path: # This can only be hit if we write a future bug raise SpackEnvironmentViewError( f"Attempting to get nonexistent view from environment. View root is at {self.root}" ) return self._view(path) def _view(self, root: str) -> fsv.SimpleFilesystemView: """Returns a view object for a given root dir.""" return fsv.SimpleFilesystemView( root, spack.store.STORE.layout, ignore_conflicts=True, projections=self.projections, link_type=self.link_type, ) def __contains__(self, spec): """Is the spec described by the view descriptor Note: This does not claim the spec is already linked in the view. It merely checks that the spec is selected if a select operation is specified and is not excluded if an exclude operator is specified. """ if self.select: if not self.select_fn(spec): return False if self.exclude: if not self.exclude_fn(spec): return False return True def specs_for_view(self, concrete_roots: List[Spec]) -> List[Spec]: """Flatten the DAGs of the concrete roots, keep only unique, selected, and installed specs in topological order from root to leaf.""" if self.link == "all": deptype = dt.LINK | dt.RUN elif self.link == "run": deptype = dt.RUN else: deptype = dt.NONE specs = traverse.traverse_nodes( concrete_roots, order="topo", deptype=deptype, key=traverse.by_dag_hash ) # Filter selected, installed specs with spack.store.STORE.db.read_transaction(): result = [s for s in specs if s in self and s.installed] return self._exclude_duplicate_runtimes(result) def regenerate(self, env: "Environment") -> None: if self.groups is None: concrete_roots = env.concrete_roots() else: concrete_roots = [c for g in self.groups for _, c in env.concretized_specs_by(group=g)] specs = self.specs_for_view(concrete_roots) # To ensure there are no conflicts with packages being installed # that cannot be resolved or have repos that have been removed # we always regenerate the view from scratch. # We will do this by hashing the view contents and putting the view # in a directory by hash, and then having a symlink to the real # view in the root. The real root for a view at /dirname/basename # will be /dirname/._basename_. # This allows for atomic swaps when we update the view # cache the roots because the way we determine which is which does # not work while we are updating new_root = self._next_root(specs) old_root = self._current_root if new_root == old_root: tty.debug(f"View at {self.root} does not need regeneration.") return _error_on_nonempty_view_dir(new_root) # construct view at new_root if specs: tty.msg(f"Updating view at {self.root}") view = self.view(new=new_root) root_dirname = os.path.dirname(self.root) tmp_symlink_name = os.path.join(root_dirname, "._view_link") # Remove self.root if is it an empty dir, since we need a symlink there. Note that rmdir # fails if self.root is a symlink. try: os.rmdir(self.root) except (FileNotFoundError, NotADirectoryError): pass except OSError as e: if e.errno == errno.ENOTEMPTY: msg = "it is a non-empty directory" elif e.errno == errno.EACCES: msg = "of insufficient permissions" else: raise raise SpackEnvironmentViewError( f"The environment view in {self.root} cannot not be created because {msg}." ) from e # Create a new view try: fs.mkdirp(new_root) view.add_specs(*specs) # create symlink from tmp_symlink_name to new_root if os.path.exists(tmp_symlink_name): os.unlink(tmp_symlink_name) symlink(new_root, tmp_symlink_name) # mv symlink atomically over root symlink to old_root fs.rename(tmp_symlink_name, self.root) except Exception as e: # Clean up new view and temporary symlink on any failure. try: shutil.rmtree(new_root, ignore_errors=True) os.unlink(tmp_symlink_name) except OSError: pass # Give an informative error message for the typical error case: two specs, same package # project to same prefix. if isinstance(e, ConflictingSpecsError): spec_a = e.args[0].format(color=clr.get_color_when()) spec_b = e.args[1].format(color=clr.get_color_when()) raise SpackEnvironmentViewError( f"The environment view in {self.root} could not be created, " "because the following two specs project to the same prefix:\n" f" {spec_a}, and\n" f" {spec_b}.\n" " To resolve this issue:\n" " a. use `concretization:unify:true` to ensure there is only one " "package per spec in the environment, or\n" " b. disable views with `view:false`, or\n" " c. create custom view projections." ) from e raise # Remove the old root when it's in the same folder as the new root. This guards # against removal of an arbitrary path when the original symlink in self.root # was not created by the environment, but by the user. if ( old_root and os.path.exists(old_root) and os.path.samefile(os.path.dirname(new_root), os.path.dirname(old_root)) ): try: shutil.rmtree(old_root) except OSError as e: msg = "Failed to remove old view at %s\n" % old_root msg += str(e) tty.warn(msg) def _exclude_duplicate_runtimes(self, specs: List[Spec]) -> List[Spec]: """Stably filter out duplicates of "runtime" tagged packages, keeping only latest.""" # Maps packages tagged "runtime" to the spec with latest version. latest: Dict[str, Spec] = {} for s in specs: if "runtime" not in getattr(s.package, "tags", ()): continue elif s.name not in latest or latest[s.name].version < s.version: latest[s.name] = s return [x for x in specs if x.name not in latest or latest[x.name] is x] def env_subdir_path(manifest_dir: Union[str, pathlib.Path]) -> str: """Path to where the environment stores repos, logs, views, configs. Args: manifest_dir: directory containing the environment manifest file Returns: directory the environment uses to manage its files """ return os.path.join(str(manifest_dir), env_subdir_name) class ConcretizedRootInfo: """Data on root specs that have been concretized""" __slots__ = ("root", "hash", "new", "group") def __init__( self, *, root_spec: spack.spec.Spec, root_hash: str, new: bool = False, group: str ): self.root = root_spec self.hash = root_hash self.new = new self.group = group def __str__(self): return f"{self.root} -> {self.hash} [new={self.new}]" def __eq__(self, other: object) -> bool: return ( isinstance(other, ConcretizedRootInfo) and self.root == other.root and self.hash == other.hash and self.new == other.new and self.group == other.group ) def __hash__(self) -> int: return hash((self.root, self.hash, self.new, self.group)) @staticmethod def from_info_dict(info_dict: Dict[str, str]) -> "ConcretizedRootInfo": # Lockfile versions < 7 don't have the "group" attribute return ConcretizedRootInfo( root_spec=Spec(info_dict["spec"]), root_hash=info_dict["hash"], new=False, group=info_dict.get("group", DEFAULT_USER_SPEC_GROUP), ) class Environment: """A Spack environment, which bundles together configuration and a list of specs.""" def __init__(self, manifest_dir: Union[str, pathlib.Path]) -> None: """An environment can be constructed from a directory containing a "spack.yaml" file, and optionally a consistent "spack.lock" file. Args: manifest_dir: directory with the "spack.yaml" associated with the environment """ self.path = os.path.abspath(str(manifest_dir)) self.name = environment_name(self.path) self.env_subdir_path = env_subdir_path(self.path) self.txlock = lk.Lock(self._transaction_lock_path) self._unify = None self.views: Dict[str, ViewDescriptor] = {} #: Parser for spec lists self._spec_lists_parser = SpecListParser() #: Specs from "spack.yaml" self.spec_lists: Dict[str, SpecList] = {} #: Information on concretized roots self.concretized_roots: List[ConcretizedRootInfo] = [] #: Concretized specs by hash self.specs_by_hash: Dict[str, Spec] = {} #: Repository for this environment (memoized) self._repo = None #: Environment root dirs for concrete (lockfile) included environments self.included_concrete_env_root_dirs: List[str] = [] #: First-level included concretized spec data from/to the lockfile. self.included_concrete_spec_data: Dict[str, Dict[str, List[str]]] = {} #: Roots from included environments from the last concretization, keyed by env path self.included_concretized_roots: Dict[str, List[ConcretizedRootInfo]] = {} #: Concretized specs by hash from the included environments self.included_specs_by_hash: Dict[str, Dict[str, Spec]] = {} #: Previously active environment self._previous_active = None self._dev_specs = None # Load the manifest file contents into memory self._load_manifest_file() def _load_manifest_file(self): """Instantiate and load the manifest file contents into memory.""" with lk.ReadTransaction(self.txlock): self.manifest = EnvironmentManifestFile(self.path, self.name) with self.manifest.use_config(): self._read() @contextlib.contextmanager def config_override_for_group(self, *, group: str): key = self.manifest._ensure_group_exists(group=group) internal_scope = self.manifest.config_override(group=key) if internal_scope is None: # No internal scope tty.debug( f"[{__name__}] No configuration override necessary for the '{group}' group " f"in the environment at {self.manifest_path}" ) yield return try: tty.debug( f"[{__name__}] Overriding the configuration for the '{group}' group defined " f"in {self.manifest_path} before concretization" ) spack.config.CONFIG.push_scope( internal_scope, priority=ConfigScopePriority.ENVIRONMENT_SPEC_GROUPS ) yield finally: spack.config.CONFIG.remove_scope(internal_scope.name) def __getstate__(self): state = self.__dict__.copy() state.pop("txlock", None) state.pop("_repo", None) state.pop("repo_token", None) state.pop("store_token", None) return state def __setstate__(self, state): self.__dict__.update(state) self.txlock = lk.Lock(self._transaction_lock_path) self._repo = None def _re_read(self): """Reinitialize the environment object.""" self.clear() self._load_manifest_file() def _read(self): self._construct_state_from_manifest() if os.path.exists(self.lock_path): with open(self.lock_path, encoding="utf-8") as f: read_lock_version = self._read_lockfile(f)["_meta"]["lockfile-version"] if read_lock_version == 1: tty.debug(f"Storing backup of {self.lock_path} at {self._lock_backup_v1_path}") shutil.copy(self.lock_path, self._lock_backup_v1_path) def write_transaction(self): """Get a write lock context manager for use in a ``with`` block.""" return lk.WriteTransaction(self.txlock, acquire=self._re_read) def _process_view(self, env_view: Optional[Union[bool, str, Dict]]): """Process view option(s), which can be boolean, string, or None. A boolean environment view option takes precedence over any that may be included. So ``view: True`` results in the default view only. And ``view: False`` means the environment will have no view. Args: env_view: view option provided in the manifest or configuration """ def add_view(name, values): """Add the view with the name and the string or dict values.""" if isinstance(values, str): self.views[name] = ViewDescriptor(self.path, values) elif isinstance(values, dict): self.views[name] = ViewDescriptor.from_dict(self.path, values) else: tty.error(f"Cannot add view named {name} for {type(values)} values {values}") # If the configuration specifies 'view: False' then we are done # processing views. If this is called with the environment's view # view (versus an included view), then there are to be NO views. if env_view is False: return # If the configuration specifies 'view: True' then only the default # view will be created for the environment and we are done processing # views. if env_view is True: add_view(default_view_name, self.view_path_default) return # Otherwise, the configuration has a subdirectory or dictionary. if isinstance(env_view, str): add_view(default_view_name, env_view) elif env_view: for name, values in env_view.items(): add_view(name, values) # If we reach this point without an explicit view option then we # provide the default view. if self.views == dict(): self.views[default_view_name] = ViewDescriptor(self.path, self.view_path_default) def _load_concrete_include_data(self): """Load concrete include specs data from included concrete directories.""" if self.included_concrete_env_root_dirs: if os.path.exists(self.lock_path): with open(self.lock_path, encoding="utf-8") as f: data = self._read_lockfile(f) if lockfile_include_key in data: self.included_concrete_spec_data = data[lockfile_include_key] else: self.include_concrete_envs() def _process_included_lockfiles(self): """Extract and load into memory included lock file data.""" includes = self.manifest[TOP_LEVEL_KEY].get(lockfile_include_key, []) if includes: tty.warn( f"Use of '{lockfile_include_key}' in manifest files " f"is deprecated. The key should be '{manifest_include_name}' " f"and the path should end with '{lockfile_name}'. Run " f"'spack env update {self.name}' to update the manifest." ) includes = [os.path.join(inc, lockfile_name) for inc in includes] includes += self.manifest[TOP_LEVEL_KEY].get(manifest_include_name, []) if not includes: return # Expand config and environment variables for concrete environments, # indicated by the inclusion of lock files. self.included_concrete_env_root_dirs = [] for entry in includes: include = spack.config.included_path(entry) if isinstance(include, spack.config.GitIncludePaths): # Git includes must be cloned first; paths are relative to the # clone destination, not to the manifest directory. destination = include._clone(self.manifest.env_config_scope) if destination is None: continue resolved = [os.path.join(destination, p) for p in include.paths] else: resolved = [ spack.util.path.canonicalize_path(p, default_wd=self.path) for p in include.paths ] for path in resolved: if os.path.basename(path) != lockfile_name: continue tty.debug(f"Adding {path} to the concrete environment root directories") self.included_concrete_env_root_dirs.append(os.path.dirname(path)) # Cache concrete environments for required lock files. self._load_concrete_include_data() def _construct_state_from_manifest(self): """Set up user specs and views from the manifest file.""" self.views = {} self._sync_speclists() self._process_view(spack.config.get("view", True)) self._process_included_lockfiles() def _sync_speclists(self): self._spec_lists_parser = SpecListParser( toolchains=spack.config.CONFIG.get("toolchains", {}) ) self.spec_lists = {} self.spec_lists.update( self._spec_lists_parser.parse_definitions( data=spack.config.CONFIG.get("definitions", []) ) ) for group in self.manifest.groups(): tty.debug(f"[{__name__}]: Synchronizing user specs from the '{group}' group", level=2) key = self._user_specs_key(group=group) self.spec_lists[key] = self._spec_lists_parser.parse_user_specs( name=key, yaml_list=self.manifest.user_specs(group=group) ) def _user_specs_key(self, *, group: Optional[str] = None) -> str: if group is None or group == DEFAULT_USER_SPEC_GROUP: return USER_SPECS_KEY return f"{USER_SPECS_KEY}:{group}" @property def user_specs(self) -> SpecList: return self.user_specs_by(group=DEFAULT_USER_SPEC_GROUP) def user_specs_by(self, *, group: Optional[str]) -> SpecList: """Returns a dictionary of user specs keyed by their group.""" key = self._user_specs_key(group=group) return self.spec_lists[key] def explicit_roots(self): for x in self.concretized_roots: if self.manifest.is_explicit(group=x.group): yield x @property def dev_specs(self): dev_specs = {} dev_config = spack.config.get("develop", {}) for name, entry in dev_config.items(): local_entry = {"spec": str(entry["spec"])} # default path is the spec name if "path" not in entry: local_entry["path"] = name else: local_entry["path"] = entry["path"] dev_specs[name] = local_entry return dev_specs @property def included_user_specs(self) -> SpecList: """Included concrete user (or root) specs from last concretization.""" spec_list = SpecList() if not self.included_concrete_env_root_dirs: return spec_list def add_root_specs(included_concrete_specs): # add specs from the include *and* any nested includes it may have for env, info in included_concrete_specs.items(): for root_list in info["roots"]: spec_list.add(root_list["spec"]) if lockfile_include_key in info: add_root_specs(info[lockfile_include_key]) add_root_specs(self.included_concrete_spec_data) return spec_list def clear(self): """Clear the contents of the environment""" self.spec_lists = {} self._dev_specs = {} self.concretized_roots = [] self.specs_by_hash = {} # concretized specs by hash self.included_concrete_spec_data = {} # concretized specs from lockfile of included envs self.included_concretized_roots = {} # root specs of the included envs, keyed by env path self.included_specs_by_hash = {} # concretized specs by hash from the included envs self.invalidate_repository_cache() self._previous_active = None # previously active environment self.manifest.clear() @property def active(self): """True if this environment is currently active.""" return _active_environment and self.path == _active_environment.path @property def manifest_path(self): """Path to spack.yaml file in this environment.""" return os.path.join(self.path, manifest_name) @property def _transaction_lock_path(self): """The location of the lock file used to synchronize multiple processes updating the same environment. """ return os.path.join(self.env_subdir_path, "transaction_lock") @property def lock_path(self): """Path to spack.lock file in this environment.""" return os.path.join(self.path, lockfile_name) @property def _lock_backup_v1_path(self): """Path to backup of v1 lockfile before conversion to v2""" return self.lock_path + ".backup.v1" @property def repos_path(self): return os.path.join(self.env_subdir_path, "repos") @property def view_path_default(self) -> str: # default path for environment views return os.path.join(self.env_subdir_path, "view") @property def repo(self): if self._repo is None: self._repo = make_repo_path(self.repos_path) return self._repo @property def scope_name(self): """Name of the config scope of this environment's manifest file.""" return self.manifest.scope_name def include_concrete_envs(self): """Copy and save the included environments' specs internally.""" root_hash_seen = set() concrete_hash_seen = set() self.included_concrete_spec_data = {} for env_path in self.included_concrete_env_root_dirs: # Check that the environment (lockfile) exists if not is_env_dir(env_path): raise SpackEnvironmentError(f"Unable to find env at {env_path}") env = Environment(env_path) self.included_concrete_spec_data[env_path] = {"roots": [], "concrete_specs": {}} # Copy unique root specs from env for root_dict in env._concrete_roots_dict(): if root_dict["hash"] not in root_hash_seen: self.included_concrete_spec_data[env_path]["roots"].append(root_dict) root_hash_seen.add(root_dict["hash"]) # Copy unique concrete specs from env for dag_hash, spec_details in env._concrete_specs_dict().items(): if dag_hash not in concrete_hash_seen: self.included_concrete_spec_data[env_path]["concrete_specs"].update( {dag_hash: spec_details} ) concrete_hash_seen.add(dag_hash) # Copy transitive include data transitive = env.included_concrete_spec_data if transitive: self.included_concrete_spec_data[env_path][lockfile_include_key] = transitive self.unify_specs() self.write() def destroy(self): """Remove this environment from Spack entirely.""" shutil.rmtree(self.path) def add(self, user_spec, list_name=USER_SPECS_KEY) -> bool: """Add a single user_spec (non-concretized) to the Environment Returns: True if the spec was added, False if it was already present and did not need to be added """ spec = Spec(user_spec) if list_name not in self.spec_lists: raise SpackEnvironmentError(f"No list {list_name} exists in environment {self.name}") if list_name == USER_SPECS_KEY: if spec.anonymous: raise SpackEnvironmentError("cannot add anonymous specs to an environment") elif not spack.repo.PATH.exists(spec.name) and not spec.abstract_hash: virtuals = spack.repo.PATH.provider_index.providers.keys() if spec.name not in virtuals: raise SpackEnvironmentError(f"no such package: {spec.name}") list_to_change = self.spec_lists[list_name] existing = str(spec) in list_to_change.yaml_list if not existing: list_to_change.add(spec) if list_name == USER_SPECS_KEY: self.manifest.add_user_spec(str(user_spec)) else: self.manifest.add_definition(str(user_spec), list_name=list_name) self._sync_speclists() return bool(not existing) def change_existing_spec( self, change_spec: Spec, list_name: str = USER_SPECS_KEY, match_spec: Optional[Spec] = None, allow_changing_multiple_specs=False, ): """ Find the spec identified by ``match_spec`` and change it to ``change_spec``. Arguments: change_spec: defines the spec properties that need to be changed. This will not change attributes of the matched spec unless they conflict with ``change_spec``. list_name: identifies the spec list in the environment that should be modified match_spec: if set, this identifies the spec that should be changed. If not set, it is assumed we are looking for a spec with the same name as ``change_spec``. """ if not (change_spec.name or match_spec): raise ValueError( "Must specify a spec name or match spec to identify a single spec" " in the environment that will be changed (or multiple with '--all')" ) match_spec = match_spec or Spec(change_spec.name) list_to_change = self.spec_lists[list_name] if list_to_change.is_matrix: raise SpackEnvironmentError( "Cannot directly change specs in matrices:" " specify a named list that is not a matrix" ) matches = list((idx, x) for idx, x in enumerate(list_to_change) if x.satisfies(match_spec)) if len(matches) == 0: raise ValueError( "There are no specs named {0} in {1}".format(match_spec.name, list_name) ) elif len(matches) > 1 and not allow_changing_multiple_specs: raise ValueError(f"{str(match_spec)} matches multiple specs") for idx, spec in matches: override_spec = Spec.override(spec, change_spec) if list_name == USER_SPECS_KEY: self.manifest.override_user_spec(str(override_spec), idx=idx) else: self.manifest.override_definition( str(spec), override=str(override_spec), list_name=list_name ) self._sync_speclists() def remove(self, query_spec, list_name=USER_SPECS_KEY, force=False): """Remove specs from an environment that match a query_spec""" err_msg_header = ( f"Cannot remove '{query_spec}' from '{list_name}' definition " f"in {self.manifest.manifest_file}" ) query_spec = Spec(query_spec) try: list_to_change = self.spec_lists[list_name] except KeyError as e: msg = f"{err_msg_header}, since '{list_name}' does not exist" raise SpackEnvironmentError(msg) from e if not query_spec.concrete: matches = [s for s in list_to_change if s.satisfies(query_spec)] else: # concrete specs match against concrete specs in the env by dag hash. matches = [x.root for x in self.concretized_roots if query_spec.dag_hash() == x.hash] if not matches: raise SpackEnvironmentError(f"{err_msg_header}, no spec matches") old_specs = set(self.user_specs) # Remove specs from the appropriate spec list for spec in matches: if spec not in list_to_change: continue try: list_to_change.remove(spec) except SpecListError as e: msg = str(e) if force: msg += " It will be removed from the concrete specs." tty.warn(msg) else: if list_name == USER_SPECS_KEY: self.manifest.remove_user_spec(str(spec)) else: self.manifest.remove_definition(str(spec), list_name=list_name) # Recompute "definitions" and user specs self._sync_speclists() new_specs = set(self.user_specs) # If 'force', update stale concretized specs if force: stale_specs = old_specs - new_specs self.concretized_roots, removed = stable_partition( self.concretized_roots, lambda x: x.root not in stale_specs ) for x in removed: del self.specs_by_hash[x.hash] def is_develop(self, spec): """Returns true when the spec is built from local sources""" return spec.name in self.dev_specs def apply_develop(self, spec: spack.spec.Spec, path: Optional[str] = None): """Mutate concrete specs to include dev_path provenance pointing to path. This will fail if any existing concrete spec for the same package does not satisfy the given develop spec.""" selector = spack.spec.Spec(spec.name) mutator = spack.spec.Spec() if path: variant = vt.SingleValuedVariant("dev_path", path) else: variant = vt.VariantValueRemoval("dev_path") mutator.variants["dev_path"] = variant msg = ( f"Develop spec '{spec}' conflicts with concrete specs in environment." " Try again with 'spack develop --no-modify-concrete-specs'" " and run 'spack concretize --force' to apply your changes." ) self.mutate(selector, mutator, validator=spec, msg=msg) def mutate( self, selector: spack.spec.Spec, mutator: spack.spec.Spec, validator: Optional[spack.spec.Spec] = None, msg: Optional[str] = None, ): """Mutate concrete specs of an environment Mutate any spec that matches ``selector``. Invalidate caches on parents of mutated specs. If a validator spec is supplied, throw an error if a selected spec does not satisfy the validator. """ # Find all specs that this mutation applies to modify_specs = [] modified_specs = [] for dep in self.all_specs_generator(): if dep.satisfies(selector): if not dep.satisfies(validator or selector): if not msg: msg = f"spec {dep} satisfies selector {selector}" msg += f" but not validator {validator}" raise SpackEnvironmentDevelopError(msg) modify_specs.append(dep) # Manipulate selected specs for s in modify_specs: modified = s.mutate(mutator, rehash=False) if modified: modified_specs.append(s) # Identify roots modified and invalidate all dependent hashes modified_roots = [] for parent in traverse.traverse_nodes(modified_specs, direction="parents"): # record whether this parent is a root before we modify the hash if parent.dag_hash() in self.specs_by_hash: modified_roots.append((parent, parent.dag_hash())) # modify the parent to invalidate hashes parent._mark_root_concrete(False) parent.clear_caches() # Compute new hashes and update the env list of specs hash_mutations = {} for root, old_hash in modified_roots: # New hash must be computed after we finalize concretization root._finalize_concretization() new_hash = root.dag_hash() self.specs_by_hash.pop(old_hash) self.specs_by_hash[new_hash] = root hash_mutations[old_hash] = new_hash for x in self.concretized_roots: if x.hash in hash_mutations: x.hash = hash_mutations[x.hash] if modified_roots: self.write() def concretize( self, *, force: Optional[bool] = None, tests: Union[bool, Sequence[str]] = False ) -> Sequence[SpecPair]: """Concretize user_specs in this environment. Only concretizes specs that haven't been concretized yet unless force is ``True``. This only modifies the environment in memory. ``write()`` will write out a lockfile containing concretized specs. Arguments: force: re-concretize ALL specs, even those that were already concretized; defaults to ``spack.config.get("concretizer:force")`` tests: False to run no tests, True to test all packages, or a list of package names to run tests for some Returns: List of specs that have been concretized. Each entry is a tuple of the user spec and the corresponding concretized spec. """ return EnvironmentConcretizer(self).concretize(force=force, tests=tests) def sync_concretized_specs(self) -> None: """Removes concrete specs that no longer correlate to a user spec""" if not self.concretized_roots: return to_deconcretize, user_specs = [], self._all_user_specs_with_group() for x in self.concretized_roots: if (x.group, x.root) not in user_specs: to_deconcretize.append(x) for x in to_deconcretize: self.deconcretize_by_user_spec(x.root, group=x.group) def _all_user_specs_with_group(self) -> Set[Tuple[str, Spec]]: result = set() for group in self.manifest.groups(): result.update([(group, x) for x in self.user_specs_by(group=group)]) return result def clear_concretized_specs(self) -> None: """Clears the currently concretized specs""" self.concretized_roots = [] self.specs_by_hash = {} def deconcretize_by_hash(self, dag_hash: str) -> None: """Removes a concrete spec from the environment concretization""" self.concretized_roots = [x for x in self.concretized_roots if x.hash != dag_hash] self._maybe_remove_dag_hash(dag_hash) def deconcretize_by_user_spec( self, spec: spack.spec.Spec, *, group: Optional[str] = None ) -> None: """Removes a user spec from the environment concretization Arguments: spec: user spec to deconcretize group: group of the spec to remove. If not specified, the spec is removed from the default group """ group = group or DEFAULT_USER_SPEC_GROUP # spec has to be a root of the environment discarded, self.concretized_roots = stable_partition( self.concretized_roots, lambda x: x.group == group and x.root == spec ) assert len({x.hash for x in discarded}) == 1, ( "More than one hash associated with a single user spec" ) dag_hash = discarded[0].hash self._maybe_remove_dag_hash(dag_hash) def _maybe_remove_dag_hash(self, dag_hash: str): # If this was the only user spec that concretized to this concrete spec, remove it if not self.user_spec_with_hash(dag_hash) and dag_hash in self.specs_by_hash: # if we deconcretized a dependency that doesn't correspond to a root, it won't be here. del self.specs_by_hash[dag_hash] def user_spec_with_hash(self, dag_hash: str) -> bool: """Returns True if any user spec is associated with a concrete spec with the given hash""" return any(x.hash == dag_hash for x in self.concretized_roots) def unify_specs(self) -> None: # Keep the information on new specs by copying the concretized roots old_concretized_roots = self.concretized_roots self._read_lockfile_dict(self._to_lockfile_dict()) self.concretized_roots = old_concretized_roots @property def default_view(self): if not self.has_view(default_view_name): raise SpackEnvironmentError(f"{self.name} does not have a default view enabled") return self.views[default_view_name] def has_view(self, view_name: str) -> bool: return view_name in self.views def update_default_view(self, path_or_bool: Union[str, bool]) -> None: """Updates the path of the default view. If the argument passed as input is False the default view is deleted, if present. The manifest will have an entry ``view: false``. If the argument passed as input is True a default view is created, if not already present. The manifest will have an entry ``view: true``. If a default view is already declared, it will be left untouched. If the argument passed as input is a path a default view pointing to that path is created, if not present already. If a default view is already declared, only its "root" will be changed. Args: path_or_bool: either True, or False or a path """ view_path = self.view_path_default if path_or_bool is True else path_or_bool # We don't have the view, and we want to remove it if default_view_name not in self.views and path_or_bool is False: return # We want to enable the view, but we have it already if default_view_name in self.views and path_or_bool is True: return # We have the view, and we want to set it to the same path if default_view_name in self.views and self.default_view.root == view_path: return self.delete_default_view() if path_or_bool is False: self.views.pop(default_view_name, None) self.manifest.remove_default_view() return # If we had a default view already just update its path, # else create a new one and add it to views if default_view_name in self.views: self.default_view.update_root(view_path) else: assert isinstance(view_path, str), f"expected str for 'view_path', but got {view_path}" self.views[default_view_name] = ViewDescriptor(self.path, view_path) self.manifest.set_default_view(self._default_view_as_yaml()) def delete_default_view(self) -> None: """Deletes the default view associated with this environment.""" if default_view_name not in self.views: return try: view = pathlib.Path(self.default_view.root) shutil.rmtree(view.resolve()) view.unlink() except FileNotFoundError as e: msg = f"[ENVIRONMENT] error trying to delete the default view: {str(e)}" tty.debug(msg) def regenerate_views(self): if not self.views: tty.debug("Skip view update, this environment does not maintain a view") return for view in self.views.values(): view.regenerate(self) def check_views(self): """Checks if the environments default view can be activated.""" try: # This is effectively a no-op, but it touches all packages in the # default view if they are installed. for view_name, view in self.views.items(): for spec in self.concrete_roots(): if spec in view and spec.package and spec.installed: msg = '{0} in view "{1}"' tty.debug(msg.format(spec.name, view_name)) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.warn(e) tty.warn( "Environment %s includes out of date packages or repos. " "Loading the environment view will require reconcretization." % self.name ) def _env_modifications_for_view( self, view: ViewDescriptor, reverse: bool = False ) -> spack.util.environment.EnvironmentModifications: try: with spack.store.STORE.db.read_transaction(): installed_roots = [s for s in self.concrete_roots() if s.installed] mods = uenv.environment_modifications_for_specs(*installed_roots, view=view) except Exception as e: # Failing to setup spec-specific changes shouldn't be a hard error. tty.warn( f"could not {'unload' if reverse else 'load'} runtime environment due " f"to {e.__class__.__name__}: {e}" ) return spack.util.environment.EnvironmentModifications() return mods.reversed() if reverse else mods def add_view_to_env( self, env_mod: spack.util.environment.EnvironmentModifications, view: str ) -> spack.util.environment.EnvironmentModifications: """Collect the environment modifications to activate an environment using the provided view. Removes duplicate paths. Args: env_mod: the environment modifications object that is modified. view: the name of the view to activate.""" descriptor = self.views.get(view) if not descriptor: return env_mod env_mod.extend(uenv.unconditional_environment_modifications(descriptor)) env_mod.extend(self._env_modifications_for_view(descriptor)) # deduplicate paths from specs mapped to the same location for env_var in env_mod.group_by_name(): env_mod.prune_duplicate_paths(env_var) return env_mod def rm_view_from_env( self, env_mod: spack.util.environment.EnvironmentModifications, view: str ) -> spack.util.environment.EnvironmentModifications: """Collect the environment modifications to deactivate an environment using the provided view. Reverses the action of ``add_view_to_env``. Args: env_mod: the environment modifications object that is modified. view: the name of the view to deactivate.""" descriptor = self.views.get(view) if not descriptor: return env_mod env_mod.extend(uenv.unconditional_environment_modifications(descriptor).reversed()) env_mod.extend(self._env_modifications_for_view(descriptor, reverse=True)) return env_mod def add_concrete_spec( self, spec: spack.spec.Spec, concrete: spack.spec.Spec, *, new: bool = True, group: Optional[str] = None, ): """Called when a new concretized spec is added to the environment. This ensures that all internal data structures are kept in sync. Arguments: spec: user spec that resulted in the concrete spec concrete: spec concretized within this environment new: whether to write this spec's package to the env repo on write() """ assert concrete.concrete h = concrete.dag_hash() group = group or DEFAULT_USER_SPEC_GROUP self.concretized_roots.append( ConcretizedRootInfo(root_spec=spec, root_hash=h, new=new, group=group) ) self.specs_by_hash[h] = concrete def _dev_specs_that_need_overwrite(self): """Return the hashes of all specs that need to be reinstalled due to source code change.""" changed_dev_specs = [ s for s in traverse.traverse_nodes( self.concrete_roots(), order="breadth", key=traverse.by_dag_hash ) if _is_dev_spec_and_has_changed(s) ] # Collect their hashes, and the hashes of their installed parents. # Notice: with order=breadth all changed dev specs are at depth 0, # even if they occur as parents of one another. return [ spec.dag_hash() for depth, spec in traverse.traverse_nodes( changed_dev_specs, root=True, order="breadth", depth=True, direction="parents", key=traverse.by_dag_hash, ) if depth == 0 or spec.installed ] def _partition_roots_by_install_status(self): """Partition root specs into those that do not have to be passed to the installer, and those that should be, taking into account development specs. This is done in a single read transaction per environment instead of per spec.""" with spack.store.STORE.db.read_transaction(): uninstalled, installed = stable_partition(self.concrete_roots(), _is_uninstalled) return installed, uninstalled def uninstalled_specs(self): """Return root specs that are not installed, or are installed, but are development specs themselves or have those among their dependencies.""" return self._partition_roots_by_install_status()[1] def install_all(self, **install_args): """Install all concretized specs in an environment. Note: this does not regenerate the views for the environment; that needs to be done separately with a call to write(). Args: install_args (dict): keyword install arguments """ self.install_specs(None, **install_args) def install_specs(self, specs: Optional[List[Spec]] = None, **install_args): roots = self.concrete_roots() specs = specs if specs is not None else roots # Extract reporter arguments reporter = install_args.pop("reporter", None) report_file = install_args.pop("report_file", None) # Extend the set of specs to overwrite with modified dev specs and their parents install_args["overwrite"] = { *install_args.get("overwrite", ()), *self._dev_specs_that_need_overwrite(), } # Only environment roots in explicit groups are marked explicit install_args["explicit"] = { *install_args.get("explicit", ()), *(x.hash for x in self.explicit_roots()), } builder = spack.installer_dispatch.create_installer( [spec.package for spec in specs], create_reports=reporter is not None, **install_args ) try: builder.install() finally: if reporter: if isinstance(builder.reports, dict): reporter.build_report(report_file, list(builder.reports.values())) elif isinstance(builder.reports, list): reporter.build_report(report_file, builder.reports) else: raise TypeError("builder.reports must be either a dictionary or a list") def all_specs_generator(self) -> Iterable[Spec]: """Returns a generator for all concrete specs""" return traverse.traverse_nodes(self.concrete_roots(), key=traverse.by_dag_hash) def all_specs(self) -> List[Spec]: """Returns a list of all concrete specs""" return list(self.all_specs_generator()) def all_hashes(self): """Return hashes of all specs.""" return [s.dag_hash() for s in self.all_specs_generator()] def roots(self): """Specs explicitly requested by the user *in this environment*. Yields both added and installed specs that have user specs in ``spack.yaml``. """ concretized = dict(self.concretized_specs()) for spec in self.user_specs: concrete = concretized.get(spec) yield concrete if concrete else spec def added_specs(self): """Specs that are not yet installed. Yields the user spec for non-concretized specs, and the concrete spec for already concretized but not yet installed specs. """ # use a transaction to avoid overhead of repeated calls # to `package.spec.installed` with spack.store.STORE.db.read_transaction(): concretized = dict(self.concretized_specs()) for spec in self.user_specs: concrete = concretized.get(spec) if not concrete: yield spec elif not concrete.installed: yield concrete def concretized_specs(self): """Tuples of (user spec, concrete spec) for all concrete specs.""" for x in self.concretized_roots: yield x.root, self.specs_by_hash[x.hash] yield from self.concretized_specs_from_all_included_environments() def concretized_specs_from_all_included_environments(self): seen = {(x.root, x.hash) for x in self.concretized_roots} for included_env in self.included_concretized_roots: yield from self.concretized_specs_from_included_environment(included_env, _seen=seen) def concretized_specs_from_included_environment( self, included_env: str, *, _seen: Optional[Set[Tuple[spack.spec.Spec, str]]] = None ): _seen = set() if _seen is None else _seen for x in self.included_concretized_roots[included_env]: if (x.root, x.hash) in _seen: continue _seen.add((x.root, x.hash)) yield x.root, self.included_specs_by_hash[included_env][x.hash] def concrete_roots(self): """Same as concretized_specs, except it returns the list of concrete roots *without* associated user spec""" return [root for _, root in self.concretized_specs()] def concretized_specs_by(self, *, group: str) -> Iterable[Tuple[Spec, Spec]]: """Generates all the (abstract, concrete) spec pairs for a given group""" for x in self.concretized_roots: if x.group != group: continue yield x.root, self.specs_by_hash[x.hash] def get_by_hash(self, dag_hash: str) -> List[Spec]: # If it's not a partial hash prefix we can early exit early_exit = len(dag_hash) == 32 matches = [] for spec in traverse.traverse_nodes( self.concrete_roots(), key=traverse.by_dag_hash, order="breadth" ): if spec.dag_hash().startswith(dag_hash): matches.append(spec) if early_exit: break return matches def get_one_by_hash(self, dag_hash): """Returns the single spec from the environment which matches the provided hash. Raises an AssertionError if no specs match or if more than one spec matches.""" hash_matches = self.get_by_hash(dag_hash) assert len(hash_matches) == 1 return hash_matches[0] def all_matching_specs(self, *specs: spack.spec.Spec) -> List[Spec]: """Returns all concretized specs in the environment satisfying any of the input specs""" return [ s for s in traverse.traverse_nodes(self.concrete_roots(), key=traverse.by_dag_hash) if any(s.satisfies(t) for t in specs) ] @spack.repo.autospec def matching_spec(self, spec): """ Given a spec (likely not concretized), find a matching concretized spec in the environment. The matching spec does not have to be installed in the environment, but must be concrete (specs added with ``spack add`` without an intervening ``spack concretize`` will not be matched). If there is a single root spec that matches the provided spec or a single dependency spec that matches the provided spec, then the concretized instance of that spec will be returned. If multiple root specs match the provided spec, or no root specs match and multiple dependency specs match, then this raises an error and reports all matching specs. """ env_root_to_user = {root.dag_hash(): user for user, root in self.concretized_specs()} root_matches, dep_matches = [], [] for env_spec in traverse.traverse_nodes( specs=[root for _, root in self.concretized_specs()], key=traverse.by_dag_hash, order="breadth", ): if not env_spec.satisfies(spec): continue # If the spec is concrete, then there is no possibility of multiple matches, # and we immediately return the single match if spec.concrete: return env_spec # Distinguish between environment roots and deps. Specs that are both # are classified as environment roots. user_spec = env_root_to_user.get(env_spec.dag_hash()) if user_spec: root_matches.append((env_spec, user_spec)) else: dep_matches.append(env_spec) # No matching spec if not root_matches and not dep_matches: return None # Single root spec, any number of dep specs => return root spec. if len(root_matches) == 1: return root_matches[0][0] if not root_matches and len(dep_matches) == 1: return dep_matches[0] # More than one spec matched, and either multiple roots matched or # none of the matches were roots # If multiple root specs match, it is assumed that the abstract # spec will most-succinctly summarize the difference between them # (and the user can enter one of these to disambiguate) fmt_str = "{hash:7} " + spack.spec.DEFAULT_FORMAT color = clr.get_color_when() match_strings = [ f"Root spec {abstract.format(color=color)}\n {concrete.format(fmt_str, color=color)}" for concrete, abstract in root_matches ] match_strings.extend( f"Dependency spec\n {s.format(fmt_str, color=color)}" for s in dep_matches ) matches_str = "\n".join(match_strings) raise SpackEnvironmentError( f"{spec} matches multiple specs in the environment {self.name}: \n{matches_str}" ) def removed_specs(self): """Tuples of (user spec, concrete spec) for all specs that will be removed on next concretize.""" needed = set() for s, c in self.concretized_specs(): if s in self.user_specs: for d in c.traverse(): needed.add(d) for s, c in self.concretized_specs(): for d in c.traverse(): if d not in needed: yield d def _concrete_specs_dict(self): concrete_specs = {} for s in traverse.traverse_nodes(self.specs_by_hash.values(), key=traverse.by_dag_hash): spec_dict = s.node_dict_with_hashes(hash=ht.dag_hash) # Assumes no legacy formats, since this was just created. spec_dict[ht.dag_hash.name] = s.dag_hash() concrete_specs[s.dag_hash()] = spec_dict if s.build_spec is not s: for d in s.build_spec.traverse(): build_spec_dict = d.node_dict_with_hashes(hash=ht.dag_hash) build_spec_dict[ht.dag_hash.name] = d.dag_hash() concrete_specs[d.dag_hash()] = build_spec_dict return concrete_specs def _concrete_roots_dict(self): if not self.has_groups(): return [{"hash": x.hash, "spec": str(x.root)} for x in self.concretized_roots] return [ {"hash": x.hash, "spec": str(x.root), "group": x.group} for x in self.concretized_roots ] def has_groups(self) -> bool: groups = self.manifest.groups() # True if groups != {DEFAULT_USER_SPEC_GROUP} return len(groups) != 1 or DEFAULT_USER_SPEC_GROUP not in groups def _to_lockfile_dict(self): """Create a dictionary to store a lockfile for this environment.""" lockfile_version = CURRENT_LOCKFILE_VERSION if self.has_groups() else 6 concrete_specs = self._concrete_specs_dict() root_specs = self._concrete_roots_dict() spack_dict = {"version": spack.spack_version} spack_commit = spack.get_spack_commit() if spack_commit: spack_dict["type"] = "git" spack_dict["commit"] = spack_commit else: spack_dict["type"] = "release" # this is the lockfile we'll write out data = { # metadata about the format "_meta": { "file-type": "spack-lockfile", "lockfile-version": lockfile_version, "specfile-version": spack.spec.SPECFILE_FORMAT_VERSION, }, # spack version information "spack": spack_dict, # users specs + hashes are the 'roots' of the environment "roots": root_specs, # Concrete specs by hash, including dependencies "concrete_specs": concrete_specs, } if self.included_concrete_env_root_dirs: data[lockfile_include_key] = self.included_concrete_spec_data return data def _read_lockfile(self, file_or_json): """Read a lockfile from a file or from a raw string.""" lockfile_dict = sjson.load(file_or_json) self._read_lockfile_dict(lockfile_dict) return lockfile_dict def _set_included_env_roots( self, env_name: str, env_info: Dict[str, Dict[str, Any]], included_json_specs_by_hash: Dict[str, Dict[str, Any]], ) -> Dict[str, Dict[str, Any]]: """Populates included_concretized_roots from included environment data, including any transitively nested included environments. Args: env_name: the path of the included environment env_info: included concrete environment data included_json_specs_by_hash: concrete spec data keyed by hash Returns: updated specs_by_hash """ self.included_concretized_roots[env_name] = [] def add_specs(name, info, specs_by_hash): # Add specs from the environment as well as any of its nested # environments. for root_info in info["roots"]: self.included_concretized_roots[name].append( ConcretizedRootInfo.from_info_dict(root_info) ) if "concrete_specs" in info: specs_by_hash.update(info["concrete_specs"]) if lockfile_include_key in info: for included_name, included_info in info[lockfile_include_key].items(): if included_name not in self.included_concretized_roots: self.included_concretized_roots[included_name] = [] add_specs(included_name, included_info, specs_by_hash) add_specs(env_name, env_info, included_json_specs_by_hash) return included_json_specs_by_hash def _read_lockfile_dict(self, d): """Read a lockfile dictionary into this environment.""" self.specs_by_hash = {} self.included_specs_by_hash = {} self.included_concretized_roots = {} roots = d["roots"] self.concretized_roots = [ConcretizedRootInfo.from_info_dict(r) for r in roots] json_specs_by_hash = d["concrete_specs"] included_json_specs_by_hash = {} if lockfile_include_key in d: for env_name, env_info in d[lockfile_include_key].items(): included_json_specs_by_hash.update( self._set_included_env_roots(env_name, env_info, included_json_specs_by_hash) ) current_lockfile_format = d["_meta"]["lockfile-version"] try: reader = READER_CLS[current_lockfile_format] except KeyError: msg = ( f"Spack {spack.__version__} cannot read the lockfile '{self.lock_path}', using " f"the v{current_lockfile_format} format." ) if CURRENT_LOCKFILE_VERSION < current_lockfile_format: msg += " You need to use a newer Spack version." raise SpackEnvironmentError(msg) concretized_order = [x.hash for x in self.concretized_roots] first_seen, concretized_order = self._filter_specs( reader, json_specs_by_hash, concretized_order ) for idx, spec_dag_hash in enumerate(concretized_order): self.concretized_roots[idx].hash = spec_dag_hash self.specs_by_hash[spec_dag_hash] = first_seen[spec_dag_hash] if any(self.included_concretized_roots.values()): first_seen = {} for env_name, roots in self.included_concretized_roots.items(): order = [x.hash for x in roots] filtered_spec, new_order = self._filter_specs( reader, included_json_specs_by_hash, order ) first_seen.update(filtered_spec) for idx, spec_dag_hash in enumerate(new_order): roots[idx].hash = spec_dag_hash for env_path, roots in self.included_concretized_roots.items(): self.included_specs_by_hash[env_path] = {x.hash: first_seen[x.hash] for x in roots} def _filter_specs(self, reader, json_specs_by_hash, order_concretized): # Track specs by their lockfile key. Currently, spack uses the finest # grained hash as the lockfile key, while older formats used the build # hash or a previous incarnation of the DAG hash (one that did not # include build deps or package hash). specs_by_hash = {} # Track specs by their DAG hash, allows handling DAG hash collisions first_seen = {} # First pass: Put each spec in the map ignoring dependencies for lockfile_key, node_dict in json_specs_by_hash.items(): spec = reader.from_node_dict(node_dict) if not spec._hash: # in v1 lockfiles, the hash only occurs as a key spec._hash = lockfile_key specs_by_hash[lockfile_key] = spec # Second pass: For each spec, get its dependencies from the node dict # and add them to the spec, including build specs for lockfile_key, node_dict in json_specs_by_hash.items(): name, data = reader.name_and_data(node_dict) for _, dep_hash, deptypes, _, virtuals, direct in reader.dependencies_from_node_dict( data ): specs_by_hash[lockfile_key]._add_dependency( specs_by_hash[dep_hash], depflag=dt.canonicalize(deptypes), virtuals=virtuals, direct=direct, ) if "build_spec" in node_dict: _, bhash, _ = reader.extract_build_spec_info_from_node_dict(node_dict) specs_by_hash[lockfile_key]._build_spec = specs_by_hash[bhash] # Traverse the root specs one at a time in the order they appear. # The first time we see each DAG hash, that's the one we want to # keep. This is only required as long as we support older lockfile # formats where the mapping from DAG hash to lockfile key is possibly # one-to-many. for lockfile_key in order_concretized: for s in specs_by_hash[lockfile_key].traverse(): if s.dag_hash() not in first_seen: first_seen[s.dag_hash()] = s # Now make sure concretized_order and our internal specs dict # contains the keys used by modern spack (i.e. the dag_hash # that includes build deps and package hash). order_concretized = [specs_by_hash[h_key].dag_hash() for h_key in order_concretized] return first_seen, order_concretized def write(self, regenerate: bool = True) -> None: """Writes an in-memory environment to its location on disk. Write out package files for each newly concretized spec. Also regenerate any views associated with the environment and run post-write hooks, if regenerate is True. Args: regenerate: regenerate views and run post-write hooks as well as writing if True. """ self.manifest_uptodate_or_warn() if self.specs_by_hash or self.included_concrete_env_root_dirs: self.ensure_env_directory_exists(dot_env=True) self.update_environment_repository() self.manifest.flush() # Write the lock file last. This is useful for Makefiles # with `spack.lock: spack.yaml` rules, where the target # should be newer than the prerequisite to avoid # redundant re-concretization. self.update_lockfile() else: self.ensure_env_directory_exists(dot_env=False) with fs.safe_remove(self.lock_path): self.manifest.flush() if regenerate: self.regenerate_views() for x in self.concretized_roots: x.new = False def update_lockfile(self) -> None: with fs.write_tmp_and_move(self.lock_path, encoding="utf-8") as f: sjson.dump(self._to_lockfile_dict(), stream=f) def ensure_env_directory_exists(self, dot_env: bool = False) -> None: """Ensure that the root directory of the environment exists Args: dot_env: if True also ensures that the /.env directory exists """ fs.mkdirp(self.path) if dot_env: fs.mkdirp(self.env_subdir_path) def update_environment_repository(self) -> None: """Updates the repository associated with the environment.""" new_specs = [self.specs_by_hash[x.hash] for x in self.concretized_roots if x.new] for spec in traverse.traverse_nodes(new_specs): if not spec.concrete: raise ValueError("specs passed to environment.write() must be concrete!") self._add_to_environment_repository(spec) def _add_to_environment_repository(self, spec_node: Spec) -> None: """Add the root node of the spec to the environment repository""" namespace: str = spec_node.namespace repository = spack.repo.create_or_construct( root=os.path.join(self.repos_path, namespace), namespace=namespace, package_api=spack.repo.PATH.get_repo(namespace).package_api, ) pkg_dir = repository.dirname_for_package_name(spec_node.name) fs.mkdirp(pkg_dir) spack.repo.PATH.dump_provenance(spec_node, pkg_dir) def manifest_uptodate_or_warn(self): """Emits a warning if the manifest file is not up-to-date.""" if not is_latest_format(self.manifest_path): ver = ".".join(str(s) for s in spack.spack_version_info[:2]) msg = ( 'The environment "{}" is written to disk in a deprecated format. ' "Please update it using:\n\n" "\tspack env update {}\n\n" "Note that versions of Spack older than {} may not be able to " "use the updated configuration." ) warnings.warn(msg.format(self.name, self.name, ver)) def _default_view_as_yaml(self): """This internal function assumes the default view is set""" path = self.default_view.raw_root if ( self.default_view == ViewDescriptor(self.path, self.view_path_default) and len(self.views) == 1 ): return True if self.default_view == ViewDescriptor(self.path, path) and len(self.views) == 1: return path return self.default_view.to_dict() def invalidate_repository_cache(self): self._repo = None def __enter__(self): self._previous_active = _active_environment if self._previous_active: deactivate() activate(self) return self def __exit__(self, exc_type, exc_val, exc_tb): deactivate() if self._previous_active: activate(self._previous_active) def _is_uninstalled(spec): return not spec.installed or (spec.satisfies("dev_path=*") or spec.satisfies("^dev_path=*")) class ReusableSpecsFactory: """Creates a list of SpecFilters to generate the reusable specs for the environment""" def __init__(self, *, env: Environment, group: str): self.env = env self.group = group @staticmethod def _const(specs: List[Spec]) -> Callable[[], List[Spec]]: """Returns a zero-argument callable that always returns the given list.""" return lambda: specs def __call__( self, is_usable: Callable[[Spec], bool], configuration: spack.config.Configuration ) -> List[SpecFilter]: result = [] # Specs from group dependencies _must_ be reused, regardless of configuration dependencies = self.env.manifest.needs(group=self.group) necessary_specs = [] for d in dependencies: necessary_specs.extend([x for _, x in self.env.concretized_specs_by(group=d)]) # Specs from groups listed as dependencies if necessary_specs: necessary_specs = list( traverse.traverse_nodes(necessary_specs, deptype=("link", "run")) ) result.append( SpecFilter( self._const(necessary_specs), include=[], exclude=[], is_usable=is_usable ) ) # Included environments and _this_ group, instead, are subject to configuration concretizer_yaml = configuration.get_config("concretizer") reuse_yaml = concretizer_yaml.get("reuse", False) # With no reuse don't account for previously concretized specs in _this_ group if reuse_yaml is False: return result this_group_specs = [x for _, x in self.env.concretized_specs_by(group=self.group)] included_specs = [ x for _, x in self.env.concretized_specs_from_all_included_environments() ] additional_specs = list(traverse.traverse_nodes(this_group_specs + included_specs)) if not isinstance(reuse_yaml, Mapping): result.append( SpecFilter( self._const(additional_specs), include=[], exclude=[], is_usable=is_usable ) ) return result # Here we know we have a complex reuse configuration default_include = reuse_yaml.get("include", []) default_exclude = reuse_yaml.get("exclude", []) for source in reuse_yaml.get("from", []): # We just need to take care of the environment-related parts if source["type"] != "environment": continue include = source.get("include", default_include) exclude = source.get("exclude", default_exclude) if "path" not in source: result.append( SpecFilter( self._const(additional_specs), include=include, exclude=exclude, is_usable=is_usable, ) ) continue env_dir = as_env_dir(source["path"]) if env_dir in self.env.included_concrete_env_root_dirs: spec_pairs_from_included_envs = [ x for _, x in self.env.concretized_specs_from_included_environment(env_dir) ] included_specs = list(traverse.traverse_nodes(spec_pairs_from_included_envs)) result.append( SpecFilter( self._const(included_specs), include=include, exclude=exclude, is_usable=is_usable, ) ) return result class EnvironmentConcretizer: def __init__(self, env: Environment): self.env = env def concretize( self, *, force: Optional[bool] = None, tests: Union[bool, Sequence[str]] = False ) -> List[SpecPair]: if force is None: force = spack.config.get("concretizer:force") self._prepare_environment_for_concretization(force=force) result = [] # Sort so that the ordering is deterministic, and "default" specs are first for current_group in self._order_groups(): with self.env.config_override_for_group(group=current_group): partial_result = self._concretize_single_group(group=current_group, tests=tests) result.extend(partial_result) # Unify the specs objects, so we get correct references to all parents if result: self.env.unify_specs() return result def _concretize_single_group( self, *, group: str, tests: Union[bool, Sequence[str]] ) -> List[SpecPair]: # Exit early if the set of concretized specs is the set of user specs new_user_specs, kept_user_specs = self._partition_user_specs(group=group) if not new_user_specs: return [] # Pick the right concretization strategy if group != DEFAULT_USER_SPEC_GROUP: tty.msg(f"Concretizing the '{group}' group of specs") unify = spack.config.CONFIG.get_config("concretizer").get("unify", False) factory = ReusableSpecsFactory(env=self.env, group=group) if unify == "when_possible": partial_result = self._concretize_together_where_possible( new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory ) elif unify is True: partial_result = self._concretize_together( new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory ) elif unify is False: partial_result = self._concretize_separately( new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory ) else: raise SpackEnvironmentError(f"concretization strategy not implemented [{unify}]") return partial_result def _prepare_environment_for_concretization(self, *, force: bool): """Reset the environment concrete state and ensure consistency with user specs.""" if force: self.env.clear_concretized_specs() else: self.env.sync_concretized_specs() # If a combined env, check updated spec is in the linked envs if self.env.included_concrete_env_root_dirs: self.env.include_concrete_envs() def _partition_user_specs( self, *, group: str ) -> Tuple[List[spack.spec.Spec], List[spack.spec.Spec]]: """Splits the users specs in the list of the ones to be computed, and the list of the ones to retain. """ concretized_user_specs = {x.root for x in self.env.concretized_roots if x.group == group} kept_user_specs, new_user_specs = stable_partition( self.env.user_specs_by(group=group), lambda x: x in concretized_user_specs ) kept_user_specs += self.env.included_user_specs return new_user_specs, kept_user_specs def _order_groups(self) -> List[str]: done, result = {DEFAULT_USER_SPEC_GROUP}, [DEFAULT_USER_SPEC_GROUP] all_groups = self.env.manifest.groups() remaining = all_groups - {DEFAULT_USER_SPEC_GROUP} # Validate upfront that all 'needs' references point to defined groups for group in remaining: for dep in self.env.manifest.needs(group=group): if dep not in all_groups: raise SpackEnvironmentConfigError( f"group '{group}' needs '{dep}', but '{dep}' is not a defined group", self.env.manifest.manifest_file, ) while remaining: # Check we have groups that are "ready" ready = [] for current in remaining: deps = self.env.manifest.needs(group=current) if all(d in done for d in deps): ready.append(current) # Check we can progress - if nothing is ready, there is a cycle if not ready: raise SpackEnvironmentConfigError( f"cyclic dependency detected among groups: {', '.join(sorted(remaining))}", self.env.manifest.manifest_file, ) result.extend(ready) done.update(ready) remaining.difference_update(ready) return result def _user_spec_pairs( self, user_specs_to_compute: List[Spec], user_specs_to_keep: List[Spec] ) -> List[SpecPair]: specs_to_concretize = [(s, None) for s in user_specs_to_compute] + [ (abstract, concrete) for abstract, concrete in self.env.concretized_specs() if abstract in user_specs_to_keep ] return specs_to_concretize def _concretize_together_where_possible( self, to_compute: List[Spec], to_keep: List[Spec], *, group: Optional[str] = None, tests: Union[bool, Sequence] = False, factory: ReusableSpecsFactory, ) -> List[SpecPair]: import spack.concretize specs_to_concretize = self._user_spec_pairs(to_compute, to_keep) result = spack.concretize.concretize_together_when_possible( specs_to_concretize, tests=tests, factory=factory ) result = [x for x in result if x[0] in to_compute] for abstract, concrete in result: self.env.add_concrete_spec(abstract, concrete, new=True, group=group) return result def _concretize_together( self, to_compute: List[Spec], to_keep: List[Spec], *, group: Optional[str] = None, tests: Union[bool, Sequence] = False, factory: ReusableSpecsFactory, ) -> List[SpecPair]: import spack.concretize to_concretize = self._user_spec_pairs(to_compute, to_keep) try: concrete_pairs = spack.concretize.concretize_together( to_concretize, tests=tests, factory=factory ) except spack.error.UnsatisfiableSpecError as e: # "Enhance" the error message for multiple root specs, suggest a less strict # form of concretization. if len(self.env.user_specs_by(group=group)) > 1: e.message += ". " if to_keep: e.message += ( "Couldn't concretize without changing the existing environment. " "If you are ok with changing it, try `spack concretize --force`. " ) e.message += ( "You could consider setting `concretizer:unify` to `when_possible` " "or `false` to allow multiple versions of some packages." ) raise # Return the portion of the return value that is new result = concrete_pairs[: len(to_compute)] for abstract, concrete in result: self.env.add_concrete_spec(abstract, concrete, new=True, group=group) return result def _concretize_separately( self, to_compute: List[Spec], to_keep: List[Spec], *, group: Optional[str] = None, tests: Union[bool, Sequence] = False, factory: ReusableSpecsFactory, ) -> List[SpecPair]: """Concretization strategy that concretizes separately one user spec after the other""" import spack.concretize to_concretize = [(x, None) for x in to_compute] concrete_pairs = spack.concretize.concretize_separately( to_concretize, tests=tests, factory=factory ) for abstract, concrete in concrete_pairs: self.env.add_concrete_spec(abstract, concrete, new=True, group=group) return concrete_pairs def yaml_equivalent(first, second) -> bool: """Returns whether two spack yaml items are equivalent, including overrides""" # YAML has timestamps and dates, but we don't use them yet in schemas if isinstance(first, dict): return isinstance(second, dict) and _equiv_dict(first, second) elif isinstance(first, list): return isinstance(second, list) and _equiv_list(first, second) elif isinstance(first, bool): return isinstance(second, bool) and first is second elif isinstance(first, int): return isinstance(second, int) and first == second elif first is None: return second is None else: # it's a string return isinstance(second, str) and first == second def _equiv_list(first, second): """Returns whether two spack yaml lists are equivalent, including overrides""" if len(first) != len(second): return False return all(yaml_equivalent(f, s) for f, s in zip(first, second)) def _equiv_dict(first, second): """Returns whether two spack yaml dicts are equivalent, including overrides""" if len(first) != len(second): return False same_values = all(yaml_equivalent(fv, sv) for fv, sv in zip(first.values(), second.values())) same_keys_with_same_overrides = all( fk == sk and getattr(fk, "override", False) == getattr(sk, "override", False) for fk, sk in zip(first.keys(), second.keys()) ) return same_values and same_keys_with_same_overrides def display_specs(specs: List[spack.spec.Spec], *, highlight_non_defaults: bool = False) -> None: """Displays a list of specs traversed breadth-first, covering nodes, with install status. Args: specs: list of specs to be displayed highlight_non_defaults: if True, highlights non-default versions and variants in the specs being displayed """ tree_string = spack.spec.tree( specs, format=spack.spec.DISPLAY_FORMAT, hashes=True, hashlen=7, status_fn=spack.spec.Spec.install_status, highlight_version_fn=( spack.package_base.non_preferred_version if highlight_non_defaults else None ), highlight_variant_fn=( spack.package_base.non_default_variant if highlight_non_defaults else None ), key=traverse.by_dag_hash, ) print(tree_string) def make_repo_path(root): """Make a RepoPath from the repo subdirectories in an environment.""" repos = ( spack.repo.from_path(os.path.dirname(p)) for p in glob.glob(os.path.join(root, "**", "repo.yaml"), recursive=True) ) return spack.repo.RepoPath(*repos) def manifest_file(env_name_or_dir): """Return the absolute path to a manifest file given the environment name or directory. Args: env_name_or_dir (str): either the name of a valid environment or a directory where a manifest file resides Raises: AssertionError: if the environment is not found """ env_dir = None if is_env_dir(env_name_or_dir): env_dir = os.path.abspath(env_name_or_dir) elif exists(env_name_or_dir): env_dir = os.path.abspath(root(env_name_or_dir)) assert env_dir, "environment not found [env={0}]".format(env_name_or_dir) return os.path.join(env_dir, manifest_name) def update_yaml(manifest, backup_file): """Update a manifest file from an old format to the current one. Args: manifest (str): path to a manifest file backup_file (str): file where to copy the original manifest Returns: True if the manifest was updated, False otherwise. Raises: AssertionError: in case anything goes wrong during the update """ # Check if the environment needs update with open(manifest, encoding="utf-8") as f: data = syaml.load(f) top_level_key = _top_level_key(data) needs_update = spack.schema.env.update(data[top_level_key]) if not needs_update: msg = "No update needed [manifest={0}]".format(manifest) tty.debug(msg) return False # Copy environment to a backup file and update it msg = ( 'backup file "{0}" already exists on disk. Check its content ' "and remove it before trying to update again." ) assert not os.path.exists(backup_file), msg.format(backup_file) shutil.copy(manifest, backup_file) with open(manifest, "w", encoding="utf-8") as f: syaml.dump_config(data, f) return True def _top_level_key(data): """Return the top level key used in this environment Args: data (dict): raw yaml data of the environment Returns: Either 'spack' or 'env' """ msg = 'cannot find top level attribute "spack" or "env" in the environment' assert any(x in data for x in ("spack", "env")), msg if "spack" in data: return "spack" return "env" def is_latest_format(manifest): """Return False if the manifest file exists and is not in the latest schema format. Args: manifest (str): manifest file to be analyzed """ try: with open(manifest, encoding="utf-8") as f: data = syaml.load(f) except OSError: return True top_level_key = _top_level_key(data) changed = spack.schema.env.update(data[top_level_key]) return not changed @contextlib.contextmanager def no_active_environment(): """Deactivate the active environment for the duration of the context. Has no effect when there is no active environment.""" env = active_environment() try: deactivate() yield finally: # TODO: we don't handle `use_env_repo` here. if env: activate(env) def initialize_environment_dir( environment_dir: Union[str, pathlib.Path], envfile: Optional[Union[str, pathlib.Path]] ) -> None: """Initialize an environment directory starting from an envfile. Files with suffix .json or .lock are considered lockfiles. Files with any other name are considered manifest files. Args: environment_dir: directory where the environment should be placed envfile: manifest file or lockfile used to initialize the environment Raises: SpackEnvironmentError: if the directory can't be initialized """ environment_dir = pathlib.Path(environment_dir) target_lockfile = environment_dir / lockfile_name target_manifest = environment_dir / manifest_name if target_manifest.exists(): msg = f"cannot initialize environment, {target_manifest} already exists" raise SpackEnvironmentError(msg) if target_lockfile.exists(): msg = f"cannot initialize environment, {target_lockfile} already exists" raise SpackEnvironmentError(msg) def _ensure_env_dir(): try: environment_dir.mkdir(parents=True, exist_ok=True) except FileExistsError as e: msg = f"cannot initialize the environment, '{environment_dir}' already exists" raise SpackEnvironmentError(msg) from e if envfile is None: _ensure_env_dir() target_manifest.write_text(default_manifest_yaml()) return envfile = pathlib.Path(envfile) if not envfile.exists(): msg = f"cannot initialize environment, {envfile} is not a valid file" raise SpackEnvironmentError(msg) if envfile.is_dir(): # initialization file is an entire env directory if not (envfile / "spack.yaml").is_file(): msg = f"cannot initialize environment, {envfile} is not a valid environment" raise SpackEnvironmentError(msg) copy_tree(str(envfile), str(environment_dir)) return _ensure_env_dir() # When we have a lockfile we should copy that and produce a consistent default manifest if str(envfile).endswith(".lock") or str(envfile).endswith(".json"): shutil.copy(envfile, target_lockfile) # This constructor writes a spack.yaml which is consistent with the root # specs in the spack.lock try: EnvironmentManifestFile.from_lockfile(environment_dir) except Exception as e: msg = f"cannot initialize environment, '{environment_dir}' from lockfile" raise SpackEnvironmentError(msg) from e return shutil.copy(envfile, target_manifest) # Copy relative path includes that live inside the environment dir try: manifest = EnvironmentManifestFile(environment_dir) except Exception: # error handling for bad manifests is handled on other code paths return # TODO: make this recursive includes = manifest[TOP_LEVEL_KEY].get(manifest_include_name, []) paths = spack.config.paths_from_includes(includes) for path in paths: if os.path.isabs(path): continue abspath = pathlib.Path(os.path.normpath(environment_dir / path)) common_path = pathlib.Path(os.path.commonpath([environment_dir, abspath])) if common_path != environment_dir: tty.debug(f"Will not copy relative include file from outside environment: {path}") continue orig_abspath = os.path.normpath(envfile.parent / path) if os.path.isfile(orig_abspath): fs.touchp(abspath) shutil.copy(orig_abspath, abspath) continue if not os.path.exists(orig_abspath): tty.warn(f"Skipping copy of non-existent include path: '{path}'") continue if os.path.exists(abspath): tty.warn(f"Skipping copy of directory over existing path: {path}") continue shutil.copytree(orig_abspath, abspath, symlinks=True) class EnvironmentManifestFile(collections.abc.Mapping): """Manages the in-memory representation of a manifest file, and its synchronization with the actual manifest on disk. """ @staticmethod def from_lockfile(manifest_dir: Union[pathlib.Path, str]) -> "EnvironmentManifestFile": """Returns an environment manifest file compatible with the lockfile already present in the environment directory. This function also writes a spack.yaml file that is consistent with the spack.lock already existing in the directory. Args: manifest_dir: directory containing the manifest and lockfile """ # TBD: Should this be the abspath? manifest_dir = pathlib.Path(manifest_dir) lockfile = manifest_dir / lockfile_name with lockfile.open("r", encoding="utf-8") as f: data = sjson.load(f) roots = data["roots"] user_specs_by_group: Dict[str, List[str]] = {} for item in roots: # "group" is not there for Lockfile v6 and lower group = item.get("group", DEFAULT_USER_SPEC_GROUP) user_specs_by_group.setdefault(group, []).append(item["spec"]) default_content = manifest_dir / manifest_name default_content.write_text(default_manifest_yaml()) manifest = EnvironmentManifestFile(manifest_dir) for group, specs in user_specs_by_group.items(): for spec in specs: manifest.add_user_spec(spec, group=group) manifest.flush() return manifest def __init__(self, manifest_dir: Union[pathlib.Path, str], name: Optional[str] = None) -> None: self.manifest_dir = pathlib.Path(manifest_dir) self.name = name or str(manifest_dir) self.manifest_file = self.manifest_dir / manifest_name self.scope_name = f"env:{self.name}" self.config_stage_dir = os.path.join(env_subdir_path(manifest_dir), "config") #: Configuration scope associated with this environment. Note that this is not #: invalidated by a re-read of the manifest file. self._env_config_scope: Optional[spack.config.ConfigScope] = None if not self.manifest_file.exists(): msg = f"cannot find '{manifest_name}' in {self.manifest_dir}" raise SpackEnvironmentError(msg) with self.manifest_file.open(encoding="utf-8") as f: self.yaml_content = _read_yaml(f) # Maps groups to their dependencies self._groups: Dict[str, Tuple[str, ...]] = {DEFAULT_USER_SPEC_GROUP: tuple()} # Raw YAML definitions of the user specs for each group self._user_specs: Dict[str, List] = {DEFAULT_USER_SPEC_GROUP: []} # Configuration overrides for each group self._config_override: Dict[str, Any] = {DEFAULT_USER_SPEC_GROUP: None} # Whether specs in each group are marked explicit self._explicit: Dict[str, bool] = {DEFAULT_USER_SPEC_GROUP: True} self._init_user_specs() self.changed = False def _init_user_specs(self): specs_yaml = self.configuration.get(USER_SPECS_KEY, []) for item in specs_yaml: if isinstance(item, str): self._user_specs[DEFAULT_USER_SPEC_GROUP].append(item) elif isinstance(item, dict): group = item.get("group", DEFAULT_USER_SPEC_GROUP) # Error if a group is defined more than once if group != DEFAULT_USER_SPEC_GROUP and group in self._groups: raise SpackEnvironmentConfigError( f"group '{group}' defined more than once", self.manifest_file ) # Add an entry for the user specs and store group dependencies if group not in self._user_specs: self._user_specs[group] = [] self._groups[group] = tuple(item.get("needs", ())) self._config_override[group] = item.get("override", None) self._explicit[group] = item.get("explicit", True) if "matrix" in item: # Short form if the group is composed of only one matrix self._user_specs[group].append({"matrix": item["matrix"]}) elif "specs" in item: self._user_specs[group].extend(item["specs"]) def _clear_user_specs(self) -> None: self._user_specs = {DEFAULT_USER_SPEC_GROUP: []} self._groups = {DEFAULT_USER_SPEC_GROUP: tuple()} self._config_override = {DEFAULT_USER_SPEC_GROUP: None} self._explicit = {DEFAULT_USER_SPEC_GROUP: True} def _all_matches(self, user_spec: str) -> List[str]: """Maps the input string to the first equivalent user spec in the manifest, and returns it. Args: user_spec: user spec to be found Raises: ValueError: if no equivalent match is found """ result = [] for yaml_spec_str in self.configuration["specs"]: if Spec(yaml_spec_str) == Spec(user_spec): result.append(yaml_spec_str) if not result: raise ValueError(f"cannot find a spec equivalent to {user_spec}") return result def user_specs(self, *, group: Optional[str] = None) -> List: group = self._ensure_group_exists(group) return self._user_specs[group] def config_override( self, *, group: Optional[str] = None ) -> Optional[spack.config.InternalConfigScope]: group = self._ensure_group_exists(group) data = self._config_override[group] if data is None: return None return spack.config.InternalConfigScope(f"env:groups:{group}", data) def groups(self) -> KeysView: """Returns the list of groups defined in the manifest""" return self._groups.keys() def needs(self, *, group: Optional[str] = None) -> Tuple[str, ...]: """Returns the dependencies of a group of user specs.""" group = self._ensure_group_exists(group) return self._groups[group] def is_explicit(self, *, group: Optional[str] = None) -> bool: """Returns whether specs in a group are marked explicit. When False, specs in the group are installed as implicit dependencies and are eligible for garbage collection once no other spec depends on them. """ group = self._ensure_group_exists(group) return self._explicit[group] def _ensure_group_exists(self, group: Optional[str]) -> str: group = DEFAULT_USER_SPEC_GROUP if group is None else group if group not in self._groups: raise ValueError(f"user specs group '{group}' not found in {self.manifest_file}") return group def add_user_spec(self, user_spec: str, *, group: Optional[str] = None) -> None: """Appends the user spec passed as input to the list of root specs for the given group. Args: user_spec: user spec to be appended group: group where the spec should be added. If None, the default group is used. """ group = group or DEFAULT_USER_SPEC_GROUP if group == DEFAULT_USER_SPEC_GROUP: # Append to top-most specs: attribute specs_yaml = self.configuration.setdefault("specs", []) specs_yaml.append(user_spec) else: # Append to specs: attribute within a group group_in_yaml = self._get_group(group) group_in_yaml.setdefault("specs", []).append(user_spec) self._user_specs[group].append(user_spec) self.changed = True def _get_group(self, group: str) -> Dict: """Find or create the group entry in the manifest""" specs_yaml = self.configuration.setdefault("specs", []) group_entry = None for item in specs_yaml: if isinstance(item, dict) and item.get("group") == group: group_entry = item break if group_entry is None: group_entry = {"group": group, "specs": []} specs_yaml.append(group_entry) self._groups[group] = tuple() self._config_override[group] = None self._user_specs[group] = [] self._explicit[group] = True return group_entry def remove_user_spec(self, user_spec: str) -> None: """Removes the user spec passed as input from the default list of root specs Args: user_spec: user spec to be removed Raises: SpackEnvironmentError: when the user spec is not in the list """ try: for key in self._all_matches(user_spec): self.configuration["specs"].remove(key) self._user_specs[DEFAULT_USER_SPEC_GROUP].remove(key) except ValueError as e: msg = f"cannot remove {user_spec} from {self}, no such spec exists" raise SpackEnvironmentError(msg) from e self.changed = True def clear(self) -> None: """Clear all user specs from the list of root specs""" self.configuration["specs"] = [] self._clear_user_specs() self.changed = True def override_user_spec(self, user_spec: str, idx: int) -> None: """Overrides the user spec at index idx with the one passed as input. Args: user_spec: new user spec idx: index of the spec to be overridden Raises: SpackEnvironmentError: when the user spec cannot be overridden """ try: self.configuration["specs"][idx] = user_spec self._clear_user_specs() self._init_user_specs() except ValueError as e: msg = f"cannot override {user_spec} from {self}" raise SpackEnvironmentError(msg) from e self.changed = True def set_include_concrete(self, include_concrete: List[str]) -> None: """Sets the included concrete environments in the manifest to the value(s) passed as input. Args: include_concrete: list of already existing concrete environments to include """ self.configuration[lockfile_include_key] = list(include_concrete) self.changed = True def add_definition(self, user_spec: str, list_name: str) -> None: """Appends a user spec to the first active definition matching the name passed as argument. Args: user_spec: user spec to be appended list_name: name of the definition where to append Raises: SpackEnvironmentError: is no valid definition exists already """ defs = self.configuration.get("definitions", []) msg = f"cannot add {user_spec} to the '{list_name}' definition, no valid list exists" for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg): item[list_name].append(user_spec) break # "definitions" can be remote, so we need to update the global config too spack.config.CONFIG.set("definitions", defs, scope=self.scope_name) self.changed = True def remove_definition(self, user_spec: str, list_name: str) -> None: """Removes a user spec from an active definition that matches the name passed as argument. Args: user_spec: user spec to be removed list_name: name of the definition where to remove the spec from Raises: SpackEnvironmentError: if the user spec cannot be removed from the list, or the list does not exist """ defs = self.configuration.get("definitions", []) msg = f"cannot remove {user_spec} from the '{list_name}' definition, no valid list exists" for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg): try: item[list_name].remove(user_spec) break except ValueError: pass # "definitions" can be remote, so we need to update the global config too spack.config.CONFIG.set("definitions", defs, scope=self.scope_name) self.changed = True def override_definition(self, user_spec: str, *, override: str, list_name: str) -> None: """Overrides a user spec from an active definition that matches the name passed as argument. Args: user_spec: user spec to be overridden override: new spec to be used list_name: name of the definition where to override the spec Raises: SpackEnvironmentError: if the user spec cannot be overridden """ defs = self.configuration.get("definitions", []) msg = f"cannot override {user_spec} with {override} in the '{list_name}' definition" for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg): try: sub_index = item[list_name].index(user_spec) item[list_name][sub_index] = override break except ValueError: pass # "definitions" can be remote, so we need to update the global config too spack.config.CONFIG.set("definitions", defs, scope=self.scope_name) self.changed = True def _iterate_on_definitions(self, definitions, *, list_name, err_msg): """Iterates on definitions, returning the active ones matching a given name.""" def extract_name(_item): names = list(x for x in _item if x != "when") assert len(names) == 1, f"more than one name in {_item}" return names[0] for idx, item in enumerate(definitions): name = extract_name(item) if name != list_name: continue condition_str = item.get("when", "True") if not spack.spec.eval_conditional(condition_str): continue yield idx, item else: raise SpackEnvironmentError(err_msg) def set_default_view(self, view: Union[bool, str, pathlib.Path, Dict[str, str]]) -> None: """Sets the default view root in the manifest to the value passed as input. Args: view: If the value is a string or a path, it specifies the path to the view. If True the default view is used for the environment, if False there's no view. """ if isinstance(view, dict): self.configuration["view"][default_view_name].update(view) self.changed = True return if not isinstance(view, bool): view = str(view) self.configuration["view"] = view self.changed = True def remove_default_view(self) -> None: """Removes the default view from the manifest file""" view_data = self.configuration.get("view") if isinstance(view_data, collections.abc.Mapping): self.configuration["view"].pop(default_view_name) self.changed = True return self.set_default_view(view=False) def flush(self) -> None: """Synchronizes the object with the manifest file on disk.""" if not self.changed: return with fs.write_tmp_and_move(os.path.realpath(self.manifest_file)) as f: _write_yaml(self.yaml_content, f) self.changed = False @property def configuration(self): """Return the dictionaries in the pristine YAML, without the top level attribute""" return self.yaml_content[TOP_LEVEL_KEY] def __len__(self): return len(self.yaml_content) def __getitem__(self, key): return self.yaml_content[key] def __iter__(self): return iter(self.yaml_content) def __str__(self): return str(self.manifest_file) @property def env_config_scope(self) -> spack.config.ConfigScope: """The configuration scope for the environment manifest""" if self._env_config_scope is None: self._env_config_scope = spack.config.SingleFileScope( self.scope_name, str(self.manifest_file), spack.schema.env.schema, yaml_path=[TOP_LEVEL_KEY], ) ensure_no_disallowed_env_config_mods(self._env_config_scope) return self._env_config_scope def prepare_config_scope(self) -> None: """Add the manifest's scope to the global configuration search path.""" spack.config.CONFIG.push_scope( self.env_config_scope, priority=ConfigScopePriority.ENVIRONMENT ) def deactivate_config_scope(self) -> None: """Remove the manifest's scope from the global config path.""" spack.config.CONFIG.remove_scope(self.env_config_scope.name) @contextlib.contextmanager def use_config(self): """Ensure only the manifest's configuration scopes are global.""" with no_active_environment(): self.prepare_config_scope() yield self.deactivate_config_scope() def environment_path_scope(name: str, path: str) -> Optional[spack.config.ConfigScope]: """Retrieve the suitably named environment path scope Arguments: name: configuration scope name path: path to configuration file(s) Returns: list of environment scopes, if any, or None """ if exists(path): # managed environment manifest = EnvironmentManifestFile(root(path)) elif is_env_dir(path): # anonymous environment manifest = EnvironmentManifestFile(path) else: return None manifest.env_config_scope.name = f"{name}:{manifest.env_config_scope.name}" manifest.env_config_scope.writable = False return manifest.env_config_scope class SpackEnvironmentError(spack.error.SpackError): """Superclass for all errors to do with Spack environments.""" class SpackEnvironmentViewError(SpackEnvironmentError): """Class for errors regarding view generation.""" class SpackEnvironmentConfigError(SpackEnvironmentError): """Class for Spack environment-specific configuration errors.""" def __init__(self, msg, filename): super().__init__(f"{msg} in {filename}") class SpackEnvironmentDevelopError(SpackEnvironmentError): """Class for errors in applying develop information to an environment.""" ================================================ FILE: lib/spack/spack/environment/list.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import itertools from typing import Any, Dict, List, NamedTuple, Optional, Union import spack.spec import spack.util.spack_yaml import spack.variant from spack.error import SpackError from spack.spec import Spec from spack.spec_parser import expand_toolchains class SpecList: def __init__( self, *, name: str = "specs", yaml_list=None, expanded_list=None, toolchains=None ): self.name = name self.yaml_list = yaml_list[:] if yaml_list is not None else [] # Expansions can be expensive to compute and difficult to keep updated # We cache results and invalidate when self.yaml_list changes self.specs_as_yaml_list = expanded_list or [] self._constraints = None self._specs: Optional[List[Spec]] = None self._toolchains = toolchains @property def is_matrix(self): for item in self.specs_as_yaml_list: if isinstance(item, dict): return True return False @property def specs_as_constraints(self): if self._constraints is None: constraints = [] for item in self.specs_as_yaml_list: if isinstance(item, dict): # matrix of specs constraints.extend(_expand_matrix_constraints(item)) else: # individual spec constraints.append([Spec(item)]) self._constraints = constraints return self._constraints @property def specs(self) -> List[Spec]: if self._specs is None: specs: List[Spec] = [] # This could be slightly faster done directly from yaml_list, # but this way is easier to maintain. for constraint_list in self.specs_as_constraints: spec = constraint_list[0].copy() for const in constraint_list[1:]: spec.constrain(const) if self._toolchains: expand_toolchains(spec, self._toolchains) specs.append(spec) self._specs = specs return self._specs def add(self, spec: Spec): spec_str = str(spec) self.yaml_list.append(spec_str) # expanded list can be updated without invalidation if self.specs_as_yaml_list is not None: self.specs_as_yaml_list.append(spec_str) # Invalidate cache variables when we change the list self._constraints = None self._specs = None def remove(self, spec): # Get spec to remove from list remove = [ s for s in self.yaml_list if (isinstance(s, str) and not s.startswith("$")) and Spec(s) == Spec(spec) ] if not remove: msg = f"Cannot remove {spec} from SpecList {self.name}.\n" msg += f"Either {spec} is not in {self.name} or {spec} is " msg += "expanded from a matrix and cannot be removed directly." raise SpecListError(msg) # Remove may contain more than one string representation of the same spec for item in remove: self.yaml_list.remove(item) self.specs_as_yaml_list.remove(item) # invalidate cache variables when we change the list self._constraints = None self._specs = None def extend(self, other: "SpecList", copy_reference=True) -> None: self.yaml_list.extend(other.yaml_list) self.specs_as_yaml_list.extend(other.specs_as_yaml_list) self._constraints = None self._specs = None def __len__(self): return len(self.specs) def __getitem__(self, key): return self.specs[key] def __iter__(self): return iter(self.specs) def _expand_matrix_constraints(matrix_config): # recurse so we can handle nested matrices expanded_rows = [] for row in matrix_config["matrix"]: new_row = [] for r in row: if isinstance(r, dict): # Flatten the nested matrix into a single row of constraints new_row.extend( [ [" ".join([str(c) for c in expanded_constraint_list])] for expanded_constraint_list in _expand_matrix_constraints(r) ] ) else: new_row.append([r]) expanded_rows.append(new_row) excludes = matrix_config.get("exclude", []) # only compute once sigil = matrix_config.get("sigil", "") results = [] for combo in itertools.product(*expanded_rows): # Construct a combined spec to test against excludes flat_combo = [Spec(constraint) for constraints in combo for constraint in constraints] test_spec = flat_combo[0].copy() for constraint in flat_combo[1:]: test_spec.constrain(constraint) # Abstract variants don't have normal satisfaction semantics # Convert all variants to concrete types. # This method is best effort, so all existing variants will be # converted before any error is raised. # Catch exceptions because we want to be able to operate on # abstract specs without needing package information try: spack.spec.substitute_abstract_variants(test_spec) except spack.variant.UnknownVariantError: pass # Resolve abstract hashes for exclusion criteria if any(test_spec.lookup_hash().satisfies(x) for x in excludes): continue if sigil: flat_combo[0] = Spec(sigil + str(flat_combo[0])) # Add to list of constraints results.append(flat_combo) return results def _sigilify(item, sigil): if isinstance(item, dict): if sigil: item["sigil"] = sigil return item else: return sigil + item class Definition(NamedTuple): name: str yaml_list: List[Union[str, Dict]] when: Optional[str] class SpecListParser: """Parse definitions and user specs from data in environments""" def __init__(self, *, toolchains=None): self.definitions: Dict[str, SpecList] = {} self._toolchains = toolchains def parse_definitions(self, *, data: List[Dict[str, Any]]) -> Dict[str, SpecList]: definitions_from_yaml: Dict[str, List[Definition]] = {} for item in data: value = self._parse_yaml_definition(item) definitions_from_yaml.setdefault(value.name, []).append(value) self.definitions = {} self._build_definitions(definitions_from_yaml) return self.definitions def parse_user_specs(self, *, name, yaml_list) -> SpecList: definition = Definition(name=name, yaml_list=yaml_list, when=None) return self._speclist_from_definitions(name, [definition]) def _parse_yaml_definition(self, yaml_entry) -> Definition: when_string = yaml_entry.get("when") if (when_string and len(yaml_entry) > 2) or (not when_string and len(yaml_entry) > 1): mark = spack.util.spack_yaml.get_mark_from_yaml_data(yaml_entry) attributes = ", ".join(x for x in yaml_entry if x != "when") error_msg = f"definition must have a single attribute, got many: {attributes}" raise SpecListError(f"{mark.name}:{mark.line + 1}: {error_msg}") for name, yaml_list in yaml_entry.items(): if name == "when": continue return Definition(name=name, yaml_list=yaml_list, when=when_string) # If we are here, it means only "when" is in the entry mark = spack.util.spack_yaml.get_mark_from_yaml_data(yaml_entry) error_msg = "definition must have a single attribute, got none" raise SpecListError(f"{mark.name}:{mark.line + 1}: {error_msg}") def _build_definitions(self, definitions_from_yaml: Dict[str, List[Definition]]): for name, definitions in definitions_from_yaml.items(): self.definitions[name] = self._speclist_from_definitions(name, definitions) def _speclist_from_definitions(self, name, definitions) -> SpecList: combined_yaml_list = [] for def_part in definitions: if def_part.when is not None and not spack.spec.eval_conditional(def_part.when): continue combined_yaml_list.extend(def_part.yaml_list) expanded_list = self._expand_yaml_list(combined_yaml_list) return SpecList( name=name, yaml_list=combined_yaml_list, expanded_list=expanded_list, toolchains=self._toolchains, ) def _expand_yaml_list(self, raw_yaml_list): result = [] for item in raw_yaml_list: if isinstance(item, str) and item.startswith("$"): result.extend(self._expand_reference(item)) continue value = item if isinstance(item, dict): value = self._expand_yaml_matrix(item) result.append(value) return result def _expand_reference(self, item: str): sigil, name = "", item[1:] if name.startswith("^") or name.startswith("%"): sigil, name = name[0], name[1:] if name not in self.definitions: mark = spack.util.spack_yaml.get_mark_from_yaml_data(item) error_msg = f"trying to expand the name '{name}', which is not defined yet" raise UndefinedReferenceError(f"{mark.name}:{mark.line + 1}: {error_msg}") value = self.definitions[name].specs_as_yaml_list if not sigil: return value return [_sigilify(x, sigil) for x in value] def _expand_yaml_matrix(self, matrix_yaml): extra_attributes = set(matrix_yaml) - {"matrix", "exclude"} if extra_attributes: mark = spack.util.spack_yaml.get_mark_from_yaml_data(matrix_yaml) error_msg = f"extra attributes in spec matrix: {','.join(sorted(extra_attributes))}" raise SpecListError(f"{mark.name}:{mark.line + 1}: {error_msg}") if "matrix" not in matrix_yaml: mark = spack.util.spack_yaml.get_mark_from_yaml_data(matrix_yaml) error_msg = "matrix is missing the 'matrix' attribute" raise SpecListError(f"{mark.name}:{mark.line + 1}: {error_msg}") # Assume data has been validated against the YAML schema result = {"matrix": [self._expand_yaml_list(row) for row in matrix_yaml["matrix"]]} if "exclude" in matrix_yaml: result["exclude"] = matrix_yaml["exclude"] return result class SpecListError(SpackError): """Error class for all errors related to SpecList objects.""" class UndefinedReferenceError(SpecListError): """Error class for undefined references in Spack stacks.""" class InvalidSpecConstraintError(SpecListError): """Error class for invalid spec constraints at concretize time.""" ================================================ FILE: lib/spack/spack/environment/shell.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import textwrap from typing import Optional import spack.config import spack.environment as ev import spack.llnl.util.tty as tty import spack.repo import spack.schema.environment import spack.store from spack.llnl.util.tty.color import colorize from spack.util.environment import EnvironmentModifications def activate_header(env, shell, prompt=None, view: Optional[str] = None): # Construct the commands to run cmds = "" if shell == "csh": # TODO: figure out how to make color work for csh cmds += "setenv SPACK_ENV %s;\n" % env.path if view: cmds += "setenv SPACK_ENV_VIEW %s;\n" % view cmds += 'alias despacktivate "spack env deactivate";\n' if prompt: cmds += "if (! $?SPACK_OLD_PROMPT ) " cmds += 'setenv SPACK_OLD_PROMPT "${prompt}";\n' cmds += 'set prompt="%s ${prompt}";\n' % prompt elif shell == "fish": if "color" in os.getenv("TERM", "") and prompt: prompt = colorize("@G{%s} " % prompt, color=True) cmds += "set -gx SPACK_ENV %s;\n" % env.path if view: cmds += "set -gx SPACK_ENV_VIEW %s;\n" % view cmds += "function despacktivate;\n" cmds += " spack env deactivate;\n" cmds += "end;\n" # # NOTE: We're not changing the fish_prompt function (which is fish's # solution to the PS1 variable) here. This is a bit fiddly, and easy to # screw up => spend time reasearching a solution. Feedback welcome. # elif shell == "bat": # TODO: Color cmds += 'set "SPACK_ENV=%s"\n' % env.path if view: cmds += 'set "SPACK_ENV_VIEW=%s"\n' % view if prompt: old_prompt = os.environ.get("SPACK_OLD_PROMPT") if not old_prompt: old_prompt = os.environ.get("PROMPT") cmds += f'set "SPACK_OLD_PROMPT={old_prompt}"\n' cmds += f'set "PROMPT={prompt} $P$G"\n' elif shell == "pwsh": cmds += "$Env:SPACK_ENV='%s'\n" % env.path if view: cmds += "$Env:SPACK_ENV_VIEW='%s'\n" % view if prompt: cmds += ( "function global:prompt { $pth = $(Convert-Path $(Get-Location))" ' | Split-Path -leaf; if(!"$Env:SPACK_OLD_PROMPT") ' '{$Env:SPACK_OLD_PROMPT="[spack] PS $pth>"}; ' '"%s PS $pth>"}\n' % prompt ) else: bash_color_prompt = colorize(f"@G{{{prompt}}}", color=True, enclose=True) zsh_color_prompt = colorize(f"@G{{{prompt}}}", color=True, enclose=False, zsh=True) cmds += "export SPACK_ENV=%s;\n" % env.path if view: cmds += "export SPACK_ENV_VIEW=%s;\n" % view cmds += "alias despacktivate='spack env deactivate';\n" if prompt: cmds += textwrap.dedent( rf""" if [ -z ${{SPACK_OLD_PS1+x}} ]; then if [ -z ${{PS1+x}} ]; then PS1='$$$$'; fi; export SPACK_OLD_PS1="${{PS1}}"; fi; if [ -n "${{TERM:-}}" ] && [ "${{TERM#*color}}" != "${{TERM}}" ] && \ [ -n "${{BASH:-}}" ]; then export PS1="{bash_color_prompt} ${{PS1}}"; elif [ -n "${{TERM:-}}" ] && [ "${{TERM#*color}}" != "${{TERM}}" ] && \ [ -n "${{ZSH_NAME:-}}" ]; then export PS1="{zsh_color_prompt} ${{PS1}}"; else export PS1="{prompt} ${{PS1}}"; fi """ ).lstrip("\n") return cmds def deactivate_header(shell): cmds = "" if shell == "csh": cmds += "unsetenv SPACK_ENV;\n" cmds += "unsetenv SPACK_ENV_VIEW;\n" cmds += "if ( $?SPACK_OLD_PROMPT ) " cmds += ' eval \'set prompt="$SPACK_OLD_PROMPT" &&' cmds += " unsetenv SPACK_OLD_PROMPT';\n" cmds += "unalias despacktivate;\n" elif shell == "fish": cmds += "set -e SPACK_ENV;\n" cmds += "set -e SPACK_ENV_VIEW;\n" cmds += "functions -e despacktivate;\n" # # NOTE: Not changing fish_prompt (above) => no need to restore it here. # elif shell == "bat": # TODO: Color cmds += 'set "SPACK_ENV="\n' cmds += 'set "SPACK_ENV_VIEW="\n' # TODO: despacktivate old_prompt = os.environ.get("SPACK_OLD_PROMPT") if old_prompt: cmds += f'set "PROMPT={old_prompt}"\n' cmds += 'set "SPACK_OLD_PROMPT="\n' elif shell == "pwsh": cmds += "Set-Item -Path Env:SPACK_ENV\n" cmds += "Set-Item -Path Env:SPACK_ENV_VIEW\n" cmds += ( "function global:prompt { $pth = $(Convert-Path $(Get-Location))" ' | Split-Path -leaf; $spack_prompt = "[spack] $pth >"; ' 'if("$Env:SPACK_OLD_PROMPT") {$spack_prompt=$Env:SPACK_OLD_PROMPT};' " $spack_prompt}\n" ) else: cmds += "if [ ! -z ${SPACK_ENV+x} ]; then\n" cmds += "unset SPACK_ENV; export SPACK_ENV;\n" cmds += "fi;\n" cmds += "if [ ! -z ${SPACK_ENV_VIEW+x} ]; then\n" cmds += "unset SPACK_ENV_VIEW; export SPACK_ENV_VIEW;\n" cmds += "fi;\n" cmds += "alias despacktivate > /dev/null 2>&1 && unalias despacktivate;\n" cmds += "if [ ! -z ${SPACK_OLD_PS1+x} ]; then\n" cmds += " if [ \"$SPACK_OLD_PS1\" = '$$$$' ]; then\n" cmds += " unset PS1; export PS1;\n" cmds += " else\n" cmds += ' export PS1="$SPACK_OLD_PS1";\n' cmds += " fi;\n" cmds += " unset SPACK_OLD_PS1; export SPACK_OLD_PS1;\n" cmds += "fi;\n" return cmds def activate( env: ev.Environment, use_env_repo=False, view: Optional[str] = "default" ) -> EnvironmentModifications: """Activate an environment and append environment modifications To activate an environment, we add its configuration scope to the existing Spack configuration, and we set active to the current environment. Arguments: env: the environment to activate use_env_repo: use the packages exactly as they appear in the environment's repository view: generate commands to add runtime environment variables for named view Returns: spack.util.environment.EnvironmentModifications: Environment variables modifications to activate environment.""" ev.activate(env, use_env_repo=use_env_repo) env_mods = EnvironmentModifications() # # NOTE in the fish-shell: Path variables are a special kind of variable # used to support colon-delimited path lists including PATH, CDPATH, # MANPATH, PYTHONPATH, etc. All variables that end in PATH (case-sensitive) # become PATH variables. # env_vars_yaml = spack.config.get("env_vars", None) if env_vars_yaml: env_mods.extend(spack.schema.environment.parse(env_vars_yaml)) try: if view and env.has_view(view): with spack.store.STORE.db.read_transaction(): env.add_view_to_env(env_mods, view) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.error(e) tty.die( "Environment view is broken due to a missing package or repo.\n", " To activate without views enabled, activate with:\n", " spack env activate -V {0}\n".format(env.name), " To remove it and resolve the issue, force concretize with the command:\n", " spack -e {0} concretize --force".format(env.name), ) return env_mods def deactivate() -> EnvironmentModifications: """Deactivate an environment and collect corresponding environment modifications. Note: unloads the environment in its current state, not in the state it was loaded in, meaning that specs that were removed from the spack environment after activation are not unloaded. Returns: Environment variables modifications to activate environment. """ env_mods = EnvironmentModifications() active = ev.active_environment() if active is None: return env_mods with active.manifest.use_config(): env_vars_yaml = spack.config.get("env_vars", None) if env_vars_yaml: env_mods.extend(spack.schema.environment.parse(env_vars_yaml).reversed()) active_view = os.getenv(ev.spack_env_view_var) if active_view and active.has_view(active_view): try: with spack.store.STORE.db.read_transaction(): active.rm_view_from_env(env_mods, active_view) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.warn(e) tty.warn( "Could not fully deactivate view due to missing package " "or repo, shell environment may be corrupt." ) ev.deactivate() return env_mods ================================================ FILE: lib/spack/spack/error.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import sys from typing import Optional import spack.llnl.util.tty as tty #: at what level we should write stack traces or short error messages #: this is module-scoped because it needs to be set very early debug = 0 #: whether to show a backtrace when an error is printed, enabled with ``--backtrace``. SHOW_BACKTRACE = False class SpackAPIWarning(UserWarning): """Warning that formats with file and line number.""" class SpackError(Exception): """This is the superclass for all Spack errors. Subclasses can be found in the modules they have to do with. """ def __init__(self, message: str, long_message: Optional[str] = None) -> None: super().__init__() self.message = message self._long_message = long_message # for exceptions raised from child build processes, we save the # traceback as a string and print it in the parent. self.traceback = None # we allow exceptions to print debug info via print_context() # before they are caught at the top level. If they *haven't* # printed context early, we do it by default when die() is # called, so we need to remember whether it's been called. self.printed = False @property def long_message(self): return self._long_message def print_context(self): """Print extended debug information about this exception. This is usually printed when the top-level Spack error handler calls ``die()``, but it can be called separately beforehand if a lower-level error handler needs to print error context and continue without raising the exception to the top level. """ if self.printed: return # basic debug message tty.error(self.message) if self.long_message: sys.stderr.write(self.long_message) sys.stderr.write("\n") # stack trace, etc. in debug mode. if debug: if self.traceback: # exception came from a build child, already got # traceback in child, so print it. sys.stderr.write(self.traceback) else: # run parent exception hook. sys.excepthook(*sys.exc_info()) sys.stderr.flush() self.printed = True def die(self): self.print_context() sys.exit(1) def __str__(self): if self._long_message: return f"{self.message}\n {self._long_message}" return self.message def __repr__(self): qualified_name = type(self).__module__ + "." + type(self).__name__ return f"{qualified_name}({repr(self.message)}, {repr(self.long_message)})" def __reduce__(self): return type(self), (self.message, self.long_message) class UnsupportedPlatformError(SpackError): """Raised by packages when a platform is not supported""" def __init__(self, message): super().__init__(message) class NoLibrariesError(SpackError): """Raised when package libraries are requested but cannot be found""" def __init__(self, message_or_name, prefix=None): super().__init__( message_or_name if prefix is None else "Unable to locate {0} libraries in {1}".format(message_or_name, prefix) ) class NoHeadersError(SpackError): """Raised when package headers are requested but cannot be found""" class SpecError(SpackError): """Superclass for all errors that occur while constructing specs.""" class UnsatisfiableSpecError(SpecError): """ Raised when a spec conflicts with package constraints. For original concretizer, provide the requirement that was violated when raising. """ def __init__(self, provided, required, constraint_type): # This is only the entrypoint for old concretizer errors super().__init__("%s does not satisfy %s" % (provided, required)) self.provided = provided self.required = required self.constraint_type = constraint_type class FetchError(SpackError): """Superclass for fetch-related errors.""" class NoSuchPatchError(SpackError): """Raised when a patch file doesn't exist.""" class PatchDirectiveError(SpackError): """Raised when the wrong arguments are suppled to the patch directive.""" class PatchLookupError(NoSuchPatchError): """Raised when a patch file cannot be located from sha256.""" class SpecSyntaxError(Exception): """Base class for Spec syntax errors""" class PackageError(SpackError): """Raised when something is wrong with a package definition.""" def __init__(self, message, long_msg=None): super().__init__(message, long_msg) class NoURLError(PackageError): """Raised when someone tries to build a URL for a package with no URLs.""" def __init__(self, cls): super().__init__("Package %s has no version with a URL." % cls.__name__) class InstallError(SpackError): """Raised when something goes wrong during install or uninstall. The error can be annotated with a ``pkg`` attribute to allow the caller to get the package for which the exception was raised. """ def __init__(self, message, long_msg=None, pkg=None): super().__init__(message, long_msg) self.pkg = pkg class ConfigError(SpackError): """Superclass for all Spack config related errors.""" class StopPhase(SpackError): """Pickle-able exception to control stopped builds.""" def __reduce__(self): return _make_stop_phase, (self.message, self.long_message) def _make_stop_phase(msg, long_msg): return StopPhase(msg, long_msg) class MirrorError(SpackError): """Superclass of all mirror-creation related errors.""" def __init__(self, msg, long_msg=None): super().__init__(msg, long_msg) class NoChecksumException(SpackError): """ Raised if file fails checksum verification. """ def __init__(self, path, size, contents, algorithm, expected, computed): super().__init__( f"{algorithm} checksum failed for {path}", f"Expected {expected} but got {computed}. " f"File size = {size} bytes. Contents = {contents!r}", ) class CompilerError(SpackError): """Raised if something goes wrong when probing or querying a compiler.""" class SpecFilenameError(SpecError): """Raised when a spec file name is invalid.""" class NoSuchSpecFileError(SpecFilenameError): """Raised when a spec file doesn't exist.""" ================================================ FILE: lib/spack/spack/extensions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Service functions and classes to implement the hooks for Spack's command extensions. """ import glob import importlib import os import re import sys import types from pathlib import Path from typing import List import spack.config import spack.error import spack.llnl.util.lang import spack.util.path _extension_regexp = re.compile(r"spack-(\w[-\w]*)$") def _python_name(cmd_name): return cmd_name.replace("-", "_") def extension_name(path): """Returns the name of the extension in the path passed as argument. Args: path (str): path where the extension resides Returns: The extension name. Raises: ExtensionNamingError: if path does not match the expected format for a Spack command extension. """ regexp_match = re.search(_extension_regexp, os.path.basename(os.path.normpath(path))) if not regexp_match: raise ExtensionNamingError(path) return regexp_match.group(1) def load_command_extension(command, path): """Loads a command extension from the path passed as argument. Args: command (str): name of the command (contains ``-``, not ``_``). path (str): base path of the command extension Returns: A valid module if found and loadable; None if not found. Module loading exceptions are passed through. """ extension = _python_name(extension_name(path)) # Compute the name of the module we search, exit early if already imported cmd_package = "{0}.{1}.cmd".format(__name__, extension) python_name = _python_name(command) module_name = "{0}.{1}".format(cmd_package, python_name) if module_name in sys.modules: return sys.modules[module_name] # Compute the absolute path of the file to be loaded, along with the # name of the python module where it will be stored cmd_path = os.path.join(path, extension, "cmd", python_name + ".py") # Short circuit if the command source file does not exist if not os.path.exists(cmd_path): return None ensure_extension_loaded(extension, path=path) module = importlib.import_module(module_name) sys.modules[module_name] = module return module def ensure_extension_loaded(extension, *, path): def ensure_package_creation(name): package_name = "{0}.{1}".format(__name__, name) if package_name in sys.modules: return parts = [path] + name.split(".") + ["__init__.py"] init_file = os.path.join(*parts) if os.path.exists(init_file): m = spack.llnl.util.lang.load_module_from_file(package_name, init_file) else: m = types.ModuleType(package_name) # Setting __path__ to give spack extensions the # ability to import from their own tree, see: # # https://docs.python.org/3/reference/import.html#package-path-rules # m.__path__ = [os.path.dirname(init_file)] sys.modules[package_name] = m # Create a searchable package for both the root folder of the extension # and the subfolder containing the commands ensure_package_creation(extension) ensure_package_creation(extension + ".cmd") def load_extension(name: str) -> str: """Loads a single extension into the ``spack.extensions`` package. Args: name: name of the extension """ extension_root = path_for_extension(name, paths=get_extension_paths()) ensure_extension_loaded(name, path=extension_root) commands = glob.glob( os.path.join(extension_root, extension_name(extension_root), "cmd", "*.py") ) commands = [os.path.basename(x).rstrip(".py") for x in commands] for command in commands: load_command_extension(command, extension_root) return extension_root def get_extension_paths(): """Return the list of canonicalized extension paths from config:extensions.""" extension_paths = spack.config.get("config:extensions") or [] extension_paths.extend(extension_paths_from_entry_points()) paths = [spack.util.path.canonicalize_path(p) for p in extension_paths] return paths @spack.llnl.util.lang.memoized def extension_paths_from_entry_points() -> List[str]: """Load extensions from a Python package's entry points. A python package can register entry point metadata so that Spack can find its extensions by adding the following to the project's pyproject.toml: .. code-block:: toml [project.entry-points."spack.extensions"] baz = "baz:get_spack_extensions" The function ``get_spack_extensions`` returns paths to the package's spack extensions This function assumes that the state of entry points doesn't change from the first time it's called. E.g., it doesn't support any new installation of packages between two calls. """ extension_paths: List[str] = [] for entry_point in spack.llnl.util.lang.get_entry_points(group="spack.extensions"): hook = entry_point.load() if callable(hook): paths = hook() or [] if isinstance(paths, (Path, str)): extension_paths.append(str(paths)) else: extension_paths.extend(paths) return extension_paths def get_command_paths(): """Return the list of paths where to search for command files.""" command_paths = [] extension_paths = get_extension_paths() for path in extension_paths: extension = _python_name(extension_name(path)) command_paths.append(os.path.join(path, extension, "cmd")) return command_paths def path_for_extension(target_name: str, *, paths: List[str]) -> str: """Return the test root dir for a given extension. Args: target_name (str): name of the extension to test *paths: paths where the extensions reside Returns: Root directory where tests should reside or None """ for path in paths: name = extension_name(path) if name == target_name: return path else: raise OSError('extension "{0}" not found'.format(target_name)) def get_module(cmd_name): """Imports the extension module for a particular command name and returns it. Args: cmd_name (str): name of the command for which to get a module (contains ``-``, not ``_``). """ # If built-in failed the import search the extension # directories in order extensions = get_extension_paths() for folder in extensions: module = load_command_extension(cmd_name, folder) if module: return module return None def get_template_dirs(): """Returns the list of directories where to search for templates in extensions. """ extension_dirs = get_extension_paths() extensions = [os.path.join(x, "templates") for x in extension_dirs] return extensions class ExtensionNamingError(spack.error.SpackError): """Exception class thrown when a configured extension does not follow the expected naming convention. """ def __init__(self, path): super().__init__("{0} does not match the format for a Spack extension path.".format(path)) ================================================ FILE: lib/spack/spack/externals.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This module turns the configuration data in the ``packages`` section into a list of concrete specs. This is mainly done by the ``ExternalSpecsParser`` class, which is responsible for: 1. Transforming an intermediate representation of the YAML configuration into a set of nodes 2. Ensuring the dependency specifications are not ambiguous 3. Inferring missing information about the external specs (e.g. architecture, deptypes) 4. Wiring up the external specs to their dependencies The helper function ``extract_dicts_from_configuration`` is used to transform the configuration into the intermediate representation. """ import re import uuid import warnings from typing import Any, Callable, Dict, List, NamedTuple, Tuple, Union from spack.vendor.typing_extensions import TypedDict import spack.archspec import spack.deptypes import spack.repo import spack.spec from spack.error import SpackError from spack.llnl.util import tty class DependencyDict(TypedDict, total=False): id: str spec: str deptypes: spack.deptypes.DepTypes virtuals: str class ExternalDict(TypedDict, total=False): """Dictionary representation of an external spec. This representation mostly follows the one used in the configuration files, with a few exceptions needed to support specific features. """ spec: str prefix: str modules: List[str] extra_attributes: Dict[str, Any] id: str dependencies: List[DependencyDict] # Target requirement from configuration. Not in the external schema required_target: str def node_from_dict(external_dict: ExternalDict) -> spack.spec.Spec: """Returns an external spec node from a dictionary representation.""" extra_attributes = external_dict.get("extra_attributes", {}) result = spack.spec.Spec( # Allow `@x.y.z` instead of `@=x.y.z` str(spack.spec.parse_with_version_concrete(external_dict["spec"])), external_path=external_dict.get("prefix"), external_modules=external_dict.get("modules"), ) if not result.versions.concrete: raise ExternalSpecError( f"The external spec '{external_dict['spec']}' doesn't have a concrete version" ) result.extra_attributes = extra_attributes if "required_target" in external_dict: result.constrain(f"target={external_dict['required_target']}") return result def complete_architecture(node: spack.spec.Spec) -> None: """Completes a node with architecture information. Undefined targets are set to the default host target family (e.g. ``x86_64``). The operating system and platform are set based on the current host. """ if node.architecture: if not node.architecture.target: node.architecture.target = spack.archspec.HOST_TARGET_FAMILY node.architecture.complete_with_defaults() else: node.constrain(spack.spec.Spec.default_arch()) node.architecture.target = spack.archspec.HOST_TARGET_FAMILY node.namespace = spack.repo.PATH.repo_for_pkg(node.name).namespace for flag_type in spack.spec.FlagMap.valid_compiler_flags(): node.compiler_flags.setdefault(flag_type, []) def complete_variants_and_architecture(node: spack.spec.Spec) -> None: """Completes a node with variants and architecture information. Architecture is completed first, delegating to ``complete_architecture``. Variants are then added to the node, using their default value. """ complete_architecture(node) pkg_class = spack.repo.PATH.get_pkg_class(node.name) variants_dict = pkg_class.variants.copy() changed = True while variants_dict and changed: changed = False items = list(variants_dict.items()) # copy b/c loop modifies dict for when, variants_by_name in items: if not node.satisfies(when): continue variants_dict.pop(when) for name, vdef in variants_by_name.items(): if name not in node.variants: # Cannot use Spec.constrain, because we lose information on the variant type node.variants[name] = vdef.make_default() elif ( node.variants[name].type != vdef.variant_type and len(node.variants[name].values) == 1 ): # Spec parsing defaults to MULTI for non-boolean variants. Correct the type # using the package definition, preserving the user-specified value. existing = node.variants[name] corrected = vdef.make_variant(*existing.values) node.variants.substitute(corrected) changed = True def extract_dicts_from_configuration(packages_yaml) -> List[ExternalDict]: """Transforms the packages.yaml configuration into a list of external dictionaries. The default required target is extracted from ``packages:all:require``, if present. Any package-specific required target overrides the default. """ result = [] default_required_target = "" if "all" in packages_yaml: default_required_target = _required_target(packages_yaml["all"]) for name, entry in packages_yaml.items(): pkg_required_target = _required_target(entry) or default_required_target partial_result = [current for current in entry.get("externals", [])] if pkg_required_target: for partial in partial_result: partial["required_target"] = pkg_required_target result.extend(partial_result) return result def _line_info(config_dict: Any) -> str: result = getattr(config_dict, "line_info", "") return "" if not result else f" [{result}]" _TARGET_RE = re.compile(r"target=([^\s:]+)") def _required_target(entry) -> str: """Parses the YAML configuration for a single external spec and returns the required target if defined. Returns an empty string otherwise. """ if "require" not in entry: return "" requirements = entry["require"] if not isinstance(requirements, list): requirements = [requirements] results = [] for requirement in requirements: if not isinstance(requirement, str): continue matches = _TARGET_RE.match(requirement) if matches: results.append(matches.group(1)) if len(results) == 1: return results[0] return "" class ExternalSpecAndConfig(NamedTuple): spec: spack.spec.Spec config: ExternalDict class ExternalSpecsParser: """Transforms a list of external dicts into a list of specs.""" def __init__( self, external_dicts: List[ExternalDict], *, complete_node: Callable[[spack.spec.Spec], None] = complete_variants_and_architecture, allow_nonexisting: bool = True, ): """Initializes a class to manage and process external specifications in ``packages.yaml``. Args: external_dicts: list of ExternalDict objects to provide external specifications. complete_node: a callable that completes a node with missing variants, targets, etc. Defaults to `complete_architecture`. allow_nonexisting: whether to allow non-existing packages. Defaults to True. Raises: spack.repo.UnknownPackageError: if a package does not exist, and allow_nonexisting is False. """ self.external_dicts = external_dicts self.specs_by_external_id: Dict[str, ExternalSpecAndConfig] = {} self.specs_by_name: Dict[str, List[ExternalSpecAndConfig]] = {} self.nodes: List[spack.spec.Spec] = [] self.allow_nonexisting = allow_nonexisting # Fill the data structures above (can be done lazily) self.complete_node = complete_node self._parse() def _parse(self) -> None: # Parse all nodes without creating edges among them self._parse_all_nodes() # Map dependencies specified as specs to a single id self._ensure_dependencies_have_single_id() # Attach dependencies to externals self._create_edges() # Mark the specs as concrete for node in self.nodes: node._finalize_concretization() def _create_edges(self): for eid, entry in self.specs_by_external_id.items(): current_node, current_dict = entry.spec, entry.config line_info = _line_info(current_dict) spec_str = current_dict["spec"] # Compute the dependency types for this spec pkg_class, deptypes_by_package = spack.repo.PATH.get_pkg_class(current_node.name), {} for when, by_name in pkg_class.dependencies.items(): if not current_node.satisfies(when): continue for name, dep in by_name.items(): if name not in deptypes_by_package: deptypes_by_package[name] = dep.depflag deptypes_by_package[name] |= dep.depflag for dependency_dict in current_dict.get("dependencies", []): dependency_id = dependency_dict.get("id") if not dependency_id: raise ExternalDependencyError( f"A dependency for {spec_str} does not have an external id{line_info}" ) elif dependency_id not in self.specs_by_external_id: raise ExternalDependencyError( f"A dependency for {spec_str} has an external id " f"{dependency_id} that cannot be found in packages.yaml{line_info}" ) dependency_node = self.specs_by_external_id[dependency_id].spec # Compute dependency types and virtuals depflag = spack.deptypes.NONE if "deptypes" in dependency_dict: depflag = spack.deptypes.canonicalize(dependency_dict["deptypes"]) virtuals: Tuple[str, ...] = () if "virtuals" in dependency_dict: virtuals = tuple(dependency_dict["virtuals"].split(",")) # Infer dependency types and virtuals if the user didn't specify them if depflag == spack.deptypes.NONE and not virtuals: # Infer the deptype if only '%' was used in the spec inferred_virtuals = [] for name, current_flag in deptypes_by_package.items(): if not dependency_node.intersects(name): continue depflag |= current_flag if spack.repo.PATH.is_virtual(name): inferred_virtuals.append(name) virtuals = tuple(inferred_virtuals) elif depflag == spack.deptypes.NONE: depflag = spack.deptypes.DEFAULT current_node._add_dependency(dependency_node, depflag=depflag, virtuals=virtuals) def _ensure_dependencies_have_single_id(self): for eid, entry in self.specs_by_external_id.items(): current_node, current_dict = entry.spec, entry.config spec_str = current_dict["spec"] line_info = _line_info(current_dict) if current_node.dependencies() and "dependencies" in current_dict: raise ExternalSpecError( f"the spec {spec_str} cannot specify dependencies both in the root spec and" f"in the 'dependencies' field{line_info}" ) # Transform inline entries like 'mpich %gcc' to a canonical form using 'dependencies' for edge in current_node.edges_to_dependencies(): entry: DependencyDict = {"spec": str(edge.spec)} # Handle entries with more options specified if edge.depflag != 0: entry["deptypes"] = spack.deptypes.flag_to_tuple(edge.depflag) if edge.virtuals: entry["virtuals"] = ",".join(edge.virtuals) current_dict.setdefault("dependencies", []).append(entry) current_node.clear_edges() # Map a spec: to id: for dependency_dict in current_dict.get("dependencies", []): if "id" in dependency_dict: continue if "spec" not in dependency_dict: raise ExternalDependencyError( f"the spec {spec_str} needs to specify either the id or the spec " f"of its dependencies{line_info}" ) query_spec = spack.spec.Spec(dependency_dict["spec"]) candidates = [ x for x in self.specs_by_name.get(query_spec.name, []) if x.spec.satisfies(query_spec) ] if len(candidates) == 0: raise ExternalDependencyError( f"the spec '{spec_str}' depends on '{query_spec}', but there is no such " f"external spec in packages.yaml{line_info}" ) elif len(candidates) > 1: candidates_str = ( f" [candidates are {', '.join([str(x.spec) for x in candidates])}]" ) raise ExternalDependencyError( f"the spec '{spec_str}' depends on '{query_spec}', but there are multiple " f"external specs that could satisfy the request{candidates_str}{line_info}" ) dependency_dict["id"] = candidates[0].config["id"] def _parse_all_nodes(self) -> None: """Parses all the nodes from the external dicts but doesn't add any edge.""" for external_dict in self.external_dicts: line_info = _line_info(external_dict) try: node = node_from_dict(external_dict) except spack.spec.UnsatisfiableArchitectureSpecError: spec_str, target_str = external_dict["spec"], external_dict["required_target"] tty.debug( f"[{__name__}]{line_info} Skipping external spec '{spec_str}' because it " f"cannot be constrained with the required target '{target_str}'." ) continue except ExternalSpecError as e: warnings.warn(f"{e}{line_info}") continue package_exists = spack.repo.PATH.exists(node.name) # If we allow non-existing packages, just continue if not package_exists and self.allow_nonexisting: continue if not package_exists and not self.allow_nonexisting: raise ExternalSpecError(f"Package '{node.name}' does not exist{line_info}") eid = external_dict.setdefault("id", str(uuid.uuid4())) if eid in self.specs_by_external_id: other_node = self.specs_by_external_id[eid] other_line_info = _line_info(other_node.config) raise DuplicateExternalError( f"Specs {node} and {other_node.spec} cannot have the same external id {eid}" f"{line_info}{other_line_info}" ) self.complete_node(node) # Add a Python dependency to Python extensions that don't specify it pkg_class = spack.repo.PATH.get_pkg_class(node.name) if ( "dependencies" not in external_dict and not node.dependencies() and any([c.__name__ == "PythonExtension" for c in pkg_class.__mro__]) ): warnings.warn( f"Spack is trying attach a Python dependency to '{node}'. This feature is " f"deprecated, and will be removed in v1.2. Please make the dependency " f"explicit in your configuration." ) external_dict.setdefault("dependencies", []).append({"spec": "python"}) # Normalize internally so that each node has a unique id spec_and_config = ExternalSpecAndConfig(spec=node, config=external_dict) self.specs_by_external_id[eid] = spec_and_config self.specs_by_name.setdefault(node.name, []).append(spec_and_config) self.nodes.append(node) def get_specs_for_package(self, package_name: str) -> List[spack.spec.Spec]: """Returns the external specs for a given package name.""" result = self.specs_by_name.get(package_name, []) return [x.spec for x in result] def all_specs(self) -> List[spack.spec.Spec]: """Returns all the external specs.""" return self.nodes def query(self, query: Union[str, spack.spec.Spec]) -> List[spack.spec.Spec]: """Returns the external specs matching a query spec.""" result = [] for node in self.nodes: if node.satisfies(query): result.append(node) return result def external_spec(config: ExternalDict) -> spack.spec.Spec: """Returns an external spec from a dictionary representation.""" return ExternalSpecsParser([config]).all_specs()[0] class DuplicateExternalError(SpackError): """Raised when a duplicate external is detected.""" class ExternalDependencyError(SpackError): """Raised when a dependency on an external package is specified wrongly.""" class ExternalSpecError(SpackError): """Raised when a dependency on an external package is specified wrongly.""" ================================================ FILE: lib/spack/spack/fetch_strategy.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ Fetch strategies are used to download source code into a staging area in order to build it. They need to define the following methods: ``fetch()`` This should attempt to download/check out source from somewhere. ``check()`` Apply a checksum to the downloaded source code, e.g. for an archive. May not do anything if the fetch method was safe to begin with. ``expand()`` Expand (e.g., an archive) downloaded file to source, with the standard stage source path as the destination directory. ``reset()`` Restore original state of downloaded code. Used by clean commands. This may just remove the expanded source and re-expand an archive, or it may run something like git reset ``--hard``. ``archive()`` Archive a source directory, e.g. for creating a mirror. """ import copy import functools import hashlib import http.client import os import re import shutil import sys import time import urllib.parse import urllib.request from pathlib import PurePath from typing import Callable, List, Mapping, Optional, Type import spack.config import spack.error import spack.llnl.url import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.oci.opener import spack.util.archive import spack.util.crypto as crypto import spack.util.executable import spack.util.git import spack.util.url as url_util import spack.util.web as web_util import spack.version from spack.llnl.string import comma_and, quote from spack.llnl.util.filesystem import get_single_file, mkdirp, symlink, temp_cwd, working_dir from spack.util.compression import decompressor_for from spack.util.executable import CommandNotFoundError, Executable, which #: List of all fetch strategies, created by FetchStrategy metaclass. all_strategies: List[Type["FetchStrategy"]] = [] def _needs_stage(fun): """Many methods on fetch strategies require a stage to be set using set_stage(). This decorator adds a check for self.stage.""" @functools.wraps(fun) def wrapper(self, *args, **kwargs): if not self.stage: raise NoStageError(fun) return fun(self, *args, **kwargs) return wrapper def _ensure_one_stage_entry(stage_path): """Ensure there is only one stage entry in the stage path.""" stage_entries = os.listdir(stage_path) assert len(stage_entries) == 1 return os.path.join(stage_path, stage_entries[0]) def fetcher(cls): """Decorator used to register fetch strategies.""" all_strategies.append(cls) return cls class FetchStrategy: """Superclass of all fetch strategies.""" #: The URL attribute must be specified either at the package class #: level, or as a keyword argument to ``version()``. It is used to #: distinguish fetchers for different versions in the package DSL. url_attr: Optional[str] = None #: Optional attributes can be used to distinguish fetchers when : #: classes have multiple ``url_attrs`` at the top-level. # optional attributes in version() args. optional_attrs: List[str] = [] def __init__(self, **kwargs): # The stage is initialized late, so that fetch strategies can be # constructed at package construction time. This is where things # will be fetched. self.stage = None # Enable or disable caching for this strategy based on # 'no_cache' option from version directive. self.cache_enabled = not kwargs.pop("no_cache", False) self.package = None def set_package(self, package): self.package = package # Subclasses need to implement these methods def fetch(self): """Fetch source code archive or repo. Returns: bool: True on success, False on failure. """ def check(self): """Checksum the archive fetched by this FetchStrategy.""" def expand(self): """Expand the downloaded archive into the stage source path.""" def reset(self): """Revert to freshly downloaded state. For archive files, this may just re-expand the archive. """ def archive(self, destination): """Create an archive of the downloaded data for a mirror. For downloaded files, this should preserve the checksum of the original file. For repositories, it should just create an expandable tarball out of the downloaded repository. """ @property def cachable(self): """Whether fetcher is capable of caching the resource it retrieves. This generally is determined by whether the resource is identifiably associated with a specific package version. Returns: bool: True if can cache, False otherwise. """ def source_id(self): """A unique ID for the source. It is intended that a human could easily generate this themselves using the information available to them in the Spack package. The returned value is added to the content which determines the full hash for a package using :class:`str`. """ raise NotImplementedError def mirror_id(self): """This is a unique ID for a source that is intended to help identify reuse of resources across packages. It is unique like source-id, but it does not include the package name and is not necessarily easy for a human to create themselves. """ raise NotImplementedError def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" @classmethod def matches(cls, args): """Predicate that matches fetch strategies to arguments of the version directive. Args: args: arguments of the version directive """ return cls.url_attr in args @fetcher class BundleFetchStrategy(FetchStrategy): """ Fetch strategy associated with bundle, or no-code, packages. Having a basic fetch strategy is a requirement for executing post-install hooks. Consequently, this class provides the API but does little more than log messages. TODO: Remove this class by refactoring resource handling and the link between composite stages and composite fetch strategies (see #11981). """ #: There is no associated URL keyword in ``version()`` for no-code #: packages but this property is required for some strategy-related #: functions (e.g., check_pkg_attributes). url_attr = "" def fetch(self): """Simply report success -- there is no code to fetch.""" return True @property def cachable(self): """Report False as there is no code to cache.""" return False def source_id(self): """BundlePackages don't have a source id.""" return "" def mirror_id(self): """BundlePackages don't have a mirror id.""" def _format_speed(total_bytes: int, elapsed: float) -> str: """Return a human-readable average download speed string.""" elapsed = 1 if elapsed <= 0 else elapsed # avoid divide by zero speed = total_bytes / elapsed if speed >= 1e9: return f"{speed / 1e9:6.1f} GB/s" elif speed >= 1e6: return f"{speed / 1e6:6.1f} MB/s" elif speed >= 1e3: return f"{speed / 1e3:6.1f} KB/s" return f"{speed:6.1f} B/s" def _format_bytes(total_bytes: int) -> str: """Return a human-readable total bytes string.""" if total_bytes >= 1e9: return f"{total_bytes / 1e9:7.2f} GB" elif total_bytes >= 1e6: return f"{total_bytes / 1e6:7.2f} MB" elif total_bytes >= 1e3: return f"{total_bytes / 1e3:7.2f} KB" return f"{total_bytes:7.2f} B" class FetchProgress: #: Characters to rotate in the spinner. spinner = ["|", "/", "-", "\\"] def __init__( self, total_bytes: Optional[int] = None, enabled: bool = True, get_time: Callable[[], float] = time.time, ) -> None: """Initialize a FetchProgress instance. Args: total_bytes: Total number of bytes to download, if known. enabled: Whether to print progress information. get_time: Function to get the current time.""" #: Number of bytes downloaded so far. self.current_bytes = 0 #: Delta time between progress prints self.delta = 0.1 #: Whether to print progress information. self.enabled = enabled #: Function to get the current time. self.get_time = get_time #: Time of last progress print to limit output self.last_printed = 0.0 #: Time of start of download self.start_time = get_time() if enabled else 0.0 #: Total number of bytes to download, if known. self.total_bytes = total_bytes if total_bytes and total_bytes > 0 else 0 #: Index of spinner character to print (used if total bytes is unknown) self.index = 0 @classmethod def from_headers( cls, headers: Mapping[str, str], enabled: bool = True, get_time: Callable[[], float] = time.time, ) -> "FetchProgress": """Create a FetchProgress instance from HTTP headers.""" # headers.get is case-insensitive if it's from a HTTPResponse object. content_length = headers.get("Content-Length") try: total_bytes = int(content_length) if content_length else None except ValueError: total_bytes = None return cls(total_bytes=total_bytes, enabled=enabled, get_time=get_time) def advance(self, num_bytes: int, out=sys.stdout) -> None: if not self.enabled: return self.current_bytes += num_bytes self.print(out=out) def print(self, final: bool = False, out=sys.stdout) -> None: if not self.enabled: return current_time = self.get_time() if self.last_printed + self.delta < current_time or final: self.last_printed = current_time # print a newline if this is the final update maybe_newline = "\n" if final else "" # if we know the total bytes, show a percentage, otherwise a spinner if self.total_bytes > 0: percentage = min(100 * self.current_bytes / self.total_bytes, 100.0) percent_or_spinner = f"[{percentage:3.0f}%] " else: # only show the spinner if we are not at 100% if final: percent_or_spinner = "[100%] " else: percent_or_spinner = f"[ {self.spinner[self.index]} ] " self.index = (self.index + 1) % len(self.spinner) print( f"\r {percent_or_spinner}{_format_bytes(self.current_bytes)} " f"@ {_format_speed(self.current_bytes, current_time - self.start_time)}" f"{maybe_newline}", end="", flush=True, file=out, ) @fetcher class URLFetchStrategy(FetchStrategy): """URLFetchStrategy pulls source code from a URL for an archive, check the archive against a checksum, and decompresses the archive. The destination for the resulting file(s) is the standard stage path. """ url_attr = "url" # these are checksum types. The generic 'checksum' is deprecated for # specific hash names, but we need it for backward compatibility optional_attrs = [*crypto.hashes.keys(), "checksum"] def __init__(self, *, url: str, checksum: Optional[str] = None, **kwargs) -> None: super().__init__(**kwargs) self.url = url self.mirrors = kwargs.get("mirrors", []) # digest can be set as the first argument, or from an explicit # kwarg by the hash name. self.digest: Optional[str] = checksum for h in self.optional_attrs: if h in kwargs: self.digest = kwargs[h] self.expand_archive: bool = kwargs.get("expand", True) self.extra_options: dict = kwargs.get("fetch_options", {}) self._curl: Optional[Executable] = None self.extension: Optional[str] = kwargs.get("extension", None) self._effective_url: Optional[str] = None @property def curl(self) -> Executable: if not self._curl: self._curl = web_util.require_curl() return self._curl def source_id(self): return self.digest def mirror_id(self): if not self.digest: return None # The filename is the digest. A directory is also created based on # truncating the digest to avoid creating a directory with too many # entries return os.path.sep.join(["archive", self.digest[:2], self.digest]) @property def candidate_urls(self): return [self.url] + (self.mirrors or []) @_needs_stage def fetch(self): if self.archive_file: tty.debug(f"Already downloaded {self.archive_file}") return errors: List[Exception] = [] for url in self.candidate_urls: try: self._fetch_from_url(url) break except FailedDownloadError as e: errors.extend(e.exceptions) else: raise FailedDownloadError(*errors) if not self.archive_file: raise FailedDownloadError( RuntimeError(f"Missing archive {self.archive_file} after fetching") ) def _fetch_from_url(self, url): fetch_method = spack.config.get("config:url_fetch_method", "urllib") if fetch_method.startswith("curl"): return self._fetch_curl(url, config_args=fetch_method.split()[1:]) else: return self._fetch_urllib(url) def _check_headers(self, headers): # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle # redirects properly. content_types = re.findall(r"Content-Type:[^\r\n]+", headers, flags=re.IGNORECASE) if content_types and "text/html" in content_types[-1]: msg = ( f"The contents of {self.archive_file or 'the archive'} fetched from {self.url} " " looks like HTML. This can indicate a broken URL, or an internet gateway issue." ) if self._effective_url != self.url: msg += f" The URL redirected to {self._effective_url}." tty.warn(msg) @_needs_stage def _fetch_urllib(self, url, chunk_size=65536, retries=5): """Fetch a URL using urllib, with retries on transient errors and progress reporting.""" save_file = self.stage.save_filename part_file = save_file + ".part" request = urllib.request.Request( url, headers={"User-Agent": web_util.SPACK_USER_AGENT, "Accept": "*/*"} ) response_headers_str = None for attempt in range(retries): try: with web_util.urlopen(request) as response: tty.verbose(f"Fetching {url}") progress = FetchProgress.from_headers( response.headers, enabled=sys.stdout.isatty() ) with open(part_file, "wb") as f: while True: chunk = response.read(chunk_size) if not chunk: break f.write(chunk) progress.advance(len(chunk)) progress.print(final=True) # Capture metadata before context manager closes the connection if isinstance(response, http.client.HTTPResponse): self._effective_url = response.geturl() response_headers_str = str(response.headers) os.replace(part_file, save_file) break # success: exit retry loop except Exception as e: # clean up archive on failure. if self.archive_file: os.remove(self.archive_file) if os.path.lexists(part_file): os.remove(part_file) # Raise if this was the last attempt, or if the error was not transient. if (attempt + 1 == retries) or not web_util.is_transient_error(e): raise FailedDownloadError(e) from e tty.debug(f"Retrying fetch (attempt {attempt + 1}): {e}") time.sleep(2**attempt) # Save the redirected URL for error messages. Sometimes we're redirected to an arbitrary # mirror that is broken, leading to spurious download failures. In that case it's helpful # for users to know which URL was actually fetched. self._check_headers(response_headers_str) @_needs_stage def _fetch_curl(self, url, config_args=[]): save_file = None partial_file = None if self.stage.save_filename: save_file = self.stage.save_filename partial_file = self.stage.save_filename + ".part" tty.verbose(f"Fetching {url}") if partial_file: save_args = [ "-C", "-", # continue partial downloads "-o", partial_file, ] # use a .part file else: save_args = ["-O"] timeout = 0 cookie_args = [] if self.extra_options: cookie = self.extra_options.get("cookie") if cookie: cookie_args.append("-j") # junk cookies cookie_args.append("-b") # specify cookie cookie_args.append(cookie) timeout = self.extra_options.get("timeout") base_args = web_util.base_curl_fetch_args(url, timeout) curl_args = config_args + save_args + base_args + cookie_args # Run curl but grab the mime type from the http headers curl = self.curl with working_dir(self.stage.path): headers = curl(*curl_args, output=str, fail_on_error=False) if curl.returncode != 0: # clean up archive on failure. if self.archive_file: os.remove(self.archive_file) if partial_file and os.path.lexists(partial_file): os.remove(partial_file) try: web_util.check_curl_code(curl.returncode) except spack.error.FetchError as e: raise FailedDownloadError(e) from e self._check_headers(headers) if save_file and (partial_file is not None): fs.rename(partial_file, save_file) @property # type: ignore # decorated properties unsupported in mypy @_needs_stage def archive_file(self): """Path to the source archive within this stage directory.""" return self.stage.archive_file @property def cachable(self): return self.cache_enabled and bool(self.digest) @_needs_stage def expand(self): if not self.expand_archive: tty.debug( "Staging unexpanded archive {0} in {1}".format( self.archive_file, self.stage.source_path ) ) if not self.stage.expanded: mkdirp(self.stage.source_path) dest = os.path.join(self.stage.source_path, os.path.basename(self.archive_file)) shutil.move(self.archive_file, dest) return tty.debug("Staging archive: {0}".format(self.archive_file)) if not self.archive_file: raise NoArchiveFileError( "Couldn't find archive file", "Failed on expand() for URL %s" % self.url ) # TODO: replace this by mime check. if not self.extension: self.extension = spack.llnl.url.determine_url_file_extension(self.url) if self.stage.expanded: tty.debug("Source already staged to %s" % self.stage.source_path) return decompress = decompressor_for(self.archive_file, self.extension) # Below we assume that the command to decompress expand the # archive in the current working directory with fs.exploding_archive_catch(self.stage): decompress(self.archive_file) def archive(self, destination): """Just moves this archive to the destination.""" if not self.archive_file: raise NoArchiveFileError("Cannot call archive() before fetching.") web_util.push_to_url( self.archive_file, url_util.path_to_file_url(destination), keep_original=True ) @_needs_stage def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" if not self.digest: raise NoDigestError(f"Attempt to check {self.__class__.__name__} with no digest.") verify_checksum(self.archive_file, self.digest, self.url, self._effective_url) @_needs_stage def reset(self): """ Removes the source path if it exists, then re-expands the archive. """ if not self.archive_file: raise NoArchiveFileError( f"Tried to reset {self.__class__.__name__} before fetching", f"Failed on reset() for URL{self.url}", ) # Remove everything but the archive from the stage for filename in os.listdir(self.stage.path): abspath = os.path.join(self.stage.path, filename) if abspath != self.archive_file: shutil.rmtree(abspath, ignore_errors=True) # Expand the archive again self.expand() def __repr__(self): return f"{self.__class__.__name__}<{self.url}>" def __str__(self): return self.url @fetcher class CacheURLFetchStrategy(URLFetchStrategy): """The resource associated with a cache URL may be out of date.""" @_needs_stage def fetch(self): path = url_util.file_url_string_to_path(self.url) # check whether the cache file exists. if not os.path.isfile(path): raise NoCacheError(f"No cache of {path}") # remove old symlink if one is there. filename = self.stage.save_filename if os.path.lexists(filename): os.remove(filename) # Symlink to local cached archive. symlink(path, filename) # Remove link if checksum fails, or subsequent fetchers will assume they don't need to # download. if self.digest: try: self.check() except ChecksumError: os.remove(self.archive_file) raise # Notify the user how we fetched. tty.msg(f"Using cached archive: {path}") class OCIRegistryFetchStrategy(URLFetchStrategy): def __init__(self, *, url: str, checksum: Optional[str] = None, **kwargs): super().__init__(url=url, checksum=checksum, **kwargs) self._urlopen = kwargs.get("_urlopen", spack.oci.opener.urlopen) @_needs_stage def fetch(self): file = self.stage.save_filename if os.path.lexists(file): os.remove(file) try: response = self._urlopen(self.url) tty.verbose(f"Fetching {self.url}") with open(file, "wb") as f: shutil.copyfileobj(response, f) except OSError as e: # clean up archive on failure. if self.archive_file: os.remove(self.archive_file) if os.path.lexists(file): os.remove(file) raise FailedDownloadError(e) from e class VCSFetchStrategy(FetchStrategy): """Superclass for version control system fetch strategies. Like all fetchers, VCS fetchers are identified by the attributes passed to the ``version`` directive. The optional_attrs for a VCS fetch strategy represent types of revisions, e.g. tags, branches, commits, etc. The required attributes (git, svn, etc.) are used to specify the URL and to distinguish a VCS fetch strategy from a URL fetch strategy. """ def __init__(self, **kwargs): super().__init__(**kwargs) # Set a URL based on the type of fetch strategy. self.url = kwargs.get(self.url_attr, None) if not self.url: raise ValueError(f"{self.__class__} requires {self.url_attr} argument.") for attr in self.optional_attrs: setattr(self, attr, kwargs.get(attr, None)) @_needs_stage def check(self): tty.debug(f"No checksum needed when fetching with {self.url_attr}") @_needs_stage def expand(self): tty.debug(f"Source fetched with {self.url_attr} is already expanded.") @_needs_stage def archive(self, destination, *, exclude: Optional[str] = None): assert spack.llnl.url.extension_from_path(destination) == "tar.gz" assert self.stage.source_path.startswith(self.stage.path) # We need to prepend this dir name to every entry of the tarfile top_level_dir = PurePath(self.stage.srcdir or os.path.basename(self.stage.source_path)) with working_dir(self.stage.source_path), spack.util.archive.gzip_compressed_tarfile( destination ) as (tar, _, _): spack.util.archive.reproducible_tarfile_from_prefix( tar=tar, prefix=".", skip=lambda entry: entry.name == exclude, path_to_name=lambda path: (top_level_dir / PurePath(path)).as_posix(), ) def __str__(self): return f"VCS: {self.url}" def __repr__(self): return f"{self.__class__}<{self.url}>" @fetcher class GoFetchStrategy(VCSFetchStrategy): """Fetch strategy that employs the ``go get`` infrastructure. Use like this in a package:: version("name", go="github.com/monochromegane/the_platinum_searcher/...") Go get does not natively support versions, they can be faked with git. The fetched source will be moved to the standard stage sourcepath directory during the expand step. """ url_attr = "go" def __init__(self, **kwargs): # Discards the keywords in kwargs that may conflict with the next # call to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) super().__init__(**forwarded_args) self._go = None @property def go_version(self): vstring = self.go("version", output=str).split(" ")[2] return spack.version.Version(vstring) @property def go(self): if not self._go: self._go = which("go", required=True) return self._go @_needs_stage def fetch(self): tty.debug("Getting go resource: {0}".format(self.url)) with working_dir(self.stage.path): try: os.mkdir("go") except OSError: pass env = dict(os.environ) env["GOPATH"] = os.path.join(os.getcwd(), "go") self.go("get", "-v", "-d", self.url, env=env) def archive(self, destination): super().archive(destination, exclude=".git") @_needs_stage def expand(self): tty.debug("Source fetched with %s is already expanded." % self.url_attr) # Move the directory to the well-known stage source path repo_root = _ensure_one_stage_entry(self.stage.path) shutil.move(repo_root, self.stage.source_path) @_needs_stage def reset(self): with working_dir(self.stage.source_path): self.go("clean") def __str__(self): return "[go] %s" % self.url @fetcher class GitFetchStrategy(VCSFetchStrategy): """ Fetch strategy that gets source code from a git repository. Use like this in a package:: version("name", git="https://github.com/project/repo.git") Optionally, you can provide a branch, or commit to check out, e.g.:: version("1.1", git="https://github.com/project/repo.git", tag="v1.1") You can use these three optional attributes in addition to ``git``: * ``branch``: Particular branch to build from (default is the repository's default branch) * ``tag``: Particular tag to check out * ``commit``: Particular commit hash in the repo Repositories are cloned into the standard stage source path directory. """ url_attr = "git" optional_attrs = [ "tag", "branch", "commit", "submodules", "get_full_repo", "submodules_delete", "git_sparse_paths", "skip_checkout", ] def __init__(self, **kwargs): self.commit: Optional[str] = None self.tag: Optional[str] = None self.branch: Optional[str] = None # Discards the keywords in kwargs that may conflict with the next call # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) super().__init__(**forwarded_args) self._git = None self.submodules = kwargs.get("submodules", False) self.submodules_delete = kwargs.get("submodules_delete", False) self.get_full_repo = kwargs.get("get_full_repo", False) self.git_sparse_paths = kwargs.get("git_sparse_paths", None) # skipping checkout with a blobless clone is an efficient way to traverse meta-data # see https://bhupesh.me/minimalist-guide-git-clone/ self.skip_checkout = kwargs.get("skip_checkout", False) @property def git_version(self): return GitFetchStrategy.version_from_git(self.git) @staticmethod def version_from_git(git_exe): """Given a git executable, return the Version (this will fail if the output cannot be parsed into a valid Version). """ version_string = ".".join(map(str, git_exe.version)) return spack.version.Version(version_string) @property def git(self): if not self._git: try: self._git = spack.util.git.git(required=True) except CommandNotFoundError as exc: tty.error(str(exc)) raise # Disable advice for a quieter fetch # https://github.com/git/git/blob/master/Documentation/RelNotes/1.7.2.txt if self.git_version >= spack.version.Version("1.7.2"): self._git.add_default_arg("-c", "advice.detachedHead=false") # If the user asked for insecure fetching, make that work # with git as well. if not spack.config.get("config:verify_ssl"): self._git.add_default_env("GIT_SSL_NO_VERIFY", "true") return self._git @property def cachable(self): return self.cache_enabled and bool(self.commit) def source_id(self): # TODO: tree-hash would secure download cache and mirrors, commit only secures checkouts. # TODO(psakiev): Tree-hash is part of the commit SHA computation, question comment validity return self.commit def mirror_id(self): if self.commit: provenance_id = self.commit repo_path = urllib.parse.urlparse(self.url).path if self.git_sparse_paths: sparse_paths = [] if callable(self.git_sparse_paths): sparse_paths.extend(self.git_sparse_paths()) else: sparse_paths.extend(self.git_sparse_paths) sparse_string = "_".join(sparse_paths) sparse_hash = hashlib.sha1(sparse_string.encode("utf-8")).hexdigest() provenance_id = f"{provenance_id}_{sparse_hash}" result = os.path.sep.join(["git", repo_path, provenance_id]) return result def _repo_info(self): args = "" if self.commit: args = f" at commit {self.commit}" elif self.tag: args = f" at tag {self.tag}" elif self.branch: args = f" on branch {self.branch}" return f"{self.url}{args}" @_needs_stage def fetch(self): if self.stage.expanded: tty.debug(f"Already fetched {self.stage.source_path}") return self._clone_src() self.submodule_operations() def bare_clone(self, dest: str) -> None: """ Execute a bare clone for metadata only Requires a destination since bare cloning does not provide source and shouldn't be used for staging. """ # Default to spack source path tty.debug(f"Cloning git repository: {self._repo_info()}") git = self.git debug = spack.config.get("config:debug") # We don't need to worry about which commit/branch/tag is checked out clone_args = ["clone", "--bare"] if not debug: clone_args.append("--quiet") clone_args.extend([self.url, dest]) git(*clone_args) def _clone_src(self) -> None: """Clone a repository to a path using git.""" # Default to spack source path dest = self.stage.source_path tty.debug(f"Cloning git repository: {self._repo_info()}") depth = None if self.get_full_repo else 1 name = self.package.name if self.package else None checkout_ref = self.commit or self.tag or self.branch fetch_ref = self.tag or self.branch kwargs = {"debug": spack.config.get("config:debug"), "git_exe": self.git, "dest": name} # TODO(psakievich) The use of the minimal clone need clearer justification via package API # or something. There is a trade space of storage minimization vs available git information # that grows to non-trivial proportions for larger projects minimal_clone = self.commit and name and not self.get_full_repo with temp_cwd(ignore_cleanup_errors=True): if minimal_clone: try: spack.util.git.git_init_fetch(self.url, self.commit, depth, **kwargs) except spack.util.executable.ProcessError: spack.util.git.git_clone( self.url, fetch_ref, self.get_full_repo, depth, **kwargs ) else: spack.util.git.git_clone(self.url, fetch_ref, self.get_full_repo, depth, **kwargs) repo_name = get_single_file(".") kwargs["dest"] = repo_name if not self.skip_checkout: spack.util.git.git_checkout(checkout_ref, self.git_sparse_paths, **kwargs) if self.stage: self.stage.srcdir = repo_name shutil.copytree(repo_name, dest, symlinks=True) return def submodule_operations(self): dest = self.stage.source_path git = self.git if self.submodules_delete: with working_dir(dest): for submodule_to_delete in self.submodules_delete: args = ["rm", submodule_to_delete] if not spack.config.get("config:debug"): args.insert(1, "--quiet") git(*args) # Init submodules if the user asked for them. git_commands = [] submodules = self.submodules if callable(submodules): submodules = submodules(self.package) if submodules: if isinstance(submodules, str): submodules = [submodules] git_commands.append(["submodule", "init", "--"] + submodules) git_commands.append(["submodule", "update", "--recursive"]) elif submodules: git_commands.append(["submodule", "update", "--init", "--recursive"]) if not git_commands: return with working_dir(dest): for args in git_commands: if not spack.config.get("config:debug"): args.insert(1, "--quiet") git(*args) @_needs_stage def reset(self): with working_dir(self.stage.source_path): co_args = ["checkout", "."] clean_args = ["clean", "-f"] if spack.config.get("config:debug"): co_args.insert(1, "--quiet") clean_args.insert(1, "--quiet") self.git(*co_args) self.git(*clean_args) def __str__(self): return f"[git] {self._repo_info()}" @fetcher class CvsFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a CVS repository. Use like this in a package:: version("name", cvs=":pserver:anonymous@www.example.com:/cvsroot%module=modulename") Optionally, you can provide a branch and/or a date for the URL:: version( "name", cvs=":pserver:anonymous@www.example.com:/cvsroot%module=modulename", branch="branchname", date="date" ) Repositories are checked out into the standard stage source path directory. """ url_attr = "cvs" optional_attrs = ["branch", "date"] def __init__(self, **kwargs): # Discards the keywords in kwargs that may conflict with the next call # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) super().__init__(**forwarded_args) self._cvs = None if self.branch is not None: self.branch = str(self.branch) if self.date is not None: self.date = str(self.date) @property def cvs(self): if not self._cvs: self._cvs = which("cvs", required=True) return self._cvs @property def cachable(self): return self.cache_enabled and (bool(self.branch) or bool(self.date)) def source_id(self): if not (self.branch or self.date): # We need a branch or a date to make a checkout reproducible return None id = "id" if self.branch: id += "-branch=" + self.branch if self.date: id += "-date=" + self.date return id def mirror_id(self): if not (self.branch or self.date): # We need a branch or a date to make a checkout reproducible return None # Special-case handling because this is not actually a URL elements = self.url.split(":") final = elements[-1] elements = final.split("/") # Everything before the first slash is a port number elements = elements[1:] result = os.path.sep.join(["cvs"] + elements) if self.branch: result += "%branch=" + self.branch if self.date: result += "%date=" + self.date return result @_needs_stage def fetch(self): if self.stage.expanded: tty.debug("Already fetched {0}".format(self.stage.source_path)) return tty.debug("Checking out CVS repository: {0}".format(self.url)) with temp_cwd(): url, module = self.url.split("%module=") # Check out files args = ["-z9", "-d", url, "checkout"] if self.branch is not None: args.extend(["-r", self.branch]) if self.date is not None: args.extend(["-D", self.date]) args.append(module) self.cvs(*args) # Rename repo repo_name = get_single_file(".") self.stage.srcdir = repo_name shutil.move(repo_name, self.stage.source_path) def _remove_untracked_files(self): """Removes untracked files in a CVS repository.""" with working_dir(self.stage.source_path): status = self.cvs("-qn", "update", output=str) for line in status.split("\n"): if re.match(r"^[?]", line): path = line[2:].strip() if os.path.isfile(path): os.unlink(path) def archive(self, destination): super().archive(destination, exclude="CVS") @_needs_stage def reset(self): self._remove_untracked_files() with working_dir(self.stage.source_path): self.cvs("update", "-C", ".") def __str__(self): return "[cvs] %s" % self.url @fetcher class SvnFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a subversion repository. Use like this in a package:: version("name", svn="http://www.example.com/svn/trunk") Optionally, you can provide a revision for the URL:: version("name", svn="http://www.example.com/svn/trunk", revision="1641") Repositories are checked out into the standard stage source path directory. """ url_attr = "svn" optional_attrs = ["revision"] def __init__(self, **kwargs): # Discards the keywords in kwargs that may conflict with the next call # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) super().__init__(**forwarded_args) self._svn = None if self.revision is not None: self.revision = str(self.revision) @property def svn(self): if not self._svn: self._svn = which("svn", required=True) return self._svn @property def cachable(self): return self.cache_enabled and bool(self.revision) def source_id(self): return self.revision def mirror_id(self): if self.revision: repo_path = urllib.parse.urlparse(self.url).path result = os.path.sep.join(["svn", repo_path, self.revision]) return result @_needs_stage def fetch(self): if self.stage.expanded: tty.debug("Already fetched {0}".format(self.stage.source_path)) return tty.debug("Checking out subversion repository: {0}".format(self.url)) args = ["checkout", "--force", "--quiet"] if self.revision: args += ["-r", self.revision] args.extend([self.url]) with temp_cwd(): self.svn(*args) repo_name = get_single_file(".") self.stage.srcdir = repo_name shutil.move(repo_name, self.stage.source_path) def _remove_untracked_files(self): """Removes untracked files in an svn repository.""" with working_dir(self.stage.source_path): status = self.svn("status", "--no-ignore", output=str) self.svn("status", "--no-ignore") for line in status.split("\n"): if not re.match("^[I?]", line): continue path = line[8:].strip() if os.path.isfile(path): os.unlink(path) elif os.path.isdir(path): shutil.rmtree(path, ignore_errors=True) def archive(self, destination): super().archive(destination, exclude=".svn") @_needs_stage def reset(self): self._remove_untracked_files() with working_dir(self.stage.source_path): self.svn("revert", ".", "-R") def __str__(self): return "[svn] %s" % self.url @fetcher class HgFetchStrategy(VCSFetchStrategy): """ Fetch strategy that gets source code from a Mercurial repository. Use like this in a package:: version("name", hg="https://jay.grs.rwth-aachen.de/hg/lwm2") Optionally, you can provide a branch, or revision to check out, e.g.:: version("torus", hg="https://jay.grs.rwth-aachen.de/hg/lwm2", branch="torus") You can use the optional ``revision`` attribute to check out a branch, tag, or particular revision in hg. To prevent non-reproducible builds, using a moving target like a branch is discouraged. * ``revision``: Particular revision, branch, or tag. Repositories are cloned into the standard stage source path directory. """ url_attr = "hg" optional_attrs = ["revision"] def __init__(self, **kwargs): # Discards the keywords in kwargs that may conflict with the next call # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) super().__init__(**forwarded_args) self._hg = None @property def hg(self): """ Returns: Executable: the hg executable """ if not self._hg: self._hg = which("hg", required=True) # When building PythonPackages, Spack automatically sets # PYTHONPATH. This can interfere with hg, which is a Python # script. Unset PYTHONPATH while running hg. self._hg.add_default_env("PYTHONPATH", "") return self._hg @property def cachable(self): return self.cache_enabled and bool(self.revision) def source_id(self): return self.revision def mirror_id(self): if self.revision: repo_path = urllib.parse.urlparse(self.url).path result = os.path.sep.join(["hg", repo_path, self.revision]) return result @_needs_stage def fetch(self): if self.stage.expanded: tty.debug("Already fetched {0}".format(self.stage.source_path)) return args = [] if self.revision: args.append("at revision %s" % self.revision) tty.debug("Cloning mercurial repository: {0} {1}".format(self.url, args)) args = ["clone"] if not spack.config.get("config:verify_ssl"): args.append("--insecure") if self.revision: args.extend(["-r", self.revision]) args.extend([self.url]) with temp_cwd(): self.hg(*args) repo_name = get_single_file(".") self.stage.srcdir = repo_name shutil.move(repo_name, self.stage.source_path) def archive(self, destination): super().archive(destination, exclude=".hg") @_needs_stage def reset(self): with working_dir(self.stage.path): source_path = self.stage.source_path scrubbed = "scrubbed-source-tmp" args = ["clone"] if self.revision: args += ["-r", self.revision] args += [source_path, scrubbed] self.hg(*args) shutil.rmtree(source_path, ignore_errors=True) shutil.move(scrubbed, source_path) def __str__(self): return f"[hg] {self.url}" @fetcher class S3FetchStrategy(URLFetchStrategy): """FetchStrategy that pulls from an S3 bucket.""" url_attr = "s3" @_needs_stage def fetch(self): if not self.url.startswith("s3://"): raise spack.error.FetchError( f"{self.__class__.__name__} can only fetch from s3:// urls." ) if self.archive_file: tty.debug(f"Already downloaded {self.archive_file}") return self._fetch_urllib(self.url) if not self.archive_file: raise FailedDownloadError( RuntimeError(f"Missing archive {self.archive_file} after fetching") ) @fetcher class GCSFetchStrategy(URLFetchStrategy): """FetchStrategy that pulls from a GCS bucket.""" url_attr = "gs" @_needs_stage def fetch(self): if not self.url.startswith("gs"): raise spack.error.FetchError( f"{self.__class__.__name__} can only fetch from gs:// urls." ) if self.archive_file: tty.debug(f"Already downloaded {self.archive_file}") return self._fetch_urllib(self.url) if not self.archive_file: raise FailedDownloadError( RuntimeError(f"Missing archive {self.archive_file} after fetching") ) @fetcher class FetchAndVerifyExpandedFile(URLFetchStrategy): """Fetch strategy that verifies the content digest during fetching, as well as after expanding it.""" def __init__(self, url, archive_sha256: str, expanded_sha256: str): super().__init__(url=url, checksum=archive_sha256) self.expanded_sha256 = expanded_sha256 def expand(self): """Verify checksum after expanding the archive.""" # Expand the archive super().expand() # Ensure a single patch file. src_dir = self.stage.source_path files = os.listdir(src_dir) if len(files) != 1: raise ChecksumError(self, f"Expected a single file in {src_dir}.") verify_checksum( os.path.join(src_dir, files[0]), self.expanded_sha256, self.url, self._effective_url ) def verify_checksum(file: str, digest: str, url: str, effective_url: Optional[str]) -> None: checker = crypto.Checker(digest) if not checker.check(file): # On failure, provide some information about the file size and # contents, so that we can quickly see what the issue is (redirect # was not followed, empty file, text instead of binary, ...) size, contents = fs.filesummary(file) long_msg = ( f"Expected {digest} but got {checker.sum}. " f"File size = {size} bytes. Contents = {contents!r}. " f"URL = {url}" ) if effective_url and effective_url != url: long_msg += f", redirected to = {effective_url}" raise ChecksumError(f"{checker.hash_name} checksum failed for {file}", long_msg) def stable_target(fetcher): """Returns whether the fetcher target is expected to have a stable checksum. This is only true if the target is a preexisting archive file.""" if isinstance(fetcher, URLFetchStrategy) and fetcher.cachable: return True return False def from_url(url: str) -> URLFetchStrategy: """Given a URL, find an appropriate fetch strategy for it. Currently just gives you a URLFetchStrategy that uses curl. TODO: make this return appropriate fetch strategies for other types of URLs. """ return URLFetchStrategy(url=url) def from_kwargs(**kwargs) -> FetchStrategy: """Construct an appropriate FetchStrategy from the given keyword arguments. Args: **kwargs: dictionary of keyword arguments, e.g. from a ``version()`` directive in a package. Returns: The fetch strategy that matches the args, based on attribute names (e.g., ``git``, ``hg``, etc.) Raises: spack.error.FetchError: If no ``fetch_strategy`` matches the args. """ for fetcher in all_strategies: if fetcher.matches(kwargs): return fetcher(**kwargs) raise InvalidArgsError(**kwargs) def check_pkg_attributes(pkg): """Find ambiguous top-level fetch attributes in a package. Currently this only ensures that two or more VCS fetch strategies are not specified at once. """ # a single package cannot have URL attributes for multiple VCS fetch # strategies *unless* they are the same attribute. conflicts = set([s.url_attr for s in all_strategies if hasattr(pkg, s.url_attr)]) # URL isn't a VCS fetch method. We can use it with a VCS method. conflicts -= set(["url"]) if len(conflicts) > 1: raise FetcherConflict( "Package %s cannot specify %s together. Pick at most one." % (pkg.name, comma_and(quote(conflicts))) ) def _check_version_attributes(fetcher, pkg, version): """Ensure that the fetcher for a version is not ambiguous. This assumes that we have already determined the fetcher for the specific version using ``for_package_version()`` """ all_optionals = set(a for s in all_strategies for a in s.optional_attrs) args = pkg.versions[version] extra = set(args) - set(fetcher.optional_attrs) - set([fetcher.url_attr, "no_cache"]) extra.intersection_update(all_optionals) if extra: legal_attrs = [fetcher.url_attr] + list(fetcher.optional_attrs) raise FetcherConflict( "%s version '%s' has extra arguments: %s" % (pkg.name, version, comma_and(quote(extra))), "Valid arguments for a %s fetcher are: \n %s" % (fetcher.url_attr, comma_and(quote(legal_attrs))), ) def _extrapolate(pkg, version): """Create a fetcher from an extrapolated URL for this version.""" try: return URLFetchStrategy(url=pkg.url_for_version(version), fetch_options=pkg.fetch_options) except spack.error.NoURLError: raise ExtrapolationError( f"Can't extrapolate a URL for version {version} because " f"package {pkg.name} defines no URLs" ) def _from_merged_attrs(fetcher, pkg, version): """Create a fetcher from merged package and version attributes.""" if fetcher.url_attr == "url": mirrors = pkg.all_urls_for_version(version) url = mirrors[0] mirrors = mirrors[1:] attrs = {fetcher.url_attr: url, "mirrors": mirrors} else: url = getattr(pkg, fetcher.url_attr) attrs = {fetcher.url_attr: url} attrs["fetch_options"] = pkg.fetch_options attrs.update(pkg.versions[version]) if fetcher.url_attr == "git": pkg_attr_list = ["submodules", "git_sparse_paths"] for pkg_attr in pkg_attr_list: if hasattr(pkg, pkg_attr): attrs.setdefault(pkg_attr, getattr(pkg, pkg_attr)) return fetcher(**attrs) def for_package_version(pkg, version=None): saved_versions = None if version is not None: saved_versions = pkg.spec.versions try: return _for_package_version(pkg, version) finally: if saved_versions is not None: pkg.spec.versions = saved_versions def _for_package_version(pkg, version=None): """Determine a fetch strategy based on the arguments supplied to version() in the package description.""" # No-code packages have a custom fetch strategy to work around issues # with resource staging. if not pkg.has_code: return BundleFetchStrategy() check_pkg_attributes(pkg) if version is not None: assert not pkg.spec.concrete, "concrete specs should not pass the 'version=' argument" # Specs are initialized with the universe range, if no version information is given, # so here we make sure we always match the version passed as argument if not isinstance(version, spack.version.StandardVersion): version = spack.version.Version(version) version_list = spack.version.VersionList() version_list.add(version) pkg.spec.versions = version_list else: version = pkg.version # if it's a commit, we must use a GitFetchStrategy commit_var = pkg.spec.variants.get("commit", None) commit = commit_var.value if commit_var else None tag = None if isinstance(version, spack.version.GitVersion) or commit: git_url = pkg.version_or_package_attr("git", version) if not git_url: raise spack.error.FetchError( f"Cannot fetch git version for {pkg.name}. Package has no 'git' attribute" ) if isinstance(version, spack.version.GitVersion): # Populate the version with comparisons to other commits from spack.version.git_ref_lookup import GitRefLookup version.attach_lookup(GitRefLookup(pkg.name)) if not commit and version.is_commit: commit = version.ref version_meta_data = pkg.versions.get(version.std_version) else: version_meta_data = pkg.versions.get(version) # For GitVersion, we have no way to determine whether a ref is a branch or tag # Fortunately, we handle branches and tags identically, except tags are # handled slightly more conservatively for older versions of git. # We call all non-commit refs tags in this context, at the cost of a slight # performance hit for branches on older versions of git. # Branches cannot be cached, so we tell the fetcher not to cache tags/branches # TODO(psakiev) eventually we should only need to clone based on the commit # commit stashed on version if version_meta_data: if not commit: commit = version_meta_data.get("commit") tag = version_meta_data.get("tag") or version_meta_data.get("branch") kwargs = {"commit": commit, "tag": tag, "no_cache": bool(not commit)} kwargs["git"] = git_url kwargs["submodules"] = pkg.version_or_package_attr("submodules", version, False) kwargs["git_sparse_paths"] = pkg.version_or_package_attr("git_sparse_paths", version, None) kwargs["get_full_repo"] = pkg.version_or_package_attr("get_full_repo", version, False) # if the ref_version is a known version from the package, use that version's # attributes ref_version = getattr(pkg.version, "ref_version", None) if ref_version: kwargs["git"] = pkg.version_or_package_attr("git", ref_version) kwargs["submodules"] = pkg.version_or_package_attr("submodules", ref_version, False) fetcher = GitFetchStrategy(**kwargs) return fetcher # If it's not a known version, try to extrapolate one by URL if version not in pkg.versions: return _extrapolate(pkg, version) # Set package args first so version args can override them args = {"fetch_options": pkg.fetch_options} # Grab a dict of args out of the package version dict args.update(pkg.versions[version]) # If the version specifies a `url_attr` directly, use that. for fetcher in all_strategies: if fetcher.url_attr in args: _check_version_attributes(fetcher, pkg, version) if fetcher.url_attr == "git" and hasattr(pkg, "submodules"): args.setdefault("submodules", pkg.submodules) return fetcher(**args) # if a version's optional attributes imply a particular fetch # strategy, and we have the `url_attr`, then use that strategy. for fetcher in all_strategies: if hasattr(pkg, fetcher.url_attr) or fetcher.url_attr == "url": optionals = fetcher.optional_attrs if optionals and any(a in args for a in optionals): _check_version_attributes(fetcher, pkg, version) return _from_merged_attrs(fetcher, pkg, version) # if the optional attributes tell us nothing, then use any `url_attr` # on the package. This prefers URL vs. VCS, b/c URLFetchStrategy is # defined first in this file. for fetcher in all_strategies: if hasattr(pkg, fetcher.url_attr): _check_version_attributes(fetcher, pkg, version) return _from_merged_attrs(fetcher, pkg, version) raise InvalidArgsError(pkg, version, **args) def from_url_scheme(url: str, **kwargs) -> FetchStrategy: """Finds a suitable FetchStrategy by matching its url_attr with the scheme in the given url.""" parsed_url = urllib.parse.urlparse(url, scheme="file") scheme_mapping = kwargs.get("scheme_mapping") or { "file": "url", "http": "url", "https": "url", "ftp": "url", "ftps": "url", } scheme = parsed_url.scheme scheme = scheme_mapping.get(scheme, scheme) for fetcher in all_strategies: url_attr = getattr(fetcher, "url_attr", None) if url_attr and url_attr == scheme: return fetcher(url=url, **kwargs) raise ValueError(f'No FetchStrategy found for url with scheme: "{parsed_url.scheme}"') def from_list_url(pkg): """If a package provides a URL which lists URLs for resources by version, this can can create a fetcher for a URL discovered for the specified package's version.""" if pkg.list_url: try: versions = pkg.fetch_remote_versions() try: # get a URL, and a checksum if we have it url_from_list = versions[pkg.version] checksum = None # try to find a known checksum for version, from the package version = pkg.version if version in pkg.versions: args = pkg.versions[version] checksum = next( (v for k, v in args.items() if k in crypto.hashes), args.get("checksum") ) # construct a fetcher return URLFetchStrategy( url=url_from_list, checksum=checksum, fetch_options=pkg.fetch_options ) except KeyError as e: tty.debug(e) tty.msg("Cannot find version %s in url_list" % pkg.version) except BaseException as e: # TODO: Don't catch BaseException here! Be more specific. tty.debug(e) tty.msg("Could not determine url from list_url.") class FsCache: def __init__(self, root): self.root = os.path.abspath(root) def store(self, fetcher, relative_dest): # skip fetchers that aren't cachable if not fetcher.cachable: return # Don't store things that are already cached. if isinstance(fetcher, CacheURLFetchStrategy): return dst = os.path.join(self.root, relative_dest) mkdirp(os.path.dirname(dst)) fetcher.archive(dst) def fetcher(self, target_path: str, digest: Optional[str], **kwargs) -> CacheURLFetchStrategy: path = os.path.join(self.root, target_path) url = url_util.path_to_file_url(path) return CacheURLFetchStrategy(url=url, checksum=digest, **kwargs) def destroy(self): shutil.rmtree(self.root, ignore_errors=True) class NoCacheError(spack.error.FetchError): """Raised when there is no cached archive for a package.""" class FailedDownloadError(spack.error.FetchError): """Raised when a download fails.""" def __init__(self, *exceptions: Exception): super().__init__("Failed to download") self.exceptions = exceptions class NoArchiveFileError(spack.error.FetchError): """Raised when an archive file is expected but none exists.""" class NoDigestError(spack.error.FetchError): """Raised after attempt to checksum when URL has no digest.""" class ExtrapolationError(spack.error.FetchError): """Raised when we can't extrapolate a version for a package.""" class FetcherConflict(spack.error.FetchError): """Raised for packages with invalid fetch attributes.""" class InvalidArgsError(spack.error.FetchError): """Raised when a version can't be deduced from a set of arguments.""" def __init__(self, pkg=None, version=None, **args): msg = "Could not guess a fetch strategy" if pkg: msg += " for {pkg}".format(pkg=pkg) if version: msg += "@{version}".format(version=version) long_msg = "with arguments: {args}".format(args=args) super().__init__(msg, long_msg) class ChecksumError(spack.error.FetchError): """Raised when archive fails to checksum.""" class NoStageError(spack.error.FetchError): """Raised when fetch operations are called before set_stage().""" def __init__(self, method): super().__init__("Must call FetchStrategy.set_stage() before calling %s" % method.__name__) ================================================ FILE: lib/spack/spack/filesystem_view.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import functools as ft import itertools import os import re import shutil import stat import sys import tempfile from typing import Callable, Dict, List, Optional from spack.vendor.typing_extensions import Literal import spack.config import spack.directory_layout import spack.projections import spack.relocate import spack.schema.projections import spack.spec import spack.store import spack.util.spack_json as s_json import spack.util.spack_yaml as s_yaml from spack.error import SpackError from spack.llnl.string import comma_or from spack.llnl.util import tty from spack.llnl.util.filesystem import ( mkdirp, remove_dead_links, remove_empty_directories, symlink, visit_directory_tree, ) from spack.llnl.util.lang import index_by, match_predicate from spack.llnl.util.link_tree import ( ConflictingSpecsError, DestinationMergeVisitor, LinkTree, MergeConflictSummary, SingleMergeConflictError, SourceMergeVisitor, ) from spack.llnl.util.tty.color import colorize __all__ = ["FilesystemView", "YamlFilesystemView"] _projections_path = ".spack/projections.yaml" LinkCallbackType = Callable[[str, str, "FilesystemView", Optional[spack.spec.Spec]], None] def view_symlink(src: str, dst: str, *args, **kwargs) -> None: symlink(src, dst) def view_hardlink(src: str, dst: str, *args, **kwargs) -> None: os.link(src, dst) def view_copy( src: str, dst: str, view: "FilesystemView", spec: Optional[spack.spec.Spec] = None ) -> None: """ Copy a file from src to dst. Use spec and view to generate relocations """ shutil.copy2(src, dst, follow_symlinks=False) # No need to relocate if no metadata or external. if not spec or spec.external: return # Order of this dict is somewhat irrelevant prefix_to_projection = { str(s.prefix): view.get_projection_for_spec(s) for s in spec.traverse(root=True, order="breadth") if not s.external } src_stat = os.lstat(src) # TODO: change this into a bulk operation instead of a per-file operation if stat.S_ISLNK(src_stat.st_mode): spack.relocate.relocate_links(links=[dst], prefix_to_prefix=prefix_to_projection) elif spack.relocate.is_binary(dst): spack.relocate.relocate_text_bin(binaries=[dst], prefix_to_prefix=prefix_to_projection) else: prefix_to_projection[spack.store.STORE.layout.root] = view._root spack.relocate.relocate_text(files=[dst], prefix_to_prefix=prefix_to_projection) # The os module on Windows does not have a chown function. if sys.platform != "win32": try: os.chown(dst, src_stat.st_uid, src_stat.st_gid) except OSError: tty.debug(f"Can't change the permissions for {dst}") #: Type alias for link types LinkType = Literal["hardlink", "hard", "copy", "relocate", "add", "symlink", "soft"] CanonicalLinkType = Literal["hardlink", "copy", "symlink"] #: supported string values for `link_type` in an env, mapped to canonical values _LINK_TYPES: Dict[LinkType, CanonicalLinkType] = { "hardlink": "hardlink", "hard": "hardlink", "copy": "copy", "relocate": "copy", "add": "symlink", "symlink": "symlink", "soft": "symlink", } _VALID_LINK_TYPES = sorted(set(_LINK_TYPES.values())) def canonicalize_link_type(link_type: LinkType) -> CanonicalLinkType: """Return canonical""" canonical = _LINK_TYPES.get(link_type) if not canonical: raise ValueError( f"Invalid link type: '{link_type}. Must be one of {comma_or(_VALID_LINK_TYPES)}'" ) return canonical def function_for_link_type(link_type: LinkType) -> LinkCallbackType: link_type = canonicalize_link_type(link_type) if link_type == "hardlink": return view_hardlink elif link_type == "symlink": return view_symlink elif link_type == "copy": return view_copy assert False, "invalid link type" class FilesystemView: """ Governs a filesystem view that is located at certain root-directory. Packages are linked from their install directories into a common file hierarchy. In distributed filesystems, loading each installed package separately can lead to slow-downs due to too many directories being traversed. This can be circumvented by loading all needed modules into a common directory structure. """ def __init__( self, root: str, layout: spack.directory_layout.DirectoryLayout, *, projections: Optional[Dict] = None, ignore_conflicts: bool = False, verbose: bool = False, link_type: LinkType = "symlink", ): """ Initialize a filesystem view under the given ``root`` directory with corresponding directory ``layout``. Files are linked by method ``link`` (spack.llnl.util.filesystem.symlink by default). """ self._root = root self.layout = layout self.projections = {} if projections is None else projections self.ignore_conflicts = ignore_conflicts self.verbose = verbose # Setup link function to include view self.link_type = link_type self._link = function_for_link_type(link_type) def link(self, src: str, dst: str, spec: Optional[spack.spec.Spec] = None) -> None: self._link(src, dst, self, spec) def add_specs(self, *specs: spack.spec.Spec, **kwargs) -> None: """ Add given specs to view. Should accept ``with_dependencies`` as keyword argument (default True) to indicate whether or not dependencies should be activated as well. Should except an ``exclude`` keyword argument containing a list of regexps that filter out matching spec names. This method should make use of ``activate_standalone``. """ raise NotImplementedError def add_standalone(self, spec: spack.spec.Spec) -> bool: """ Add (link) a standalone package into this view. """ raise NotImplementedError def check_added(self, spec: spack.spec.Spec) -> bool: """ Check if the given concrete spec is active in this view. """ raise NotImplementedError def remove_specs(self, *specs: spack.spec.Spec, **kwargs) -> None: """ Removes given specs from view. Should accept ``with_dependencies`` as keyword argument (default True) to indicate whether or not dependencies should be deactivated as well. Should accept ``with_dependents`` as keyword argument (default True) to indicate whether or not dependents on the deactivated specs should be removed as well. Should except an ``exclude`` keyword argument containing a list of regexps that filter out matching spec names. This method should make use of ``deactivate_standalone``. """ raise NotImplementedError def remove_standalone(self, spec: spack.spec.Spec) -> None: """ Remove (unlink) a standalone package from this view. """ raise NotImplementedError def get_projection_for_spec(self, spec: spack.spec.Spec) -> str: """ Get the projection in this view for a spec. """ raise NotImplementedError def get_all_specs(self) -> List[spack.spec.Spec]: """ Get all specs currently active in this view. """ raise NotImplementedError def get_spec(self, spec: spack.spec.Spec) -> Optional[spack.spec.Spec]: """ Return the actual spec linked in this view (i.e. do not look it up in the database by name). ``spec`` can be a name or a spec from which the name is extracted. As there can only be a single version active for any spec the name is enough to identify the spec in the view. If no spec is present, returns None. """ raise NotImplementedError def print_status(self, *specs: spack.spec.Spec, **kwargs) -> None: """ Print a short summary about the given specs, detailing whether.. * ..they are active in the view. * ..they are active but the activated version differs. * ..they are not active in the view. Takes ``with_dependencies`` keyword argument so that the status of dependencies is printed as well. """ raise NotImplementedError class YamlFilesystemView(FilesystemView): """ Filesystem view to work with a yaml based directory layout. """ def __init__( self, root: str, layout: spack.directory_layout.DirectoryLayout, *, projections: Optional[Dict] = None, ignore_conflicts: bool = False, verbose: bool = False, link_type: LinkType = "symlink", ): super().__init__( root, layout, projections=projections, ignore_conflicts=ignore_conflicts, verbose=verbose, link_type=link_type, ) # Super class gets projections from the kwargs # YAML specific to get projections from YAML file self.projections_path = os.path.join(self._root, _projections_path) if not self.projections: # Read projections file from view self.projections = self.read_projections() elif not os.path.exists(self.projections_path): # Write projections file to new view self.write_projections() else: # Ensure projections are the same from each source # Read projections file from view if self.projections != self.read_projections(): raise ConflictingProjectionsError( f"View at {self._root} has projections file" " which does not match projections passed manually." ) self._croot = colorize_root(self._root) + " " def write_projections(self): if self.projections: mkdirp(os.path.dirname(self.projections_path)) with open(self.projections_path, "w", encoding="utf-8") as f: f.write(s_yaml.dump_config({"projections": self.projections})) def read_projections(self): if os.path.exists(self.projections_path): with open(self.projections_path, "r", encoding="utf-8") as f: projections_data = s_yaml.load(f) spack.config.validate(projections_data, spack.schema.projections.schema) return projections_data["projections"] else: return {} def add_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) specs = set(specs) if kwargs.get("with_dependencies", True): specs.update(get_dependencies(specs)) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) conflicts = self.get_conflicts(*specs) if conflicts: for s, v in conflicts: self.print_conflict(v, s) return for s in specs: self.add_standalone(s) def add_standalone(self, spec): if spec.external: tty.warn(f"{self._croot}Skipping external package: {colorize_spec(spec)}") return True if self.check_added(spec): tty.warn(f"{self._croot}Skipping already linked package: {colorize_spec(spec)}") return True self.merge(spec) self.link_meta_folder(spec) if self.verbose: tty.info(f"{self._croot}Linked package: {colorize_spec(spec)}") return True def merge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate(self.layout.hidden_file_regexes, ignore) # check for dir conflicts conflicts = tree.find_dir_conflicts(view_dst, ignore_file) merge_map = tree.get_file_map(view_dst, ignore_file) if not self.ignore_conflicts: conflicts.extend(pkg.view_file_conflicts(self, merge_map)) if conflicts: raise SingleMergeConflictError(conflicts[0]) # merge directories with the tree tree.merge_directories(view_dst, ignore_file) pkg.add_files_to_view(self, merge_map) def unmerge(self, spec, ignore=None): pkg = spec.package view_source = pkg.view_source() view_dst = pkg.view_destination(self) tree = LinkTree(view_source) ignore = ignore or (lambda f: False) ignore_file = match_predicate(self.layout.hidden_file_regexes, ignore) merge_map = tree.get_file_map(view_dst, ignore_file) pkg.remove_files_from_view(self, merge_map) # now unmerge the directory tree tree.unmerge_directories(view_dst, ignore_file) def remove_files(self, files): def needs_file(spec, file): # convert the file we want to remove to a source in this spec projection = self.get_projection_for_spec(spec) relative_path = os.path.relpath(file, projection) test_path = os.path.join(spec.prefix, relative_path) # check if this spec owns a file of that name (through the # manifest in the metadata dir, which we have in the view). manifest_file = os.path.join( self.get_path_meta_folder(spec), spack.store.STORE.layout.manifest_file_name ) try: with open(manifest_file, "r", encoding="utf-8") as f: manifest = s_json.load(f) except OSError: # if we can't load it, assume it doesn't know about the file. manifest = {} return test_path in manifest specs = self.get_all_specs() for file in files: if not os.path.lexists(file): tty.warn(f"Tried to remove {file} which does not exist") continue # remove if file is not owned by any other package in the view # This will only be false if two packages are merged into a prefix # and have a conflicting file # check all specs for whether they own the file. That include the spec # we are currently removing, as we remove files before unlinking the # metadata directory. if len([s for s in specs if needs_file(s, file)]) <= 1: tty.debug(f"Removing file {file}") os.remove(file) def check_added(self, spec): assert spec.concrete return spec == self.get_spec(spec) def remove_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) with_dependents = kwargs.get("with_dependents", True) with_dependencies = kwargs.get("with_dependencies", False) # caller can pass this in, as get_all_specs() is expensive all_specs = kwargs.get("all_specs", None) or set(self.get_all_specs()) specs = set(specs) if with_dependencies: specs = get_dependencies(specs) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) to_deactivate = specs to_keep = all_specs - to_deactivate dependents = find_dependents(to_keep, to_deactivate) if with_dependents: # remove all packages depending on the ones to remove if len(dependents) > 0: tty.warn( self._croot + "The following dependents will be removed: %s" % ", ".join((s.name for s in dependents)) ) to_deactivate.update(dependents) elif len(dependents) > 0: tty.warn( self._croot + "The following packages will be unusable: %s" % ", ".join((s.name for s in dependents)) ) # Determine the order that packages should be removed from the view; # dependents come before their dependencies. to_deactivate_sorted = list() depmap = dict() for spec in to_deactivate: depmap[spec] = set(d for d in spec.traverse(root=False) if d in to_deactivate) while depmap: for spec in [s for s, d in depmap.items() if not d]: to_deactivate_sorted.append(spec) for s in depmap.keys(): depmap[s].discard(spec) depmap.pop(spec) to_deactivate_sorted.reverse() # Ensure that the sorted list contains all the packages assert set(to_deactivate_sorted) == to_deactivate # Remove the packages from the view for spec in to_deactivate_sorted: self.remove_standalone(spec) self._purge_empty_directories() def remove_standalone(self, spec): """ Remove (unlink) a standalone package from this view. """ if not self.check_added(spec): tty.warn(f"{self._croot}Skipping package not linked in view: {spec.name}") return self.unmerge(spec) self.unlink_meta_folder(spec) if self.verbose: tty.info(f"{self._croot}Removed package: {colorize_spec(spec)}") def get_projection_for_spec(self, spec): """ Return the projection for a spec in this view. Relies on the ordering of projections to avoid ambiguity. """ spec = spack.spec.Spec(spec) locator_spec = spec if spec.package.extendee_spec: locator_spec = spec.package.extendee_spec proj = spack.projections.get_projection(self.projections, locator_spec) if proj: return os.path.join(self._root, locator_spec.format_path(proj)) return self._root def get_all_specs(self): md_dirs = [] for root, dirs, files in os.walk(self._root): if spack.store.STORE.layout.metadata_dir in dirs: md_dirs.append(os.path.join(root, spack.store.STORE.layout.metadata_dir)) specs = [] for md_dir in md_dirs: if os.path.exists(md_dir): for name_dir in os.listdir(md_dir): filename = os.path.join( md_dir, name_dir, spack.store.STORE.layout.spec_file_name ) spec = get_spec_from_file(filename) if spec: specs.append(spec) return specs def get_conflicts(self, *specs): """ Return list of tuples (, ) where the spec active in the view differs from the one to be activated. """ in_view = map(self.get_spec, specs) return [(s, v) for s, v in zip(specs, in_view) if v is not None and s != v] def get_path_meta_folder(self, spec): "Get path to meta folder for either spec or spec name." return os.path.join( self.get_projection_for_spec(spec), spack.store.STORE.layout.metadata_dir, getattr(spec, "name", spec), ) def get_spec(self, spec): dotspack = self.get_path_meta_folder(spec) filename = os.path.join(dotspack, spack.store.STORE.layout.spec_file_name) return get_spec_from_file(filename) def link_meta_folder(self, spec): src = spack.store.STORE.layout.metadata_path(spec) tgt = self.get_path_meta_folder(spec) tree = LinkTree(src) # there should be no conflicts when linking the meta folder tree.merge(tgt, link=self.link) def print_conflict(self, spec_active, spec_specified, level="error"): "Singular print function for spec conflicts." cprint = getattr(tty, level) color = sys.stdout.isatty() linked = tty.color.colorize(" (@gLinked@.)", color=color) specified = tty.color.colorize("(@rSpecified@.)", color=color) cprint( f"{self._croot}Package conflict detected:\n" f"{linked} {colorize_spec(spec_active)}\n" f"{specified} {colorize_spec(spec_specified)}" ) def print_status(self, *specs, **kwargs): if kwargs.get("with_dependencies", False): specs = set(get_dependencies(specs)) specs = sorted(specs, key=lambda s: s.name) in_view = list(map(self.get_spec, specs)) for s, v in zip(specs, in_view): if not v: tty.error(f"{self._croot}Package not linked: {s.name}") elif s != v: self.print_conflict(v, s, level="warn") in_view = list(filter(None, in_view)) if len(specs) > 0: tty.msg(f"Packages linked in {self._croot[:-1]}:") # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ("architecture", "compiler")) # Traverse the index and print out each package for i, (architecture, compiler) in enumerate(sorted(index)): if i > 0: print() header = ( f"{spack.spec.ARCHITECTURE_COLOR}{{{architecture}}} " f"/ {spack.spec.COMPILER_COLOR}{{{compiler}}}" ) tty.hline(colorize(header), char="-") specs = index[(architecture, compiler)] specs.sort() abbreviated = [ s.cformat("{name}{@version}{compiler_flags}{variants}{%compiler}") for s in specs ] # Print one spec per line along with prefix path width = max(len(s) for s in abbreviated) width += 2 format = " %%-%ds%%s" % width for abbrv, s in zip(abbreviated, specs): prefix = "" if self.verbose: prefix = colorize("@K{%s}" % s.dag_hash(7)) print(prefix + (format % (abbrv, self.get_projection_for_spec(s)))) else: tty.warn(self._croot + "No packages found.") def _purge_empty_directories(self): remove_empty_directories(self._root) def _purge_broken_links(self): remove_dead_links(self._root) def clean(self): self._purge_broken_links() self._purge_empty_directories() def unlink_meta_folder(self, spec): path = self.get_path_meta_folder(spec) assert os.path.exists(path) shutil.rmtree(path) class SimpleFilesystemView(FilesystemView): """A simple and partial implementation of FilesystemView focused on performance and immutable views, where specs cannot be removed after they were added.""" def _sanity_check_view_projection(self, specs): """A very common issue is that we end up with two specs of the same package, that project to the same prefix. We want to catch that as early as possible and give a sensible error to the user. Here we use the metadata dir (.spack) projection as a quick test to see whether two specs in the view are going to clash. The metadata dir is used because it's always added by Spack with identical files, so a guaranteed clash that's easily verified.""" seen = {} for current_spec in specs: metadata_dir = self.relative_metadata_dir_for_spec(current_spec) conflicting_spec = seen.get(metadata_dir) if conflicting_spec: raise ConflictingSpecsError(current_spec, conflicting_spec) seen[metadata_dir] = current_spec def add_specs(self, *specs, **kwargs) -> None: """Link a root-to-leaf topologically ordered list of specs into the view.""" assert all((s.concrete for s in specs)) if len(specs) == 0: return # Drop externals specs = [s for s in specs if not s.external] self._sanity_check_view_projection(specs) # Ignore spack meta data folder. def skip_list(file): return os.path.basename(file) == spack.store.STORE.layout.metadata_dir # Determine if the root is on a case-insensitive filesystem normalize_paths = is_folder_on_case_insensitive_filesystem(self._root) visitor = SourceMergeVisitor(ignore=skip_list, normalize_paths=normalize_paths) # Gather all the directories to be made and files to be linked for spec in specs: src_prefix = spec.package.view_source() visitor.set_projection(self.get_relative_projection_for_spec(spec)) visit_directory_tree(src_prefix, visitor) # Check for conflicts in destination dir. visit_directory_tree(self._root, DestinationMergeVisitor(visitor)) # Throw on fatal dir-file conflicts. if visitor.fatal_conflicts: raise MergeConflictSummary(visitor.fatal_conflicts) # Inform about file-file conflicts. if visitor.file_conflicts: if self.ignore_conflicts: tty.debug(f"{len(visitor.file_conflicts)} file conflicts") else: raise MergeConflictSummary(visitor.file_conflicts) tty.debug(f"Creating {len(visitor.directories)} dirs and {len(visitor.files)} links") # Make the directory structure for dst in visitor.directories: os.mkdir(os.path.join(self._root, dst)) # Link the files using a "merge map": full src => full dst merge_map_per_prefix = self._source_merge_visitor_to_merge_map(visitor) for spec in specs: merge_map = merge_map_per_prefix.get(spec.package.view_source(), None) if not merge_map: # Not every spec may have files to contribute. continue spec.package.add_files_to_view(self, merge_map, skip_if_exists=False) # Finally create the metadata dirs. self.link_metadata(specs) def _source_merge_visitor_to_merge_map(self, visitor: SourceMergeVisitor): # For compatibility with add_files_to_view, we have to create a # merge_map of the form join(src_root, src_rel) => join(dst_root, dst_rel), # but our visitor.files format is dst_rel => (src_root, src_rel). # We exploit that visitor.files is an ordered dict, and files per source # prefix are contiguous. source_root = lambda item: item[1][0] per_source = itertools.groupby(visitor.files.items(), key=source_root) return { src_root: { os.path.join(src_root, src_rel): os.path.join(self._root, dst_rel) for dst_rel, (_, src_rel) in group } for src_root, group in per_source } def relative_metadata_dir_for_spec(self, spec): return os.path.join( self.get_relative_projection_for_spec(spec), spack.store.STORE.layout.metadata_dir, spec.name, ) def link_metadata(self, specs): metadata_visitor = SourceMergeVisitor() for spec in specs: src_prefix = os.path.join( spec.package.view_source(), spack.store.STORE.layout.metadata_dir ) proj = self.relative_metadata_dir_for_spec(spec) metadata_visitor.set_projection(proj) visit_directory_tree(src_prefix, metadata_visitor) # Check for conflicts in destination dir. visit_directory_tree(self._root, DestinationMergeVisitor(metadata_visitor)) # Throw on dir-file conflicts -- unlikely, but who knows. if metadata_visitor.fatal_conflicts: raise MergeConflictSummary(metadata_visitor.fatal_conflicts) # We are strict here for historical reasons if metadata_visitor.file_conflicts: raise MergeConflictSummary(metadata_visitor.file_conflicts) for dst in metadata_visitor.directories: os.mkdir(os.path.join(self._root, dst)) for dst_relpath, (src_root, src_relpath) in metadata_visitor.files.items(): self.link(os.path.join(src_root, src_relpath), os.path.join(self._root, dst_relpath)) def get_relative_projection_for_spec(self, spec): # Extensions are placed by their extendee, not by their own spec if spec.package.extendee_spec: spec = spec.package.extendee_spec p = spack.projections.get_projection(self.projections, spec) return spec.format_path(p) if p else "" def get_projection_for_spec(self, spec): """ Return the projection for a spec in this view. Relies on the ordering of projections to avoid ambiguity. """ spec = spack.spec.Spec(spec) if spec.package.extendee_spec: spec = spec.package.extendee_spec proj = spack.projections.get_projection(self.projections, spec) if proj: return os.path.join(self._root, spec.format_path(proj)) return self._root ##################### # utility functions # ##################### def get_spec_from_file(filename) -> Optional[spack.spec.Spec]: try: with open(filename, "r", encoding="utf-8") as f: return spack.spec.Spec.from_yaml(f) except OSError: return None def colorize_root(root): colorize = ft.partial(tty.color.colorize, color=sys.stdout.isatty()) pre, post = map(colorize, "@M[@. @M]@.".split()) return "".join([pre, root, post]) def colorize_spec(spec): "Colorize spec output if in TTY." if sys.stdout.isatty(): return spec.cshort_spec else: return spec.short_spec def find_dependents(all_specs, providers, deptype="run"): """ Return a set containing all those specs from all_specs that depend on providers at the given dependency type. """ dependents = set() for s in all_specs: for dep in s.traverse(deptype=deptype): if dep in providers: dependents.add(s) return dependents def filter_exclude(specs, exclude): "Filter specs given sequence of exclude regex" to_exclude = [re.compile(e) for e in exclude] def keep(spec): for e in to_exclude: if e.match(spec.name): return False return True return filter(keep, specs) def get_dependencies(specs): "Get set of dependencies (includes specs)" retval = set() set(map(retval.update, (set(s.traverse()) for s in specs))) return retval class ConflictingProjectionsError(SpackError): """Raised when a view has a projections file and is given one manually.""" def is_folder_on_case_insensitive_filesystem(path: str) -> bool: with tempfile.NamedTemporaryFile(dir=path, prefix=".sentinel") as sentinel: return os.path.exists(os.path.join(path, os.path.basename(sentinel.name).upper())) ================================================ FILE: lib/spack/spack/graph.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) r"""Functions for graphing DAGs of dependencies. This file contains code for graphing DAGs of software packages (i.e. Spack specs). There are two main functions you probably care about: :func:`graph_ascii` will output a colored graph of a spec in ascii format, kind of like the graph git shows with ``git log --graph``, e.g. .. code-block:: text o mpileaks |\ | |\ | o | callpath |/| | | |\| | |\ \ | | |\ \ | | | | o adept-utils | |_|_|/| |/| | | | o | | | | mpi / / / / | | o | dyninst | |/| | |/|/| | | | |/ | o | libdwarf |/ / o | libelf / o boost :func:`graph_dot` will output a graph of a spec (or multiple specs) in dot format. """ import enum import sys from typing import List, Optional, Set, TextIO, Tuple import spack.deptypes as dt import spack.llnl.util.tty.color import spack.spec import spack.tengine import spack.traverse from spack.solver.input_analysis import create_graph_analyzer def find(seq, predicate): """Find index in seq for which predicate is True. Searches the sequence and returns the index of the element for which the predicate evaluates to True. Returns -1 if the predicate does not evaluate to True for any element in seq. """ for i, elt in enumerate(seq): if predicate(elt): return i return -1 class _GraphLineState(enum.Enum): """Names of different graph line states.""" NODE = enum.auto() COLLAPSE = enum.auto() MERGE_RIGHT = enum.auto() EXPAND_RIGHT = enum.auto() BACK_EDGE = enum.auto() class AsciiGraph: def __init__(self): # These can be set after initialization or after a call to # graph() to change behavior. self.node_character = "o" self.debug = False self.indent = 0 self.depflag = dt.ALL # These are colors in the order they'll be used for edges. # See spack.llnl.util.tty.color for details on color characters. self.colors = "rgbmcyRGBMCY" # Internal vars are used in the graph() function and are initialized there self._name_to_color = None # Node name to color self._out = None # Output stream self._frontier = None # frontier self._prev_state = None # State of previous line self._prev_index = None # Index of expansion point of prev line self._pos = None def _indent(self): self._out.write(self.indent * " ") def _write_edge(self, string, index, sub=0): """Write a colored edge to the output stream.""" # Ignore empty frontier entries (they're just collapsed) if not self._frontier[index]: return name = self._frontier[index][sub] edge = f"@{self._name_to_color[name]}{{{string}}}" self._out.write(edge) def _connect_deps(self, i, deps, label=None): """Connect dependencies to existing edges in the frontier. ``deps`` are to be inserted at position i in the frontier. This routine determines whether other open edges should be merged with (if there are other open edges pointing to the same place) or whether they should just be inserted as a completely new open edge. Open edges that are not fully expanded (i.e. those that point at multiple places) are left intact. Parameters: label -- optional debug label for the connection. Returns: True if the deps were connected to another edge (i.e. the frontier did not grow) and False if the deps were NOT already in the frontier (i.e. they were inserted and the frontier grew). """ if len(deps) == 1 and deps in self._frontier: j = self._frontier.index(deps) # convert a right connection into a left connection if i < j: self._frontier.pop(j) self._frontier.insert(i, deps) return self._connect_deps(j, deps, label) collapse = True if self._prev_state == _GraphLineState.EXPAND_RIGHT: # Special case where previous line expanded and i is off by 1. self._back_edge_line([], j, i + 1, True, label + "-1.5 " + str((i + 1, j))) collapse = False else: # Previous node also expanded here, so i is off by one. if self._prev_state == _GraphLineState.NODE and self._prev_index < i: i += 1 if i - j > 1: # We need two lines to connect if distance > 1 self._back_edge_line([], j, i, True, label + "-1 " + str((i, j))) collapse = False self._back_edge_line([j], -1, -1, collapse, label + "-2 " + str((i, j))) return True if deps: self._frontier.insert(i, deps) return False return False def _set_state(self, state, index, label=None): self._prev_state = state self._prev_index = index if self.debug: self._out.write(" " * 20) self._out.write(f"{str(self._prev_state) if self._prev_state else '':<20}") self._out.write(f"{str(label) if label else '':<20}") self._out.write(f"{self._frontier}") def _back_edge_line(self, prev_ends, end, start, collapse, label=None): """Write part of a backwards edge in the graph. Writes single- or multi-line backward edges in an ascii graph. For example, a single line edge:: | | | | o | | | | |/ / <-- single-line edge connects two nodes. | | | o | Or a multi-line edge (requires two calls to back_edge):: | | | | o | | |_|_|/ / <-- multi-line edge crosses vertical edges. |/| | | | o | | | | Also handles "pipelined" edges, where the same line contains parts of multiple edges:: o start | |_|_|_|/| |/| | |_|/| <-- this line has parts of 2 edges. | | |/| | | o o Arguments: prev_ends -- indices in frontier of previous edges that need to be finished on this line. end -- end of the current edge on this line. start -- start index of the current edge. collapse -- whether the graph will be collapsing (i.e. whether to slant the end of the line or keep it straight) label -- optional debug label to print after the line. """ def advance(to_pos, edges): """Write edges up to .""" for i in range(self._pos, to_pos): for e in edges(): self._write_edge(*e) self._pos += 1 flen = len(self._frontier) self._pos = 0 self._indent() for p in prev_ends: advance(p, lambda: [("| ", self._pos)]) advance(p + 1, lambda: [("|/", self._pos)]) if end >= 0: advance(end + 1, lambda: [("| ", self._pos)]) advance(start - 1, lambda: [("|", self._pos), ("_", end)]) else: advance(start - 1, lambda: [("| ", self._pos)]) if start >= 0: advance(start, lambda: [("|", self._pos), ("/", end)]) if collapse: advance(flen, lambda: [(" /", self._pos)]) else: advance(flen, lambda: [("| ", self._pos)]) self._set_state(_GraphLineState.BACK_EDGE, end, label) self._out.write("\n") def _node_label(self, node): return node.format("{name}@@{version}{/hash:7}") def _node_line(self, index, node): """Writes a line with a node at index.""" self._indent() for c in range(index): self._write_edge("| ", c) self._out.write(f"{self.node_character} ") for c in range(index + 1, len(self._frontier)): self._write_edge("| ", c) self._out.write(self._node_label(node)) self._set_state(_GraphLineState.NODE, index) self._out.write("\n") def _collapse_line(self, index): """Write a collapsing line after a node was added at index.""" self._indent() for c in range(index): self._write_edge("| ", c) for c in range(index, len(self._frontier)): self._write_edge(" /", c) self._set_state(_GraphLineState.COLLAPSE, index) self._out.write("\n") def _merge_right_line(self, index): """Edge at index is same as edge to right. Merge directly with '\'""" self._indent() for c in range(index): self._write_edge("| ", c) self._write_edge("|", index) self._write_edge("\\", index + 1) for c in range(index + 1, len(self._frontier)): self._write_edge("| ", c) self._set_state(_GraphLineState.MERGE_RIGHT, index) self._out.write("\n") def _expand_right_line(self, index): self._indent() for c in range(index): self._write_edge("| ", c) self._write_edge("|", index) self._write_edge("\\", index + 1) for c in range(index + 2, len(self._frontier)): self._write_edge(" \\", c) self._set_state(_GraphLineState.EXPAND_RIGHT, index) self._out.write("\n") def write(self, spec, color=None, out=None): """Write out an ascii graph of the provided spec. Arguments: spec: spec to graph. This only handles one spec at a time. out: file object to write out to (default is sys.stdout) color: whether to write in color. Default is to autodetect based on output file. """ if out is None: out = sys.stdout if color is None: color = out.isatty() self._out = spack.llnl.util.tty.color.ColorStream(out, color=color) # We'll traverse the spec in topological order as we graph it. nodes_in_topological_order = list(spec.traverse(order="topo", deptype=self.depflag)) nodes_in_topological_order.reverse() # Work on a copy to be nondestructive spec = spec.copy() # Colors associated with each node in the DAG. # Edges are colored by the node they point to. self._name_to_color = { spec.dag_hash(): self.colors[i % len(self.colors)] for i, spec in enumerate(nodes_in_topological_order) } # Frontier tracks open edges of the graph as it's written out. self._frontier = [[spec.dag_hash()]] while self._frontier: # Find an unexpanded part of frontier i = find(self._frontier, lambda f: len(f) > 1) if i >= 0: # Expand frontier until there are enough columns for all children. # Figure out how many back connections there are and # sort them so we do them in order back = [] for d in self._frontier[i]: b = find(self._frontier[:i], lambda f: f == [d]) if b != -1: back.append((b, d)) # Do all back connections in sorted order so we can # pipeline them and save space. if back: back.sort() prev_ends = [] collapse_l1 = False for j, (b, d) in enumerate(back): self._frontier[i].remove(d) if i - b > 1: collapse_l1 = any(not e for e in self._frontier) self._back_edge_line(prev_ends, b, i, collapse_l1, "left-1") del prev_ends[:] prev_ends.append(b) # Check whether we did ALL the deps as back edges, # in which case we're done. pop = not self._frontier[i] collapse_l2 = pop if collapse_l1: collapse_l2 = False if pop: self._frontier.pop(i) self._back_edge_line(prev_ends, -1, -1, collapse_l2, "left-2") elif len(self._frontier[i]) > 1: # Expand forward after doing all back connections if ( i + 1 < len(self._frontier) and len(self._frontier[i + 1]) == 1 and self._frontier[i + 1][0] in self._frontier[i] ): # We need to connect to the element to the right. # Keep lines straight by connecting directly and # avoiding unnecessary expand/contract. name = self._frontier[i + 1][0] self._frontier[i].remove(name) self._merge_right_line(i) else: # Just allow the expansion here. dep_hash = self._frontier[i].pop(0) deps = [dep_hash] self._frontier.insert(i, deps) self._expand_right_line(i) self._frontier.pop(i) self._connect_deps(i, deps, "post-expand") # Handle any remaining back edges to the right j = i + 1 while j < len(self._frontier): deps = self._frontier.pop(j) if not self._connect_deps(j, deps, "back-from-right"): j += 1 else: # Nothing to expand; add dependencies for a node. node = nodes_in_topological_order.pop() # Find the named node in the frontier and draw it. i = find(self._frontier, lambda f: node.dag_hash() in f) self._node_line(i, node) # Replace node with its dependencies self._frontier.pop(i) edges = sorted(node.edges_to_dependencies(depflag=self.depflag), reverse=True) if edges: deps = [e.spec.dag_hash() for e in edges] self._connect_deps(i, deps, "new-deps") # anywhere. elif self._frontier: self._collapse_line(i) def graph_ascii( spec, node="o", out=None, debug=False, indent=0, color=None, depflag: dt.DepFlag = dt.ALL ): graph = AsciiGraph() graph.debug = debug graph.indent = indent graph.node_character = node graph.depflag = depflag graph.write(spec, color=color, out=out) class DotGraphBuilder: """Visit edges of a graph a build DOT options for nodes and edges""" def __init__(self): self.nodes: Set[Tuple[str, str]] = set() self.edges: Set[Tuple[str, str, str]] = set() def visit(self, edge: spack.spec.DependencySpec): """Visit an edge and builds up entries to render the graph""" if edge.parent is None: self.nodes.add(self.node_entry(edge.spec)) return self.nodes.add(self.node_entry(edge.parent)) self.nodes.add(self.node_entry(edge.spec)) self.edges.add(self.edge_entry(edge)) def node_entry(self, node: spack.spec.Spec) -> Tuple[str, str]: """Return a tuple of (node_id, node_options)""" raise NotImplementedError("Need to be implemented by derived classes") def edge_entry(self, edge: spack.spec.DependencySpec) -> Tuple[str, str, str]: """Return a tuple of (parent_id, child_id, edge_options)""" raise NotImplementedError("Need to be implemented by derived classes") def context(self): """Return the context to be used to render the DOT graph template""" result = {"nodes": self.nodes, "edges": self.edges} return result def render(self) -> str: """Return a string with the output in DOT format""" environment = spack.tengine.make_environment() template = environment.get_template("misc/graph.dot") return template.render(self.context()) class SimpleDAG(DotGraphBuilder): """Simple DOT graph, with nodes colored uniformly and edges without properties""" def node_entry(self, node): format_option = "{name}{@version}{/hash:7}{%compiler}" return node.dag_hash(), f'[label="{node.format(format_option)}"]' def edge_entry(self, edge): return edge.parent.dag_hash(), edge.spec.dag_hash(), None class StaticDag(DotGraphBuilder): """DOT graph for possible dependencies""" def node_entry(self, node): return node.name, f'[label="{node.name}"]' def edge_entry(self, edge): return edge.parent.name, edge.spec.name, None class DAGWithDependencyTypes(DotGraphBuilder): """DOT graph with link,run nodes grouped together and edges colored according to the dependency types. """ def __init__(self): super().__init__() self.main_unified_space: Set[str] = set() def visit(self, edge): if edge.parent is None: for node in spack.traverse.traverse_nodes([edge.spec], deptype=dt.LINK | dt.RUN): self.main_unified_space.add(node.dag_hash()) super().visit(edge) def node_entry(self, node): node_str = node.format("{name}{@version}{/hash:7}{%compiler}") options = f'[label="{node_str}", group="build_dependencies", fillcolor="coral"]' if node.dag_hash() in self.main_unified_space: options = f'[label="{node_str}", group="main_psid"]' return node.dag_hash(), options def edge_entry(self, edge): colormap = {"build": "dodgerblue", "link": "crimson", "run": "goldenrod"} label = "" if edge.virtuals: label = f' xlabel="virtuals={",".join(edge.virtuals)}"' return ( edge.parent.dag_hash(), edge.spec.dag_hash(), f'[color="{":".join(colormap[x] for x in dt.flag_to_tuple(edge.depflag))}"' + label + "]", ) def _static_edges(specs, depflag): for spec in specs: *_, edges = create_graph_analyzer().possible_dependencies( spec.name, expand_virtuals=True, allowed_deps=depflag ) for parent_name, dependencies in edges.items(): for dependency_name in dependencies: yield spack.spec.DependencySpec( spack.spec.Spec(parent_name), spack.spec.Spec(dependency_name), depflag=depflag, virtuals=(), ) def static_graph_dot( specs: List[spack.spec.Spec], depflag: dt.DepFlag = dt.ALL, out: Optional[TextIO] = None ): """Static DOT graph with edges to all possible dependencies. Args: specs: abstract specs to be represented depflag: dependency types to consider out: optional output stream. If None sys.stdout is used """ out = out or sys.stdout builder = StaticDag() for edge in _static_edges(specs, depflag): builder.visit(edge) out.write(builder.render()) def graph_dot( specs: List[spack.spec.Spec], builder: Optional[DotGraphBuilder] = None, depflag: dt.DepFlag = dt.ALL, out: Optional[TextIO] = None, ): """DOT graph of the concrete specs passed as input. Args: specs: specs to be represented builder: builder to use to render the graph depflag: dependency types to consider out: optional output stream. If None sys.stdout is used """ if not specs: raise ValueError("Must provide specs to graph_dot") if out is None: out = sys.stdout builder = builder or SimpleDAG() for edge in spack.traverse.traverse_edges( specs, cover="edges", order="breadth", deptype=depflag ): builder.visit(edge) out.write(builder.render()) ================================================ FILE: lib/spack/spack/hash_types.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Definitions that control how Spack creates Spec hashes.""" from typing import TYPE_CHECKING, Callable, List, Optional import spack.deptypes as dt import spack.repo if TYPE_CHECKING: import spack.spec class SpecHashDescriptor: """This class defines how hashes are generated on Spec objects. Spec hashes in Spack are generated from a serialized (e.g., with YAML) representation of the Spec graph. The representation may only include certain dependency types, and it may optionally include a canonicalized hash of the package.py for each node in the graph. We currently use different hashes for different use cases.""" __slots__ = "depflag", "package_hash", "name", "attr", "override" def __init__( self, depflag: dt.DepFlag, package_hash: bool, name: str, override: Optional[Callable[["spack.spec.Spec"], str]] = None, ) -> None: self.depflag = depflag self.package_hash = package_hash self.name = name self.attr = f"_{name}" # Allow spec hashes to have an alternate computation method self.override = override def __call__(self, spec: "spack.spec.Spec") -> str: """Run this hash on the provided spec.""" return spec.spec_hash(self) def __repr__(self) -> str: return ( f"SpecHashDescriptor(depflag={self.depflag!r}, " f"package_hash={self.package_hash!r}, name={self.name!r}, override={self.override!r})" ) #: The DAG hash includes all inputs that can affect how a package is built. dag_hash = SpecHashDescriptor( depflag=dt.BUILD | dt.LINK | dt.RUN | dt.TEST, package_hash=True, name="hash" ) def _content_hash_override(spec: "spack.spec.Spec") -> str: pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) pkg = pkg_cls(spec) return pkg.content_hash() #: Package hash used as part of dag hash package_hash = SpecHashDescriptor( depflag=0, package_hash=True, name="package_hash", override=_content_hash_override ) # Deprecated hash types, no longer used, but needed to understand old serialized # spec formats full_hash = SpecHashDescriptor( depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=True, name="full_hash" ) build_hash = SpecHashDescriptor( depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=False, name="build_hash" ) HASHES: List["SpecHashDescriptor"] = [dag_hash, package_hash, full_hash, build_hash] ================================================ FILE: lib/spack/spack/hooks/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This package contains modules with hooks for various stages in the Spack install process. You can add modules here and they'll be executed by package at various times during the package lifecycle. Each hook is just a function that takes a package as a parameter. Hooks are not executed in any particular order. Currently the following hooks are supported: * ``pre_install(spec)`` * ``post_install(spec, explicit)`` * ``pre_uninstall(spec)`` * ``post_uninstall(spec)`` This can be used to implement support for things like module systems (e.g. modules, lmod, etc.) or to add other custom features. """ import importlib import types from typing import List, Optional class _HookRunner: #: Order in which hooks are executed HOOK_ORDER = [ "spack.hooks.module_file_generation", "spack.hooks.licensing", "spack.hooks.sbang", "spack.hooks.windows_runtime_linkage", "spack.hooks.drop_redundant_rpaths", "spack.hooks.absolutify_elf_sonames", "spack.hooks.permissions_setters", "spack.hooks.resolve_shared_libraries", # after all mutations to the install prefix, write metadata "spack.hooks.write_install_manifest", # after all metadata is written "spack.hooks.autopush", ] #: Contains all hook modules after first call, shared among all HookRunner objects _hooks: Optional[List[types.ModuleType]] = None def __init__(self, hook_name): self.hook_name = hook_name @property def hooks(self) -> List[types.ModuleType]: if not self._hooks: self._hooks = [importlib.import_module(module_name) for module_name in self.HOOK_ORDER] return self._hooks def __call__(self, *args, **kwargs): for module in self.hooks: if hasattr(module, self.hook_name): hook = getattr(module, self.hook_name) if hasattr(hook, "__call__"): hook(*args, **kwargs) # pre/post install and run by the install subprocess pre_install = _HookRunner("pre_install") post_install = _HookRunner("post_install") pre_uninstall = _HookRunner("pre_uninstall") post_uninstall = _HookRunner("post_uninstall") ================================================ FILE: lib/spack/spack/hooks/absolutify_elf_sonames.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import spack.bootstrap import spack.config import spack.llnl.util.tty as tty import spack.relocate from spack.llnl.util.filesystem import BaseDirectoryVisitor, visit_directory_tree from spack.llnl.util.lang import elide_list from spack.util.elf import ElfParsingError, parse_elf def is_shared_library_elf(filepath): """Return true if filepath is most a shared library. Our definition of a shared library for ELF requires: 1. a dynamic section, 2. a soname OR lack of interpreter. The problem is that PIE objects (default on Ubuntu) are ET_DYN too, and not all shared libraries have a soname... no interpreter is typically the best indicator then.""" try: with open(filepath, "rb") as f: elf = parse_elf(f, interpreter=True, dynamic_section=True) return elf.has_pt_dynamic and (elf.has_soname or not elf.has_pt_interp) except (OSError, ElfParsingError): return False class SharedLibrariesVisitor(BaseDirectoryVisitor): """Visitor that collects all shared libraries in a prefix, with the exception of an exclude list.""" def __init__(self, exclude_list): # List of file and directory names to be excluded self.exclude_list = frozenset(exclude_list) # Map from (ino, dev) -> path. We need 1 path per file, if there are hardlinks, # we don't need to store the path multiple times. self.libraries = dict() # Set of (ino, dev) pairs (excluded by symlinks). self.excluded_through_symlink = set() def visit_file(self, root, rel_path, depth): # Check if excluded basename = os.path.basename(rel_path) if basename in self.exclude_list: return filepath = os.path.join(root, rel_path) s = os.lstat(filepath) identifier = (s.st_ino, s.st_dev) # We're hitting a hardlink or symlink of an excluded lib, no need to parse. if identifier in self.libraries or identifier in self.excluded_through_symlink: return # Register the file if it's a shared lib that needs to be patched. if is_shared_library_elf(filepath): self.libraries[identifier] = rel_path def visit_symlinked_file(self, root, rel_path, depth): # We don't need to follow the symlink and parse the file, since we will hit # it by recursing the prefix anyways. We only need to check if the target # should be excluded based on the filename of the symlink. E.g. when excluding # libf.so, which is a symlink to libf.so.1.2.3, we keep track of the stat data # of the latter. basename = os.path.basename(rel_path) if basename not in self.exclude_list: return # Register the (ino, dev) pair as ignored (if the symlink is not dangling) filepath = os.path.join(root, rel_path) try: s = os.stat(filepath) except OSError: return self.excluded_through_symlink.add((s.st_ino, s.st_dev)) def before_visit_dir(self, root, rel_path, depth): # Allow skipping over directories. E.g. `/lib/stubs` can be skipped by # adding `"stubs"` to the exclude list. return os.path.basename(rel_path) not in self.exclude_list def before_visit_symlinked_dir(self, root, rel_path, depth): # Never enter symlinked dirs, since we don't want to leave the prefix, and # we'll enter the target dir inside the prefix anyways since we're recursing # everywhere. return False def get_shared_libraries_relative_paths(self): """Get the libraries that should be patched, with the excluded libraries removed.""" for identifier in self.excluded_through_symlink: self.libraries.pop(identifier, None) return [rel_path for rel_path in self.libraries.values()] def patch_sonames(patchelf, root, rel_paths): """Set the soname to the file's own path for a list of given shared libraries.""" fixed = [] for rel_path in rel_paths: filepath = os.path.join(root, rel_path) normalized = os.path.normpath(filepath) args = ["--set-soname", normalized, normalized] output = patchelf(*args, output=str, error=str, fail_on_error=False) if patchelf.returncode == 0: fixed.append(rel_path) else: # Note: treat as warning to avoid (long) builds to fail post-install. tty.warn("patchelf: failed to set soname of {}: {}".format(normalized, output.strip())) return fixed def find_and_patch_sonames(prefix, exclude_list, patchelf): # Locate all shared libraries in the prefix dir of the spec, excluding # the ones set in the non_bindable_shared_objects property. visitor = SharedLibrariesVisitor(exclude_list) visit_directory_tree(prefix, visitor) # Patch all sonames. relative_paths = visitor.get_shared_libraries_relative_paths() return patch_sonames(patchelf, prefix, relative_paths) def post_install(spec, explicit=None): # Skip if disabled if not spack.config.get("config:shared_linking:bind", False): return # Skip externals if spec.external: return # Only enable on platforms using ELF. if not spec.satisfies("platform=linux"): return # Disable this hook when bootstrapping, to avoid recursion. if spack.bootstrap.is_bootstrapping(): return # Should failing to locate patchelf be a hard error? patchelf = spack.relocate._patchelf() if not patchelf: return fixes = find_and_patch_sonames(spec.prefix, spec.package.non_bindable_shared_objects, patchelf) if not fixes: return # Unfortunately this does not end up in the build logs. tty.info( "{}: Patched {} {}: {}".format( spec.name, len(fixes), "soname" if len(fixes) == 1 else "sonames", ", ".join(elide_list(fixes, max_num=5)), ) ) ================================================ FILE: lib/spack/spack/hooks/autopush.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.binary_distribution import spack.llnl.util.tty as tty import spack.mirrors.mirror def post_install(spec, explicit): # Push package to all buildcaches with autopush==True # Do nothing if spec is an external package if spec.external: return # Do nothing if package was not installed from source pkg = spec.package if pkg.installed_from_binary_cache: return # Push the package to all autopush mirrors for mirror in spack.mirrors.mirror.MirrorCollection(binary=True, autopush=True).values(): signing_key = spack.binary_distribution.select_signing_key() if mirror.signed else None with spack.binary_distribution.make_uploader( mirror=mirror, force=True, signing_key=signing_key ) as uploader: uploader.push_or_raise([spec]) tty.msg(f"{spec.name}: Pushed to build cache: '{mirror.name}'") ================================================ FILE: lib/spack/spack/hooks/drop_redundant_rpaths.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os from typing import BinaryIO, Optional, Tuple import spack.llnl.util.tty as tty from spack.llnl.util.filesystem import BaseDirectoryVisitor, visit_directory_tree from spack.util.elf import ElfParsingError, parse_elf def should_keep(path: bytes) -> bool: """Return True iff path starts with $ (typically for $ORIGIN/${ORIGIN}) or is absolute and exists.""" return path.startswith(b"$") or (os.path.isabs(path) and os.path.lexists(path)) def _drop_redundant_rpaths(f: BinaryIO) -> Optional[Tuple[bytes, bytes]]: """Drop redundant entries from rpath. Args: f: File object to patch opened in r+b mode. Returns: A tuple of the old and new rpath if the rpath was patched, None otherwise. """ try: elf = parse_elf(f, interpreter=False, dynamic_section=True) except ElfParsingError: return None # Nothing to do. if not elf.has_rpath: return None old_rpath_str = elf.dt_rpath_str new_rpath_str = b":".join(p for p in old_rpath_str.split(b":") if should_keep(p)) # Nothing to write. if old_rpath_str == new_rpath_str: return None # Pad with 0 bytes. pad = len(old_rpath_str) - len(new_rpath_str) # This can never happen since we're filtering, but lets be defensive. if pad < 0: return None # The rpath is at a given offset in the string table used by the # dynamic section. rpath_offset = elf.pt_dynamic_strtab_offset + elf.rpath_strtab_offset f.seek(rpath_offset) f.write(new_rpath_str + b"\x00" * pad) return old_rpath_str, new_rpath_str def drop_redundant_rpaths(path: str) -> Optional[Tuple[bytes, bytes]]: """Drop redundant entries from rpath. Args: path: Path to a potential ELF file to patch. Returns: A tuple of the old and new rpath if the rpath was patched, None otherwise. """ try: with open(path, "r+b") as f: return _drop_redundant_rpaths(f) except OSError: return None class ElfFilesWithRPathVisitor(BaseDirectoryVisitor): """Visitor that collects all elf files that have an rpath""" def __init__(self): # Keep track of what hardlinked files we've already visited. self.visited = set() def visit_file(self, root, rel_path, depth): filepath = os.path.join(root, rel_path) s = os.lstat(filepath) identifier = (s.st_ino, s.st_dev) # We're hitting a hardlink or symlink of an excluded lib, no need to parse. if s.st_nlink > 1: if identifier in self.visited: return self.visited.add(identifier) result = drop_redundant_rpaths(filepath) if result is not None: old, new = result tty.debug(f"Patched rpath in {rel_path} from {old!r} to {new!r}") def visit_symlinked_file(self, root, rel_path, depth): pass def before_visit_dir(self, root, rel_path, depth): # Always enter dirs return True def before_visit_symlinked_dir(self, root, rel_path, depth): # Never enter symlinked dirs return False def post_install(spec, explicit=None): # Skip externals if spec.external: return # Only enable on platforms using ELF. if not spec.satisfies("platform=linux"): return visit_directory_tree(spec.prefix, ElfFilesWithRPathVisitor()) ================================================ FILE: lib/spack/spack/hooks/licensing.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import spack.llnl.util.tty as tty import spack.util.editor as ed from spack.llnl.util.filesystem import mkdirp, symlink def pre_install(spec): """This hook handles global license setup for licensed software.""" pkg = spec.package if pkg.license_required and not pkg.spec.external: set_up_license(pkg) def set_up_license(pkg): """Prompt the user, letting them know that a license is required. For packages that rely on license files, a global license file is created and opened for editing. For packages that rely on environment variables to point to a license, a warning message is printed. For all other packages, documentation on how to set up a license is printed.""" # If the license can be stored in a file, create one if pkg.license_files: license_path = pkg.global_license_file if not os.path.exists(license_path): # Create a new license file write_license_file(pkg, license_path) # use spack.util.executable so the editor does not hang on return here ed.editor(license_path, exec_fn=ed.executable) else: # Use already existing license file tty.msg("Found already existing license %s" % license_path) # If not a file, what about an environment variable? elif pkg.license_vars: tty.warn( "A license is required to use %s. Please set %s to the " "full pathname to the license file, or port@host if you" " store your license keys on a dedicated license server" % (pkg.name, " or ".join(pkg.license_vars)) ) # If not a file or variable, suggest a website for further info elif pkg.license_url: tty.warn( "A license is required to use %s. See %s for details" % (pkg.name, pkg.license_url) ) # If all else fails, you're on your own else: tty.warn("A license is required to use %s" % pkg.name) def write_license_file(pkg, license_path): """Writes empty license file. Comments give suggestions on alternative methods of installing a license.""" # License files linktargets = "" for f in pkg.license_files: linktargets += "\t%s\n" % f # Environment variables envvars = "" if pkg.license_vars: for varname in pkg.license_vars: envvars += "\t%s\n" % varname # Documentation url = "" if pkg.license_url: url += "\t%s\n" % pkg.license_url # Assemble. NB: pkg.license_comment will be prepended upon output. txt = """ A license is required to use package '{0}'. * If your system is already properly configured for such a license, save this file UNCHANGED. The system may be configured if: - A license file is installed in a default location. """.format(pkg.name) if envvars: txt += """\ - One of the following environment variable(s) is set for you, possibly via a module file: {0} """.format(envvars) txt += """\ * Otherwise, depending on the license you have, enter AT THE BEGINNING of this file: - the contents of your license file, or - the address(es) of your license server. After installation, the following symlink(s) will be added to point to this Spack-global file (relative to the installation prefix). {0} """.format(linktargets) if url: txt += """\ * For further information on licensing, see: {0} """.format(url) txt += """\ Recap: - You may not need to modify this file at all. - Otherwise, enter your license or server address AT THE BEGINNING. """ # Global license directory may not already exist if not os.path.exists(os.path.dirname(license_path)): os.makedirs(os.path.dirname(license_path)) # Output with open(license_path, "w", encoding="utf-8") as f: for line in txt.splitlines(): f.write("{0}{1}\n".format(pkg.license_comment, line)) f.close() def post_install(spec, explicit=None): """This hook symlinks local licenses to the global license for licensed software. """ pkg = spec.package if pkg.license_required and not pkg.spec.external: symlink_license(pkg) def symlink_license(pkg): """Create local symlinks that point to the global license file.""" target = pkg.global_license_file for filename in pkg.license_files: link_name = os.path.join(pkg.prefix, filename) link_name = os.path.abspath(link_name) license_dir = os.path.dirname(link_name) if not os.path.exists(license_dir): mkdirp(license_dir) # If example file already exists, overwrite it with a symlink if os.path.lexists(link_name): os.remove(link_name) if os.path.exists(target): symlink(target, link_name) tty.msg("Added local symlink %s to global license file" % link_name) ================================================ FILE: lib/spack/spack/hooks/module_file_generation.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from typing import Optional, Set import spack.config import spack.modules import spack.spec from spack.llnl.util import tty def _for_each_enabled( spec: spack.spec.Spec, method_name: str, explicit: Optional[bool] = None ) -> None: """Calls a method for each enabled module""" set_names: Set[str] = set(spack.config.get("modules", {}).keys()) for name in set_names: enabled = spack.config.get(f"modules:{name}:enable") if not enabled: tty.debug("NO MODULE WRITTEN: list of enabled module files is empty") continue for module_type in enabled: generator = spack.modules.module_types[module_type](spec, name, explicit) try: getattr(generator, method_name)() except RuntimeError as e: msg = "cannot perform the requested {0} operation on module files" msg += " [{1}]" tty.warn(msg.format(method_name, str(e))) def post_install(spec, explicit: bool): _for_each_enabled(spec, "write", explicit) def post_uninstall(spec): _for_each_enabled(spec, "remove") ================================================ FILE: lib/spack/spack/hooks/permissions_setters.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import spack.util.file_permissions as fp def post_install(spec, explicit=None): if not spec.external: fp.set_permissions_by_spec(spec.prefix, spec) # os.walk explicitly set not to follow links for root, dirs, files in os.walk(spec.prefix, followlinks=False): for d in dirs: if not os.path.islink(os.path.join(root, d)): fp.set_permissions_by_spec(os.path.join(root, d), spec) for f in files: if not os.path.islink(os.path.join(root, f)): fp.set_permissions_by_spec(os.path.join(root, f), spec) ================================================ FILE: lib/spack/spack/hooks/resolve_shared_libraries.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import spack.config import spack.error import spack.llnl.util.tty as tty import spack.verify_libraries from spack.llnl.util.filesystem import visit_directory_tree def post_install(spec, explicit): """Check whether shared libraries can be resolved in RPATHs.""" policy = spack.config.get("config:shared_linking:missing_library_policy", "ignore") # Currently only supported for ELF files. if policy == "ignore" or spec.external or spec.platform not in ("linux", "freebsd"): return visitor = spack.verify_libraries.ResolveSharedElfLibDepsVisitor( [*spack.verify_libraries.ALLOW_UNRESOLVED, *spec.package.unresolved_libraries] ) visit_directory_tree(spec.prefix, visitor) if not visitor.problems: return output = io.StringIO("not all executables and libraries can resolve their dependencies:\n") visitor.write(output) message = output.getvalue().strip() if policy == "error": raise CannotLocateSharedLibraries(message) tty.warn(message) class CannotLocateSharedLibraries(spack.error.SpackError): pass ================================================ FILE: lib/spack/spack/hooks/sbang.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp import os import re import shutil import stat import sys import tempfile import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.package_prefs import spack.paths import spack.spec import spack.store from spack.util.socket import _gethostname #: OS-imposed character limit for shebang line: 127 for Linux; 511 for Mac. #: Different Linux distributions have different limits, but 127 is the #: smallest among all modern versions. if sys.platform == "darwin": system_shebang_limit = 511 else: system_shebang_limit = 127 try: # searching for line '#define BINPRM_BUF_SIZE 256' in /usr/include/linux/binfmts.h # the nbr-1 is the sbang limit on the linux platform sbang_limit_re = re.compile("#define BINPRM_BUF_SIZE ([0-9]+)") with open("/usr/include/linux/binfmts.h", "r", encoding="utf-8") as f: for line in f: m = sbang_limit_re.match(line) if m: system_shebang_limit = int(m.group(1)) - 1 except Exception: # ignore any error a sane default is set already pass #: Groupdb does not exist on Windows, prevent imports #: on supported systems if sys.platform != "win32": import grp #: Spack itself also limits the shebang line to at most 4KB, which should be plenty. spack_shebang_limit = 4096 interpreter_regex = re.compile(b"#![ \t]*?([^ \t\0\n]+)") def sbang_install_path(): """Location sbang is installed within the install tree.""" sbang_root = str(spack.store.STORE.unpadded_root) install_path = os.path.join(sbang_root, "bin", "sbang") path_length = len(install_path) if path_length > system_shebang_limit: msg = ( "Install tree root is too long. Spack cannot patch shebang lines" " when script path length ({0}) exceeds limit ({1}).\n {2}" ) msg = msg.format(path_length, system_shebang_limit, install_path) raise SbangPathError(msg) return install_path def sbang_shebang_line(): """Full shebang line that should be prepended to files to use sbang. The line returned does not have a final newline (caller should add it if needed). This should be the only place in Spack that knows about what interpreter we use for ``sbang``. """ return "#!/bin/sh %s" % sbang_install_path() def get_interpreter(binary_string): # The interpreter may be preceded with ' ' and \t, is itself any byte that # follows until the first occurrence of ' ', \t, \0, \n or end of file. match = interpreter_regex.match(binary_string) return None if match is None else match.group(1) def filter_shebang(path): """ Adds a second shebang line, using sbang, at the beginning of a file, if necessary. Note: Spack imposes a relaxed shebang line limit, meaning that a newline or end of file must occur before ``spack_shebang_limit`` bytes. If not, the file is not patched. """ with open(path, "rb") as original: # If there is no shebang, we shouldn't replace anything. old_shebang_line = original.read(2) if old_shebang_line != b"#!": return False # Stop reading after b'\n'. Note that old_shebang_line includes the first b'\n'. old_shebang_line += original.readline(spack_shebang_limit - 2) # If the shebang line is short, we don't have to do anything. if len(old_shebang_line) <= system_shebang_limit: return False # Whenever we can't find a newline within the maximum number of bytes, we will # not attempt to rewrite it. In principle we could still get the interpreter if # only the arguments are truncated, but note that for PHP we need the full line # since we have to append `?>` to it. Since our shebang limit is already very # generous, it's unlikely to happen, and it should be fine to ignore. if len(old_shebang_line) == spack_shebang_limit and old_shebang_line[-1] != b"\n": return False # This line will be prepended to file new_sbang_line = (sbang_shebang_line() + "\n").encode("utf-8") # Skip files that are already using sbang. if old_shebang_line == new_sbang_line: return interpreter = get_interpreter(old_shebang_line) # If there was only whitespace we don't have to do anything. if not interpreter: return False # Store the file permissions, the patched version needs the same. saved_mode = os.stat(path).st_mode # Change non-writable files to be writable if needed. if not os.access(path, os.W_OK): os.chmod(path, saved_mode | stat.S_IWUSR) # No need to delete since we'll move it and overwrite the original. patched = tempfile.NamedTemporaryFile("wb", delete=False) patched.write(new_sbang_line) # Note that in Python this does not go out of bounds even if interpreter is a # short byte array. # Note: if the interpreter string was encoded with UTF-16, there would have # been a \0 byte between all characters of lua, node, php; meaning that it would # lead to truncation of the interpreter. So we don't have to worry about weird # encodings here, and just looking at bytes is justified. if interpreter[-4:] == b"/lua" or interpreter[-7:] == b"/luajit": # Use --! instead of #! on second line for lua. patched.write(b"--!" + old_shebang_line[2:]) elif interpreter[-5:] == b"/node": # Use //! instead of #! on second line for node.js. patched.write(b"//!" + old_shebang_line[2:]) elif interpreter[-4:] == b"/php": # Use instead of #!... on second line for php. patched.write(b"") else: patched.write(old_shebang_line) # After copying the remainder of the file, we can close the original shutil.copyfileobj(original, patched) # And close the temporary file so we can move it. patched.close() # Overwrite original file with patched file, and keep the original mode shutil.move(patched.name, path) os.chmod(path, saved_mode) return True def filter_shebangs_in_directory(directory, filenames=None): if filenames is None: filenames = os.listdir(directory) is_exe = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH for file in filenames: path = os.path.join(directory, file) # Only look at executable, non-symlink files. try: st = os.lstat(path) except OSError: continue if stat.S_ISLNK(st.st_mode) or stat.S_ISDIR(st.st_mode) or not st.st_mode & is_exe: continue # test the file for a long shebang, and filter if filter_shebang(path): tty.debug("Patched overlong shebang in %s" % path) def install_sbang(): """Ensure that ``sbang`` is installed in the root of Spack's install_tree. This is the shortest known publicly accessible path, and installing ``sbang`` here ensures that users can access the script and that ``sbang`` itself is in a short path. """ # copy in a new version of sbang if it differs from what's in spack sbang_path = sbang_install_path() if os.path.exists(sbang_path) and filecmp.cmp(spack.paths.sbang_script, sbang_path): return # make $install_tree/bin sbang_bin_dir = os.path.dirname(sbang_path) fs.mkdirp(sbang_bin_dir) # get permissions for bin dir from configuration files group_name = spack.package_prefs.get_package_group(spack.spec.Spec("all")) config_mode = spack.package_prefs.get_package_dir_permissions(spack.spec.Spec("all")) if group_name: os.chmod(sbang_bin_dir, config_mode) # Use package directory permissions else: fs.set_install_permissions(sbang_bin_dir) # set group on sbang_bin_dir if not already set (only if set in configuration) # TODO: after we drop python2 support, use shutil.chown to avoid gid lookups that # can fail for remote groups if group_name and os.stat(sbang_bin_dir).st_gid != grp.getgrnam(group_name).gr_gid: os.chown(sbang_bin_dir, os.stat(sbang_bin_dir).st_uid, grp.getgrnam(group_name).gr_gid) # copy over the fresh copy of `sbang` sbang_tmp_path = os.path.join(sbang_bin_dir, f".sbang.{_gethostname()}.{os.getpid()}.tmp") shutil.copy(spack.paths.sbang_script, sbang_tmp_path) # set permissions on `sbang` (including group if set in configuration) os.chmod(sbang_tmp_path, config_mode) if group_name: os.chown(sbang_tmp_path, os.stat(sbang_tmp_path).st_uid, grp.getgrnam(group_name).gr_gid) # Finally, move the new `sbang` into place atomically os.rename(sbang_tmp_path, sbang_path) def post_install(spec, explicit=None): """This hook edits scripts so that they call /bin/bash $spack_prefix/bin/sbang instead of something longer than the shebang limit. """ if sys.platform == "win32": return if spec.external: tty.debug("SKIP: shebang filtering [external package]") return install_sbang() for directory, _, filenames in os.walk(spec.prefix): filter_shebangs_in_directory(directory, filenames) class SbangPathError(spack.error.SpackError): """Raised when the install tree root is too long for sbang to work.""" ================================================ FILE: lib/spack/spack/hooks/windows_runtime_linkage.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) def post_install(spec, explicit=None): spec.package.windows_establish_runtime_linkage() ================================================ FILE: lib/spack/spack/hooks/write_install_manifest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.verify def post_install(spec, explicit=None): if not spec.external: spack.verify.write_manifest(spec) ================================================ FILE: lib/spack/spack/install_test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import base64 import contextlib import enum import hashlib import inspect import io import os import re import shutil import sys from collections import Counter, OrderedDict from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple, Type, Union import spack.config import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.llnl.util.tty.log import spack.paths import spack.repo import spack.report import spack.spec import spack.util.executable import spack.util.path import spack.util.spack_json as sjson from spack.error import InstallError from spack.llnl.string import plural from spack.llnl.util.lang import nullcontext from spack.llnl.util.tty.color import colorize from spack.spec import Spec from spack.util.prefix import Prefix if TYPE_CHECKING: import spack.package_base #: Stand-alone test failure info type TestFailureType = Tuple[BaseException, str] #: Name of the test suite's (JSON) lock file test_suite_filename = "test_suite.lock" #: Name of the test suite results (summary) file results_filename = "results.txt" #: Name of the Spack install phase-time test log file spack_install_test_log = "install-time-test-log.txt" ListOrStringType = Union[str, List[str]] LogType = Union[spack.llnl.util.tty.log.nixlog, spack.llnl.util.tty.log.winlog] PackageObjectOrClass = Union[ "spack.package_base.PackageBase", Type["spack.package_base.PackageBase"] ] class TestStatus(enum.Enum): """Names of different stand-alone test states.""" NO_TESTS = -1 SKIPPED = 0 FAILED = 1 PASSED = 2 def __str__(self): return f"{self.name}" def lower(self): name = f"{self.name}" return name.lower() def get_escaped_text_output(filename: str) -> List[str]: """Retrieve and escape the expected text output from the file Args: filename: path to the file Returns: escaped text lines read from the file """ with open(filename, encoding="utf-8") as f: # Ensure special characters are escaped as needed expected = f.read() # Split the lines to make it easier to debug failures when there is # a lot of output return [re.escape(ln) for ln in expected.split("\n")] def get_test_stage_dir() -> str: """Retrieves the ``config:test_stage`` path to the configured test stage root directory Returns: absolute path to the configured test stage root or, if none, the default test stage path """ return spack.util.path.canonicalize_path( spack.config.get("config:test_stage", spack.paths.default_test_path) ) def cache_extra_test_sources(pkg: "spack.package_base.PackageBase", srcs: ListOrStringType): """Copy relative source paths to the corresponding install test subdir This routine is intended as an optional install test setup helper for grabbing source files/directories during the installation process and copying them to the installation test subdirectory for subsequent use during install testing. Args: pkg: package being tested srcs: relative path for file(s) and or subdirectory(ies) located in the staged source path that are to be copied to the corresponding location(s) under the install testing directory. Raises: spack.error.InstallError: if any of the source paths are absolute or do not exist under the build stage """ errors = [] paths = [srcs] if isinstance(srcs, str) else srcs for path in paths: pre = f"Source path ('{path}')" src_path = os.path.join(pkg.stage.source_path, path) dest_path = os.path.join(install_test_root(pkg), path) if os.path.isabs(path): errors.append(f"{pre} must be relative to the build stage directory.") continue if os.path.isdir(src_path): fs.install_tree(src_path, dest_path) elif os.path.exists(src_path): fs.mkdirp(os.path.dirname(dest_path)) fs.copy(src_path, dest_path) else: errors.append(f"{pre} for the copy does not exist") if errors: raise InstallError("\n".join(errors), pkg=pkg) def check_outputs(expected: Union[list, set, str], actual: str): """Ensure the expected outputs are contained in the actual outputs. Args: expected: expected raw output string(s) actual: actual output string Raises: RuntimeError: the expected output is not found in the actual output """ expected = expected if isinstance(expected, (list, set)) else [expected] errors = [] for check in expected: if not re.search(check, actual): errors.append(f"Expected '{check}' in output '{actual}'") if errors: raise RuntimeError("\n ".join(errors)) def find_required_file( root: str, filename: str, expected: int = 1, recursive: bool = True ) -> ListOrStringType: """Find the required file(s) under the root directory. Args: root: root directory for the search filename: name of the file being located expected: expected number of files to be found under the directory (default is 1) recursive: ``True`` if subdirectories are to be recursively searched, else ``False`` (default is ``True``) Returns: the path(s), relative to root, to the required file(s) Raises: Exception: SkipTest when number of files detected does not match expected """ paths = fs.find(root, filename, recursive=recursive) num_paths = len(paths) if num_paths != expected: files = ": {}".format(", ".join(paths)) if num_paths else "" raise SkipTest( "Expected {} of {} under {} but {} found{}".format( plural(expected, "copy", "copies"), filename, root, plural(num_paths, "copy", "copies"), files, ) ) return paths[0] if expected == 1 else paths def install_test_root(pkg: "spack.package_base.PackageBase") -> str: """The install test root directory.""" return os.path.join(pkg.metadata_dir, "test") def print_message(logger: LogType, msg: str, verbose: bool = False): """Print the message to the log, optionally echoing. Args: logger: instance of the output logger (e.g. nixlog or winlog) msg: message being output verbose: ``True`` displays verbose output, ``False`` suppresses it (``False`` is default) """ if verbose: with logger.force_echo(): tty.info(msg, format="g") else: tty.info(msg, format="g") def overall_status(current_status: "TestStatus", substatuses: List["TestStatus"]) -> "TestStatus": """Determine the overall status based on the current and associated sub status values. Args: current_status: current overall status, assumed to default to PASSED substatuses: status of each test part or overall status of each test spec Returns: test status encompassing the main test and all subtests """ if current_status in [TestStatus.SKIPPED, TestStatus.NO_TESTS, TestStatus.FAILED]: return current_status skipped = 0 for status in substatuses: if status == TestStatus.FAILED: return status elif status == TestStatus.SKIPPED: skipped += 1 if skipped and skipped == len(substatuses): return TestStatus.SKIPPED return current_status class PackageTest: """The class that manages stand-alone (post-install) package tests.""" def __init__(self, pkg: "spack.package_base.PackageBase") -> None: """ Args: pkg: package being tested Raises: ValueError: if the package is not concrete """ if not pkg.spec.concrete: raise ValueError("Stand-alone tests require a concrete package") self.counts: "Counter" = Counter() # type: ignore[attr-defined] self.pkg = pkg self.test_failures: List[TestFailureType] = [] self.test_parts: OrderedDict[str, "TestStatus"] = OrderedDict() self.test_log_file: str self.pkg_id: str if pkg.test_suite: # Running stand-alone tests self.test_log_file = pkg.test_suite.log_file_for_spec(pkg.spec) self.tested_file = pkg.test_suite.tested_file_for_spec(pkg.spec) self.pkg_id = pkg.test_suite.test_pkg_id(pkg.spec) else: # Running phase-time tests for a single package whose results are # retained in the package's stage directory. pkg.test_suite = TestSuite([pkg.spec]) self.test_log_file = fs.join_path(pkg.stage.path, spack_install_test_log) self.pkg_id = pkg.spec.format("{name}-{version}-{hash:7}") # Internal logger for test part processing self._logger = None @property def logger(self) -> Optional[LogType]: """The current logger or, if none, sets to one.""" if not self._logger: self._logger = spack.llnl.util.tty.log.log_output(self.test_log_file) return self._logger @contextlib.contextmanager def test_logger(self, verbose: bool = False, externals: bool = False): """Context manager for setting up the test logger Args: verbose: Display verbose output, including echoing to stdout, otherwise suppress it externals: ``True`` for performing tests if external package, ``False`` to skip them """ fs.touch(self.test_log_file) # Otherwise log_parse complains fs.set_install_permissions(self.test_log_file) with spack.llnl.util.tty.log.log_output( self.test_log_file, verbose, append=True ) as self._logger: with self.logger.force_echo(): # type: ignore[union-attr] tty.msg("Testing package " + colorize(r"@*g{" + self.pkg_id + r"}")) # use debug print levels for log file to record commands old_debug = tty.is_debug() tty.set_debug(True) try: yield self.logger finally: # reset debug level tty.set_debug(old_debug) @property def archived_install_test_log(self) -> str: return fs.join_path(self.pkg.metadata_dir, spack_install_test_log) def archive_install_test_log(self, dest_dir: str): if os.path.exists(self.test_log_file): fs.install(self.test_log_file, self.archived_install_test_log) def add_failure(self, exception: Exception, msg: str): """Add the failure details to the current list.""" self.test_failures.append((exception, msg)) def status(self, name: str, status: "TestStatus", msg: Optional[str] = None): """Track and print the test status for the test part name.""" part_name = f"{self.pkg.__class__.__name__}::{name}" extra = "" if msg is None else f": {msg}" # Handle the special case of a test part consisting of subparts. # The containing test part can be PASSED while sub-parts (assumed # to start with the same name) may not have PASSED. This extra # check is used to ensure the containing test part is not claiming # to have passed when at least one subpart failed. substatuses = [] for pname, substatus in self.test_parts.items(): if pname != part_name and pname.startswith(part_name): substatuses.append(substatus) if substatuses: status = overall_status(status, substatuses) print(f"{status}: {part_name}{extra}") self.test_parts[part_name] = status self.counts[status] += 1 def phase_tests(self, builder, phase_name: str, method_names: List[str]): """Execute the builder's package phase-time tests. Args: builder: builder for package being tested phase_name: the name of the build-time phase (e.g., ``build``, ``install``) method_names: phase-specific callback method names """ verbose = tty.is_verbose() fail_fast = spack.config.get("config:fail_fast", False) with self.test_logger(verbose=verbose, externals=False) as logger: # Report running each of the methods in the build log print_message(logger, f"Running {phase_name}-time tests", verbose) builder.pkg.test_suite.current_test_spec = builder.pkg.spec builder.pkg.test_suite.current_base_spec = builder.pkg.spec have_tests = any(name.startswith("test_") for name in method_names) if have_tests: copy_test_files(builder.pkg, builder.pkg.spec) for name in method_names: try: fn = getattr(builder, name, None) or getattr(builder.pkg, name) except AttributeError as e: print_message(logger, f"RUN-TESTS: method not implemented [{name}]", verbose) self.add_failure(e, f"RUN-TESTS: method not implemented [{name}]") if fail_fast: break continue print_message(logger, f"RUN-TESTS: {phase_name}-time tests [{name}]", verbose) fn() if have_tests: print_message(logger, "Completed testing", verbose) # Raise any collected failures here if self.test_failures: raise TestFailure(self.test_failures) def stand_alone_tests(self, kwargs, timeout: Optional[int] = None) -> None: """Run the package's stand-alone tests. Args: kwargs (dict): arguments to be used by the test process """ import spack.build_environment # avoid circular dependency process = spack.build_environment.start_build_process( self.pkg, test_process, kwargs, timeout=timeout ) process.complete() def parts(self) -> int: """The total number of (checked) test parts.""" try: # New in Python 3.10 total = self.counts.total() # type: ignore[attr-defined] except AttributeError: nums = [n for _, n in self.counts.items()] total = sum(nums) return total def print_log_path(self): """Print the test log file path.""" log = self.archived_install_test_log if not os.path.isfile(log): log = self.test_log_file if not (log and os.path.isfile(log)): tty.debug("There is no test log file (staged or installed)") return print(f"\nSee test results at:\n {log}") def ran_tests(self) -> bool: """``True`` if ran tests, ``False`` otherwise.""" return self.parts() > self.counts[TestStatus.NO_TESTS] def summarize(self): """Collect test results summary lines for this spec.""" lines = [] lines.append("{:=^80}".format(f" SUMMARY: {self.pkg_id} ")) for name, status in self.test_parts.items(): msg = f"{name} .. {status}" lines.append(msg) summary = [f"{n} {s.lower()}" for s, n in self.counts.items() if n > 0] totals = " {} of {} parts ".format(", ".join(summary), self.parts()) lines.append(f"{totals:=^80}") return lines def write_tested_status(self): """Write the overall status to the tested file. If there any test part failures, then the tests failed. If all test parts are skipped, then the tests were skipped. If any tests passed then the tests passed; otherwise, there were not tests executed. """ status = TestStatus.NO_TESTS if self.counts[TestStatus.FAILED] > 0: status = TestStatus.FAILED else: skipped = self.counts[TestStatus.SKIPPED] if skipped and self.parts() == skipped: status = TestStatus.SKIPPED elif self.counts[TestStatus.PASSED] > 0: status = TestStatus.PASSED with open(self.tested_file, "w", encoding="utf-8") as f: f.write(f"{status.value}\n") @contextlib.contextmanager def test_part( pkg: "spack.package_base.PackageBase", test_name: str, purpose: str, work_dir: str = ".", verbose: bool = False, ): # avoid circular dependency from spack.build_environment import get_package_context, write_log_summary wdir = "." if work_dir is None else work_dir tester = pkg.tester assert test_name and test_name.startswith("test_"), ( f"Test name must start with 'test_' but {test_name} was provided" ) title = "test: {}: {}".format(test_name, purpose or "unspecified purpose") with fs.working_dir(wdir, create=True): try: context = tester.logger.force_echo if verbose else nullcontext with context(): tty.info(title, format="g") yield tester.status(test_name, TestStatus.PASSED) except SkipTest as e: tester.status(test_name, TestStatus.SKIPPED, str(e)) except (AssertionError, BaseException) as e: # print a summary of the error to the log file # so that cdash and junit reporters know about it exc_type, _, tb = sys.exc_info() tester.status(test_name, TestStatus.FAILED, str(e)) import traceback # remove the current call frame to exclude the extract_stack # call from the error stack = traceback.extract_stack()[:-1] # Format and print the stack out = traceback.format_list(stack) for line in out: print(line.rstrip("\n")) if exc_type is spack.util.executable.ProcessError or exc_type is TypeError: iostr = io.StringIO() write_log_summary(iostr, "test", tester.test_log_file, last=1) # type: ignore[assignment] m = iostr.getvalue() else: # We're below the package context, so get context from # stack instead of from traceback. # The traceback is truncated here, so we can't use it to # traverse the stack. m = "\n".join(get_package_context(tb) or "") exc = e # e is deleted after this block # If we fail fast, raise another error if spack.config.get("config:fail_fast", False): raise TestFailure([(exc, m)]) else: tester.add_failure(exc, m) def copy_test_files(pkg: "spack.package_base.PackageBase", test_spec: spack.spec.Spec): """Copy the spec's cached and custom test files to the test stage directory. Args: pkg: package being tested test_spec: spec being tested, where the spec may be virtual Raises: TestSuiteError: package must be part of an active test suite """ if pkg is None or pkg.test_suite is None: base = "Cannot copy test files" msg = ( f"{base} without a package" if pkg is None else f"{pkg.name}: {base}: test suite is missing" ) raise TestSuiteError(msg) # copy installed test sources cache into test stage dir if test_spec.concrete: cache_source = install_test_root(test_spec.package) cache_dir = pkg.test_suite.current_test_cache_dir if os.path.isdir(cache_source) and not os.path.exists(cache_dir): fs.install_tree(cache_source, cache_dir) # copy test data into test stage data dir try: pkg_cls = spack.repo.PATH.get_pkg_class(test_spec.fullname) except spack.repo.UnknownPackageError: tty.debug(f"{test_spec.name}: skipping test data copy since no package class found") return data_source = Prefix(pkg_cls.package_dir).test data_dir = pkg.test_suite.current_test_data_dir if os.path.isdir(data_source) and not os.path.exists(data_dir): # We assume data dir is used read-only # maybe enforce this later shutil.copytree(data_source, data_dir) def test_function_names(pkg: "PackageObjectOrClass", add_virtuals: bool = False) -> List[str]: """Grab the names of all non-empty test functions. Args: pkg: package or package class of interest add_virtuals: ``True`` adds test methods of provided package virtual, ``False`` only returns test functions of the package Returns: names of non-empty test functions Raises: ValueError: occurs if pkg is not a package class """ fns = test_functions(pkg, add_virtuals) return [f"{cls_name}.{fn.__name__}" for (cls_name, fn) in fns] def test_functions( pkg: "PackageObjectOrClass", add_virtuals: bool = False ) -> List[Tuple[str, Callable]]: """Grab all non-empty test functions. Args: pkg: package or package class of interest add_virtuals: ``True`` adds test methods of provided package virtual, ``False`` only returns test functions of the package Returns: list of non-empty test functions' (name, function) Raises: ValueError: occurs if pkg is not a package class """ classes = [pkg if isinstance(pkg, type) else pkg.__class__] if add_virtuals: vpkgs = virtuals(pkg) for vname in vpkgs: try: classes.append(spack.repo.PATH.get_pkg_class(vname)) except spack.repo.UnknownPackageError: tty.debug(f"{vname}: virtual does not appear to have a package file") tests = [] for clss in classes: methods = inspect.getmembers(clss, predicate=lambda x: inspect.isfunction(x)) for name, test_fn in methods: if not name.startswith("test_"): continue tests.append((clss.__name__, test_fn)) # type: ignore[union-attr] return tests def process_test_parts( pkg: "spack.package_base.PackageBase", test_specs: List[spack.spec.Spec], verbose: bool = False ): """Process test parts associated with the package. Args: pkg: package being tested test_specs: list of test specs verbose: Display verbose output (suppress by default) Raises: TestSuiteError: package must be part of an active test suite """ if pkg is None or pkg.test_suite is None: base = "Cannot process tests" msg = ( f"{base} without a package" if pkg is None else f"{pkg.name}: {base}: test suite is missing" ) raise TestSuiteError(msg) test_suite = pkg.test_suite tester = pkg.tester try: work_dir = test_suite.test_dir_for_spec(pkg.spec) for spec in test_specs: test_suite.current_test_spec = spec # grab test functions associated with the spec, which may be virtual try: tests = test_functions(spack.repo.PATH.get_pkg_class(spec.fullname)) except spack.repo.UnknownPackageError: # Some virtuals don't have a package so we don't want to report # them as not having tests when that isn't appropriate. continue if len(tests) == 0: tester.status(spec.name, TestStatus.NO_TESTS) continue # copy custom and cached test files to the test stage directory copy_test_files(pkg, spec) # Run the tests for _, test_fn in tests: with test_part( pkg, test_fn.__name__, purpose=getattr(test_fn, "__doc__"), work_dir=work_dir, verbose=verbose, ): test_fn(pkg) # If fail-fast was on, we error out above # If we collect errors, raise them in batch here if tester.test_failures: raise TestFailure(tester.test_failures) finally: if tester.ran_tests(): tester.write_tested_status() # log one more test message to provide a completion timestamp # for CDash reporting tty.msg("Completed testing") lines = tester.summarize() tty.msg("\n{}".format("\n".join(lines))) if tester.test_failures: # Print the test log file path tty.msg(f"\n\nSee test results at:\n {tester.test_log_file}") else: tty.msg("No tests to run") def test_process(pkg: "spack.package_base.PackageBase", kwargs): verbose = kwargs.get("verbose", True) externals = kwargs.get("externals", False) with pkg.tester.test_logger(verbose, externals) as logger: if pkg.spec.external and not externals: print_message(logger, "Skipped tests for external package", verbose) pkg.tester.status(pkg.spec.name, TestStatus.SKIPPED) return if not pkg.spec.installed: print_message(logger, "Skipped not installed package", verbose) pkg.tester.status(pkg.spec.name, TestStatus.SKIPPED) return # Make sure properly named build-time test methods actually run as # stand-alone tests. pkg.run_tests = True # run test methods from the package and all virtuals it provides v_names = virtuals(pkg) test_specs = [pkg.spec] + [spack.spec.Spec(v_name) for v_name in sorted(v_names)] process_test_parts(pkg, test_specs, verbose) def virtuals(pkg): """Return a list of unique virtuals for the package. Args: pkg: package of interest Returns: names of unique virtual packages """ # provided virtuals have to be deduped by name v_names = list({vspec.name for vspec in pkg.virtuals_provided}) # hack for compilers that are not dependencies (yet) # TODO: this all eventually goes away c_names = ("gcc", "intel", "intel-parallel-studio") if pkg.name in c_names: v_names.extend(["c", "cxx", "fortran"]) if pkg.spec.satisfies("llvm+clang"): v_names.extend(["c", "cxx"]) return v_names def get_all_test_suites(): """Retrieves all validly staged TestSuites Returns: list: a list of TestSuite objects, which may be empty if there are none """ stage_root = get_test_stage_dir() if not os.path.isdir(stage_root): return [] def valid_stage(d): dirpath = os.path.join(stage_root, d) return os.path.isdir(dirpath) and test_suite_filename in os.listdir(dirpath) candidates = [ os.path.join(stage_root, d, test_suite_filename) for d in os.listdir(stage_root) if valid_stage(d) ] test_suites = [TestSuite.from_file(c) for c in candidates] return test_suites def get_named_test_suites(name): """Retrieves test suites with the provided name. Returns: list: a list of matching TestSuite instances, which may be empty if none Raises: Exception: TestSuiteNameError if no name is provided """ if not name: raise TestSuiteNameError("Test suite name is required.") test_suites = get_all_test_suites() return [ts for ts in test_suites if ts.name == name] def get_test_suite(name: str) -> Optional["TestSuite"]: """Ensure there is only one matching test suite with the provided name. Returns: the name if one matching test suite, else None Raises: TestSuiteNameError: If there are more than one matching TestSuites """ suites = get_named_test_suites(name) if len(suites) > 1: raise TestSuiteNameError(f"Too many suites named '{name}'. May shadow hash.") if not suites: return None return suites[0] def write_test_suite_file(suite): """Write the test suite to its (JSON) lock file.""" with open(suite.stage.join(test_suite_filename), "w", encoding="utf-8") as f: sjson.dump(suite.to_dict(), stream=f) def write_test_summary(counts: "Counter"): """Write summary of the totals for each relevant status category. Args: counts: counts of the occurrences of relevant test status types """ summary = [f"{n} {s.lower()}" for s, n in counts.items() if n > 0] try: # New in Python 3.10 total = counts.total() # type: ignore[attr-defined] except AttributeError: nums = [n for _, n in counts.items()] total = sum(nums) if total: print("{:=^80}".format(" {} of {} ".format(", ".join(summary), plural(total, "spec")))) class TestSuite: """The class that manages specs for ``spack test run`` execution.""" def __init__(self, specs: Iterable[Spec], alias: Optional[str] = None) -> None: # copy so that different test suites have different package objects # even if they contain the same spec self.specs = [spec.copy() for spec in specs] self.current_test_spec = None # spec currently tested, can be virtual self.current_base_spec = None # spec currently running do_test self.alias = alias self._hash: Optional[str] = None self._stage: Optional[Prefix] = None self.counts: "Counter" = Counter() self.reports: List[spack.report.RequestRecord] = [] @property def name(self) -> str: """The name (alias or, if none, hash) of the test suite.""" return self.alias if self.alias else self.content_hash @property def content_hash(self) -> str: """The hash used to uniquely identify the test suite.""" if not self._hash: json_text = sjson.dump(self.to_dict()) assert json_text is not None, f"{__name__} unexpected value for 'json_text'" sha = hashlib.sha1(json_text.encode("utf-8")) b32_hash = base64.b32encode(sha.digest()).lower() b32_hash = b32_hash.decode("utf-8") self._hash = b32_hash return self._hash def __call__( self, *, remove_directory: bool = True, dirty: bool = False, fail_first: bool = False, externals: bool = False, timeout: Optional[int] = None, ): self.write_reproducibility_data() for spec in self.specs: # Setup cdash/junit/etc reports report = spack.report.RequestRecord(spec) self.reports.append(report) record = spack.report.TestRecord(spec, self.stage) report.append_record(record) record.start() try: if spec.package.test_suite: raise TestSuiteSpecError( f"Package {spec.package.name} cannot be run in two test suites at once" ) # Set up the test suite to know which test is running spec.package.test_suite = self self.current_base_spec = spec self.current_test_spec = spec # setup per-test directory in the stage dir test_dir = self.test_dir_for_spec(spec) if os.path.exists(test_dir): shutil.rmtree(test_dir) fs.mkdirp(test_dir) # run the package tests spec.package.do_test(dirty=dirty, externals=externals, timeout=timeout) # Clean up on success if remove_directory: shutil.rmtree(test_dir) status = self.test_status(spec, externals) self.counts[status] += 1 self.write_test_result(spec, status) record.succeed(externals) except SkipTest: record.skip(msg="Test marked to skip") status = TestStatus.SKIPPED self.counts[status] += 1 self.write_test_result(spec, TestStatus.SKIPPED) except BaseException as exc: record.fail(exc) status = TestStatus.FAILED self.counts[status] += 1 tty.debug(f"Test failure: {str(exc)}") if isinstance(exc, (SyntaxError, TestSuiteSpecError)): # Create the test log file and report the error. self.ensure_stage() msg = f"Testing package {self.test_pkg_id(spec)}\n{str(exc)}" _add_msg_to_file(self.log_file_for_spec(spec), msg) msg = f"Test failure: {str(exc)}" _add_msg_to_file(self.log_file_for_spec(spec), msg) self.write_test_result(spec, TestStatus.FAILED) if fail_first: break finally: spec.package.test_suite = None self.current_test_spec = None self.current_base_spec = None write_test_summary(self.counts) if self.counts[TestStatus.FAILED]: for spec in self.specs: print( "\nSee {} test results at:\n {}".format( spec.format("{name}-{version}-{hash:7}"), self.log_file_for_spec(spec) ) ) failures = self.counts[TestStatus.FAILED] if failures: raise TestSuiteFailure(failures) def test_status(self, spec: spack.spec.Spec, externals: bool) -> TestStatus: """Returns the overall test results status for the spec. Args: spec: instance of the spec under test externals: ``True`` if externals are to be tested, else ``False`` """ tests_status_file = self.tested_file_for_spec(spec) if not os.path.exists(tests_status_file): self.ensure_stage() if spec.external and not externals: status = TestStatus.SKIPPED elif not spec.installed: status = TestStatus.SKIPPED else: status = TestStatus.NO_TESTS return status with open(tests_status_file, "r", encoding="utf-8") as f: value = (f.read()).strip("\n") return TestStatus(int(value)) if value else TestStatus.NO_TESTS def ensure_stage(self) -> None: """Ensure the test suite stage directory exists.""" if not os.path.exists(self.stage): fs.mkdirp(self.stage) @property def stage(self) -> Prefix: """The root test suite stage directory""" if not self._stage: self._stage = Prefix(fs.join_path(get_test_stage_dir(), self.content_hash)) return self._stage @stage.setter def stage(self, value: Union[Prefix, str]) -> None: """Set the value of a non-default stage directory.""" self._stage = value if isinstance(value, Prefix) else Prefix(value) @property def results_file(self) -> Prefix: """The path to the results summary file.""" return self.stage.join(results_filename) @classmethod def test_pkg_id(cls, spec: Spec) -> str: """The standard install test package identifier. Args: spec: instance of the spec under test """ return spec.format_path("{name}-{version}-{hash:7}") @classmethod def test_log_name(cls, spec: Spec) -> str: """The standard log filename for a spec. Args: spec: instance of the spec under test """ return f"{cls.test_pkg_id(spec)}-test-out.txt" def log_file_for_spec(self, spec: Spec) -> Prefix: """The test log file path for the provided spec. Args: spec: instance of the spec under test """ return self.stage.join(self.test_log_name(spec)) def test_dir_for_spec(self, spec: Spec) -> Prefix: """The path to the test stage directory for the provided spec. Args: spec: instance of the spec under test """ return Prefix(self.stage.join(self.test_pkg_id(spec))) @classmethod def tested_file_name(cls, spec: Spec) -> str: """The standard test status filename for the spec. Args: spec: instance of the spec under test """ return "%s-tested.txt" % cls.test_pkg_id(spec) def tested_file_for_spec(self, spec: Spec) -> str: """The test status file path for the spec. Args: spec: instance of the spec under test """ return fs.join_path(self.stage, self.tested_file_name(spec)) @property def current_test_cache_dir(self) -> str: """Path to the test stage directory where the current spec's cached build-time files were automatically copied. Raises: TestSuiteSpecError: If there is no spec being tested """ if not (self.current_test_spec and self.current_base_spec): raise TestSuiteSpecError("Unknown test cache directory: no specs being tested") test_spec = self.current_test_spec base_spec = self.current_base_spec return self.test_dir_for_spec(base_spec).cache.join(test_spec.name) @property def current_test_data_dir(self) -> str: """Path to the test stage directory where the current spec's custom package (data) files were automatically copied. Raises: TestSuiteSpecError: If there is no spec being tested """ if not (self.current_test_spec and self.current_base_spec): raise TestSuiteSpecError("Unknown test data directory: no specs being tested") test_spec = self.current_test_spec base_spec = self.current_base_spec return self.test_dir_for_spec(base_spec).data.join(test_spec.name) def write_test_result(self, spec: Spec, result: TestStatus) -> None: """Write the spec's test result to the test suite results file. Args: spec: instance of the spec under test result: result from the spec's test execution (e.g, PASSED) """ msg = f"{self.test_pkg_id(spec)} {result}" _add_msg_to_file(self.results_file, msg) def write_reproducibility_data(self) -> None: for spec in self.specs: repo_cache_path = self.stage.repo.join(spec.name) spack.repo.PATH.dump_provenance(spec, repo_cache_path) for vspec in spec.package.virtuals_provided: repo_cache_path = self.stage.repo.join(vspec.name) if not os.path.exists(repo_cache_path): try: spack.repo.PATH.dump_provenance(vspec, repo_cache_path) except spack.repo.UnknownPackageError: pass # not all virtuals have package files write_test_suite_file(self) def to_dict(self) -> Dict[str, Any]: """Build a dictionary for the test suite. Returns: The dictionary contains entries for up to two keys. * specs: list of the test suite's specs in dictionary form * alias: the alias, or name, given to the test suite if provided """ specs = [s.to_dict() for s in self.specs] d: Dict[str, Any] = {"specs": specs} if self.alias: d["alias"] = self.alias return d @staticmethod def from_dict(d): """Instantiates a TestSuite based on a dictionary specs and an optional alias: * specs: list of the test suite's specs in dictionary form * alias: the test suite alias Returns: TestSuite: Instance created from the specs """ specs = [Spec.from_dict(spec_dict) for spec_dict in d["specs"]] alias = d.get("alias", None) return TestSuite(specs, alias) @staticmethod def from_file(filename: str) -> "TestSuite": """Instantiate a TestSuite using the specs and optional alias provided in the given file. Args: filename: The path to the JSON file containing the test suite specs and optional alias. Raises: BaseException: sjson.SpackJSONError if problem parsing the file """ try: with open(filename, encoding="utf-8") as f: data = sjson.load(f) test_suite = TestSuite.from_dict(data) content_hash = os.path.basename(os.path.dirname(filename)) test_suite._hash = content_hash return test_suite except Exception as e: raise sjson.SpackJSONError("error parsing JSON TestSuite:", e) def _add_msg_to_file(filename, msg): """Append the message to the specified file. Args: filename (str): path to the file msg (str): message to be appended to the file """ with open(filename, "a+", encoding="utf-8") as f: f.write(f"{msg}\n") class SkipTest(Exception): """Raised when a test (part) is being skipped.""" class TestFailure(spack.error.SpackError): """Raised when package tests have failed for an installation.""" def __init__(self, failures: List[TestFailureType]): # Failures are all exceptions num = len(failures) msg = "{} failed.\n".format(plural(num, "test")) for failure, message in failures: msg += "\n\n%s\n" % str(failure) msg += "\n%s\n" % message super().__init__(msg) class TestSuiteError(spack.error.SpackError): """Raised when there is an error with the test suite.""" class TestSuiteFailure(spack.error.SpackError): """Raised when one or more tests in a suite have failed.""" def __init__(self, num_failures): msg = "%d test(s) in the suite failed.\n" % num_failures super().__init__(msg) class TestSuiteSpecError(spack.error.SpackError): """Raised when there is an issue associated with the spec being tested.""" class TestSuiteNameError(spack.error.SpackError): """Raised when there is an issue with the naming of the test suite.""" ================================================ FILE: lib/spack/spack/installer.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module encapsulates package installation functionality. The PackageInstaller coordinates concurrent builds of packages for the same Spack instance by leveraging the dependency DAG and file system locks. It also proceeds with the installation of non-dependent packages of failed dependencies in order to install as many dependencies of a package as possible. Bottom-up traversal of the dependency DAG while prioritizing packages with no uninstalled dependencies allows multiple processes to perform concurrent builds of separate packages associated with a spec. File system locks enable coordination such that no two processes attempt to build the same or a failed dependency package. If a dependency package fails to install, its dependents' tasks will be removed from the installing process's queue. A failure file is also written and locked. Other processes use this file to detect the failure and dequeue its dependents. This module supports the coordination of local and distributed concurrent installations of packages in a Spack instance. """ import copy import enum import glob import heapq import io import itertools import os import shutil import sys import tempfile import time from collections import defaultdict from gzip import GzipFile from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Set, Tuple, Union from spack.vendor.typing_extensions import Literal import spack.binary_distribution as binary_distribution import spack.build_environment import spack.builder import spack.config import spack.database import spack.deptypes as dt import spack.error import spack.hooks import spack.llnl.util.filesystem as fs import spack.llnl.util.lock as lk import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.package_base import spack.package_prefs as prefs import spack.repo import spack.report import spack.rewiring import spack.store import spack.util.path import spack.util.timer as timer from spack.llnl.string import ordinal from spack.llnl.util.lang import pretty_seconds from spack.llnl.util.tty.color import colorize from spack.llnl.util.tty.log import log_output, preserve_terminal_settings from spack.url_buildcache import BuildcacheEntryError from spack.util.environment import EnvironmentModifications, dump_environment if TYPE_CHECKING: import spack.spec #: Counter to support unique spec sequencing that is used to ensure packages #: with the same priority are (initially) processed in the order in which they #: were added (see https://docs.python.org/2/library/heapq.html). _counter = itertools.count(0) _FAIL_FAST_ERR = "Terminating after first install failure" #: Type for specifying installation source modes InstallPolicy = Literal["auto", "cache_only", "source_only"] class BuildStatus(enum.Enum): """Different build (task) states.""" #: Build status indicating task has been added/queued. QUEUED = enum.auto() #: Build status indicating the spec failed to install FAILED = enum.auto() #: Build status indicating the spec is being installed (possibly by another #: process) INSTALLING = enum.auto() #: Build status indicating the spec was successfully installed INSTALLED = enum.auto() #: Build status indicating the task has been popped from the queue DEQUEUED = enum.auto() #: Build status indicating task has been removed (to maintain priority #: queue invariants). REMOVED = enum.auto() def __str__(self): return f"{self.name.lower()}" def _write_timer_json(pkg, timer, cache): extra_attributes = {"name": pkg.name, "cache": cache, "hash": pkg.spec.dag_hash()} try: with open(pkg.times_log_path, "w", encoding="utf-8") as timelog: timer.write_json(timelog, extra_attributes=extra_attributes) except Exception as e: tty.debug(str(e)) return class ExecuteResult(enum.Enum): # Task succeeded SUCCESS = enum.auto() # Task failed FAILED = enum.auto() # Task is missing build spec and will be requeued MISSING_BUILD_SPEC = enum.auto() # Task is installed upstream/external or # task is not ready for installation (locked by another process) NO_OP = enum.auto() class InstallAction(enum.Enum): #: Don't perform an install NONE = enum.auto() #: Do a standard install INSTALL = enum.auto() #: Do an overwrite install OVERWRITE = enum.auto() class InstallStatus: def __init__(self, pkg_count: int): # Counters used for showing status information self.pkg_num: int = 0 self.pkg_count: int = pkg_count self.pkg_ids: Set[str] = set() def next_pkg(self, pkg: "spack.package_base.PackageBase"): pkg_id = package_id(pkg.spec) if pkg_id not in self.pkg_ids: self.pkg_num += 1 self.pkg_ids.add(pkg_id) def set_term_title(self, text: str): if not spack.config.get("config:install_status", True): return if not sys.stdout.isatty(): return status = f"{text} {self.get_progress()}" sys.stdout.write(f"\x1b]0;Spack: {status}\x07") sys.stdout.flush() def get_progress(self) -> str: return f"[{self.pkg_num}/{self.pkg_count}]" class TermStatusLine: """ This class is used in distributed builds to inform the user that other packages are being installed by another process. """ def __init__(self, enabled: bool): self.enabled: bool = enabled self.pkg_set: Set[str] = set() self.pkg_list: List[str] = [] def add(self, pkg_id: str): """Add a package to the waiting list, and if it is new, update the status line.""" if not self.enabled or pkg_id in self.pkg_set: return self.pkg_set.add(pkg_id) self.pkg_list.append(pkg_id) tty.msg(colorize("@*{Waiting for} @*g{%s}" % pkg_id)) sys.stdout.flush() def clear(self): """Clear the status line.""" if not self.enabled: return lines = len(self.pkg_list) if lines == 0: return self.pkg_set.clear() self.pkg_list = [] # Move the cursor to the beginning of the first "Waiting for" message and clear # everything after it. sys.stdout.write(f"\x1b[{lines}F\x1b[J") sys.stdout.flush() def _check_last_phase(pkg: "spack.package_base.PackageBase") -> None: """ Ensures the specified package has a valid last phase before proceeding with its installation. The last phase is also set to None if it is the last phase of the package already. Args: pkg: the package being installed Raises: ``BadInstallPhase`` if stop_before or last phase is invalid """ phases = spack.builder.create(pkg).phases # type: ignore[attr-defined] if pkg.stop_before_phase and pkg.stop_before_phase not in phases: # type: ignore[attr-defined] raise BadInstallPhase(pkg.name, pkg.stop_before_phase) # type: ignore[attr-defined] if pkg.last_phase and pkg.last_phase not in phases: # type: ignore[attr-defined] raise BadInstallPhase(pkg.name, pkg.last_phase) # type: ignore[attr-defined] def _handle_external_and_upstream(pkg: "spack.package_base.PackageBase", explicit: bool) -> bool: """ Determine if the package is external or upstream and register it in the database if it is external package. Args: pkg: the package whose installation is under consideration explicit: the package was explicitly requested by the user Return: ``True`` if the package is not to be installed locally, otherwise ``False`` """ # For external packages the workflow is simplified, and basically # consists in module file generation and registration in the DB. if pkg.spec.external: _process_external_package(pkg, explicit) _print_installed_pkg(f"{pkg.prefix} (external {package_id(pkg.spec)})") return True if pkg.spec.installed_upstream: tty.verbose( f"{package_id(pkg.spec)} is installed in an upstream Spack instance at " f"{pkg.spec.prefix}" ) _print_installed_pkg(pkg.prefix) # This will result in skipping all post-install hooks. In the case # of modules this is considered correct because we want to retrieve # the module from the upstream Spack instance. return True return False def _do_fake_install(pkg: "spack.package_base.PackageBase") -> None: """Make a fake install directory with fake executables, headers, and libraries.""" command = pkg.name header = pkg.name library = pkg.name # Avoid double 'lib' for packages whose names already start with lib if not pkg.name.startswith("lib"): library = "lib" + library plat_shared = ".dll" if sys.platform == "win32" else ".so" plat_static = ".lib" if sys.platform == "win32" else ".a" dso_suffix = ".dylib" if sys.platform == "darwin" else plat_shared # Install fake command fs.mkdirp(pkg.prefix.bin) executable = lambda path, flags: os.open(path, flags, 0o700) open(os.path.join(pkg.prefix.bin, command), "wb", opener=executable).close() # Install fake header file fs.mkdirp(pkg.prefix.include) fs.touch(os.path.join(pkg.prefix.include, header + ".h")) # Install fake shared and static libraries fs.mkdirp(pkg.prefix.lib) for suffix in [dso_suffix, plat_static]: fs.touch(os.path.join(pkg.prefix.lib, library + suffix)) # Install fake man page fs.mkdirp(pkg.prefix.man.man1) packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec) dump_packages(pkg.spec, packages_dir) def _hms(seconds: int) -> str: """ Convert seconds to hours, minutes, seconds Args: seconds: time to be converted in seconds Return: String representation of the time as #h #m #.##s """ m, s = divmod(seconds, 60) h, m = divmod(m, 60) parts = [] if h: parts.append("%dh" % h) if m: parts.append("%dm" % m) if s: parts.append(f"{s:.2f}s") return " ".join(parts) def _log_prefix(pkg_name) -> str: """Prefix of the form "[pid]: [pkg name]: ..." when printing a status update during the build.""" pid = f"{os.getpid()}: " if tty.show_pid() else "" return f"{pid}{pkg_name}:" def _print_installed_pkg(message: str) -> None: """ Output a message with a package icon. Args: message (str): message to be output """ if tty.msg_enabled(): print(colorize("@*g{[+]} ") + spack.util.path.debug_padded_filter(message)) def print_install_test_log(pkg: "spack.package_base.PackageBase") -> None: """Output install test log file path but only if have test failures. Args: pkg: instance of the package under test """ if not pkg.run_tests or not (pkg.tester and pkg.tester.test_failures): # The tests were not run or there were no test failures return pkg.tester.print_log_path() def _print_timer(pre: str, pkg_id: str, timer: timer.BaseTimer) -> None: phases = [f"{p.capitalize()}: {_hms(timer.duration(p))}." for p in timer.phases] phases.append(f"Total: {_hms(timer.duration())}") tty.msg(f"{pre} Successfully installed {pkg_id}", " ".join(phases)) def _install_from_cache( pkg: "spack.package_base.PackageBase", explicit: bool, unsigned: Optional[bool] = False ) -> bool: """ Install the package from binary cache Args: pkg: package to install from the binary cache explicit: ``True`` if installing the package was explicitly requested by the user, otherwise, ``False`` unsigned: if ``True`` or ``False`` override the mirror signature verification defaults Return: ``True`` if the package was extract from binary cache, ``False`` otherwise """ t = timer.Timer() installed_from_cache = _try_install_from_binary_cache( pkg, explicit, unsigned=unsigned, timer=t ) if not installed_from_cache: return False t.stop() pkg_id = package_id(pkg.spec) tty.debug(f"Successfully extracted {pkg_id} from binary cache") _write_timer_json(pkg, t, True) _print_timer(pre=_log_prefix(pkg.name), pkg_id=pkg_id, timer=t) _print_installed_pkg(pkg.spec.prefix) spack.hooks.post_install(pkg.spec, explicit) return True def _process_external_package(pkg: "spack.package_base.PackageBase", explicit: bool) -> None: """ Helper function to run post install hooks and register external packages. Args: pkg: the external package explicit: if the package was requested explicitly by the user, ``False`` if it was pulled in as a dependency of an explicit package. """ assert pkg.spec.external, "Expected to post-install/register an external package." pre = f"{pkg.spec.name}@{pkg.spec.version} :" spec = pkg.spec if spec.external_modules: tty.msg(f"{pre} has external module in {spec.external_modules}") tty.debug(f"{pre} is actually installed in {spec.external_path}") else: tty.debug(f"{pre} externally installed in {spec.external_path}") try: # Check if the package was already registered in the DB. # If this is the case, then only make explicit if required. tty.debug(f"{pre} already registered in DB") record = spack.store.STORE.db.get_record(spec) if explicit and not record.explicit: spack.store.STORE.db.mark(spec, "explicit", True) except KeyError: # If not, register it and generate the module file. # For external packages we just need to run # post-install hooks to generate module files. tty.debug(f"{pre} generating module file") spack.hooks.post_install(spec, explicit) # Add to the DB tty.debug(f"{pre} registering into DB") spack.store.STORE.db.add(spec, explicit=explicit) def _process_binary_cache_tarball( pkg: "spack.package_base.PackageBase", explicit: bool, unsigned: Optional[bool], mirrors_for_spec: Optional[list] = None, timer: timer.BaseTimer = timer.NULL_TIMER, ) -> bool: """ Process the binary cache tarball. Args: pkg: the package being installed explicit: the package was explicitly requested by the user unsigned: if ``True`` or ``False`` override the mirror signature verification defaults mirrors_for_spec: Optional list of mirrors to look for the spec. obtained by calling binary_distribution.get_mirrors_for_spec(). timer: timer to keep track of binary install phases. Return: bool: ``True`` if the package was extracted from binary cache, else ``False`` """ with timer.measure("fetch"): tarball_stage = binary_distribution.download_tarball( pkg.spec.build_spec, unsigned, mirrors_for_spec ) if tarball_stage is None: return False tty.msg(f"Extracting {package_id(pkg.spec)} from binary cache") with timer.measure("install"), spack.util.path.filter_padding(): binary_distribution.extract_tarball(pkg.spec, tarball_stage, force=False, timer=timer) if pkg.spec.spliced: # overwrite old metadata with new spack.store.STORE.layout.write_spec( pkg.spec, spack.store.STORE.layout.spec_file_path(pkg.spec) ) if hasattr(pkg, "_post_buildcache_install_hook"): pkg._post_buildcache_install_hook() pkg.installed_from_binary_cache = True spack.store.STORE.db.add(pkg.spec, explicit=explicit) return True def _try_install_from_binary_cache( pkg: "spack.package_base.PackageBase", explicit: bool, unsigned: Optional[bool] = None, timer: timer.BaseTimer = timer.NULL_TIMER, ) -> bool: """ Try to extract the package from binary cache. Args: pkg: package to be extracted from binary cache explicit: the package was explicitly requested by the user unsigned: if ``True`` or ``False`` override the mirror signature verification defaults timer: timer to keep track of binary install phases. """ # Early exit if no binary mirrors are configured. if not spack.mirrors.mirror.MirrorCollection(binary=True): return False tty.debug(f"Searching for binary cache of {package_id(pkg.spec)}") with timer.measure("search"): mirrors = binary_distribution.get_mirrors_for_spec(pkg.spec, index_only=True) return _process_binary_cache_tarball( pkg, explicit, unsigned, mirrors_for_spec=mirrors, timer=timer ) def combine_phase_logs(phase_log_files: List[str], log_path: str) -> None: """ Read set or list of logs and combine them into one file. Each phase will produce it's own log, so this function aims to cat all the separate phase log output files into the pkg.log_path. It is written generally to accept some list of files, and a log path to combine them to. Args: phase_log_files: a list or iterator of logs to combine log_path: the path to combine them to """ with open(log_path, "bw") as log_file: for phase_log_file in phase_log_files: with open(phase_log_file, "br") as phase_log: shutil.copyfileobj(phase_log, log_file) def dump_packages(spec: "spack.spec.Spec", path: str) -> None: """ Dump all package information for a spec and its dependencies. This creates a package repository within path for every namespace in the spec DAG, and fills the repos with package files and patch files for every node in the DAG. Args: spec: the Spack spec whose package information is to be dumped path: the path to the build packages directory """ fs.mkdirp(path) # Copy in package.py files from any dependencies. # Note that we copy them in as they are in the *install* directory # NOT as they are in the repository, because we want a snapshot of # how *this* particular build was done. for node in spec.traverse(deptype="all"): if node is not spec: # Locate the dependency package in the install tree and find # its provenance information. source = spack.store.STORE.layout.build_packages_path(node) source_repo_root = os.path.join(source, node.namespace) # If there's no provenance installed for the package, skip it. # If it's external, skip it because it either: # 1) it wasn't built with Spack, so it has no Spack metadata # 2) it was built by another Spack instance, and we do not # (currently) use Spack metadata to associate repos with externals # built by other Spack instances. # Spack can always get something current from the builtin repo. if node.external or not os.path.isdir(source_repo_root): continue # Create a source repo and get the pkg directory out of it. try: source_repo = spack.repo.from_path(source_repo_root) source_pkg_dir = source_repo.dirname_for_package_name(node.name) except spack.repo.RepoError as err: tty.debug(f"Failed to create source repo for {node.name}: {str(err)}") source_pkg_dir = None tty.warn(f"Warning: Couldn't copy in provenance for {node.name}") # Create a destination repository pkg_api = spack.repo.PATH.get_repo(node.namespace).package_api repo_root = os.path.join(path, node.namespace) if pkg_api < (2, 0) else path repo = spack.repo.create_or_construct( repo_root, namespace=node.namespace, package_api=pkg_api ) # Get the location of the package in the dest repo. dest_pkg_dir = repo.dirname_for_package_name(node.name) if node is spec: spack.repo.PATH.dump_provenance(node, dest_pkg_dir) elif source_pkg_dir: fs.install_tree(source_pkg_dir, dest_pkg_dir) def get_dependent_ids(spec: "spack.spec.Spec") -> List[str]: """ Return a list of package ids for the spec's dependents Args: spec: Concretized spec Returns: list of package ids """ return [package_id(d) for d in spec.dependents()] def install_msg(name: str, pid: int, install_status: InstallStatus) -> str: """ Colorize the name/id of the package being installed Args: name: Name/id of the package being installed pid: id of the installer process Return: Colorized installing message """ pre = f"{pid}: " if tty.show_pid() else "" post = ( " @*{%s}" % install_status.get_progress() if install_status and spack.config.get("config:install_status", True) else "" ) return pre + colorize("@*{Installing} @*g{%s}%s" % (name, post)) def archive_install_logs(pkg: "spack.package_base.PackageBase", phase_log_dir: str) -> None: """ Copy install logs to their destination directory(ies) Args: pkg: the package that was built and installed phase_log_dir: path to the archive directory """ # Copy a compressed version of the install log with open(pkg.log_path, "rb") as f, open(pkg.install_log_path, "wb") as g: # Use GzipFile directly so we can omit filename / mtime in header gzip_file = GzipFile(filename="", mode="wb", compresslevel=6, mtime=0, fileobj=g) shutil.copyfileobj(f, gzip_file) gzip_file.close() # Archive the install-phase test log, if present pkg.archive_install_test_log() def log(pkg: "spack.package_base.PackageBase") -> None: """ Copy provenance into the install directory on success Args: pkg: the package that was built and installed """ packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec) # Remove first if we're overwriting another build try: # log and env install paths are inside this shutil.rmtree(packages_dir) except Exception as e: # FIXME : this potentially catches too many things... tty.debug(e) archive_install_logs(pkg, os.path.dirname(packages_dir)) # Archive the environment modifications for the build. fs.install(pkg.env_mods_path, pkg.install_env_path) if os.path.exists(pkg.configure_args_path): # Archive the args used for the build fs.install(pkg.configure_args_path, pkg.install_configure_args_path) # Finally, archive files that are specific to each package with fs.working_dir(pkg.stage.path): errors = io.StringIO() target_dir = os.path.join( spack.store.STORE.layout.metadata_path(pkg.spec), "archived-files" ) for glob_expr in spack.builder.create(pkg).archive_files: # Check that we are trying to copy things that are # in the stage tree (not arbitrary files) abs_expr = os.path.realpath(glob_expr) if os.path.realpath(pkg.stage.path) not in abs_expr: errors.write(f"[OUTSIDE SOURCE PATH]: {glob_expr}\n") continue # Now that we are sure that the path is within the correct # folder, make it relative and check for matches if os.path.isabs(glob_expr): glob_expr = os.path.relpath(glob_expr, pkg.stage.path) files = glob.glob(glob_expr) for f in files: try: target = os.path.join(target_dir, f) # We must ensure that the directory exists before # copying a file in fs.mkdirp(os.path.dirname(target)) fs.install(f, target) except Exception as e: tty.debug(e) # Here try to be conservative, and avoid discarding # the whole install procedure because of copying a # single file failed errors.write(f"[FAILED TO ARCHIVE]: {f}") if errors.getvalue(): error_file = os.path.join(target_dir, "errors.txt") fs.mkdirp(target_dir) with open(error_file, "w", encoding="utf-8") as err: err.write(errors.getvalue()) tty.warn(f"Errors occurred when archiving files.\n\tSee: {error_file}") dump_packages(pkg.spec, packages_dir) def package_id(spec: "spack.spec.Spec") -> str: """A "unique" package identifier for installation purposes The identifier is used to track tasks, locks, install, and failure statuses. The identifier needs to distinguish between combinations of compilers and packages for combinatorial environments. Args: pkg: the package from which the identifier is derived """ if not spec.concrete: raise ValueError("Cannot provide a unique, readable id when the spec is not concretized.") return f"{spec.name}-{spec.version}-{spec.dag_hash()}" class BuildRequest: """Class for representing an installation request.""" def __init__(self, pkg: "spack.package_base.PackageBase", install_args: dict): """ Instantiate a build request for a package. Args: pkg: the package to be built and installed install_args: the install arguments associated with ``pkg`` """ # Ensure dealing with a package that has a concrete spec if not isinstance(pkg, spack.package_base.PackageBase): raise ValueError(f"{str(pkg)} must be a package") self.pkg = pkg if not self.pkg.spec.concrete: raise ValueError(f"{self.pkg.name} must have a concrete spec") self.pkg.stop_before_phase = install_args.get("stop_before") # type: ignore[attr-defined] # noqa: E501 self.pkg.last_phase = install_args.get("stop_at") # type: ignore[attr-defined] # Cache the package id for convenience self.pkg_id = package_id(pkg.spec) # Save off the original install arguments plus standard defaults # since they apply to the requested package *and* dependencies. self.install_args = install_args if install_args else {} self._add_default_args() # Cache overwrite information self.overwrite = set(self.install_args.get("overwrite", [])) self.overwrite_time = time.time() # Save off dependency package ids for quick checks since traversals # are not able to return full dependents for all packages across # environment specs. self.dependencies = set( package_id(d) for d in self.pkg.spec.dependencies(deptype=self.get_depflags(self.pkg)) if package_id(d) != self.pkg_id ) def __repr__(self) -> str: """Return a formal representation of the build request.""" rep = f"{self.__class__.__name__}(" for attr, value in self.__dict__.items(): rep += f"{attr}={value.__repr__()}, " return f"{rep.strip(', ')})" def __str__(self) -> str: """Return a printable version of the build request.""" return f"package={self.pkg.name}, install_args={self.install_args}" def _add_default_args(self) -> None: """Ensure standard install options are set to at least the default.""" for arg, default in [ ("context", "build"), # installs *always* build ("dependencies_policy", "auto"), ("dirty", False), ("fail_fast", False), ("fake", False), ("install_deps", True), ("install_package", True), ("install_source", False), ("root_policy", "auto"), ("keep_prefix", False), ("keep_stage", False), ("restage", False), ("skip_patch", False), ("tests", False), ("unsigned", None), ("verbose", False), ]: _ = self.install_args.setdefault(arg, default) def get_depflags(self, pkg: "spack.package_base.PackageBase") -> int: """Determine the required dependency types for the associated package. Args: pkg: explicit or implicit package being installed Returns: tuple: required dependency type(s) for the package """ depflag = dt.LINK | dt.RUN include_build_deps = self.install_args.get("include_build_deps") if self.pkg_id == package_id(pkg.spec): policy = self.install_args.get("root_policy", "auto") else: policy = self.install_args.get("dependencies_policy", "auto") # Include build dependencies if pkg is going to be built from sources, or # if build deps are explicitly requested. if include_build_deps or not ( policy == "cache_only" or pkg.spec.installed and pkg.spec.dag_hash() not in self.overwrite ): depflag |= dt.BUILD if self.run_tests(pkg): depflag |= dt.TEST return depflag def has_dependency(self, dep_id) -> bool: """Returns ``True`` if the package id represents a known dependency of the requested package, ``False`` otherwise.""" return dep_id in self.dependencies def run_tests(self, pkg: "spack.package_base.PackageBase") -> bool: """Determine if the tests should be run for the provided packages Args: pkg: explicit or implicit package being installed Returns: bool: ``True`` if they should be run; ``False`` otherwise """ tests = self.install_args.get("tests", False) return tests is True or (tests and pkg.name in tests) @property def spec(self) -> "spack.spec.Spec": """The specification associated with the package.""" return self.pkg.spec def traverse_dependencies(self, spec=None, visited=None) -> Iterator["spack.spec.Spec"]: """Yield any dependencies of the appropriate type(s)""" # notice: deptype is not constant across nodes, so we cannot use # spec.traverse_edges(deptype=...). if spec is None: spec = self.spec if visited is None: visited = set() for dep in spec.dependencies(deptype=self.get_depflags(spec.package)): hash = dep.dag_hash() if hash in visited: continue visited.add(hash) # In Python 3: yield from self.traverse_dependencies(dep, visited) for s in self.traverse_dependencies(dep, visited): yield s yield dep class Task: """Base class for representing a task for a package.""" success_result: Optional[ExecuteResult] = None error_result: Optional[BaseException] = None no_op: bool = False def __init__( self, pkg: "spack.package_base.PackageBase", request: BuildRequest, *, compiler: bool = False, start_time: float = 0.0, attempts: int = 0, status: BuildStatus = BuildStatus.QUEUED, installed: Set[str] = set(), ): """ Instantiate a task for a package. Args: pkg: the package to be built and installed request: the associated install request start_time: the initial start time for the package, in seconds attempts: the number of attempts to install the package, which should be 0 when the task is initially instantiated status: the installation status installed: the (string) identifiers of packages that have been installed so far Raises: ``InstallError`` if the build status is incompatible with the task ``TypeError`` if provided an argument of the wrong type ``ValueError`` if provided an argument with the wrong value or state """ # Ensure dealing with a package that has a concrete spec if not isinstance(pkg, spack.package_base.PackageBase): raise TypeError(f"{str(pkg)} must be a package") self.pkg = pkg if not self.pkg.spec.concrete: raise ValueError(f"{self.pkg.name} must have a concrete spec") # The "unique" identifier for the task's package self.pkg_id = package_id(self.pkg.spec) # The explicit build request associated with the package if not isinstance(request, BuildRequest): raise TypeError(f"{request} is not a valid build request") self.request = request # Report for tracking install success/failure record_cls = self.request.install_args.get("record_cls", spack.report.InstallRecord) self.record = record_cls(self.pkg.spec) # Initialize the status to an active state. The status is used to # ensure priority queue invariants when tasks are "removed" from the # queue. if not isinstance(status, BuildStatus): raise TypeError(f"{status} is not a valid build status") # The initial build task cannot have status "removed". if attempts == 0 and status == BuildStatus.REMOVED: raise spack.error.InstallError( f"Cannot create a task for {self.pkg_id} with status '{status}'", pkg=pkg ) self.status = status # cache the PID, which is used for distributed build messages in self.execute self.pid = os.getpid() # The initial start time for processing the spec self.start_time = start_time if not isinstance(installed, set): raise TypeError( f"BuildTask constructor requires 'installed' be a 'set', " f"not '{installed.__class__.__name__}'." ) # Set of dependents, which needs to include the requesting package # to support tracking of parallel, multi-spec, environment installs. self.dependents = set(get_dependent_ids(self.pkg.spec)) tty.debug(f"Pkg id {self.pkg_id} has the following dependents:") for dep_id in self.dependents: tty.debug(f"- {dep_id}") # Set of dependencies # # Be consistent wrt use of dependents and dependencies. That is, # if use traverse for transitive dependencies, then must remove # transitive dependents on failure. self.dependencies = set( package_id(d) for d in self.pkg.spec.dependencies(deptype=self.request.get_depflags(self.pkg)) if package_id(d) != self.pkg_id ) # List of uninstalled dependencies, which is used to establish # the priority of the task. self.uninstalled_deps = set( pkg_id for pkg_id in self.dependencies if pkg_id not in installed ) # Ensure key sequence-related properties are updated accordingly. self.attempts = attempts self._update() # initialize cache variables self._install_action = None def start(self): """Start the work of this task.""" raise NotImplementedError def poll(self) -> bool: """Check if child process has information ready to receive.""" raise NotImplementedError def complete(self) -> ExecuteResult: """Complete the work of this task.""" raise NotImplementedError def __eq__(self, other): return self.key == other.key def __ge__(self, other): return self.key >= other.key def __gt__(self, other): return self.key > other.key def __le__(self, other): return self.key <= other.key def __lt__(self, other): return self.key < other.key def __ne__(self, other): return self.key != other.key def __repr__(self) -> str: """Returns a formal representation of the task.""" rep = f"{self.__class__.__name__}(" for attr, value in self.__dict__.items(): rep += f"{attr}={value.__repr__()}, " return f"{rep.strip(', ')})" def __str__(self) -> str: """Returns a printable version of the task.""" dependencies = f"#dependencies={len(self.dependencies)}" return "priority={0}, status={1}, start_time={2}, {3}".format( self.priority, self.status, self.start_time, dependencies ) def _update(self) -> None: """Update properties associated with a new instance of a task.""" # Number of times the task has/will be queued self.attempts = self.attempts + 1 # Ensure the task gets a unique sequence number to preserve the # order in which it is added. self.sequence = next(_counter) def add_dependent(self, pkg_id: str) -> None: """ Ensure the package is in this task's ``dependents`` list. Args: pkg_id: package identifier of the dependent package """ if pkg_id != self.pkg_id and pkg_id not in self.dependents: tty.debug(f"Adding {pkg_id} as a dependent of {self.pkg_id}") self.dependents.add(pkg_id) def add_dependency(self, pkg_id, installed=False): """ Ensure the package is in this task's ``dependencies`` list. Args: pkg_id (str): package identifier of the dependency package installed (bool): install status of the dependency package """ if pkg_id != self.pkg_id and pkg_id not in self.dependencies: tty.debug(f"Adding {pkg_id} as a dependency of {self.pkg_id}") self.dependencies.add(pkg_id) if not installed: self.uninstalled_deps.add(pkg_id) def flag_installed(self, installed: List[str]) -> None: """ Ensure the dependency is not considered to still be uninstalled. Args: installed: the identifiers of packages that have been installed so far """ now_installed = self.uninstalled_deps & set(installed) for pkg_id in now_installed: self.uninstalled_deps.remove(pkg_id) tty.debug( f"{self.pkg_id}: Removed {pkg_id} from uninstalled deps list: " f"{self.uninstalled_deps}", level=2, ) def _setup_install_dir(self, pkg: "spack.package_base.PackageBase") -> None: """ Create and ensure proper access controls for the install directory. Write a small metadata file with the current spack environment. Args: pkg: the package to be built and installed """ # Move to a module level method. if not os.path.exists(pkg.spec.prefix): path = spack.util.path.debug_padded_filter(pkg.spec.prefix) tty.debug(f"Creating the installation directory {path}") spack.store.STORE.layout.create_install_directory(pkg.spec) else: # Set the proper group for the prefix group = prefs.get_package_group(pkg.spec) if group: fs.chgrp(pkg.spec.prefix, group) # Set the proper permissions. # This has to be done after group because changing groups blows # away the sticky group bit on the directory mode = os.stat(pkg.spec.prefix).st_mode perms = prefs.get_package_dir_permissions(pkg.spec) if mode != perms: os.chmod(pkg.spec.prefix, perms) # Ensure the metadata path exists as well fs.mkdirp(spack.store.STORE.layout.metadata_path(pkg.spec), mode=perms) # Always write host environment - we assume this can change spack.store.STORE.layout.write_host_environment(pkg.spec) @property def install_action(self): if not self._install_action: self._install_action = self.get_install_action() return self._install_action def get_install_action(self: "Task") -> InstallAction: """ Determine whether the installation should be overwritten (if it already exists) or skipped (if has been handled by another process). If the package has not been installed yet, this will indicate that the installation should proceed as normal (i.e. no need to transactionally preserve the old prefix). """ # If we don't have to overwrite, do a normal install if self.pkg.spec.dag_hash() not in self.request.overwrite: return InstallAction.INSTALL # If it's not installed, do a normal install as well rec, installed = check_db(self.pkg.spec) if not installed: return InstallAction.INSTALL # Ensure install_tree projections have not changed. assert rec and self.pkg.prefix == rec.path # If another process has overwritten this, we shouldn't install at all if rec.installation_time >= self.request.overwrite_time: return InstallAction.NONE # If the install prefix is missing, warn about it, and proceed with # normal install. if not os.path.exists(self.pkg.prefix): tty.debug("Missing installation to overwrite") return InstallAction.INSTALL # Otherwise, do an actual overwrite install. We backup the original # install directory, put the old prefix # back on failure return InstallAction.OVERWRITE @property def explicit(self) -> bool: return self.pkg.spec.dag_hash() in self.request.install_args.get("explicit", []) @property def is_build_request(self) -> bool: """The package was requested directly""" return self.pkg == self.request.pkg @property def install_policy(self) -> InstallPolicy: if self.is_build_request: return self.request.install_args.get("root_policy", "auto") else: return self.request.install_args.get("dependencies_policy", "auto") @property def key(self) -> Tuple[int, int]: """The key is the tuple (# uninstalled dependencies, sequence).""" return (self.priority, self.sequence) def next_attempt(self, installed) -> "Task": """Create a new, updated task for the next installation attempt.""" task = copy.copy(self) task._update() task.start_time = self.start_time or time.time() task.flag_installed(installed) return task @property def priority(self): """The priority is based on the remaining uninstalled dependencies.""" return len(self.uninstalled_deps) def terminate(self) -> None: """End any processes and clean up any resources allocated by this Task. By default this is a no-op. """ def check_db(spec: "spack.spec.Spec") -> Tuple[Optional[spack.database.InstallRecord], bool]: """Determine if the spec is flagged as installed in the database Args: spec: spec whose database install status is being checked Return: Tuple of optional database record, and a boolean installed_in_db that's ``True`` iff the spec is considered installed """ try: rec = spack.store.STORE.db.get_record(spec) installed_in_db = rec.installed if rec else False except KeyError: # KeyError is raised if there is no matching spec in the database # (versus no matching specs that are installed). rec = None installed_in_db = False return rec, installed_in_db class BuildTask(Task): """Class for representing a build task for a package.""" process_handle: Optional["spack.build_environment.BuildProcess"] = None started: bool = False no_op: bool = False tmpdir = None backup_dir = None def start(self): """Attempt to use the binary cache to install requested spec and/or dependency if requested. Otherwise, start a process for of the requested spec and/or dependency represented by the BuildTask.""" self.record.start() if self.install_action == InstallAction.OVERWRITE: self.tmpdir = tempfile.mkdtemp(dir=os.path.dirname(self.pkg.prefix), prefix=".backup") self.backup_dir = os.path.join(self.tmpdir, "backup") os.rename(self.pkg.prefix, self.backup_dir) assert not self.started, "Cannot start a task that has already been started." self.started = True self.start_time = self.start_time or time.time() install_args = self.request.install_args unsigned = install_args.get("unsigned") pkg, pkg_id = self.pkg, self.pkg_id tests = install_args.get("tests") pkg.run_tests = tests is True or tests and pkg.name in tests # Use the binary cache to install if requested, # save result to be handled in BuildTask.complete() # TODO: change binary installs to occur in subprocesses rather than the main Spack process policy = self.install_policy if policy != "source_only": if _install_from_cache(pkg, self.explicit, unsigned): self.success_result = ExecuteResult.SUCCESS return elif policy == "cache_only": self.error_result = spack.error.InstallError( "No binary found when cache-only was specified", pkg=pkg ) return else: tty.msg(f"No binary for {pkg_id} found: installing from source") # if there's an error result, don't start a new process, and leave if self.error_result is not None: return # Create stage object now and let it be serialized for the child process. That # way monkeypatch in tests works correctly. pkg.stage self._setup_install_dir(pkg) # Create a child process to do the actual installation. self._start_build_process() def _start_build_process(self): self.process_handle = spack.build_environment.start_build_process( self.pkg, build_process, self.request.install_args ) # Identify the child process self.child_pid = self.process_handle.pid def poll(self): """Check if task has successfully executed, caused an InstallError, or the child process has information ready to receive.""" assert self.started or self.no_op, ( "Can't call `poll()` before `start()` or identified no-operation task" ) return self.no_op or self.success_result or self.error_result or self.process_handle.poll() def succeed(self): self.record.succeed() # delete the temporary backup for an overwrite # see spack.llnl.util.filesystem.restore_directory_transaction if self.install_action == InstallAction.OVERWRITE: shutil.rmtree(self.tmpdir, ignore_errors=True) def fail(self, inner_exception): self.record.fail(inner_exception) if self.install_action != InstallAction.OVERWRITE: raise inner_exception # restore the overwrite directory from backup # see spack.llnl.util.filesystem.restore_directory_transaction try: if os.path.exists(self.pkg.prefix): shutil.rmtree(self.pkg.prefix) os.rename(self.backup_dir, self.pkg.prefix) except Exception as outer_exception: raise fs.CouldNotRestoreDirectoryBackup(inner_exception, outer_exception) raise inner_exception def complete(self): """ Complete the installation of the requested spec and/or dependency represented by the build task. """ assert self.started or self.no_op, ( "Can't call `complete()` before `start()` or identified no-operation task" ) pkg = self.pkg self.status = BuildStatus.INSTALLING # If task has been identified as a no operation, # return ExecuteResult.NOOP if self.no_op: # This is one exit point that does not need to call # self.succeed/fail. Job is either a no_op (external, upstream) # or requeued. return ExecuteResult.NO_OP # If installing a package from binary cache is successful, # return ExecuteResult.SUCCESS if self.success_result is not None: self.succeed() return self.success_result # If an error arises from installing a package, # raise spack.error.InstallError if self.error_result is not None: self.fail(self.error_result) # hook that allows tests to inspect the Package before installation # see _unit_test_check() docs. if not pkg._unit_test_check(): self.succeed() return ExecuteResult.FAILED try: # Check if the task's child process has completed spack.package_base.PackageBase._verbose = self.process_handle.complete() # Note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. spack.store.STORE.db.add(pkg.spec, explicit=self.explicit) except spack.error.StopPhase as e: # A StopPhase exception means that do_install was asked to # stop early from clients, and is not an error at this point pid = f"{self.pid}: " if tty.show_pid() else "" tty.debug(f"{pid}{str(e)}") tty.debug(f"Package stage directory: {pkg.stage.source_path}") except (Exception, KeyboardInterrupt, SystemExit) as e: self.fail(e) self.succeed() return ExecuteResult.SUCCESS def terminate(self) -> None: """Terminate any processes this task still has running.""" if self.process_handle: self.process_handle.terminate() class MockBuildProcess: def complete(self) -> bool: return True def terminate(self) -> None: pass class FakeBuildTask(BuildTask): """Blocking BuildTask executed directly in the main thread. Used for --fake installs.""" process_handle = MockBuildProcess() # type: ignore[assignment] def _start_build_process(self): build_process(self.pkg, self.request.install_args) def poll(self): return True class RewireTask(Task): """Class for representing a rewire task for a package.""" def start(self): self.record.start() def poll(self): return True def complete(self): """Execute rewire task Rewire tasks are executed by either rewiring self.package.spec.build_spec that is already installed or downloading and rewiring a binary for the it. If not available installed or as binary, return ExecuteResult.MISSING_BUILD_SPEC. This will prompt the Installer to requeue the task with a dependency on the BuildTask to install self.pkg.spec.build_spec """ oldstatus = self.status self.status = BuildStatus.INSTALLING if not self.pkg.spec.build_spec.installed: try: install_args = self.request.install_args unsigned = install_args.get("unsigned") success = _process_binary_cache_tarball( self.pkg, explicit=self.explicit, unsigned=unsigned ) if not success: tty.msg( "Failed to find binary for build spec, requeuing {self.pkg.spec} with" "dependency install task for its build spec" ) self.status = oldstatus return ExecuteResult.MISSING_BUILD_SPEC _print_installed_pkg(self.pkg.prefix) self.record.succeed() return ExecuteResult.SUCCESS except BaseException as e: tty.error(f"Failed to rewire {self.pkg.spec} from binary. {e}") self.status = oldstatus return ExecuteResult.MISSING_BUILD_SPEC try: spack.rewiring.rewire_node(self.pkg.spec, self.explicit) _print_installed_pkg(self.pkg.prefix) self.record.succeed() return ExecuteResult.SUCCESS except BaseException as e: self.record.fail(e) class PackageInstaller: """ Class for managing the install process for a Spack instance based on a bottom-up DAG approach. This installer can coordinate concurrent batch and interactive, local and distributed (on a shared file system) builds for the same Spack instance. """ def __init__( self, packages: List["spack.package_base.PackageBase"], *, dirty: bool = False, explicit: Union[Set[str], bool] = False, overwrite: Optional[Union[List[str], Set[str]]] = None, fail_fast: bool = False, fake: bool = False, include_build_deps: bool = False, install_deps: bool = True, install_package: bool = True, install_source: bool = False, keep_prefix: bool = False, keep_stage: bool = False, restage: bool = False, skip_patch: bool = False, stop_at: Optional[str] = None, stop_before: Optional[str] = None, tests: Union[bool, List[str], Set[str]] = False, unsigned: Optional[bool] = None, verbose: bool = False, concurrent_packages: Optional[int] = None, root_policy: InstallPolicy = "auto", dependencies_policy: InstallPolicy = "auto", create_reports: bool = False, ) -> None: """ Arguments: explicit: Set of package hashes to be marked as installed explicitly in the db. If True, the specs from ``packages`` are marked explicit, while their dependencies are not. fail_fast: Fail if any dependency fails to install; otherwise, the default is to install as many dependencies as possible (i.e., best effort installation). fake: Don't really build; install fake stub files instead. install_deps: Install dependencies before installing this package install_source: By default, source is not installed, but for debugging it might be useful to keep it around. keep_prefix: Keep install prefix on failure. By default, destroys it. keep_stage: By default, stage is destroyed only if there are no exceptions during build. Set to True to keep the stage even with exceptions. restage: Force spack to restage the package source. skip_patch: Skip patch stage of build if True. stop_before: stop execution before this installation phase (or None) stop_at: last installation phase to be executed (or None) tests: False to run no tests, True to test all packages, or a list of package names to run tests for some verbose: Display verbose build output (by default, suppresses it) concurrent_packages: Max packages to be built concurrently root_policy: ``"auto"``, ``"cache_only"``, ``"source_only"``. dependencies_policy: ``"auto"``, ``"cache_only"``, ``"source_only"``. create_reports: whether to generate reports for each install """ if sys.platform == "win32": # No locks on Windows, we should always use 1 process # TODO: perhaps raise an error instead and update cmd-line interface # to omit this option on Windows for now concurrent_packages = 1 if isinstance(explicit, bool): explicit = {pkg.spec.dag_hash() for pkg in packages} if explicit else set() if concurrent_packages is None: concurrent_packages = spack.config.get("config:concurrent_packages", default=1) # The value 0 means no concurrency in the old installer. if concurrent_packages == 0: concurrent_packages = 1 self.concurrent_packages = concurrent_packages install_args = { "dependencies_policy": dependencies_policy, "dirty": dirty, "explicit": explicit, "fail_fast": fail_fast, "fake": fake, "include_build_deps": include_build_deps, "install_deps": install_deps, "install_package": install_package, "install_source": install_source, "keep_prefix": keep_prefix, "keep_stage": keep_stage, "overwrite": overwrite or [], "root_policy": root_policy, "restage": restage, "skip_patch": skip_patch, "stop_at": stop_at, "stop_before": stop_before, "tests": tests, "unsigned": unsigned, "verbose": verbose, "concurrent_packages": self.concurrent_packages, } # List of build requests self.build_requests = [BuildRequest(pkg, install_args) for pkg in packages] # When no reporter is configured, use NullInstallRecord to skip log file reads. if not create_reports: for br in self.build_requests: br.install_args["record_cls"] = spack.report.NullInstallRecord # Priority queue of tasks self.build_pq: List[Tuple[Tuple[int, int], Task]] = [] # Mapping of unique package ids to task self.build_tasks: Dict[str, Task] = {} # Cache of package locks for failed packages, keyed on package's ids self.failed: Dict[str, Optional[lk.Lock]] = {} # Cache the PID for distributed build messaging self.pid: int = os.getpid() # Cache of installed packages' unique ids self.installed: Set[str] = set() # Data store layout self.layout = spack.store.STORE.layout # Locks on specs being built, keyed on the package's unique id self.locks: Dict[str, Tuple[str, Optional[lk.Lock]]] = {} # Cache fail_fast option to ensure if one build request asks to fail # fast then that option applies to all build requests. self.fail_fast = False # Initializing all_dependencies to empty. This will be set later in _init_queue. self.all_dependencies: Dict[str, Set[str]] = {} # Maximum number of concurrent packages to build self.max_active_tasks = self.concurrent_packages # Reports on install success/failure if create_reports: self.reports: Dict[str, spack.report.RequestRecord] = {} for build_request in self.build_requests: # Skip reporting for already installed specs request_record = spack.report.RequestRecord(build_request.pkg.spec) request_record.skip_installed() self.reports[build_request.pkg_id] = request_record else: self.reports = { br.pkg_id: spack.report.NullRequestRecord() for br in self.build_requests } def __repr__(self) -> str: """Returns a formal representation of the package installer.""" rep = f"{self.__class__.__name__}(" for attr, value in self.__dict__.items(): rep += f"{attr}={value.__repr__()}, " return f"{rep.strip(', ')})" def __str__(self) -> str: """Returns a printable version of the package installer.""" requests = f"#requests={len(self.build_requests)}" tasks = f"#tasks={len(self.build_tasks)}" failed = f"failed ({len(self.failed)}) = {self.failed}" installed = f"installed ({len(self.installed)}) = {self.installed}" return f"{self.pid}: {requests}; {tasks}; {installed}; {failed}" def _add_init_task( self, pkg: "spack.package_base.PackageBase", request: BuildRequest, all_deps: Dict[str, Set[str]], ) -> None: """ Creates and queues the initial task for the package. Args: pkg: the package to be built and installed request: the associated install request all_deps: dictionary of all dependencies and associated dependents """ cls: type[Task] = BuildTask if pkg.spec.spliced: cls = RewireTask elif request.install_args.get("fake"): cls = FakeBuildTask task = cls(pkg, request=request, status=BuildStatus.QUEUED, installed=self.installed) for dep_id in task.dependencies: all_deps[dep_id].add(package_id(pkg.spec)) self._push_task(task) def _check_deps_status(self, request: BuildRequest) -> None: """Check the install status of the requested package Args: request: the associated install request """ err = "Cannot proceed with {0}: {1}" for dep in request.traverse_dependencies(): dep_pkg = dep.package dep_id = package_id(dep) # Check for failure since a prefix lock is not required if spack.store.STORE.failure_tracker.has_failed(dep): action = "'spack install' the dependency" msg = f"{dep_id} is marked as an install failure: {action}" raise spack.error.InstallError(err.format(request.pkg_id, msg), pkg=dep_pkg) # Attempt to get a read lock to ensure another process does not # uninstall the dependency while the requested spec is being # installed ltype, lock = self._ensure_locked("read", dep_pkg) if lock is None: msg = f"{dep_id} is write locked by another process" raise spack.error.InstallError(err.format(request.pkg_id, msg), pkg=request.pkg) # Flag external and upstream packages as being installed if dep_pkg.spec.external or dep_pkg.spec.installed_upstream: self._flag_installed(dep_pkg) continue # Check the database to see if the dependency has been installed # and flag as such if appropriate rec, installed_in_db = check_db(dep) if ( rec and installed_in_db and ( dep.dag_hash() not in request.overwrite or rec.installation_time > request.overwrite_time ) ): tty.debug(f"Flagging {dep_id} as installed per the database") self._flag_installed(dep_pkg) else: lock.release_read() def _prepare_for_install(self, task: Task) -> None: """ Check the database and leftover installation directories/files and prepare for a new install attempt for an uninstalled package. Preparation includes cleaning up installation and stage directories and ensuring the database is up-to-date. Args: task: the task whose associated package is being checked """ install_args = task.request.install_args keep_prefix = install_args.get("keep_prefix") # Make sure the package is ready to be locally installed. self._ensure_install_ready(task.pkg) # Skip file system operations if we've already gone through them for # this spec. if task.pkg_id in self.installed: # Already determined the spec has been installed return # Determine if the spec is flagged as installed in the database rec, installed_in_db = check_db(task.pkg.spec) if not installed_in_db: # Ensure there is no other installed spec with the same prefix dir if spack.store.STORE.db.is_occupied_install_prefix(task.pkg.spec.prefix): task.error_result = spack.error.InstallError( f"Install prefix collision for {task.pkg_id}", long_msg=f"Prefix directory {task.pkg.spec.prefix} already " "used by another installed spec.", pkg=task.pkg, ) return # Make sure the installation directory is in the desired state # for uninstalled specs. if os.path.isdir(task.pkg.spec.prefix): if not keep_prefix and task.install_action != InstallAction.OVERWRITE: task.pkg.remove_prefix() else: tty.debug(f"{task.pkg_id} is partially installed") if ( rec and installed_in_db and ( rec.spec.dag_hash() not in task.request.overwrite or rec.installation_time > task.request.overwrite_time ) ): self._update_installed(task) # Only update the explicit entry once for the explicit package if task.explicit and not rec.explicit: spack.store.STORE.db.mark(task.pkg.spec, "explicit", True) def _cleanup_all_tasks(self) -> None: """Cleanup all tasks to include releasing their locks.""" for pkg_id in self.locks: self._release_lock(pkg_id) for pkg_id in self.failed: self._cleanup_failed(pkg_id) ids = list(self.build_tasks) for pkg_id in ids: try: self._remove_task(pkg_id) except Exception: pass def _cleanup_failed(self, pkg_id: str) -> None: """ Cleanup any failed markers for the package Args: pkg_id (str): identifier for the failed package """ lock = self.failed.get(pkg_id, None) if lock is not None: err = "{0} exception when removing failure tracking for {1}: {2}" try: tty.verbose(f"Removing failure mark on {pkg_id}") lock.release_write() except Exception as exc: tty.warn(err.format(exc.__class__.__name__, pkg_id, str(exc))) def _cleanup_task(self, pkg: "spack.package_base.PackageBase") -> None: """ Cleanup the task for the spec Args: pkg: the package being installed """ self._remove_task(package_id(pkg.spec)) # Ensure we have a read lock to prevent others from uninstalling the # spec during our installation. self._ensure_locked("read", pkg) def _ensure_install_ready(self, pkg: "spack.package_base.PackageBase") -> None: """ Ensure the package is ready to install locally, which includes already locked. Args: pkg: the package being locally installed """ pkg_id = package_id(pkg.spec) pre = f"{pkg_id} cannot be installed locally:" # External packages cannot be installed locally. if pkg.spec.external: raise ExternalPackageError(f"{pre} is external") # Upstream packages cannot be installed locally. if pkg.spec.installed_upstream: raise UpstreamPackageError(f"{pre} is upstream") # The package must have a prefix lock at this stage. if pkg_id not in self.locks: raise InstallLockError(f"{pre} not locked") def _ensure_locked( self, lock_type: str, pkg: "spack.package_base.PackageBase" ) -> Tuple[str, Optional[lk.Lock]]: """ Add a prefix lock of the specified type for the package spec If the lock exists, then adjust accordingly. That is, read locks will be upgraded to write locks if a write lock is requested and write locks will be downgraded to read locks if a read lock is requested. The lock timeout for write locks is deliberately near zero seconds in order to ensure the current process proceeds as quickly as possible to the next spec. Args: lock_type: 'read' for a read lock, 'write' for a write lock pkg: the package whose spec is being installed Return: (lock_type, lock) tuple where lock will be None if it could not be obtained """ assert lock_type in ["read", "write"], ( f'"{lock_type}" is not a supported package management lock type' ) pkg_id = package_id(pkg.spec) ltype, lock = self.locks.get(pkg_id, (lock_type, None)) if lock and ltype == lock_type: return ltype, lock desc = f"{lock_type} lock" msg = "{0} a {1} on {2} with timeout {3}" err = "Failed to {0} a {1} for {2} due to {3}: {4}" if lock_type == "read": # Wait until the other process finishes if there are no more # tasks with priority 0 (i.e., with no uninstalled # dependencies). no_p0 = len(self.build_tasks) == 0 or not self._next_is_pri0() timeout = None if no_p0 else 3.0 else: timeout = 1e-9 # Near 0 to iterate through install specs quickly try: if lock is None: tty.debug(msg.format("Acquiring", desc, pkg_id, pretty_seconds(timeout or 0))) op = "acquire" lock = spack.store.STORE.prefix_locker.lock(pkg.spec, timeout) if timeout != lock.default_timeout: tty.warn(f"Expected prefix lock timeout {timeout}, not {lock.default_timeout}") if lock_type == "read": lock.acquire_read() else: lock.acquire_write() elif lock_type == "read": # write -> read # Only get here if the current lock is a write lock, which # must be downgraded to be a read lock # Retain the original lock timeout, which is in the lock's # default_timeout setting. tty.debug( msg.format( "Downgrading to", desc, pkg_id, pretty_seconds(lock.default_timeout or 0) ) ) op = "downgrade to" lock.downgrade_write_to_read() else: # read -> write # Only get here if the current lock is a read lock, which # must be upgraded to be a write lock tty.debug(msg.format("Upgrading to", desc, pkg_id, pretty_seconds(timeout or 0))) op = "upgrade to" lock.upgrade_read_to_write(timeout) tty.debug(f"{pkg_id} is now {lock_type} locked") except (lk.LockDowngradeError, lk.LockTimeoutError) as exc: tty.debug(err.format(op, desc, pkg_id, exc.__class__.__name__, str(exc))) return (lock_type, None) except (Exception, KeyboardInterrupt, SystemExit) as exc: tty.error(err.format(op, desc, pkg_id, exc.__class__.__name__, str(exc))) self._cleanup_all_tasks() raise self.locks[pkg_id] = (lock_type, lock) return self.locks[pkg_id] def _requeue_with_build_spec_tasks(self, task): """Requeue the task and its missing build spec dependencies""" # Full install of the build_spec is necessary because it didn't already exist somewhere spec = task.pkg.spec for dep in spec.build_spec.traverse(): dep_pkg = dep.package dep_id = package_id(dep) if dep_id not in self.build_tasks: self._add_init_task(dep_pkg, task.request, self.all_dependencies) # Clear any persistent failure markings _unless_ they are # associated with another process in this parallel build # of the spec. spack.store.STORE.failure_tracker.clear(dep, force=False) # Queue the build spec. build_pkg_id = package_id(spec.build_spec) build_spec_task = self.build_tasks[build_pkg_id] spec_pkg_id = package_id(spec) spec_task = task.next_attempt(self.installed) spec_task.status = BuildStatus.QUEUED # Convey a build spec as a dependency of a deployed spec. build_spec_task.add_dependent(spec_pkg_id) spec_task.add_dependency(build_pkg_id) self._push_task(spec_task) def _add_tasks(self, request: BuildRequest, all_deps): """Add tasks to the priority queue for the given build request. It also tracks all dependents associated with each dependency in order to ensure proper tracking of uninstalled dependencies. Args: request (BuildRequest): the associated install request all_deps (defaultdict(set)): dictionary of all dependencies and associated dependents """ tty.debug(f"Initializing the build queue for {request.pkg.name}") # Ensure not attempting to perform an installation when user didn't # want to go that far for the requested package. try: _check_last_phase(request.pkg) except BadInstallPhase as err: tty.warn(f"Installation request refused: {str(err)}") return install_deps = request.install_args.get("install_deps") if install_deps: for dep in request.traverse_dependencies(): dep_pkg = dep.package dep_id = package_id(dep) if dep_id not in self.build_tasks: self._add_init_task(dep_pkg, request, all_deps=all_deps) # Clear any persistent failure markings _unless_ they are # associated with another process in this parallel build # of the spec. spack.store.STORE.failure_tracker.clear(dep, force=False) install_package = request.install_args.get("install_package") if install_package and request.pkg_id not in self.build_tasks: # Be sure to clear any previous failure spack.store.STORE.failure_tracker.clear(request.spec, force=True) # If not installing dependencies, then determine their # installation status before proceeding if not install_deps: self._check_deps_status(request) # Now add the package itself, if appropriate self._add_init_task(request.pkg, request, all_deps=all_deps) # Ensure if one request is to fail fast then all requests will. fail_fast = bool(request.install_args.get("fail_fast")) self.fail_fast = self.fail_fast or fail_fast def _complete_task(self, task: Task, install_status: InstallStatus) -> None: """ Complete the installation of the requested spec and/or dependency represented by the task. Args: task: the installation task for a package install_status: the installation status for the package """ try: rc = task.complete() except BaseException: self.reports[task.request.pkg_id].append_record(task.record) raise if rc == ExecuteResult.MISSING_BUILD_SPEC: self._requeue_with_build_spec_tasks(task) elif rc == ExecuteResult.NO_OP: pass else: # if rc == ExecuteResult.SUCCESS or rc == ExecuteResult.FAILED self._update_installed(task) self.reports[task.request.pkg_id].append_record(task.record) def _next_is_pri0(self) -> bool: """ Determine if the next task has priority 0 Return: True if it does, False otherwise """ # Leverage the fact that the first entry in the queue is the next # one that will be processed task = self.build_pq[0][1] return task.priority == 0 def _clear_removed_tasks(self): """Get rid of any tasks in the queue with status 'BuildStatus.REMOVED'""" while self.build_pq and self.build_pq[0][1].status == BuildStatus.REMOVED: heapq.heappop(self.build_pq) def _peek_ready_task(self) -> Optional[Task]: """ Return the first ready task in the queue, or None if there are no ready tasks. """ self._clear_removed_tasks() if not self.build_pq: return None task = self.build_pq[0][1] return task if task.priority == 0 else None def _tasks_installing_in_other_spack(self) -> bool: """Whether any tasks in the build queue are installing in other spack processes.""" return any(task.status == BuildStatus.INSTALLING for _, task in self.build_pq) def _pop_task(self) -> Task: """Pop the first task off the queue and return it. Raise an index error if the queue is empty.""" self._clear_removed_tasks() if not self.build_pq: raise IndexError("Attempt to pop empty queue") _, task = heapq.heappop(self.build_pq) del self.build_tasks[task.pkg_id] task.status = BuildStatus.DEQUEUED return task def _pop_ready_task(self) -> Optional[Task]: """ Pop the first ready task off the queue and return it. Return None if no ready task. """ if self._peek_ready_task(): return self._pop_task() return None def _push_task(self, task: Task) -> None: """ Push (or queue) the specified task for the package. Source: Customization of "add_task" function at docs.python.org/2/library/heapq.html Args: task: the installation task for a package """ msg = "{0} a task for {1} with status '{2}'" skip = "Skipping requeue of task for {0}: {1}" # Ensure do not (re-)queue installed or failed packages whose status # may have been determined by a separate process. if task.pkg_id in self.installed: tty.debug(skip.format(task.pkg_id, "installed")) return if task.pkg_id in self.failed: tty.debug(skip.format(task.pkg_id, "failed")) return # Remove any associated task since its sequence will change self._remove_task(task.pkg_id) desc = ( "Queueing" if task.attempts == 1 else f"Requeuing ({ordinal(task.attempts)} attempt)" ) tty.debug(msg.format(desc, task.pkg_id, task.status)) # Now add the new task to the queue with a new sequence number to # ensure it is the last entry popped with the same priority. This # is necessary in case we are re-queueing a task whose priority # was decremented due to the installation of one of its dependencies. self.build_tasks[task.pkg_id] = task heapq.heappush(self.build_pq, (task.key, task)) def _release_lock(self, pkg_id: str) -> None: """ Release any lock on the package Args: pkg_id (str): identifier for the package whose lock is be released """ if pkg_id in self.locks: err = "{0} exception when releasing {1} lock for {2}: {3}" msg = "Releasing {0} lock on {1}" ltype, lock = self.locks[pkg_id] if lock is not None: try: tty.debug(msg.format(ltype, pkg_id)) if ltype == "read": lock.release_read() else: lock.release_write() except Exception as exc: tty.warn(err.format(exc.__class__.__name__, ltype, pkg_id, str(exc))) def _remove_task(self, pkg_id: str) -> Optional[Task]: """ Mark the existing package task as being removed and return it. Raises KeyError if not found. Source: Variant of function at docs.python.org/2/library/heapq.html Args: pkg_id: identifier for the package to be removed """ if pkg_id in self.build_tasks: tty.debug(f"Removing task for {pkg_id} from list") task = self.build_tasks.pop(pkg_id) task.status = BuildStatus.REMOVED return task else: return None def _requeue_task(self, task: Task, install_status: InstallStatus) -> None: """ Requeues a task that appears to be in progress by another process. Args: task (Task): the installation task for a package """ if task.status not in [BuildStatus.INSTALLED, BuildStatus.INSTALLING]: tty.debug( f"{install_msg(task.pkg_id, self.pid, install_status)} " "in progress by another process" ) new_task = task.next_attempt(self.installed) new_task.status = BuildStatus.INSTALLING self._push_task(new_task) def _update_failed( self, task: Task, mark: bool = False, exc: Optional[BaseException] = None ) -> None: """ Update the task and transitive dependents as failed; optionally mark externally as failed; and remove associated tasks. Args: task: the task for the failed package mark: ``True`` if the package and its dependencies are to be marked as "failed", otherwise, ``False`` exc: optional exception if associated with the failure """ pkg_id = task.pkg_id err = "" if exc is None else f": {str(exc)}" tty.debug(f"Flagging {pkg_id} as failed{err}") if mark: self.failed[pkg_id] = spack.store.STORE.failure_tracker.mark(task.pkg.spec) else: self.failed[pkg_id] = None task.status = BuildStatus.FAILED for dep_id in task.dependents: if dep_id in self.build_tasks: tty.warn(f"Skipping build of {dep_id} since {pkg_id} failed") # Ensure the dependent's uninstalled dependents are # up-to-date and their tasks removed. dep_task = self.build_tasks[dep_id] self._update_failed(dep_task, mark) self._remove_task(dep_id) else: tty.debug(f"No task for {dep_id} to skip since {pkg_id} failed") def _update_installed(self, task: Task) -> None: """ Mark the task as installed and ensure dependent tasks are aware. Args: task: the task for the installed package """ task.status = BuildStatus.INSTALLED self._flag_installed(task.pkg, task.dependents) def _flag_installed( self, pkg: "spack.package_base.PackageBase", dependent_ids: Optional[Set[str]] = None ) -> None: """ Flag the package as installed and ensure known by all tasks of known dependents. Args: pkg: Package that has been installed locally, externally or upstream dependent_ids: set of the package's dependent ids, or None if the dependent ids are limited to those maintained in the package (dependency DAG) """ pkg_id = package_id(pkg.spec) if pkg_id in self.installed: # Already determined the package has been installed return tty.debug(f"Flagging {pkg_id} as installed") self.installed.add(pkg_id) # Update affected dependents dependent_ids = dependent_ids or get_dependent_ids(pkg.spec) for dep_id in set(dependent_ids): tty.debug(f"Removing {pkg_id} from {dep_id}'s uninstalled dependencies.") if dep_id in self.build_tasks: # Ensure the dependent's uninstalled dependencies are # up-to-date. This will require requeuing the task. dep_task = self.build_tasks[dep_id] self._push_task(dep_task.next_attempt(self.installed)) else: tty.debug(f"{dep_id} has no task to update for {pkg_id}'s success") def _init_queue(self) -> None: """Initialize the build queue from the list of build requests.""" all_dependencies: Dict[str, Set[str]] = defaultdict(set) tty.debug("Initializing the build queue from the build requests") for request in self.build_requests: self._add_tasks(request, all_dependencies) # Add any missing dependents to ensure proper uninstalled dependency # tracking when installing multiple specs tty.debug("Ensure all dependencies know all dependents across specs") for dep_id in all_dependencies: if dep_id in self.build_tasks: dependents = all_dependencies[dep_id] task = self.build_tasks[dep_id] for dependent_id in dependents.difference(task.dependents): task.add_dependent(dependent_id) self.all_dependencies = all_dependencies def start_task( self, task: Task, install_status: InstallStatus, term_status: TermStatusLine ) -> None: """Attempt to start a package installation.""" pkg, pkg_id, spec = task.pkg, task.pkg_id, task.pkg.spec install_status.next_pkg(pkg) # install_status.set_term_title(f"Processing {task.pkg.name}") tty.debug(f"Processing {pkg_id}: task={task}") # Debug task.record.start() # Skip the installation if the spec is not being installed locally # (i.e., if external or upstream) BUT flag it as installed since # some package likely depends on it. if _handle_external_and_upstream(pkg, task.explicit): term_status.clear() self._flag_installed(pkg, task.dependents) task.no_op = True return # Flag a failed spec. Do not need an (install) prefix lock since # assume using a separate (failed) prefix lock file. if pkg_id in self.failed or spack.store.STORE.failure_tracker.has_failed(spec): term_status.clear() tty.warn(f"{pkg_id} failed to install") self._update_failed(task) if self.fail_fast: task.error_result = spack.error.InstallError(_FAIL_FAST_ERR, pkg=pkg) # Attempt to get a write lock. If we can't get the lock then # another process is likely (un)installing the spec or has # determined the spec has already been installed (though the # other process may be hung). install_status.set_term_title(f"Acquiring lock for {task.pkg.name}") term_status.add(pkg_id) ltype, lock = self._ensure_locked("write", pkg) if lock is None: # Attempt to get a read lock instead. If this fails then # another process has a write lock so must be (un)installing # the spec (or that process is hung). ltype, lock = self._ensure_locked("read", pkg) # Requeue the spec if we cannot get at least a read lock so we # can check the status presumably established by another process # -- failed, installed, or uninstalled -- on the next pass. if lock is None: self._requeue_task(task, install_status) task.no_op = True return term_status.clear() # Take a timestamp with the overwrite argument to allow checking # whether another process has already overridden the package. if task.request.overwrite and task.explicit: task.request.overwrite_time = time.time() # install_status.set_term_title(f"Preparing {task.pkg.name}") self._prepare_for_install(task) # Flag an already installed package if pkg_id in self.installed: # Downgrade to a read lock to preclude other processes from # uninstalling the package until we're done installing its # dependents. ltype, lock = self._ensure_locked("read", pkg) if lock is not None: self._update_installed(task) path = spack.util.path.debug_padded_filter(pkg.prefix) _print_installed_pkg(path) else: # At this point we've failed to get a write or a read # lock, which means another process has taken a write # lock between our releasing the write and acquiring the # read. # # Requeue the task so we can re-check the status # established by the other process -- failed, installed, # or uninstalled -- on the next pass. self.installed.remove(pkg_id) self._requeue_task(task, install_status) task.no_op = True return # Having a read lock on an uninstalled pkg may mean another # process completed an uninstall of the software between the # time we failed to acquire the write lock and the time we # took the read lock. # # Requeue the task so we can check the status presumably # established by the other process -- failed, installed, or # uninstalled -- on the next pass. if ltype == "read": lock.release_read() self._requeue_task(task, install_status) task.no_op = True return # Proceed with the installation since we have an exclusive write # lock on the package. install_status.set_term_title(f"Installing {task.pkg.name}") action = task.install_action if action in (InstallAction.INSTALL, InstallAction.OVERWRITE): # Start a child process for a task that's ready to be installed. task.start() tty.msg(install_msg(pkg_id, self.pid, install_status)) def complete_task(self, task: Task, install_status: InstallStatus) -> Optional[Tuple]: """Attempts to complete a package installation.""" pkg, pkg_id = task.pkg, task.pkg_id install_args = task.request.install_args keep_prefix = install_args.get("keep_prefix") action = task.install_action try: self._complete_task(task, install_status) # If we installed then we should keep the prefix stop_before_phase = getattr(pkg, "stop_before_phase", None) last_phase = getattr(pkg, "last_phase", None) keep_prefix = keep_prefix or (stop_before_phase is None and last_phase is None) except KeyboardInterrupt as exc: # The build has been terminated with a Ctrl-C so terminate # regardless of the number of remaining specs. tty.error(f"Failed to install {pkg.name} due to {exc.__class__.__name__}: {str(exc)}") raise except BuildcacheEntryError as exc: if task.install_policy == "cache_only": raise # Checking hash on downloaded binary failed. tty.error( f"Failed to install {pkg.name} from binary cache due " f"to {str(exc)}: Requeuing to install from source." ) # this overrides a full method, which is ugly. task.install_policy = "source_only" # type: ignore[misc] self._requeue_task(task, install_status) return None # Overwrite install exception handling except fs.CouldNotRestoreDirectoryBackup as e: spack.store.STORE.db.remove(task.pkg.spec) tty.error( f"Recovery of install dir of {task.pkg.name} failed due to " f"{e.outer_exception.__class__.__name__}: {str(e.outer_exception)}. " "The spec is now uninstalled." ) # Unwrap the actual installation exception. raise e.inner_exception except (Exception, SystemExit) as exc: # Overwrite process exception handling self._update_failed(task, True, exc) # Best effort installs suppress the exception and mark the # package as a failure. if not isinstance(exc, spack.error.SpackError) or not exc.printed: # type: ignore[union-attr] # noqa: E501 exc.printed = True # type: ignore[union-attr] # SpackErrors can be printed by the build process or at # lower levels -- skip printing if already printed. # TODO: sort out this and SpackError.print_context() tty.error( f"Failed to install {pkg.name} due to {exc.__class__.__name__}: {str(exc)}" ) # Terminate if requested to do so on the first failure. if self.fail_fast: raise spack.error.InstallError(f"{_FAIL_FAST_ERR}: {str(exc)}", pkg=pkg) from exc # Terminate when a single build request has failed, or summarize errors later. if task.is_build_request: if len(self.build_requests) == 1: raise return (pkg, pkg_id, str(exc)) finally: # Remove the install prefix if anything went wrong during # install. if not keep_prefix and action != InstallAction.OVERWRITE: pkg.remove_prefix() # Perform basic task cleanup for the installed spec to # include downgrading the write to a read lock if pkg.spec.installed: self._cleanup_task(pkg) # mark installed if we haven't yet - may be discovering installed for the first time self._update_installed(task) return None def install(self) -> None: """Install the requested package(s) and/or associated dependencies.""" # ensure that build processes do not permanently bork terminal settings with preserve_terminal_settings(sys.stdin): self._install() def _install(self) -> None: """Helper with main implementation of ``install()``. We need to wrap the installation routine with a context manager for preserving keyboard sanity. Wrappers go in ``install()``. This does the real work. """ self._init_queue() failed_build_requests = [] install_status = InstallStatus(len(self.build_pq)) active_tasks: List[Task] = [] # Only enable the terminal status line when we're in a tty without debug info # enabled, so that the output does not get cluttered. term_status = TermStatusLine( enabled=sys.stdout.isatty() and tty.msg_enabled() and not tty.is_debug() ) # While a task is ready or tasks are running while self._peek_ready_task() or active_tasks or self._tasks_installing_in_other_spack(): # While there's space for more active tasks to start while len(active_tasks) < self.max_active_tasks: task = self._pop_ready_task() if not task: # no ready tasks break active_tasks.append(task) try: # Attempt to start the task's package installation self.start_task(task, install_status, term_status) except BaseException as e: # Delegating any exception that happens in start_task() to be # handled in complete_task() task.error_result = e # 10 ms to avoid busy waiting time.sleep(0.01) # Check if any tasks have completed and add to list done = [task for task in active_tasks if task.poll()] try: # Iterate through the done tasks and complete them for task in done: # If complete_task does not return None, the build request failed failure = self.complete_task(task, install_status) if failure: failed_build_requests.append(failure) active_tasks.remove(task) except Exception: # Terminate any active child processes if there's an installation error for task in active_tasks: task.terminate() active_tasks.clear() # they're all done now raise self._clear_removed_tasks() if self.build_pq: task = self._pop_task() assert task.priority != 0, "Found ready task after _peek_ready_task returned None" # If the spec has uninstalled dependencies # and no active tasks running, then there must be # a bug in the code (e.g., priority queue or uninstalled # dependencies handling). So terminate under the assumption # that all subsequent task will have non-zero priorities or may # be dependencies of this task. term_status.clear() tty.error( f"Detected uninstalled dependencies for {task.pkg_id}: {task.uninstalled_deps}" ) left = [dep_id for dep_id in task.uninstalled_deps if dep_id not in self.installed] if not left: tty.warn(f"{task.pkg_id} does NOT actually have any uninstalled deps left") dep_str = "dependencies" if task.priority > 1 else "dependency" raise spack.error.InstallError( f"Cannot proceed with {task.pkg_id}: {task.priority} uninstalled " f"{dep_str}: {','.join(task.uninstalled_deps)}", pkg=task.pkg, ) # Cleanup, which includes releasing all of the read locks self._cleanup_all_tasks() # Ensure we properly report if one or more explicit specs failed # or were not installed when should have been. missing = [ (request.pkg, request.pkg_id) for request in self.build_requests if request.install_args.get("install_package") and request.pkg_id not in self.installed ] if failed_build_requests or missing: for _, pkg_id, err in failed_build_requests: tty.error(f"{pkg_id}: {err}") for _, pkg_id in missing: tty.error(f"{pkg_id}: Package was not installed") if len(failed_build_requests) > 0: pkg = failed_build_requests[0][0] ids = [pkg_id for _, pkg_id, _ in failed_build_requests] tty.debug( "Associating installation failure with first failed " f"explicit package ({ids[0]}) from {', '.join(ids)}" ) elif len(missing) > 0: pkg = missing[0][0] ids = [pkg_id for _, pkg_id in missing] tty.debug( "Associating installation failure with first " f"missing package ({ids[0]}) from {', '.join(ids)}" ) raise spack.error.InstallError( "Installation request failed. Refer to reported errors for failing package(s).", pkg=pkg, ) class BuildProcessInstaller: """This class implements the part installation that happens in the child process.""" def __init__(self, pkg: "spack.package_base.PackageBase", install_args: dict): """Create a new BuildProcessInstaller. It is assumed that the lifecycle of this object is the same as the child process in the build. Arguments: pkg: the package being installed. install_args: arguments to the installer from parent process. """ self.pkg = pkg # whether to do a fake install self.fake = install_args.get("fake", False) # whether to install source code with the package self.install_source = install_args.get("install_source", False) is_develop = pkg.spec.is_develop # whether to keep the build stage after installation # Note: user commands do not have an explicit choice to disable # keeping stages (i.e., we have a --keep-stage option, but not # a --destroy-stage option), so we can override a default choice # to destroy self.keep_stage = is_develop or install_args.get("keep_stage", False) # whether to restage self.restage = (not is_develop) and install_args.get("restage", False) # whether to skip the patch phase self.skip_patch = install_args.get("skip_patch", False) # whether to enable echoing of build output initially or not self.verbose = bool(install_args.get("verbose", False)) # whether installation was explicitly requested by the user self.explicit = pkg.spec.dag_hash() in install_args.get("explicit", []) # env before starting installation self.unmodified_env = install_args.get("unmodified_env", {}) # env modifications by Spack self.env_mods = install_args.get("env_modifications", EnvironmentModifications()) # timer for build phases self.timer = timer.Timer() # If we are using a padded path, filter the output to compress padded paths padding = spack.config.get("config:install_tree:padded_length", None) self.filter_fn = spack.util.path.padding_filter if padding else None # info/debug information self.pre = _log_prefix(pkg.name) self.pkg_id = package_id(pkg.spec) def run(self) -> bool: """Main entry point from ``build_process`` to kick off install in child.""" stage = self.pkg.stage stage.keep = self.keep_stage with stage: if self.restage: stage.destroy() self.timer.start("stage") if not self.fake: if not self.skip_patch: self.pkg.do_patch() else: self.pkg.do_stage() self.timer.stop("stage") tty.debug( f"{self.pre} Building {self.pkg_id} [{self.pkg.build_system_class}]" # type: ignore[attr-defined] # noqa: E501 ) # get verbosity from install parameter or saved value self.echo = self.verbose if spack.package_base.PackageBase._verbose is not None: self.echo = spack.package_base.PackageBase._verbose # Run the pre-install hook in the child process after # the directory is created. spack.hooks.pre_install(self.pkg.spec) if self.fake: _do_fake_install(self.pkg) else: if self.install_source: self._install_source() self._real_install() # Run post install hooks before build stage is removed. self.timer.start("post-install") spack.hooks.post_install(self.pkg.spec, self.explicit) self.timer.stop("post-install") # Stop the timer and save results self.timer.stop() _write_timer_json(self.pkg, self.timer, False) print_install_test_log(self.pkg) _print_timer(pre=self.pre, pkg_id=self.pkg_id, timer=self.timer) _print_installed_pkg(self.pkg.prefix) # preserve verbosity across runs return self.echo def _install_source(self) -> None: """Install source code from stage into share/pkg/src if necessary.""" pkg = self.pkg if not os.path.isdir(pkg.stage.source_path): return src_target = os.path.join(pkg.spec.prefix, "share", pkg.name, "src") tty.debug(f"{self.pre} Copying source to {src_target}") fs.install_tree(pkg.stage.source_path, src_target) def _real_install(self) -> None: pkg = self.pkg # Do the real install in the source directory. with fs.working_dir(pkg.stage.source_path): # Save the build environment in a file before building. dump_environment(pkg.env_path) # Save just the changes to the environment. This file can be # safely installed, since it does not contain secret variables. with open(pkg.env_mods_path, "w", encoding="utf-8") as env_mods_file: mods = self.env_mods.shell_modifications(explicit=True, env=self.unmodified_env) env_mods_file.write(mods) for attr in ("configure_args", "cmake_args"): try: configure_args = getattr(pkg, attr)() configure_args = " ".join(configure_args) with open(pkg.configure_args_path, "w", encoding="utf-8") as args_file: args_file.write(configure_args) break except Exception: pass # cache debug settings debug_level = tty.debug_level() # Spawn a daemon that reads from a pipe and redirects # everything to log_path, and provide the phase for logging builder = spack.builder.create(pkg) for i, phase_fn in enumerate(builder): # Keep a log file for each phase log_dir = os.path.dirname(pkg.log_path) log_file = "spack-build-%02d-%s-out.txt" % (i + 1, phase_fn.name.lower()) log_file = os.path.join(log_dir, log_file) try: # DEBUGGING TIP - to debug this section, insert an IPython # embed here, and run the sections below without log capture log_contextmanager = log_output( log_file, self.echo, debug=True, filter_fn=self.filter_fn ) with log_contextmanager as logger: # Redirect stdout and stderr to daemon pipe with logger.force_echo(): inner_debug_level = tty.debug_level() tty.set_debug(debug_level) tty.msg(f"{self.pre} Executing phase: '{phase_fn.name}'") tty.set_debug(inner_debug_level) # Catch any errors to report to logging self.timer.start(phase_fn.name) phase_fn.execute() self.timer.stop(phase_fn.name) except BaseException: combine_phase_logs(pkg.phase_log_files, pkg.log_path) raise # We assume loggers share echo True/False self.echo = logger.echo # After log, we can get all output/error files from the package stage combine_phase_logs(pkg.phase_log_files, pkg.log_path) log(pkg) def build_process(pkg: "spack.package_base.PackageBase", install_args: dict) -> bool: """Perform the installation/build of the package. This runs in a separate child process, and has its own process and python module space set up by build_environment.start_build_process(). This essentially wraps an instance of ``BuildProcessInstaller`` so that we can more easily create one in a subprocess. This function's return value is returned to the parent process. Arguments: pkg: the package being installed. install_args: arguments to installer from parent process. """ installer = BuildProcessInstaller(pkg, install_args) # don't print long padded paths in executable debug output. with spack.util.path.filter_padding(): return installer.run() def deprecate(spec: "spack.spec.Spec", deprecator: "spack.spec.Spec", link_fn) -> None: """Deprecate this package in favor of deprecator spec""" # Here we assume we don't deprecate across different stores, and that same hash # means same binary artifacts if spec.dag_hash() == deprecator.dag_hash(): return # We can't really have control over external specs, and cannot link anything in their place if spec.external: return # Install deprecator if it isn't installed already if not spack.store.STORE.db.query(deprecator): PackageInstaller([deprecator.package], explicit=True).install() old_deprecator = spack.store.STORE.db.deprecator(spec) if old_deprecator: # Find this spec file from its old deprecation specfile = spack.store.STORE.layout.deprecated_file_path(spec, old_deprecator) else: specfile = spack.store.STORE.layout.spec_file_path(spec) # copy spec metadata to "deprecated" dir of deprecator depr_specfile = spack.store.STORE.layout.deprecated_file_path(spec, deprecator) fs.mkdirp(os.path.dirname(depr_specfile)) shutil.copy2(specfile, depr_specfile) # Any specs deprecated in favor of this spec are re-deprecated in favor of its new deprecator for deprecated in spack.store.STORE.db.specs_deprecated_by(spec): deprecate(deprecated, deprecator, link_fn) # Now that we've handled metadata, uninstall and replace with link spack.package_base.PackageBase.uninstall_by_spec(spec, force=True, deprecator=deprecator) link_fn(deprecator.prefix, spec.prefix) class BadInstallPhase(spack.error.InstallError): def __init__(self, pkg_name, phase): super().__init__(f"'{phase}' is not a valid phase for package {pkg_name}") class ExternalPackageError(spack.error.InstallError): """Raised by install() when a package is only for external use.""" class InstallLockError(spack.error.InstallError): """Raised during install when something goes wrong with package locking.""" class UpstreamPackageError(spack.error.InstallError): """Raised during install when something goes wrong with an upstream package.""" ================================================ FILE: lib/spack/spack/installer_dispatch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import sys from typing import TYPE_CHECKING, List, Optional, Set, Union from spack.vendor.typing_extensions import Literal import spack.config import spack.traverse if TYPE_CHECKING: import spack.installer import spack.new_installer import spack.package_base def create_installer( packages: List["spack.package_base.PackageBase"], *, dirty: bool = False, explicit: Union[Set[str], bool] = False, overwrite: Optional[Union[List[str], Set[str]]] = None, fail_fast: bool = False, fake: bool = False, include_build_deps: bool = False, install_deps: bool = True, install_package: bool = True, install_source: bool = False, keep_prefix: bool = False, keep_stage: bool = False, restage: bool = True, skip_patch: bool = False, stop_at: Optional[str] = None, stop_before: Optional[str] = None, tests: Union[bool, List[str], Set[str]] = False, unsigned: Optional[bool] = None, verbose: bool = False, concurrent_packages: Optional[int] = None, root_policy: Literal["auto", "cache_only", "source_only"] = "auto", dependencies_policy: Literal["auto", "cache_only", "source_only"] = "auto", create_reports: bool = False, ) -> Union["spack.installer.PackageInstaller", "spack.new_installer.PackageInstaller"]: """Create an installer based on the current configuration and feature support.""" use_old_installer = ( sys.platform == "win32" or spack.config.get("config:installer", "new") == "old" ) # Use the old installer if splicing is used. if not use_old_installer: specs = [pkg.spec for pkg in packages] for s in spack.traverse.traverse_nodes(specs): if s.build_spec is not s: use_old_installer = True break if use_old_installer: from spack.installer import PackageInstaller # type: ignore else: from spack.new_installer import PackageInstaller # type: ignore return PackageInstaller( packages, dirty=dirty, explicit=explicit, overwrite=overwrite, fail_fast=fail_fast, fake=fake, include_build_deps=include_build_deps, install_deps=install_deps, install_package=install_package, install_source=install_source, keep_prefix=keep_prefix, keep_stage=keep_stage, restage=restage, skip_patch=skip_patch, stop_at=stop_at, stop_before=stop_before, tests=tests, unsigned=unsigned, verbose=verbose, concurrent_packages=concurrent_packages, root_policy=root_policy, dependencies_policy=dependencies_policy, create_reports=create_reports, ) ================================================ FILE: lib/spack/spack/llnl/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/llnl/path.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Path primitives that just require Python standard library.""" import functools import sys from typing import List, Optional from urllib.parse import urlparse class Path: """Enum to identify the path-style.""" unix: int = 0 windows: int = 1 platform_path: int = windows if sys.platform == "win32" else unix def format_os_path(path: str, mode: int = Path.unix) -> str: """Formats the input path to use consistent, platform specific separators. Absolute paths are converted between drive letters and a prepended ``/`` as per platform requirement. Parameters: path: the path to be normalized, must be a string or expose the replace method. mode: the path file separator style to normalize the passed path to. Default is unix style, i.e. ``/`` """ if not path: return path if mode == Path.windows: path = path.replace("/", "\\") else: path = path.replace("\\", "/") return path def convert_to_posix_path(path: str) -> str: """Converts the input path to POSIX style.""" return format_os_path(path, mode=Path.unix) def convert_to_platform_path(path: str) -> str: """Converts the input path to the current platform's native style.""" return format_os_path(path, mode=Path.platform_path) def path_to_os_path(*parameters: str) -> List[str]: """Takes an arbitrary number of positional parameters, converts each argument of type string to use a normalized filepath separator, and returns a list of all values. """ def _is_url(path_or_url: str) -> bool: if "\\" in path_or_url: return False url_tuple = urlparse(path_or_url) return bool(url_tuple.scheme) and len(url_tuple.scheme) > 1 result = [] for item in parameters: if isinstance(item, str) and not _is_url(item): item = convert_to_platform_path(item) result.append(item) return result def _system_path_filter(_func=None, arg_slice: Optional[slice] = None): """Filters function arguments to account for platform path separators. Optional slicing range can be specified to select specific arguments This decorator takes all (or a slice) of a method's positional arguments and normalizes usage of filepath separators on a per platform basis. Note: `**kwargs`, urls, and any type that is not a string are ignored so in such cases where path normalization is required, that should be handled by calling path_to_os_path directly as needed. Parameters: arg_slice: a slice object specifying the slice of arguments in the decorated method over which filepath separators are normalized """ def holder_func(func): @functools.wraps(func) def path_filter_caller(*args, **kwargs): args = list(args) if arg_slice: args[arg_slice] = path_to_os_path(*args[arg_slice]) else: args = path_to_os_path(*args) return func(*args, **kwargs) return path_filter_caller if _func: return holder_func(_func) return holder_func def _noop_decorator(_func=None, arg_slice: Optional[slice] = None): return _func if _func else lambda x: x if sys.platform == "win32": system_path_filter = _system_path_filter else: system_path_filter = _noop_decorator def sanitize_win_longpath(path: str) -> str: """Strip Windows extended path prefix from strings Returns sanitized string. no-op if extended path prefix is not present""" return path.lstrip("\\\\?\\") ================================================ FILE: lib/spack/spack/llnl/string.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """String manipulation functions that do not have other dependencies than Python standard library """ from typing import List, Optional, Sequence def comma_list(sequence: Sequence[str], article: str = "") -> str: if type(sequence) is not list: sequence = list(sequence) if not sequence: return "" if len(sequence) == 1: return sequence[0] out = ", ".join(str(s) for s in sequence[:-1]) if len(sequence) != 2: out += "," # oxford comma out += " " if article: out += article + " " out += str(sequence[-1]) return out def comma_or(sequence: Sequence[str]) -> str: """Return a string with all the elements of the input joined by comma, but the last one (which is joined by ``"or"``). """ return comma_list(sequence, "or") def comma_and(sequence: List[str]) -> str: """Return a string with all the elements of the input joined by comma, but the last one (which is joined by ``"and"``). """ return comma_list(sequence, "and") def ordinal(number: int) -> str: """Return the ordinal representation (1st, 2nd, 3rd, etc.) for the provided number. Args: number: int to convert to ordinal number Returns: number's corresponding ordinal """ idx = (number % 10) << 1 tens = number % 100 // 10 suffix = "th" if tens == 1 or idx > 6 else "thstndrd"[idx : idx + 2] return f"{number}{suffix}" def quote(sequence: List[str], q: str = "'") -> List[str]: """Quotes each item in the input list with the quote character passed as second argument.""" return [f"{q}{e}{q}" for e in sequence] def plural(n: int, singular: str, plural: Optional[str] = None, show_n: bool = True) -> str: """Pluralize word by adding an s if n != 1. Arguments: n: number of things there are singular: singular form of word plural: optional plural form, for when it's not just singular + 's' show_n: whether to include n in the result string (default True) Returns: "1 thing" if n == 1 or "n things" if n != 1 """ number = f"{n} " if show_n else "" if n == 1: return f"{number}{singular}" elif plural is not None: return f"{number}{plural}" else: return f"{number}{singular}s" ================================================ FILE: lib/spack/spack/llnl/url.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """URL primitives that just require Python standard library.""" import itertools import os import re from typing import Optional, Set, Tuple from urllib.parse import urlsplit, urlunsplit # Archive extensions allowed in Spack PREFIX_EXTENSIONS = ("tar", "TAR") EXTENSIONS = ("gz", "bz2", "xz", "Z") NO_TAR_EXTENSIONS = ("zip", "tgz", "tbz2", "tbz", "txz", "whl") # Add PREFIX_EXTENSIONS and EXTENSIONS last so that .tar.gz is matched *before* .tar or .gz ALLOWED_ARCHIVE_TYPES = ( tuple(".".join(ext) for ext in itertools.product(PREFIX_EXTENSIONS, EXTENSIONS)) + PREFIX_EXTENSIONS + EXTENSIONS + NO_TAR_EXTENSIONS ) CONTRACTION_MAP = {"tgz": "tar.gz", "txz": "tar.xz", "tbz": "tar.bz2", "tbz2": "tar.bz2"} def find_list_urls(url: str) -> Set[str]: r"""Find good list URLs for the supplied URL. By default, returns the dirname of the archive path. Provides special treatment for the following websites, which have a unique list URL different from the dirname of the download URL: ========= ======================================================= GitHub ``https://github.com///releases`` GitLab ``https://gitlab.\*///tags`` BitBucket ``https://bitbucket.org///downloads/?tab=tags`` CRAN ``https://\*.r-project.org/src/contrib/Archive/`` PyPI ``https://pypi.org/simple//`` LuaRocks ``https://luarocks.org/modules//`` ========= ======================================================= Note: this function is called by ``spack versions``, ``spack checksum``, and ``spack create``, but not by ``spack fetch`` or ``spack install``. Parameters: url (str): The download URL for the package Returns: set: One or more list URLs for the package """ url_types = [ # GitHub # e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz (r"(.*github\.com/[^/]+/[^/]+)", lambda m: m.group(1) + "/releases"), # GitLab API endpoint # e.g. https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2 ( r"(.*gitlab[^/]+)/api/v4/projects/([^/]+)%2F([^/]+)", lambda m: m.group(1) + "/" + m.group(2) + "/" + m.group(3) + "/tags", ), # GitLab non-API endpoint # e.g. https://gitlab.dkrz.de/k202009/libaec/uploads/631e85bcf877c2dcaca9b2e6d6526339/libaec-1.0.0.tar.gz (r"(.*gitlab[^/]+/(?!api/v4/projects)[^/]+/[^/]+)", lambda m: m.group(1) + "/tags"), # BitBucket # e.g. https://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2 (r"(.*bitbucket.org/[^/]+/[^/]+)", lambda m: m.group(1) + "/downloads/?tab=tags"), # CRAN # e.g. https://cran.r-project.org/src/contrib/Rcpp_0.12.9.tar.gz # e.g. https://cloud.r-project.org/src/contrib/rgl_0.98.1.tar.gz ( r"(.*\.r-project\.org/src/contrib)/([^_]+)", lambda m: m.group(1) + "/Archive/" + m.group(2), ), # PyPI # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip # e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl ( r"(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)", lambda m: "https://pypi.org/simple/" + m.group(1) + "/", ), # LuaRocks # e.g. https://luarocks.org/manifests/gvvaughan/lpeg-1.0.2-1.src.rock # e.g. https://luarocks.org/manifests/openresty/lua-cjson-2.1.0-1.src.rock ( r"luarocks[^/]+/(?:modules|manifests)/(?P[^/]+)/" + r"(?P.+?)-[0-9.-]*\.src\.rock", lambda m: ( "https://luarocks.org/modules/" + m.group("org") + "/" + m.group("name") + "/" ), ), ] list_urls = {os.path.dirname(url)} for pattern, fun in url_types: match = re.search(pattern, url) if match: list_urls.add(fun(match)) return list_urls def strip_query_and_fragment(url: str) -> Tuple[str, str]: """Strips query and fragment from a url, then returns the base url and the suffix. Args: url: URL to be stripped Raises: ValueError: when there is any error parsing the URL """ components = urlsplit(url) stripped = components[:3] + (None, None) query, frag = components[3:5] suffix = "" if query: suffix += "?" + query if frag: suffix += "#" + frag return urlunsplit(stripped), suffix SOURCEFORGE_RE = re.compile(r"(.*(?:sourceforge\.net|sf\.net)/.*)(/download)$") def split_url_on_sourceforge_suffix(url: str) -> Tuple[str, ...]: """If the input is a sourceforge URL, returns base URL and ``/download`` suffix. Otherwise, returns the input URL and an empty string. """ match = SOURCEFORGE_RE.search(url) if match is not None: return match.groups() return url, "" def has_extension(path_or_url: str, ext: str) -> bool: """Returns true if the extension in input is present in path, false otherwise.""" prefix, _ = split_url_on_sourceforge_suffix(path_or_url) if not ext.startswith(r"\."): ext = rf"\.{ext}$" if re.search(ext, prefix): return True return False def extension_from_path(path_or_url: Optional[str]) -> Optional[str]: """Tries to match an allowed archive extension to the input. Returns the first match, or None if no match was found. Raises: ValueError: if the input is None """ if path_or_url is None: raise ValueError("Can't call extension() on None") for t in ALLOWED_ARCHIVE_TYPES: if has_extension(path_or_url, t): return t return None def remove_extension(path_or_url: str, *, extension: str) -> str: """Returns the input with the extension removed""" suffix = rf"\.{extension}$" return re.sub(suffix, "", path_or_url) def check_and_remove_ext(path: str, *, extension: str) -> str: """Returns the input path with the extension removed, if the extension is present in path. Otherwise, returns the input unchanged. """ if not has_extension(path, extension): return path path, _ = split_url_on_sourceforge_suffix(path) return remove_extension(path, extension=extension) def strip_extension(path_or_url: str, *, extension: Optional[str] = None) -> str: """If a path contains the extension in input, returns the path stripped of the extension. Otherwise, returns the input path. If extension is None, attempts to strip any allowed extension from path. """ if extension is None: for t in ALLOWED_ARCHIVE_TYPES: if has_extension(path_or_url, ext=t): extension = t break else: return path_or_url return check_and_remove_ext(path_or_url, extension=extension) def split_url_extension(url: str) -> Tuple[str, ...]: """Some URLs have a query string, e.g.: 1. ``https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true`` 2. ``http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz`` 3. ``https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0`` In (1), the query string needs to be stripped to get at the extension, but in (2) & (3), the filename is IN a single final query argument. This strips the URL into three pieces: ``prefix``, ``ext``, and ``suffix``. The suffix contains anything that was stripped off the URL to get at the file extension. In (1), it will be ``'?raw=true'``, but in (2), it will be empty. In (3) the suffix is a parameter that follows after the file extension, e.g.: 1. ``('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')`` 2. ``('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin', '.tar.gz', None)`` 3. ``('https://gitlab.kitware.com/vtk/vtk/repository/archive', '.tar.bz2', '?ref=v7.0.0')`` """ # noqa: E501 # Strip off sourceforge download suffix. # e.g. https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download prefix, suffix = split_url_on_sourceforge_suffix(url) ext = extension_from_path(prefix) if ext is not None: prefix = strip_extension(prefix) return prefix, ext, suffix try: prefix, suf = strip_query_and_fragment(prefix) except ValueError: # FIXME: tty.debug("Got error parsing path %s" % path) # Ignore URL parse errors here return url, "" ext = extension_from_path(prefix) prefix = strip_extension(prefix) suffix = suf + suffix if ext is None: ext = "" return prefix, ext, suffix def strip_version_suffixes(path_or_url: str) -> str: """Some tarballs contain extraneous information after the version: * ``bowtie2-2.2.5-source`` * ``libevent-2.0.21-stable`` * ``cuda_8.0.44_linux.run`` These strings are not part of the version number and should be ignored. This function strips those suffixes off and returns the remaining string. The goal is that the version is always the last thing in ``path``: * ``bowtie2-2.2.5`` * ``libevent-2.0.21`` * ``cuda_8.0.44`` Args: path_or_url: The filename or URL for the package Returns: The ``path`` with any extraneous suffixes removed """ # NOTE: This could be done with complicated regexes in parse_version_offset # NOTE: The problem is that we would have to add these regexes to the end # NOTE: of every single version regex. Easier to just strip them off # NOTE: permanently suffix_regexes = [ # Download type r"[Ii]nstall", r"all", r"code", r"[Ss]ources?", r"file", r"full", r"single", r"with[a-zA-Z_-]+", r"rock", r"src(_0)?", r"public", r"bin", r"binary", r"run", r"[Uu]niversal", r"jar", r"complete", r"dynamic", r"oss", r"gem", r"tar", r"sh", # Download version r"release", r"bin", r"stable", r"[Ff]inal", r"rel", r"orig", r"dist", r"\+", # License r"gpl", # Arch # Needs to come before and after OS, appears in both orders r"ia32", r"intel", r"amd64", r"linux64", r"x64", r"64bit", r"x86[_-]64", r"i586_64", r"x86", r"i[36]86", r"ppc64(le)?", r"armv?(7l|6l|64)", # Other r"cpp", r"gtk", r"incubating", # OS r"[Ll]inux(_64)?", r"LINUX", r"[Uu]ni?x", r"[Ss]un[Oo][Ss]", r"[Mm]ac[Oo][Ss][Xx]?", r"[Oo][Ss][Xx]", r"[Dd]arwin(64)?", r"[Aa]pple", r"[Ww]indows", r"[Ww]in(64|32)?", r"[Cc]ygwin(64|32)?", r"[Mm]ingw", r"centos", # Arch # Needs to come before and after OS, appears in both orders r"ia32", r"intel", r"amd64", r"linux64", r"x64", r"64bit", r"x86[_-]64", r"i586_64", r"x86", r"i[36]86", r"ppc64(le)?", r"armv?(7l|6l|64)?", # PyPI wheels r"-(?:py|cp)[23].*", ] for regex in suffix_regexes: # Remove the suffix from the end of the path # This may be done multiple times path_or_url = re.sub(r"[._-]?" + regex + "$", "", path_or_url) return path_or_url def expand_contracted_extension(extension: str) -> str: """Returns the expanded version of a known contracted extension. This function maps extensions like ``.tgz`` to ``.tar.gz``. On unknown extensions, return the input unmodified. """ extension = extension.strip(".") return CONTRACTION_MAP.get(extension, extension) def expand_contracted_extension_in_path( path_or_url: str, *, extension: Optional[str] = None ) -> str: """Returns the input path or URL with any contraction extension expanded. Args: path_or_url: path or URL to be expanded extension: if specified, only attempt to expand that extension """ extension = extension or extension_from_path(path_or_url) if extension is None: return path_or_url expanded = expand_contracted_extension(extension) if expanded != extension: return re.sub(rf"{extension}", rf"{expanded}", path_or_url) return path_or_url def compression_ext_from_compressed_archive(extension: str) -> Optional[str]: """Returns compression extension for a compressed archive""" extension = expand_contracted_extension(extension) for ext in EXTENSIONS: if ext in extension: return ext return None def strip_compression_extension(path_or_url: str, ext: Optional[str] = None) -> str: """Strips the compression extension from the input, and returns it. For instance, ``"foo.tgz"`` becomes ``"foo.tar"``. If no extension is given, try a default list of extensions. Args: path_or_url: input to be stripped ext: if given, extension to be stripped """ if not extension_from_path(path_or_url): return path_or_url expanded_path = expand_contracted_extension_in_path(path_or_url) candidates = [ext] if ext is not None else EXTENSIONS for current_extension in candidates: modified_path = check_and_remove_ext(expanded_path, extension=current_extension) if modified_path != expanded_path: return modified_path return expanded_path def allowed_archive(path_or_url: str) -> bool: """Returns true if the input is a valid archive, False otherwise.""" return ( False if not path_or_url else any(path_or_url.endswith(t) for t in ALLOWED_ARCHIVE_TYPES) ) def determine_url_file_extension(path: str) -> str: """This returns the type of archive a URL refers to. This is sometimes confusing because of URLs like: (1) ``https://github.com/petdance/ack/tarball/1.93_02`` Where the URL doesn't actually contain the filename. We need to know what type it is so that we can appropriately name files in mirrors. """ match = re.search(r"github.com/.+/(zip|tar)ball/", path) if match: if match.group(1) == "zip": return "zip" elif match.group(1) == "tar": return "tar.gz" prefix, ext, suffix = split_url_extension(path) return ext ================================================ FILE: lib/spack/spack/llnl/util/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/llnl/util/argparsewriter.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import abc import argparse import io import re import sys from argparse import ArgumentParser from typing import IO, Any, Iterable, List, Optional, Sequence, Tuple, Union class Command: """Parsed representation of a command from argparse. This is a single command from an argparse parser. ``ArgparseWriter`` creates these and returns them from ``parse()``, and it passes one of these to each call to ``format()`` so that we can take an action for a single command. """ def __init__( self, prog: str, description: Optional[str], usage: str, positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]], optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]], subcommands: List[Tuple[ArgumentParser, str, str]], ) -> None: """Initialize a new Command instance. Args: prog: Program name. description: Command description. usage: Command usage. positionals: List of positional arguments. optionals: List of optional arguments. subcommands: List of subcommand parsers. """ self.prog = prog self.description = description self.usage = usage self.positionals = positionals self.optionals = optionals self.subcommands = subcommands # NOTE: The only reason we subclass argparse.HelpFormatter is to get access to self._expand_help(), # ArgparseWriter is not intended to be used as a formatter_class. class ArgparseWriter(argparse.HelpFormatter, abc.ABC): """Analyze an argparse ArgumentParser for easy generation of help.""" def __init__(self, prog: str, out: IO = sys.stdout, aliases: bool = False) -> None: """Initialize a new ArgparseWriter instance. Args: prog: Program name. out: File object to write to. aliases: Whether or not to include subparsers for aliases. """ super().__init__(prog) self.level = 0 self.prog = prog self.out = out self.aliases = aliases def parse(self, parser: ArgumentParser, prog: str) -> Command: """Parse the parser object and return the relevant components. Args: parser: Command parser. prog: Program name. Returns: Information about the command from the parser. """ self.parser = parser split_prog = parser.prog.split(" ") split_prog[-1] = prog prog = " ".join(split_prog) description = parser.description fmt = parser._get_formatter() actions = parser._actions groups = parser._mutually_exclusive_groups usage = fmt._format_usage(None, actions, groups, "").strip() # Go through actions and split them into optionals, positionals, and subcommands optionals = [] positionals = [] subcommands = [] for action in actions: if action.option_strings: flags = action.option_strings dest_flags = fmt._format_action_invocation(action) nargs = action.nargs help = ( self._expand_help(action) if action.help and action.help != argparse.SUPPRESS else "" ) help = help.split("\n")[0] if action.choices is not None: dest = [str(choice) for choice in action.choices] else: dest = [action.dest] optionals.append((flags, dest, dest_flags, nargs, help)) elif isinstance(action, argparse._SubParsersAction): for subaction in action._choices_actions: subparser = action._name_parser_map[subaction.dest] help = ( self._expand_help(subaction) if subaction.help and action.help != argparse.SUPPRESS else "" ) help = help.split("\n")[0] subcommands.append((subparser, subaction.dest, help)) # Look for aliases of the form 'name (alias, ...)' if self.aliases and isinstance(subaction.metavar, str): match = re.match(r"(.*) \((.*)\)", subaction.metavar) if match: aliases = match.group(2).split(", ") for alias in aliases: subparser = action._name_parser_map[alias] help = ( self._expand_help(subaction) if subaction.help and action.help != argparse.SUPPRESS else "" ) help = help.split("\n")[0] subcommands.append((subparser, alias, help)) else: args = fmt._format_action_invocation(action) help = ( self._expand_help(action) if action.help and action.help != argparse.SUPPRESS else "" ) help = help.split("\n")[0] positionals.append((args, action.choices, action.nargs, help)) return Command(prog, description, usage, positionals, optionals, subcommands) @abc.abstractmethod def format(self, cmd: Command) -> str: """Return the string representation of a single node in the parser tree. Override this in subclasses to define how each subcommand should be displayed. Args: cmd: Parsed information about a command or subcommand. Returns: String representation of this subcommand. """ def _write(self, parser: ArgumentParser, prog: str, level: int = 0) -> None: """Recursively write a parser. Args: parser: Command parser. prog: Program name. level: Current level. """ self.level = level cmd = self.parse(parser, prog) self.out.write(self.format(cmd)) for subparser, prog, help in cmd.subcommands: self._write(subparser, prog, level=level + 1) def write(self, parser: ArgumentParser) -> None: """Write out details about an ArgumentParser. Args: parser: Command parser. """ try: self._write(parser, self.prog) except BrokenPipeError: # Swallow pipe errors pass _rst_levels = ["=", "-", "^", "~", ":", "`"] class ArgparseRstWriter(ArgparseWriter): """Write argparse output as rst sections.""" def __init__( self, prog: str, out: IO = sys.stdout, aliases: bool = False, rst_levels: Sequence[str] = _rst_levels, ) -> None: """Initialize a new ArgparseRstWriter instance. Args: prog: Program name. out: File object to write to. aliases: Whether or not to include subparsers for aliases. rst_levels: List of characters for rst section headings. """ super().__init__(prog, out, aliases) self.rst_levels = rst_levels def format(self, cmd: Command) -> str: """Return the string representation of a single node in the parser tree. Args: cmd: Parsed information about a command or subcommand. Returns: String representation of a node. """ string = io.StringIO() string.write(self.begin_command(cmd.prog)) if cmd.description: string.write(self.description(cmd.description)) string.write(self.usage(cmd.usage)) if cmd.positionals: string.write(self.begin_positionals()) for args, choices, nargs, help in cmd.positionals: string.write(self.positional(args, help)) string.write(self.end_positionals()) if cmd.optionals: string.write(self.begin_optionals()) for flags, dest, dest_flags, nargs, help in cmd.optionals: string.write(self.optional(dest_flags, help)) string.write(self.end_optionals()) if cmd.subcommands: string.write(self.begin_subcommands(cmd.subcommands)) return string.getvalue() def begin_command(self, prog: str) -> str: """Text to print before a command. Args: prog: Program name. Returns: Text before a command. """ return """ ---- .. _{0}: {1} {2} """.format(prog.replace(" ", "-"), prog, self.rst_levels[self.level] * len(prog)) def description(self, description: str) -> str: """Description of a command. Args: description: Command description. Returns: Description of a command. """ return description + "\n\n" def usage(self, usage: str) -> str: """Example usage of a command. Args: usage: Command usage. Returns: Usage of a command. """ return """\ .. code-block:: console {0} """.format(usage) def begin_positionals(self) -> str: """Text to print before positional arguments. Returns: Positional arguments header. """ return "\n**Positional arguments**\n\n" def positional(self, name: str, help: str) -> str: """Description of a positional argument. Args: name: Argument name. help: Help text. Returns: Positional argument description. """ return """\ ``{0}`` {1} """.format(name, help) def end_positionals(self) -> str: """Text to print after positional arguments. Returns: Positional arguments footer. """ return "" def begin_optionals(self) -> str: """Text to print before optional arguments. Returns: Optional arguments header. """ return "\n**Optional arguments**\n\n" def optional(self, opts: str, help: str) -> str: """Description of an optional argument. Args: opts: Optional argument. help: Help text. Returns: Optional argument description. """ return """\ ``{0}`` {1} """.format(opts, help) def end_optionals(self) -> str: """Text to print after optional arguments. Returns: Optional arguments footer. """ return "" def begin_subcommands(self, subcommands: List[Tuple[ArgumentParser, str, str]]) -> str: """Table with links to other subcommands. Arguments: subcommands: List of subcommands. Returns: Subcommand linking text. """ string = """ **Subcommands** .. hlist:: :columns: 4 """ for cmd, _, _ in subcommands: prog = re.sub(r"^[^ ]* ", "", cmd.prog) string += " * :ref:`{0} <{1}>`\n".format(prog, cmd.prog.replace(" ", "-")) return string + "\n" ================================================ FILE: lib/spack/spack/llnl/util/filesystem.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import collections.abc import errno import fnmatch import glob import hashlib import io import itertools import numbers import os import pathlib import posixpath import re import shutil import stat import subprocess import sys import tempfile from contextlib import contextmanager from itertools import accumulate from typing import ( IO, Callable, Deque, Dict, Generator, Iterable, List, Match, Optional, Sequence, Set, Tuple, Union, ) from spack.llnl.path import path_to_os_path, sanitize_win_longpath, system_path_filter from spack.llnl.util import lang, tty from spack.llnl.util.lang import dedupe, fnmatch_translate_multiple, memoized if sys.platform != "win32": import grp import pwd else: import win32security from win32file import CreateHardLink __all__ = [ "FileFilter", "FileList", "HeaderList", "LibraryList", "ancestor", "can_access", "change_sed_delimiter", "copy_mode", "filter_file", "find", "find_first", "find_headers", "find_all_headers", "find_libraries", "find_system_libraries", "force_remove", "force_symlink", "getuid", "chgrp", "chmod_x", "copy", "install", "copy_tree", "install_tree", "is_exe", "join_path", "library_extensions", "mkdirp", "partition_path", "prefixes", "remove_dead_links", "remove_directory_contents", "remove_if_dead_link", "remove_linked_tree", "rename", "set_executable", "set_install_permissions", "touch", "touchp", "traverse_tree", "unset_executable_mode", "working_dir", "keep_modification_time", "BaseDirectoryVisitor", "visit_directory_tree", ] Path = Union[str, pathlib.Path] if sys.version_info < (3, 7, 4): # monkeypatch shutil.copystat to fix PermissionError when copying read-only # files on Lustre when using Python < 3.7.4 def copystat(src, dst, follow_symlinks=True): """Copy file metadata Copy the permission bits, last access time, last modification time, and flags from `src` to `dst`. On Linux, copystat() also copies the "extended attributes" where possible. The file contents, owner, and group are unaffected. `src` and `dst` are path names given as strings. If the optional flag `follow_symlinks` is not set, symlinks aren't followed if and only if both `src` and `dst` are symlinks. """ def _nop(args, ns=None, follow_symlinks=None): pass # follow symlinks (aka don't not follow symlinks) follow = follow_symlinks or not (islink(src) and islink(dst)) if follow: # use the real function if it exists def lookup(name): return getattr(os, name, _nop) else: # use the real function only if it exists # *and* it supports follow_symlinks def lookup(name): fn = getattr(os, name, _nop) if sys.version_info >= (3, 3): if fn in os.supports_follow_symlinks: # novermin return fn return _nop st = lookup("stat")(src, follow_symlinks=follow) mode = stat.S_IMODE(st.st_mode) lookup("utime")(dst, ns=(st.st_atime_ns, st.st_mtime_ns), follow_symlinks=follow) # We must copy extended attributes before the file is (potentially) # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. shutil._copyxattr(src, dst, follow_symlinks=follow) try: lookup("chmod")(dst, mode, follow_symlinks=follow) except NotImplementedError: # if we got a NotImplementedError, it's because # * follow_symlinks=False, # * lchown() is unavailable, and # * either # * fchownat() is unavailable or # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. # (it returned ENOSUP.) # therefore we're out of options--we simply cannot chown the # symlink. give up, suppress the error. # (which is what shutil always did in this circumstance.) pass if hasattr(st, "st_flags"): try: lookup("chflags")(dst, st.st_flags, follow_symlinks=follow) except OSError as why: for err in "EOPNOTSUPP", "ENOTSUP": if hasattr(errno, err) and why.errno == getattr(errno, err): break else: raise shutil.copystat = copystat def polite_path(components: Iterable[str]): """ Given a list of strings which are intended to be path components, generate a path, and format each component to avoid generating extra path entries. For example all "/", "\", and ":" characters will be replaced with "_". Other characters like "=" will also be replaced. """ return os.path.join(*[polite_filename(x) for x in components]) @memoized def _polite_antipattern(): # A regex of all the characters we don't want in a filename return re.compile(r"[^A-Za-z0-9_+.-]") def polite_filename(filename: str) -> str: """ Replace generally problematic filename characters with underscores. This differs from sanitize_filename in that it is more aggressive in changing characters in the name. For example it removes "=" which can confuse path parsing in external tools. """ # This character set applies for both Windows and Linux. It does not # account for reserved filenames in Windows. return _polite_antipattern().sub("_", filename) if sys.platform == "win32": def _getuid_win32() -> Union[str, int]: """Returns os getuid on non Windows On Windows returns 0 for admin users, login string otherwise This is in line with behavior from get_owner_uid which always returns the login string on Windows """ import ctypes # If not admin, use the string name of the login as a unique ID if ctypes.windll.shell32.IsUserAnAdmin() == 0: return os.getlogin() return 0 getuid = _getuid_win32 else: getuid = os.getuid @system_path_filter def _win_rename(src, dst): # On Windows, os.rename will fail if the destination file already exists # os.replace is the same as os.rename on POSIX and is MoveFileExW w/ # the MOVEFILE_REPLACE_EXISTING flag on Windows # Windows invocation is abstracted behind additional logic handling # remaining cases of divergent behavior across platforms # os.replace will still fail if on Windows (but not POSIX) if the dst # is a symlink to a directory (all other cases have parity Windows <-> Posix) if os.path.islink(dst) and os.path.isdir(os.path.realpath(dst)): if os.path.samefile(src, dst): # src and dst are the same # do nothing and exit early return # If dst exists and is a symlink to a directory # we need to remove dst and then perform rename/replace # this is safe to do as there's no chance src == dst now os.remove(dst) os.replace(src, dst) @system_path_filter def msdos_escape_parens(path): """MS-DOS interprets parens as grouping parameters even in a quoted string""" if sys.platform == "win32": return path.replace("(", "^(").replace(")", "^)") else: return path @system_path_filter def path_contains_subdirectory(path: str, root: str) -> bool: """Check if the path is a subdirectory of the root directory. Note: this is a symbolic check, and does not resolve symlinks.""" norm_root = os.path.abspath(root).rstrip(os.path.sep) + os.path.sep norm_path = os.path.abspath(path).rstrip(os.path.sep) + os.path.sep return norm_path.startswith(norm_root) #: This generates the library filenames that may appear on any OS. library_extensions = ["a", "la", "so", "tbd", "dylib"] def possible_library_filenames(library_names): """Given a collection of library names like 'libfoo', generate the set of library filenames that may be found on the system (e.g. libfoo.so). """ lib_extensions = library_extensions return set( ".".join((lib, extension)) for lib, extension in itertools.product(library_names, lib_extensions) ) def paths_containing_libs(paths, library_names): """Given a collection of filesystem paths, return the list of paths that which include one or more of the specified libraries. """ required_lib_fnames = possible_library_filenames(library_names) rpaths_to_include = [] paths = path_to_os_path(*paths) for path in paths: fnames = set(os.listdir(path)) if fnames & required_lib_fnames: rpaths_to_include.append(path) return rpaths_to_include def filter_file( regex: str, repl: Union[str, Callable[[Match], str]], *filenames: str, string: bool = False, backup: bool = False, ignore_absent: bool = False, start_at: Optional[str] = None, stop_at: Optional[str] = None, encoding: Optional[str] = "utf-8", ) -> None: r"""Like ``sed``, but uses Python regular expressions. Filters every line of each file through regex and replaces the file with a filtered version. Preserves mode of filtered files. As with :func:`re.sub`, ``repl`` can be either a string or a callable. If it is a callable, it is passed the match object and should return a suitable replacement string. If it is a string, it can contain ``\1``, ``\2``, etc. to represent back-substitution as sed would allow. Args: regex: The regular expression to search for repl: The string to replace matches with *filenames: One or more files to search and replace string: Treat regex as a plain string. Default it False backup: Make backup file(s) suffixed with ``~``. Default is False ignore_absent: Ignore any files that don't exist. Default is False start_at: Marker used to start applying the replacements. If a text line matches this marker filtering is started at the next line. All contents before the marker and the marker itself are copied verbatim. Default is to start filtering from the first line of the file. stop_at: Marker used to stop scanning the file further. If a text line matches this marker filtering is stopped and the rest of the file is copied verbatim. Default is to filter until the end of the file. encoding: The encoding to use when reading and writing the files. Default is None, which uses the system's default encoding. """ # Allow strings to use \1, \2, etc. for replacement, like sed if not callable(repl): unescaped = repl.replace(r"\\", "\\") def replace_groups_with_groupid(m: Match) -> str: def groupid_to_group(x): return m.group(int(x.group(1))) return re.sub(r"\\([1-9])", groupid_to_group, unescaped) repl = replace_groups_with_groupid if string: regex = re.escape(regex) regex_compiled = re.compile(regex) for path in path_to_os_path(*filenames): if ignore_absent and not os.path.exists(path): tty.debug(f'FILTER FILE: file "{path}" not found. Skipping to next file.') continue else: tty.debug(f'FILTER FILE: {path} [replacing "{regex}"]') fd, temp_path = tempfile.mkstemp( prefix=f"{os.path.basename(path)}.", dir=os.path.dirname(path) ) os.close(fd) shutil.copy(path, temp_path) errored = False try: # Open as a text file and filter until the end of the file is reached, or we found a # marker in the line if it was specified. To avoid translating line endings (\n to # \r\n and vice-versa) use newline="". with open( temp_path, mode="r", errors="surrogateescape", newline="", encoding=encoding ) as input_file, open( path, mode="w", errors="surrogateescape", newline="", encoding=encoding ) as output_file: if start_at is None and stop_at is None: # common case, avoids branching in loop for line in input_file: output_file.write(re.sub(regex_compiled, repl, line)) else: # state is -1 before start_at; 0 between; 1 after stop_at state = 0 if start_at is None else -1 for line in input_file: if state == 0: if stop_at == line.strip(): state = 1 else: line = re.sub(regex_compiled, repl, line) elif state == -1 and start_at == line.strip(): state = 0 output_file.write(line) except BaseException: # restore the original file os.rename(temp_path, path) errored = True raise finally: if not errored and not backup: os.unlink(temp_path) class FileFilter: """ Convenience class for repeatedly applying :func:`filter_file` to one or more files. This class allows you to specify a set of filenames and then call :meth:`filter` multiple times to perform search-and-replace operations using Python regular expressions, similar to ``sed``. Example usage:: foo_c = FileFilter("foo.c") foo_c.filter(r"#define FOO", "#define BAR") foo_c.filter(r"old_func", "new_func") """ def __init__(self, *filenames): self.filenames = filenames def filter( self, regex: str, repl: Union[str, Callable[[Match], str]], string: bool = False, backup: bool = False, ignore_absent: bool = False, start_at: Optional[str] = None, stop_at: Optional[str] = None, ) -> None: return filter_file( regex, repl, *self.filenames, string=string, backup=backup, ignore_absent=ignore_absent, start_at=start_at, stop_at=stop_at, ) def change_sed_delimiter(old_delim: str, new_delim: str, *filenames: str) -> None: """Find all sed search/replace commands and change the delimiter. e.g., if the file contains seds that look like ``'s///'``, you can call ``change_sed_delimiter('/', '@', file)`` to change the delimiter to ``'@'``. Note that this routine will fail if the delimiter is ``'`` or ``"``. Handling those is left for future work. Parameters: old_delim: The delimiter to search for new_delim: The delimiter to replace with *filenames: One or more files to search and replace """ assert len(old_delim) == 1 assert len(new_delim) == 1 # TODO: handle these cases one day? assert old_delim != '"' assert old_delim != "'" assert new_delim != '"' assert new_delim != "'" whole_lines = "^s@([^@]*)@(.*)@[gIp]$" whole_lines = whole_lines.replace("@", old_delim) single_quoted = r"'s@((?:\\'|[^@'])*)@((?:\\'|[^'])*)@[gIp]?'" single_quoted = single_quoted.replace("@", old_delim) double_quoted = r'"s@((?:\\"|[^@"])*)@((?:\\"|[^"])*)@[gIp]?"' double_quoted = double_quoted.replace("@", old_delim) repl = r"s@\1@\2@g" repl = repl.replace("@", new_delim) filenames = path_to_os_path(*filenames) for f in filenames: filter_file(whole_lines, repl, f) filter_file(single_quoted, "'%s'" % repl, f) filter_file(double_quoted, '"%s"' % repl, f) @contextmanager def exploding_archive_catch(stage): # Check for an exploding tarball, i.e. one that doesn't expand to # a single directory. If the tarball *didn't* explode, move its # contents to the staging source directory & remove the container # directory. If the tarball did explode, just rename the tarball # directory to the staging source directory. # # NOTE: The tar program on Mac OS X will encode HFS metadata in # hidden files, which can end up *alongside* a single top-level # directory. We initially ignore presence of hidden files to # accommodate these "semi-exploding" tarballs but ensure the files # are copied to the source directory. # Expand all tarballs in their own directory to contain # exploding tarballs. tarball_container = os.path.join(stage.path, "spack-expanded-archive") mkdirp(tarball_container) orig_dir = os.getcwd() os.chdir(tarball_container) try: yield # catch an exploding archive on successful extraction os.chdir(orig_dir) exploding_archive_handler(tarball_container, stage) except Exception as e: # return current directory context to previous on failure os.chdir(orig_dir) raise e @system_path_filter def exploding_archive_handler(tarball_container, stage): """ Args: tarball_container: where the archive was expanded to stage: Stage object referencing filesystem location where archive is being expanded """ files = os.listdir(tarball_container) non_hidden = [f for f in files if not f.startswith(".")] if len(non_hidden) == 1: src = os.path.join(tarball_container, non_hidden[0]) if os.path.isdir(src): stage.srcdir = non_hidden[0] shutil.move(src, stage.source_path) if len(files) > 1: files.remove(non_hidden[0]) for f in files: src = os.path.join(tarball_container, f) dest = os.path.join(stage.path, f) shutil.move(src, dest) os.rmdir(tarball_container) else: # This is a non-directory entry (e.g., a patch file) so simply # rename the tarball container to be the source path. shutil.move(tarball_container, stage.source_path) else: shutil.move(tarball_container, stage.source_path) @system_path_filter(arg_slice=slice(1)) def get_owner_uid(path, err_msg=None) -> Union[str, int]: """Returns owner UID of path destination On non Windows this is the value of st_uid On Windows this is the login string associated with the owning user. """ if not os.path.exists(path): mkdirp(path, mode=stat.S_IRWXU) p_stat = os.stat(path) if p_stat.st_mode & stat.S_IRWXU != stat.S_IRWXU: tty.error( "Expected {0} to support mode {1}, but it is {2}".format( path, stat.S_IRWXU, p_stat.st_mode ) ) raise OSError(errno.EACCES, err_msg.format(path, path) if err_msg else "") else: p_stat = os.stat(path) if sys.platform != "win32": owner_uid = p_stat.st_uid else: sid = win32security.GetFileSecurity( path, win32security.OWNER_SECURITY_INFORMATION ).GetSecurityDescriptorOwner() owner_uid = win32security.LookupAccountSid(None, sid)[0] return owner_uid @system_path_filter def set_install_permissions(path): """Set appropriate permissions on the installed file.""" # If this points to a file maintained in a Spack prefix, it is assumed that # this function will be invoked on the target. If the file is outside a # Spack-maintained prefix, the permissions should not be modified. if islink(path): return if os.path.isdir(path): os.chmod(path, 0o755) else: os.chmod(path, 0o644) def group_ids(uid: Optional[int] = None) -> List[int]: """Get group ids that a uid is a member of. Arguments: uid: id of user, or None for current user Returns: gids of groups the user is a member of """ if sys.platform == "win32": tty.warn("Function is not supported on Windows") return [] if uid is None: uid = getuid() pwd_entry = pwd.getpwuid(uid) user = pwd_entry.pw_name # user's primary group id may not be listed in grp (i.e. /etc/group) # you have to check pwd for that, so start the list with that gids = [pwd_entry.pw_gid] return sorted(set(gids + [g.gr_gid for g in grp.getgrall() if user in g.gr_mem])) @system_path_filter(arg_slice=slice(1)) def chgrp(path, group, follow_symlinks=True): """Implement the bash chgrp function on a single path""" if sys.platform == "win32": raise OSError("Function 'chgrp' is not supported on Windows") if isinstance(group, str): gid = grp.getgrnam(group).gr_gid else: gid = group if os.stat(path).st_gid == gid: return if follow_symlinks: os.chown(path, -1, gid) else: os.lchown(path, -1, gid) @system_path_filter(arg_slice=slice(1)) def chmod_x(entry, perms): """Implements chmod, treating all executable bits as set using the chmod utility's ``+X`` option. """ mode = os.stat(entry).st_mode if os.path.isfile(entry): if not mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH): perms &= ~stat.S_IXUSR perms &= ~stat.S_IXGRP perms &= ~stat.S_IXOTH os.chmod(entry, perms) @system_path_filter def copy_mode(src, dest): """Set the mode of dest to that of src unless it is a link.""" if islink(dest): return src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH os.chmod(dest, dest_mode) @system_path_filter def unset_executable_mode(path): mode = os.stat(path).st_mode mode &= ~stat.S_IXUSR mode &= ~stat.S_IXGRP mode &= ~stat.S_IXOTH os.chmod(path, mode) @system_path_filter def copy(src: str, dest: str, _permissions: bool = False) -> None: """Copy the file(s) ``src`` to the file or directory ``dest``. If ``dest`` specifies a directory, the file will be copied into ``dest`` using the base filename from ``src``. ``src`` may contain glob characters. Parameters: src: the file(s) to copy dest: the destination file or directory _permissions: for internal use only Raises: OSError: if ``src`` does not match any files or directories ValueError: if ``src`` matches multiple files but ``dest`` is not a directory """ if _permissions: tty.debug(f"Installing {src} to {dest}") else: tty.debug(f"Copying {src} to {dest}") files = glob.glob(src) if not files: raise OSError(f"No such file or directory: '{src}'") if len(files) > 1 and not os.path.isdir(dest): raise ValueError(f"'{src}' matches multiple files but '{dest}' is not a directory") for src in files: # Expand dest to its eventual full path if it is a directory. dst = dest if os.path.isdir(dest): dst = join_path(dest, os.path.basename(src)) shutil.copy(src, dst) if _permissions: set_install_permissions(dst) copy_mode(src, dst) @system_path_filter def install(src: str, dest: str) -> None: """Install the file(s) ``src`` to the file or directory ``dest``. Same as :py:func:`copy` with the addition of setting proper permissions on the installed file. Parameters: src: the file(s) to install dest: the destination file or directory Raises: OSError: if ``src`` does not match any files or directories ValueError: if ``src`` matches multiple files but ``dest`` is not a directory """ copy(src, dest, _permissions=True) @system_path_filter def copy_tree( src: str, dest: str, symlinks: bool = True, ignore: Optional[Callable[[str], bool]] = None, _permissions: bool = False, ): """Recursively copy an entire directory tree rooted at ``src``. If the destination directory ``dest`` does not already exist, it will be created as well as missing parent directories. ``src`` may contain glob characters. If *symlinks* is true, symbolic links in the source tree are represented as symbolic links in the new tree and the metadata of the original links will be copied as far as the platform allows; if false, the contents and metadata of the linked files are copied to the new tree. If *ignore* is set, then each path relative to ``src`` will be passed to this function; the function returns whether that path should be skipped. Parameters: src: the directory to copy dest: the destination directory symlinks: whether or not to preserve symlinks ignore: function indicating which files to ignore _permissions: for internal use only Raises: OSError: if ``src`` does not match any files or directories ValueError: if ``src`` is a parent directory of ``dest`` """ if _permissions: tty.debug("Installing {0} to {1}".format(src, dest)) else: tty.debug("Copying {0} to {1}".format(src, dest)) abs_dest = os.path.abspath(dest) if not abs_dest.endswith(os.path.sep): abs_dest += os.path.sep files = glob.glob(src) if not files: raise OSError("No such file or directory: '{0}'".format(src), errno.ENOENT) # For Windows hard-links and junctions, the source path must exist to make a symlink. Add # all symlinks to this list while traversing the tree, then when finished, make all # symlinks at the end. links = [] for src in files: abs_src = os.path.abspath(src) if not abs_src.endswith(os.path.sep): abs_src += os.path.sep # Stop early to avoid unnecessary recursion if being asked to copy # from a parent directory. if abs_dest.startswith(abs_src): raise ValueError( "Cannot copy ancestor directory {0} into {1}".format(abs_src, abs_dest) ) mkdirp(abs_dest) for s, d in traverse_tree( abs_src, abs_dest, order="pre", follow_links=not symlinks, ignore=ignore, follow_nonexisting=True, ): if islink(s): link_target = resolve_link_target_relative_to_the_link(s) if symlinks: target = readlink(s) if os.path.isabs(target): def escaped_path(path): return path.replace("\\", r"\\") new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target) if new_target != target: tty.debug("Redirecting link {0} to {1}".format(target, new_target)) target = new_target links.append((target, d, s)) continue elif os.path.isdir(link_target): mkdirp(d) else: shutil.copyfile(s, d) else: if os.path.isdir(s): mkdirp(d) else: shutil.copy2(s, d) if _permissions: set_install_permissions(d) copy_mode(s, d) for target, d, s in links: symlink(target, d) if _permissions: set_install_permissions(d) copy_mode(s, d) @system_path_filter def install_tree( src: str, dest: str, symlinks: bool = True, ignore: Optional[Callable[[str], bool]] = None ): """Recursively install an entire directory tree rooted at ``src``. Same as :py:func:`copy_tree` with the addition of setting proper permissions on the installed files and directories. Parameters: src: the directory to install dest: the destination directory symlinks: whether or not to preserve symlinks ignore: function indicating which files to ignore Raises: OSError: if ``src`` does not match any files or directories ValueError: if ``src`` is a parent directory of ``dest`` """ copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True) @system_path_filter def is_exe(path) -> bool: """Returns :obj:`True` iff the specified path exists, is a regular file, and has executable permissions for the current process.""" return os.path.isfile(path) and os.access(path, os.X_OK) def has_shebang(path) -> bool: """Returns whether a path has a shebang line. Returns False if the file cannot be opened.""" try: with open(path, "rb") as f: return f.read(2) == b"#!" except OSError: return False @system_path_filter def is_nonsymlink_exe_with_shebang(path): """Returns whether the path is an executable regular file with a shebang. Returns False too when the path is a symlink to a script, and also when the file cannot be opened.""" try: st = os.lstat(path) except OSError: return False # Should not be a symlink if stat.S_ISLNK(st.st_mode): return False # Should be executable if not st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH): return False return has_shebang(path) @system_path_filter(arg_slice=slice(1)) def chgrp_if_not_world_writable(path, group): """chgrp path to group if path is not world writable""" mode = os.stat(path).st_mode if not mode & stat.S_IWOTH: chgrp(path, group) def mkdirp( *paths: str, mode: Optional[int] = None, group: Optional[Union[str, int]] = None, default_perms: Optional[str] = None, ): """Creates a directory, as well as parent directories if needed. Arguments: paths: paths to create with mkdirp mode: optional permissions to set on the created directory -- use OS default if not provided group: optional group for permissions of final created directory -- use OS default if not provided. Only used if world write permissions are not set default_perms: one of ``"parents"`` or ``"args"``. The default permissions that are set for directories that are not themselves an argument for mkdirp. ``"parents"`` means intermediate directories get the permissions of their direct parent directory, ``"args"`` means intermediate get the same permissions specified in the arguments to mkdirp -- default value is ``"args"`` """ default_perms = default_perms or "args" paths = path_to_os_path(*paths) for path in paths: if not os.path.exists(path): try: last_parent, intermediate_folders = longest_existing_parent(path) # create folders os.makedirs(path) # leaf folder permissions if mode is not None: os.chmod(path, mode) if group: chgrp_if_not_world_writable(path, group) if mode is not None: os.chmod(path, mode) # reset sticky grp bit post chgrp # for intermediate folders, change mode just for newly created # ones and if mode_intermediate has been specified, otherwise # intermediate folders list is not populated at all and default # OS mode will be used if default_perms == "args": intermediate_mode = mode intermediate_group = group elif default_perms == "parents": stat_info = os.stat(last_parent) intermediate_mode = stat_info.st_mode intermediate_group = stat_info.st_gid else: msg = "Invalid value: '%s'. " % default_perms msg += "Choose from 'args' or 'parents'." raise ValueError(msg) for intermediate_path in reversed(intermediate_folders): if intermediate_mode is not None: os.chmod(intermediate_path, intermediate_mode) if intermediate_group is not None: chgrp_if_not_world_writable(intermediate_path, intermediate_group) if intermediate_mode is not None: os.chmod( intermediate_path, intermediate_mode ) # reset sticky bit after except OSError as e: if e.errno != errno.EEXIST or not os.path.isdir(path): raise e elif not os.path.isdir(path): raise OSError(errno.EEXIST, "File already exists", path) def longest_existing_parent(path: str) -> Tuple[str, List[str]]: """Return the last existing parent and a list of all intermediate directories to be created for the directory passed as input. Args: path: directory to be created """ # detect missing intermediate folders intermediate_folders = [] last_parent = "" intermediate_path = os.path.dirname(path) while intermediate_path: if os.path.lexists(intermediate_path): last_parent = intermediate_path break intermediate_folders.append(intermediate_path) intermediate_path = os.path.dirname(intermediate_path) return last_parent, intermediate_folders @system_path_filter def force_remove(*paths: str) -> None: """Remove files without printing errors. Like ``rm -f``, does NOT remove directories.""" for path in paths: try: os.remove(path) except OSError: pass @contextmanager @system_path_filter def working_dir(dirname: str, *, create: bool = False): """Context manager to change the current working directory to ``dirname``. Args: dirname: the directory to change to create: if :obj:`True`, create the directory if it does not exist Example usage:: with working_dir("/path/to/dir"): # do something in /path/to/dir pass """ if create: mkdirp(dirname) orig_dir = os.getcwd() os.chdir(dirname) try: yield finally: os.chdir(orig_dir) class CouldNotRestoreDirectoryBackup(RuntimeError): def __init__(self, inner_exception, outer_exception): self.inner_exception = inner_exception self.outer_exception = outer_exception @contextmanager @system_path_filter def replace_directory_transaction(directory_name): """Temporarily renames a directory in the same parent dir. If the operations executed within the context manager don't raise an exception, the renamed directory is deleted. If there is an exception, the move is undone. Args: directory_name (path): absolute path of the directory name Returns: temporary directory where ``directory_name`` has been moved """ # Check the input is indeed a directory with absolute path. # Raise before anything is done to avoid moving the wrong directory directory_name = os.path.abspath(directory_name) assert os.path.isdir(directory_name), "Not a directory: " + directory_name # Note: directory_name is normalized here, meaning the trailing slash is dropped, # so dirname is the directory's parent not the directory itself. tmpdir = tempfile.mkdtemp(dir=os.path.dirname(directory_name), prefix=".backup") # We have to jump through hoops to support Windows, since # os.rename(directory_name, tmpdir) errors there. backup_dir = os.path.join(tmpdir, "backup") os.rename(directory_name, backup_dir) tty.debug("Directory moved [src={0}, dest={1}]".format(directory_name, backup_dir)) try: yield backup_dir except (Exception, KeyboardInterrupt, SystemExit) as inner_exception: # Try to recover the original directory, if this fails, raise a # composite exception. try: # Delete what was there, before copying back the original content if os.path.exists(directory_name): shutil.rmtree(directory_name) os.rename(backup_dir, directory_name) except Exception as outer_exception: raise CouldNotRestoreDirectoryBackup(inner_exception, outer_exception) tty.debug("Directory recovered [{0}]".format(directory_name)) raise else: # Otherwise delete the temporary directory shutil.rmtree(tmpdir, ignore_errors=True) tty.debug("Temporary directory deleted [{0}]".format(tmpdir)) @system_path_filter def hash_directory(directory, ignore=[]): """Hashes recursively the content of a directory. Args: directory (path): path to a directory to be hashed Returns: hash of the directory content """ assert os.path.isdir(directory), '"directory" must be a directory!' md5_hash = hashlib.md5() # Adapted from https://stackoverflow.com/a/3431835/771663 for root, dirs, files in os.walk(directory): for name in sorted(files): filename = os.path.join(root, name) if filename not in ignore: # TODO: if caching big files becomes an issue, convert this to # TODO: read in chunks. Currently it's used only for testing # TODO: purposes. with open(filename, "rb") as f: md5_hash.update(f.read()) return md5_hash.hexdigest() @contextmanager @system_path_filter def write_tmp_and_move(filename: str, *, encoding: Optional[str] = None): """Write to a temporary file, then move into place.""" dirname = os.path.dirname(filename) basename = os.path.basename(filename) tmp = os.path.join(dirname, ".%s.tmp" % basename) with open(tmp, "w", encoding=encoding) as f: yield f shutil.move(tmp, filename) @system_path_filter def touch(path): """Creates an empty file at the specified path.""" if sys.platform == "win32": perms = os.O_WRONLY | os.O_CREAT else: perms = os.O_WRONLY | os.O_CREAT | os.O_NONBLOCK | os.O_NOCTTY fd = None try: fd = os.open(path, perms) os.utime(path, None) finally: if fd is not None: os.close(fd) @system_path_filter def touchp(path): """Like ``touch``, but creates any parent directories needed for the file.""" mkdirp(os.path.dirname(path)) touch(path) @system_path_filter def force_symlink(src: str, dest: str) -> None: """Create a symlink at ``dest`` pointing to ``src``. Similar to ``ln -sf``.""" try: symlink(src, dest) except OSError: os.remove(dest) symlink(src, dest) @system_path_filter def join_path(prefix, *args) -> str: """Alias for :func:`os.path.join`""" path = str(prefix) for elt in args: path = os.path.join(path, str(elt)) return path @system_path_filter def ancestor(dir, n=1): """Get the nth ancestor of a directory.""" parent = os.path.abspath(dir) for i in range(n): parent = os.path.dirname(parent) return parent @system_path_filter def get_single_file(directory): fnames = os.listdir(directory) if len(fnames) != 1: raise ValueError("Expected exactly 1 file, got {0}".format(str(len(fnames)))) return fnames[0] @system_path_filter def windows_sfn(path: os.PathLike): """Returns 8.3 Filename (SFN) representation of path 8.3 Filenames (SFN or short filename) is a file naming convention used prior to Win95 that Windows still (and will continue to) support. This convention caps filenames at 8 characters, and most importantly does not allow for spaces in addition to other specifications. The scheme is generally the same as a normal Windows file scheme, but all spaces are removed and the filename is capped at 6 characters. The remaining characters are replaced with ~N where N is the number file in a directory that a given file represents i.e. Program Files and Program Files (x86) would be PROGRA~1 and PROGRA~2 respectively. Further, all file/directory names are all caps (although modern Windows is case insensitive in practice). Conversion is accomplished by fileapi.h GetShortPathNameW Returns paths in 8.3 Filename form Note: this method is a no-op on Linux Args: path: Path to be transformed into SFN (8.3 filename) format """ # This should not be run-able on linux/macos if sys.platform != "win32": return path path = str(path) import ctypes k32 = ctypes.WinDLL("kernel32", use_last_error=True) # Method with null values returns size of short path name sz = k32.GetShortPathNameW(path, None, 0) # stub Windows types TCHAR[LENGTH] TCHAR_arr = ctypes.c_wchar * sz ret_str = TCHAR_arr() k32.GetShortPathNameW(path, ctypes.byref(ret_str), sz) return ret_str.value @contextmanager def temp_cwd(ignore_cleanup_errors=False): tmp_dir = tempfile.mkdtemp() try: with working_dir(tmp_dir): yield tmp_dir finally: kwargs = {} if sys.platform == "win32" or ignore_cleanup_errors: kwargs["ignore_errors"] = False kwargs["onerror"] = readonly_file_handler(ignore_errors=True) shutil.rmtree(tmp_dir, **kwargs) @system_path_filter def can_access(file_name): """True if the current process has read and write access to the file.""" return os.access(file_name, os.R_OK | os.W_OK) @system_path_filter def traverse_tree( source_root: str, dest_root: str, rel_path: str = "", *, order: str = "pre", ignore: Optional[Callable[[str], bool]] = None, follow_nonexisting: bool = True, follow_links: bool = False, ): """Traverse two filesystem trees simultaneously. Walks the LinkTree directory in pre or post order. Yields each file in the source directory with a matching path from the dest directory, along with whether the file is a directory. e.g., for this tree:: root/ a/ file1 file2 b/ file3 When called on dest, this yields:: ("root", "dest") ("root/a", "dest/a") ("root/a/file1", "dest/a/file1") ("root/a/file2", "dest/a/file2") ("root/b", "dest/b") ("root/b/file3", "dest/b/file3") Keyword Arguments: order (str): Whether to do pre- or post-order traversal. Accepted values are ``"pre"`` and ``"post"`` ignore (typing.Callable): function indicating which files to ignore. This will also ignore symlinks if they point to an ignored file (regardless of whether the symlink is explicitly ignored); note this only supports one layer of indirection (i.e. if you have x -> y -> z, and z is ignored but x/y are not, then y would be ignored but not x). To avoid this, make sure the ignore function also ignores the symlink paths too. follow_nonexisting (bool): Whether to descend into directories in ``src`` that do not exit in ``dest``. Default is True follow_links (bool): Whether to descend into symlinks in ``src`` """ if order not in ("pre", "post"): raise ValueError("Order must be 'pre' or 'post'.") # List of relative paths to ignore under the src root. ignore = ignore or (lambda filename: False) # Don't descend into ignored directories if ignore(rel_path): return source_path = os.path.join(source_root, rel_path) dest_path = os.path.join(dest_root, rel_path) # preorder yields directories before children if order == "pre": yield (source_path, dest_path) for f in os.listdir(source_path): source_child = os.path.join(source_path, f) dest_child = os.path.join(dest_path, f) rel_child = os.path.join(rel_path, f) # If the source path is a link and the link's source is ignored, then ignore the link too, # but only do this if the ignore is defined. if ignore is not None: if islink(source_child) and not follow_links: target = readlink(source_child) all_parents = accumulate(target.split(os.sep), lambda x, y: os.path.join(x, y)) if any(map(ignore, all_parents)): tty.warn( f"Skipping {source_path} because the source or a part of the source's " f"path is included in the ignores." ) continue # Treat as a directory # TODO: for symlinks, os.path.isdir looks for the link target. If the # target is relative to the link, then that may not resolve properly # relative to our cwd - see resolve_link_target_relative_to_the_link if os.path.isdir(source_child) and (follow_links or not islink(source_child)): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): tuples = traverse_tree( source_root, dest_root, rel_child, order=order, ignore=ignore, follow_nonexisting=follow_nonexisting, follow_links=follow_links, ) for t in tuples: yield t # Treat as a file. elif not ignore(os.path.join(rel_path, f)): yield (source_child, dest_child) if order == "post": yield (source_path, dest_path) class BaseDirectoryVisitor: """Base class and interface for :py:func:`visit_directory_tree`.""" def visit_file(self, root: str, rel_path: str, depth: int) -> None: """Handle the non-symlink file at ``os.path.join(root, rel_path)`` Parameters: root: root directory rel_path: relative path to current file from ``root`` depth (int): depth of current file from the ``root`` directory""" pass def visit_symlinked_file(self, root: str, rel_path: str, depth) -> None: """Handle the symlink to a file at ``os.path.join(root, rel_path)``. Note: ``rel_path`` is the location of the symlink, not to what it is pointing to. The symlink may be dangling. Parameters: root: root directory rel_path: relative path to current symlink from ``root`` depth: depth of current symlink from the ``root`` directory""" pass def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool: """Return True from this function to recurse into the directory at os.path.join(root, rel_path). Return False in order not to recurse further. Parameters: root: root directory rel_path: relative path to current directory from ``root`` depth: depth of current directory from the ``root`` directory Returns: bool: ``True`` when the directory should be recursed into. ``False`` when not""" return False def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool: """Return ``True`` to recurse into the symlinked directory and ``False`` in order not to. Note: ``rel_path`` is the path to the symlink itself. Following symlinked directories blindly can cause infinite recursion due to cycles. Parameters: root: root directory rel_path: relative path to current symlink from ``root`` depth: depth of current symlink from the ``root`` directory Returns: bool: ``True`` when the directory should be recursed into. ``False`` when not""" return False def after_visit_dir(self, root: str, rel_path: str, depth: int) -> None: """Called after recursion into ``rel_path`` finished. This function is not called when ``rel_path`` was not recursed into. Parameters: root: root directory rel_path: relative path to current directory from ``root`` depth: depth of current directory from the ``root`` directory""" pass def after_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> None: """Called after recursion into ``rel_path`` finished. This function is not called when ``rel_path`` was not recursed into. Parameters: root: root directory rel_path: relative path to current symlink from ``root`` depth: depth of current symlink from the ``root`` directory""" pass def visit_directory_tree( root: str, visitor: BaseDirectoryVisitor, rel_path: str = "", depth: int = 0 ): """Recurses the directory root depth-first through a visitor pattern using the interface from :py:class:`BaseDirectoryVisitor` Parameters: root: path of directory to recurse into visitor: what visitor to use rel_path: current relative path from the root depth: current depth from the root """ dir = os.path.join(root, rel_path) dir_entries = sorted(os.scandir(dir), key=lambda d: d.name) for f in dir_entries: rel_child = os.path.join(rel_path, f.name) islink = f.is_symlink() # On Windows, symlinks to directories are distinct from symlinks to files, and it is # possible to create a broken symlink to a directory (e.g. using os.symlink without # `target_is_directory=True`), invoking `isdir` on a symlink on Windows that is broken in # this manner will result in an error. In this case we can work around the issue by reading # the target and resolving the directory ourselves try: isdir = f.is_dir() except OSError as e: if sys.platform == "win32" and e.errno == errno.EACCES and islink: # if path is a symlink, determine destination and evaluate file vs directory link_target = resolve_link_target_relative_to_the_link(f) # link_target might be relative but resolve_link_target_relative_to_the_link # will ensure that if so, that it is relative to the CWD and therefore makes sense isdir = os.path.isdir(link_target) else: raise e if not isdir and not islink: # handle non-symlink files visitor.visit_file(root, rel_child, depth) elif not isdir: visitor.visit_symlinked_file(root, rel_child, depth) elif not islink and visitor.before_visit_dir(root, rel_child, depth): # Handle ordinary directories visit_directory_tree(root, visitor, rel_child, depth + 1) visitor.after_visit_dir(root, rel_child, depth) elif islink and visitor.before_visit_symlinked_dir(root, rel_child, depth): # Handle symlinked directories visit_directory_tree(root, visitor, rel_child, depth + 1) visitor.after_visit_symlinked_dir(root, rel_child, depth) @system_path_filter def set_executable(path): """Set the executable bit on a file or directory.""" mode = os.stat(path).st_mode if mode & stat.S_IRUSR: mode |= stat.S_IXUSR if mode & stat.S_IRGRP: mode |= stat.S_IXGRP if mode & stat.S_IROTH: mode |= stat.S_IXOTH os.chmod(path, mode) @system_path_filter def recursive_mtime_greater_than(path: str, time: float) -> bool: """Returns true if any file or dir recursively under `path` has mtime greater than `time`.""" # use bfs order to increase likelihood of early return queue: Deque[str] = collections.deque([path]) if os.stat(path).st_mtime > time: return True while queue: current = queue.popleft() try: entries = os.scandir(current) except OSError: continue with entries: for entry in entries: try: st = entry.stat(follow_symlinks=False) except OSError: continue if st.st_mtime > time: return True if entry.is_dir(follow_symlinks=False): queue.append(entry.path) return False @system_path_filter def remove_empty_directories(root): """Ascend up from the leaves accessible from `root` and remove empty directories. Parameters: root (str): path where to search for empty directories """ for dirpath, subdirs, files in os.walk(root, topdown=False): for sd in subdirs: sdp = os.path.join(dirpath, sd) try: os.rmdir(sdp) except OSError: pass @system_path_filter def remove_dead_links(root): """Recursively removes any dead link that is present in root. Parameters: root (str): path where to search for dead links """ for dirpath, subdirs, files in os.walk(root, topdown=False): for f in files: path = join_path(dirpath, f) remove_if_dead_link(path) @system_path_filter def remove_if_dead_link(path): """Removes the argument if it is a dead link. Parameters: path (str): The potential dead link """ if islink(path) and not os.path.exists(path): os.unlink(path) def readonly_file_handler(ignore_errors=False): # TODO: generate stages etc. with write permissions wherever # so this callback is no-longer required """ Generate callback for shutil.rmtree to handle permissions errors on Windows. Some files may unexpectedly lack write permissions even though they were generated by Spack on behalf of the user (e.g. the stage), so this callback will detect such cases and modify the permissions if that is the issue. For other errors, the fallback is either to raise (if ignore_errors is False) or ignore (if ignore_errors is True). This is only intended for Windows systems and will raise a separate error if it is ever invoked (by accident) on a non-Windows system. """ def error_remove_readonly(func, path, exc): if sys.platform != "win32": raise RuntimeError("This method should only be invoked on Windows") excvalue = exc[1] if ( sys.platform == "win32" and func in (os.rmdir, os.remove, os.unlink) and excvalue.errno == errno.EACCES ): # change the file to be readable,writable,executable: 0777 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # retry func(path) elif not ignore_errors: raise return error_remove_readonly @system_path_filter def remove_linked_tree(path: str) -> None: """Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real directory before removing the link. This method will force-delete files on Windows Parameters: path: Directory to be removed """ kwargs: dict = {"ignore_errors": True} # Windows readonly files cannot be removed by Python # directly. if sys.platform == "win32": kwargs["ignore_errors"] = False kwargs["onerror"] = readonly_file_handler(ignore_errors=True) if os.path.exists(path): if islink(path): shutil.rmtree(os.path.realpath(path), **kwargs) os.unlink(path) else: if sys.platform == "win32": # Adding this prefix allows shutil to remove long paths on windows # https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry long_path_pfx = "\\\\?\\" if not path.startswith(long_path_pfx): path = long_path_pfx + path shutil.rmtree(path, **kwargs) @contextmanager @system_path_filter def safe_remove(*files_or_dirs): """Context manager to remove the files passed as input, but restore them in case any exception is raised in the context block. Args: *files_or_dirs: glob expressions for files or directories to be removed Returns: Dictionary that maps deleted files to their temporary copy within the context block. """ # Find all the files or directories that match glob_matches = [glob.glob(x) for x in files_or_dirs] # Sort them so that shorter paths like "/foo/bar" come before # nested paths like "/foo/bar/baz.yaml". This simplifies the # handling of temporary copies below sorted_matches = sorted([os.path.abspath(x) for x in itertools.chain(*glob_matches)], key=len) # Copy files and directories in a temporary location removed, dst_root = {}, tempfile.mkdtemp() try: for id, file_or_dir in enumerate(sorted_matches): # The glob expression at the top ensures that the file/dir exists # at the time we enter the loop. Double check here since it might # happen that a previous iteration of the loop already removed it. # This is the case, for instance, if we remove the directory # "/foo/bar" before the file "/foo/bar/baz.yaml". if not os.path.exists(file_or_dir): continue # The monotonic ID is a simple way to make the filename # or directory name unique in the temporary folder basename = os.path.basename(file_or_dir) + "-{0}".format(id) temporary_path = os.path.join(dst_root, basename) shutil.move(file_or_dir, temporary_path) removed[file_or_dir] = temporary_path yield removed except BaseException: # Restore the files that were removed for original_path, temporary_path in removed.items(): shutil.move(temporary_path, original_path) raise def find_first(root: str, files: Union[Iterable[str], str], bfs_depth: int = 2) -> Optional[str]: """Find the first file matching a pattern. The following .. code-block:: console $ find /usr -name 'abc*' -o -name 'def*' -quit is equivalent to: >>> find_first("/usr", ["abc*", "def*"]) Any glob pattern supported by fnmatch can be used. The search order of this method is breadth-first over directories, until depth bfs_depth, after which depth-first search is used. Parameters: root: The root directory to start searching from files: File pattern(s) to search for bfs_depth: (advanced) parameter that specifies at which depth to switch to depth-first search. Returns: The matching file or :data:`None` when no file is found. """ if isinstance(files, str): files = [files] return FindFirstFile(root, *files, bfs_depth=bfs_depth).find() def find( root: Union[Path, Sequence[Path]], files: Union[str, Sequence[str]], recursive: bool = True, max_depth: Optional[int] = None, ) -> List[str]: """Finds all files matching the patterns from ``files`` starting from ``root``. This function returns a deterministic result for the same input and directory structure when run multiple times. Symlinked directories are followed, and unique directories are searched only once. Each matching file is returned only once at lowest depth in case multiple paths exist due to symlinked directories. Accepts any glob characters accepted by :py:func:`fnmatch.fnmatch`: ========== ==================================== Pattern Meaning ========== ==================================== ``*`` matches one or more characters ``?`` matches any single character ``[seq]`` matches any character in ``seq`` ``[!seq]`` matches any character not in ``seq`` ========== ==================================== Examples: >>> find("/usr", "*.txt", recursive=True, max_depth=2) finds all files with the extension ``.txt`` in the directory ``/usr`` and subdirectories up to depth 2. >>> find(["/usr", "/var"], ["*.txt", "*.log"], recursive=True) finds all files with the extension ``.txt`` or ``.log`` in the directories ``/usr`` and ``/var`` at any depth. >>> find("/usr", "GL/*.h", recursive=True) finds all header files in a directory GL at any depth in the directory ``/usr``. Parameters: root: One or more root directories to start searching from files: One or more filename patterns to search for recursive: if False search only root, if True descends from roots. Defaults to True. max_depth: if set, don't search below this depth. Cannot be set if recursive is False Returns a list of absolute, matching file paths. """ if isinstance(root, (str, pathlib.Path)): root = [root] elif not isinstance(root, collections.abc.Sequence): raise TypeError(f"'root' arg must be a path or a sequence of paths, not '{type(root)}']") if isinstance(files, str): files = [files] elif not isinstance(files, collections.abc.Sequence): raise TypeError(f"'files' arg must be str or a sequence of str, not '{type(files)}']") # If recursive is false, max_depth can only be None or 0 if max_depth and not recursive: raise ValueError(f"max_depth ({max_depth}) cannot be set if recursive is False") tty.debug(f"Find (max depth = {max_depth}): {root} {files}") if not recursive: max_depth = 0 elif max_depth is None: max_depth = sys.maxsize result = _find_max_depth(root, files, max_depth) tty.debug(f"Find complete: {root} {files}") return result def _log_file_access_issue(e: OSError, path: str) -> None: tty.debug(f"find must skip {path}: {e}") def _file_id(s: os.stat_result) -> Tuple[int, int]: # Note: on windows, st_ino is the file index and st_dev is the volume serial number. See # https://github.com/python/cpython/blob/3.9/Python/fileutils.c return (s.st_ino, s.st_dev) def _dedupe_files(paths: List[str]) -> List[str]: """Deduplicate files by inode and device, dropping files that cannot be accessed.""" unique_files: List[str] = [] # tuple of (inode, device) for each file without following symlinks visited: Set[Tuple[int, int]] = set() for path in paths: try: stat_info = os.lstat(path) except OSError as e: _log_file_access_issue(e, path) continue file_id = _file_id(stat_info) if file_id not in visited: unique_files.append(path) visited.add(file_id) return unique_files def _find_max_depth( roots: Sequence[Path], globs: Sequence[str], max_depth: int = sys.maxsize ) -> List[str]: """See ``find`` for the public API.""" # We optimize for the common case of simple filename only patterns: a single, combined regex # is used. For complex patterns that include path components, we use a slower glob call from # every directory we visit within max_depth. filename_only_patterns = { f"pattern_{i}": os.path.normcase(x) for i, x in enumerate(globs) if "/" not in x } complex_patterns = {f"pattern_{i}": x for i, x in enumerate(globs) if "/" in x} regex = re.compile(fnmatch_translate_multiple(filename_only_patterns)) # Ordered dictionary that keeps track of what pattern found which files matched_paths: Dict[str, List[str]] = {f"pattern_{i}": [] for i, _ in enumerate(globs)} # Ensure returned paths are always absolute roots = [os.path.abspath(r) for r in roots] # Breadth-first search queue. Each element is a tuple of (depth, dir) dir_queue: Deque[Tuple[int, str]] = collections.deque() # Set of visited directories. Each element is a tuple of (inode, device) visited_dirs: Set[Tuple[int, int]] = set() for root in roots: try: stat_root = os.stat(root) except OSError as e: _log_file_access_issue(e, root) continue dir_id = _file_id(stat_root) if dir_id not in visited_dirs: dir_queue.appendleft((0, root)) visited_dirs.add(dir_id) while dir_queue: depth, curr_dir = dir_queue.pop() try: dir_iter = os.scandir(curr_dir) except OSError as e: _log_file_access_issue(e, curr_dir) continue # Use glob.glob for complex patterns. for pattern_name, pattern in complex_patterns.items(): matched_paths[pattern_name].extend( path for path in glob.glob(os.path.join(curr_dir, pattern)) ) # List of subdirectories by path and (inode, device) tuple subdirs: List[Tuple[str, Tuple[int, int]]] = [] with dir_iter: for dir_entry in dir_iter: # Match filename only patterns if filename_only_patterns: m = regex.match(os.path.normcase(dir_entry.name)) if m: for pattern_name in filename_only_patterns: if m.group(pattern_name): matched_paths[pattern_name].append(dir_entry.path) break # Collect subdirectories if depth >= max_depth: continue try: if not dir_entry.is_dir(follow_symlinks=True): continue if sys.platform == "win32": # Note: st_ino/st_dev on DirEntry.stat are not set on Windows, so we have # to call os.stat stat_info = os.stat(dir_entry.path, follow_symlinks=True) else: stat_info = dir_entry.stat(follow_symlinks=True) except OSError as e: # Possible permission issue, or a symlink that cannot be resolved (ELOOP). _log_file_access_issue(e, dir_entry.path) continue subdirs.append((dir_entry.path, _file_id(stat_info))) # Enqueue subdirectories in a deterministic order if subdirs: subdirs.sort(key=lambda s: os.path.basename(s[0])) for subdir, subdir_id in subdirs: if subdir_id not in visited_dirs: dir_queue.appendleft((depth + 1, subdir)) visited_dirs.add(subdir_id) # Sort the matched paths for deterministic output for paths in matched_paths.values(): paths.sort() all_matching_paths = [path for paths in matched_paths.values() for path in paths] # We only dedupe files if we have any complex patterns, since only they can match the same file # multiple times return _dedupe_files(all_matching_paths) if complex_patterns else all_matching_paths # Utilities for libraries and headers class FileList(collections.abc.Sequence): """Sequence of absolute paths to files. Provides a few convenience methods to manipulate file paths. """ def __init__(self, files: Union[str, Iterable[str]]) -> None: if isinstance(files, str): files = [files] self.files = list(dedupe(files)) @property def directories(self) -> List[str]: """Stable de-duplication of the directories where the files reside. >>> l = LibraryList(["/dir1/liba.a", "/dir2/libb.a", "/dir1/libc.a"]) >>> l.directories ["/dir1", "/dir2"] >>> h = HeaderList(["/dir1/a.h", "/dir1/b.h", "/dir2/c.h"]) >>> h.directories ["/dir1", "/dir2"] Returns: A list of directories """ return list(dedupe(os.path.dirname(x) for x in self.files if os.path.dirname(x))) @property def basenames(self) -> List[str]: """Stable de-duplication of the base-names in the list >>> l = LibraryList(["/dir1/liba.a", "/dir2/libb.a", "/dir3/liba.a"]) >>> l.basenames ["liba.a", "libb.a"] >>> h = HeaderList(["/dir1/a.h", "/dir2/b.h", "/dir3/a.h"]) >>> h.basenames ["a.h", "b.h"] Returns: A list of base-names """ return list(dedupe(os.path.basename(x) for x in self.files)) def __getitem__(self, item): cls = type(self) if isinstance(item, numbers.Integral): return self.files[item] return cls(self.files[item]) def __add__(self, other): return self.__class__(dedupe(self.files + list(other))) def __radd__(self, other): return self.__add__(other) def __eq__(self, other): return self.files == other.files def __len__(self): return len(self.files) def joined(self, separator: str = " ") -> str: return separator.join(self.files) def __repr__(self): return self.__class__.__name__ + "(" + repr(self.files) + ")" def __str__(self): return self.joined() class HeaderList(FileList): """Sequence of absolute paths to headers. Provides a few convenience methods to manipulate header paths and get commonly used compiler flags or names. """ # Make sure to only match complete words, otherwise path components such # as "xinclude" will cause false matches. # Avoid matching paths such as /include/something/detail/include, # e.g. in the CUDA Toolkit which ships internal libc++ headers. include_regex = re.compile(r"(.*?)(\binclude\b)(.*)") def __init__(self, files): super().__init__(files) self._macro_definitions = [] self._directories = None @property def directories(self) -> List[str]: """Directories to be searched for header files.""" values = self._directories if values is None: values = self._default_directories() return list(dedupe(values)) @directories.setter def directories(self, value): value = value or [] # Accept a single directory as input if isinstance(value, str): value = [value] self._directories = [path_to_os_path(os.path.normpath(x))[0] for x in value] def _default_directories(self): """Default computation of directories based on the list of header files. """ dir_list = super().directories values = [] for d in dir_list: # If the path contains a subdirectory named 'include' then stop # there and don't add anything else to the path. m = self.include_regex.match(d) value = os.path.join(*m.group(1, 2)) if m else d values.append(value) return values @property def headers(self) -> List[str]: """Stable de-duplication of the headers. Returns: A list of header files """ return self.files @property def names(self) -> List[str]: """Stable de-duplication of header names in the list without extensions >>> h = HeaderList(["/dir1/a.h", "/dir2/b.h", "/dir3/a.h"]) >>> h.names ["a", "b"] Returns: A list of files without extensions """ names = [] for x in self.basenames: name = x # Valid extensions include: [".cuh", ".hpp", ".hh", ".h"] for ext in [".cuh", ".hpp", ".hh", ".h"]: i = name.rfind(ext) if i != -1: names.append(name[:i]) break else: # No valid extension, should we still include it? names.append(name) return list(dedupe(names)) @property def include_flags(self) -> str: """Include flags >>> h = HeaderList(["/dir1/a.h", "/dir1/b.h", "/dir2/c.h"]) >>> h.include_flags "-I/dir1 -I/dir2" Returns: A joined list of include flags """ return " ".join(["-I" + x for x in self.directories]) @property def macro_definitions(self) -> str: """Macro definitions >>> h = HeaderList(["/dir1/a.h", "/dir1/b.h", "/dir2/c.h"]) >>> h.add_macro("-DBOOST_LIB_NAME=boost_regex") >>> h.add_macro("-DBOOST_DYN_LINK") >>> h.macro_definitions "-DBOOST_LIB_NAME=boost_regex -DBOOST_DYN_LINK" Returns: A joined list of macro definitions """ return " ".join(self._macro_definitions) @property def cpp_flags(self) -> str: """Include flags + macro definitions >>> h = HeaderList(["/dir1/a.h", "/dir1/b.h", "/dir2/c.h"]) >>> h.cpp_flags "-I/dir1 -I/dir2" >>> h.add_macro("-DBOOST_DYN_LINK") >>> h.cpp_flags "-I/dir1 -I/dir2 -DBOOST_DYN_LINK" Returns: A joined list of include flags and macro definitions """ cpp_flags = self.include_flags if self.macro_definitions: cpp_flags += " " + self.macro_definitions return cpp_flags def add_macro(self, macro: str) -> None: """Add a macro definition Parameters: macro: The macro to add """ self._macro_definitions.append(macro) def find_headers(headers: Union[str, List[str]], root: str, recursive: bool = False) -> HeaderList: """Returns an iterable object containing a list of full paths to headers if found. Accepts any glob characters accepted by :py:func:`fnmatch.fnmatch`: ========== ==================================== Pattern Meaning ========== ==================================== ``*`` matches one or more characters ``?`` matches any single character ``[seq]`` matches any character in ``seq`` ``[!seq]`` matches any character not in ``seq`` ========== ==================================== Parameters: headers: Header name(s) to search for root: The root directory to start searching from recursive: if :data:`False` search only root folder, if :data:`True` descends top-down from the root. Defaults to :data:`False`. Returns: The headers that have been found """ if isinstance(headers, str): headers = [headers] elif not isinstance(headers, collections.abc.Sequence): message = "{0} expects a string or sequence of strings as the " message += "first argument [got {1} instead]" message = message.format(find_headers.__name__, type(headers)) raise TypeError(message) # Construct the right suffix for the headers suffixes = [ # C "h", # C++ "hpp", "hxx", "hh", "H", "txx", "tcc", "icc", # Fortran "mod", "inc", ] # List of headers we are searching with suffixes headers = ["{0}.{1}".format(header, suffix) for header in headers for suffix in suffixes] return HeaderList(find(root, headers, recursive)) @system_path_filter def find_all_headers(root: str) -> HeaderList: """Convenience function that returns the list of all headers found in the directory passed as argument. Args: root: directory where to look recursively for header files Returns: List of all headers found in ``root`` and subdirectories. """ return find_headers("*", root=root, recursive=True) class LibraryList(FileList): """Sequence of absolute paths to libraries Provides a few convenience methods to manipulate library paths and get commonly used compiler flags or names """ @property def libraries(self) -> List[str]: """Stable de-duplication of library files. Returns: A list of library files """ return self.files @property def names(self) -> List[str]: """Stable de-duplication of library names in the list >>> l = LibraryList(["/dir1/liba.a", "/dir2/libb.a", "/dir3/liba.so"]) >>> l.names ["a", "b"] Returns: A list of library names """ names = [] for x in self.basenames: name = x if x.startswith("lib"): name = x[3:] # Valid extensions include: ['.dylib', '.so', '.a'] # on non Windows platform # Windows valid library extensions are: # ['.dll', '.lib'] valid_exts = [".dll", ".lib"] if sys.platform == "win32" else [".dylib", ".so", ".a"] for ext in valid_exts: i = name.rfind(ext) if i != -1: names.append(name[:i]) break else: # No valid extension, should we still include it? names.append(name) return list(dedupe(names)) @property def search_flags(self) -> str: """Search flags for the libraries >>> l = LibraryList(["/dir1/liba.a", "/dir2/libb.a", "/dir1/liba.so"]) >>> l.search_flags "-L/dir1 -L/dir2" Returns: A joined list of search flags """ return " ".join(["-L" + x for x in self.directories]) @property def link_flags(self) -> str: """Link flags for the libraries >>> l = LibraryList(["/dir1/liba.a", "/dir2/libb.a", "/dir1/liba.so"]) >>> l.link_flags "-la -lb" Returns: A joined list of link flags """ return " ".join(["-l" + name for name in self.names]) @property def ld_flags(self) -> str: """Search flags + link flags >>> l = LibraryList(["/dir1/liba.a", "/dir2/libb.a", "/dir1/liba.so"]) >>> l.ld_flags "-L/dir1 -L/dir2 -la -lb" Returns: A joined list of search flags and link flags """ return self.search_flags + " " + self.link_flags def find_system_libraries(libraries: Union[str, List[str]], shared: bool = True) -> LibraryList: """Searches the usual system library locations for ``libraries``. Search order is as follows: 1. ``/lib64`` 2. ``/lib`` 3. ``/usr/lib64`` 4. ``/usr/lib`` 5. ``/usr/local/lib64`` 6. ``/usr/local/lib`` Accepts any glob characters accepted by :py:func:`fnmatch.fnmatch`: ========== ==================================== Pattern Meaning ========== ==================================== ``*`` matches one or more characters ``?`` matches any single character ``[seq]`` matches any character in ``seq`` ``[!seq]`` matches any character not in ``seq`` ========== ==================================== Parameters: libraries: Library name(s) to search for shared: if :data:`True` searches for shared libraries, otherwise for static. Defaults to :data:`True`. Returns: The libraries that have been found """ if isinstance(libraries, str): libraries = [libraries] elif not isinstance(libraries, collections.abc.Sequence): message = "{0} expects a string or sequence of strings as the " message += "first argument [got {1} instead]" message = message.format(find_system_libraries.__name__, type(libraries)) raise TypeError(message) libraries_found = LibraryList([]) search_locations = [ "/lib64", "/lib", "/usr/lib64", "/usr/lib", "/usr/local/lib64", "/usr/local/lib", ] for library in libraries: for root in search_locations: result = find_libraries(library, root, shared, recursive=True) if result: libraries_found += result break return libraries_found def find_libraries( libraries: Union[str, List[str]], root: str, shared: bool = True, recursive: bool = False, runtime: bool = True, max_depth: Optional[int] = None, ) -> LibraryList: """Returns an iterable of full paths to libraries found in a root dir. Accepts any glob characters accepted by :py:func:`fnmatch.fnmatch`: ========== ==================================== Pattern Meaning ========== ==================================== ``*`` matches one or more characters ``?`` matches any single character ``[seq]`` matches any character in ``seq`` ``[!seq]`` matches any character not in ``seq`` ========== ==================================== Parameters: libraries: Library name(s) to search for root: The root directory to start searching from shared: if :data:`True` searches for shared libraries, otherwise for static. Defaults to :data:`True`. recursive: if :data:`False` search only root folder, if :data:`True` descends top-down from the root. Defaults to :data:`False`. max_depth: if set, don't search below this depth. Cannot be set if recursive is :data:`False` runtime: Windows only option, no-op elsewhere. If :data:`True`, search for runtime shared libs (``.DLL``), otherwise, search for ``.Lib`` files. If ``shared`` is :data:`False`, this has no meaning. Defaults to :data:`True`. Returns: The libraries that have been found """ if isinstance(libraries, str): libraries = [libraries] elif not isinstance(libraries, collections.abc.Sequence): message = "{0} expects a string or sequence of strings as the " message += "first argument [got {1} instead]" message = message.format(find_libraries.__name__, type(libraries)) raise TypeError(message) if sys.platform == "win32": static_ext = "lib" # For linking (runtime=False) you need the .lib files regardless of # whether you are doing a shared or static link shared_ext = "dll" if runtime else "lib" else: # Used on both Linux and macOS static_ext = "a" shared_ext = "so" # Construct the right suffix for the library if shared: # Used on both Linux and macOS suffixes = [shared_ext] if sys.platform == "darwin": # Only used on macOS suffixes.append("dylib") else: suffixes = [static_ext] # List of libraries we are searching with suffixes libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes] if not recursive: if max_depth: raise ValueError(f"max_depth ({max_depth}) cannot be set if recursive is False") # If not recursive, look for the libraries directly in root return LibraryList(find(root, libraries, recursive=False)) # To speedup the search for external packages configured e.g. in /usr, # perform first non-recursive search in root/lib then in root/lib64 and # finally search all of root recursively. The search stops when the first # match is found. common_lib_dirs = ["lib", "lib64"] if sys.platform == "win32": common_lib_dirs.extend(["bin", "Lib"]) for subdir in common_lib_dirs: dirname = join_path(root, subdir) if not os.path.isdir(dirname): continue found_libs = find(dirname, libraries, False) if found_libs: break else: found_libs = find(root, libraries, recursive=True, max_depth=max_depth) return LibraryList(found_libs) def find_all_shared_libraries( root: str, recursive: bool = False, runtime: bool = True ) -> LibraryList: """Convenience function that returns the list of all shared libraries found in the directory passed as argument. See documentation for :py:func:`find_libraries` for more information """ return find_libraries("*", root=root, shared=True, recursive=recursive, runtime=runtime) def find_all_static_libraries(root: str, recursive: bool = False) -> LibraryList: """Convenience function that returns the list of all static libraries found in the directory passed as argument. See documentation for :py:func:`find_libraries` for more information """ return find_libraries("*", root=root, shared=False, recursive=recursive) def find_all_libraries(root: str, recursive: bool = False) -> LibraryList: """Convenience function that returns the list of all libraries found in the directory passed as argument. See documentation for :py:func:`find_libraries` for more information """ return find_all_shared_libraries(root, recursive=recursive) + find_all_static_libraries( root, recursive=recursive ) @system_path_filter @memoized def can_access_dir(path): """Returns True if the argument is an accessible directory. Args: path: path to be tested Returns: True if ``path`` is an accessible directory, else False """ return os.path.isdir(path) and os.access(path, os.R_OK | os.X_OK) @system_path_filter @memoized def can_write_to_dir(path): """Return True if the argument is a directory in which we can write. Args: path: path to be tested Returns: True if ``path`` is an writeable directory, else False """ return os.path.isdir(path) and os.access(path, os.R_OK | os.X_OK | os.W_OK) @system_path_filter @memoized def files_in(*search_paths): """Returns all the files in paths passed as arguments. Caller must ensure that each path in ``search_paths`` is a directory. Args: *search_paths: directories to be searched Returns: List of (file, full_path) tuples with all the files found. """ files = [] for d in filter(can_access_dir, search_paths): files.extend( filter( lambda x: os.path.isfile(x[1]), [(f, os.path.join(d, f)) for f in os.listdir(d)] ) ) return files def is_readable_file(file_path): """Return True if the path passed as argument is readable""" return os.path.isfile(file_path) and os.access(file_path, os.R_OK) @system_path_filter def search_paths_for_executables(*path_hints): """Given a list of path hints returns a list of paths where to search for an executable. Args: *path_hints (list of paths): list of paths taken into consideration for a search Returns: A list containing the real path of every existing directory in `path_hints` and its `bin` subdirectory if it exists. """ executable_paths = [] for path in path_hints: if not os.path.isdir(path): continue path = os.path.abspath(path) executable_paths.append(path) bin_dir = os.path.join(path, "bin") if os.path.isdir(bin_dir): executable_paths.append(bin_dir) return executable_paths @system_path_filter def search_paths_for_libraries(*path_hints): """Given a list of path hints returns a list of paths where to search for a shared library. Args: *path_hints (list of paths): list of paths taken into consideration for a search Returns: A list containing the real path of every existing directory in `path_hints` and its `lib` and `lib64` subdirectory if it exists. """ library_paths = [] for path in path_hints: if not os.path.isdir(path): continue path = os.path.abspath(path) library_paths.append(path) lib_dir = os.path.join(path, "lib") if os.path.isdir(lib_dir): library_paths.append(lib_dir) lib64_dir = os.path.join(path, "lib64") if os.path.isdir(lib64_dir): library_paths.append(lib64_dir) return library_paths @system_path_filter def partition_path(path, entry=None): """ Split the prefixes of the path at the first occurrence of entry and return a 3-tuple containing a list of the prefixes before the entry, a string of the prefix ending with the entry, and a list of the prefixes after the entry. If the entry is not a node in the path, the result will be the prefix list followed by an empty string and an empty list. """ paths = prefixes(path) if entry is not None: # Derive the index of entry within paths, which will correspond to # the location of the entry in within the path. try: sep = os.sep entries = path.split(sep) if entries[0].endswith(":"): # Handle drive letters e.g. C:/ on Windows entries[0] = entries[0] + sep i = entries.index(entry) if "" in entries: i -= 1 return paths[:i], paths[i], paths[i + 1 :] except ValueError: pass return paths, "", [] @system_path_filter def prefixes(path): """ Returns a list containing the path and its ancestors, top-to-bottom. The list for an absolute path will not include an ``os.sep`` entry. For example, assuming ``os.sep`` is ``/``, given path ``/ab/cd/efg`` the resulting paths will be, in order: ``/ab``, ``/ab/cd``, and ``/ab/cd/efg`` The list for a relative path starting ``./`` will not include ``.``. For example, path ``./hi/jkl/mn`` results in a list with the following paths, in order: ``./hi``, ``./hi/jkl``, and ``./hi/jkl/mn``. On Windows, paths will be normalized to use ``/`` and ``/`` will always be used as the separator instead of ``os.sep``. Parameters: path (str): the string used to derive ancestor paths Returns: A list containing ancestor paths in order and ending with the path """ if not path: return [] sep = os.sep parts = path.strip(sep).split(sep) if path.startswith(sep): parts.insert(0, sep) elif parts[0].endswith(":"): # Handle drive letters e.g. C:/ on Windows parts[0] = parts[0] + sep paths = [os.path.join(*parts[: i + 1]) for i in range(len(parts))] try: paths.remove(sep) except ValueError: pass try: paths.remove(".") except ValueError: pass return paths @system_path_filter def remove_directory_contents(dir): """Remove all contents of a directory.""" if os.path.exists(dir): for entry in [os.path.join(dir, entry) for entry in os.listdir(dir)]: if os.path.isfile(entry) or islink(entry): os.unlink(entry) else: shutil.rmtree(entry) @contextmanager @system_path_filter def keep_modification_time(*filenames: str) -> Generator[None, None, None]: """ Context manager to keep the modification timestamps of the input files. Tolerates and has no effect on non-existent files and files that are deleted by the nested code. Example:: with keep_modification_time("file1.txt", "file2.txt"): # do something that modifies file1.txt and file2.txt Parameters: *filenames: one or more files that must have their modification timestamps unchanged """ mtimes = {} for f in filenames: if os.path.exists(f): mtimes[f] = os.path.getmtime(f) yield for f, mtime in mtimes.items(): if os.path.exists(f): os.utime(f, (os.path.getatime(f), mtime)) @contextmanager def temporary_file_position(stream): orig_pos = stream.tell() yield stream.seek(orig_pos) @contextmanager def current_file_position(stream: IO, loc: int, relative_to=io.SEEK_CUR): with temporary_file_position(stream): stream.seek(loc, relative_to) yield @contextmanager def temporary_dir( suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Optional[str] = None ): """Create a temporary directory and cd's into it. Delete the directory on exit. Takes the same arguments as tempfile.mkdtemp() """ tmp_dir = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir) try: with working_dir(tmp_dir): yield tmp_dir finally: remove_directory_contents(tmp_dir) @contextmanager def edit_in_place_through_temporary_file(file_path: str) -> Generator[str, None, None]: """Context manager for modifying ``file_path`` in place, preserving its inode and hardlinks, for functions or external tools that do not support in-place editing. Notice that this function is unsafe in that it works with paths instead of a file descriptors, but this is by design, since we assume the call site will create a new inode at the same path.""" tmp_fd, tmp_path = tempfile.mkstemp( dir=os.path.dirname(file_path), prefix=f"{os.path.basename(file_path)}." ) # windows cannot replace a file with open fds, so close since the call site needs to replace. os.close(tmp_fd) try: shutil.copyfile(file_path, tmp_path, follow_symlinks=True) yield tmp_path shutil.copyfile(tmp_path, file_path, follow_symlinks=True) finally: os.unlink(tmp_path) def filesummary(path, print_bytes=16) -> Tuple[int, bytes]: """Create a small summary of the given file. Does not error when file does not exist. Args: print_bytes (int): Number of bytes to print from start/end of file Returns: Tuple of size and byte string containing first n .. last n bytes. Size is 0 if file cannot be read.""" try: n = print_bytes with open(path, "rb") as f: size = os.fstat(f.fileno()).st_size if size <= 2 * n: short_contents = f.read(2 * n) else: short_contents = f.read(n) f.seek(-n, 2) short_contents += b"..." + f.read(n) return size, short_contents except OSError: return 0, b"" class FindFirstFile: """Uses hybrid iterative deepening to locate the first matching file. Up to depth ``bfs_depth`` it uses iterative deepening, which mimics breadth-first with the same memory footprint as depth-first search, after which it switches to ordinary depth-first search using ``os.walk``.""" def __init__(self, root: str, *file_patterns: str, bfs_depth: int = 2): """Create a small summary of the given file. Does not error when file does not exist. Args: root (str): directory in which to recursively search file_patterns (str): glob file patterns understood by fnmatch bfs_depth (int): until this depth breadth-first traversal is used, when no match is found, the mode is switched to depth-first search. """ self.root = root self.bfs_depth = bfs_depth self.match: Callable # normcase is trivial on posix regex = re.compile("|".join(fnmatch.translate(os.path.normcase(p)) for p in file_patterns)) # On case sensitive filesystems match against normcase'd paths. if os.path is posixpath: self.match = regex.match else: self.match = lambda p: regex.match(os.path.normcase(p)) def find(self) -> Optional[str]: """Run the file search Returns: str or None: path of the matching file """ self.file = None # First do iterative deepening (i.e. bfs through limited depth dfs) for i in range(self.bfs_depth + 1): if self._find_at_depth(self.root, i): return self.file # Then fall back to depth-first search return self._find_dfs() def _find_at_depth(self, path, max_depth, depth=0) -> bool: """Returns True when done. Notice search can be done either because a file was found, or because it recursed through all directories.""" try: entries = os.scandir(path) except OSError: return True done = True with entries: # At max depth we look for matching files. if depth == max_depth: for f in entries: # Exit on match if self.match(f.name): self.file = os.path.join(path, f.name) return True # is_dir should not require a stat call, so it's a good optimization. if self._is_dir(f): done = False return done # At lower depth only recurse into subdirs for f in entries: if not self._is_dir(f): continue # If any subdir is not fully traversed, we're not done yet. if not self._find_at_depth(os.path.join(path, f.name), max_depth, depth + 1): done = False # Early exit when we've found something. if self.file: return True return done def _is_dir(self, f: os.DirEntry) -> bool: """Returns True when f is dir we can enter (and not a symlink).""" try: return f.is_dir(follow_symlinks=False) except OSError: return False def _find_dfs(self) -> Optional[str]: """Returns match or None""" for dirpath, _, filenames in os.walk(self.root): for file in filenames: if self.match(file): return os.path.join(dirpath, file) return None def _windows_symlink( src: str, dst: str, target_is_directory: bool = False, *, dir_fd: Union[int, None] = None ): """On Windows with System Administrator privileges this will be a normal symbolic link via os.symlink. On Windows without privileges the link will be a junction for a directory and a hardlink for a file. On Windows the various link types are: Symbolic Link: A link to a file or directory on the same or different volume (drive letter) or even to a remote file or directory (using UNC in its path). Need System Administrator privileges to make these. Hard Link: A link to a file on the same volume (drive letter) only. Every file (file's data) has at least 1 hard link (file's name). But when this method creates a new hard link there will be 2. Deleting all hard links effectively deletes the file. Don't need System Administrator privileges. Junction: A link to a directory on the same or different volume (drive letter) but not to a remote directory. Don't need System Administrator privileges.""" source_path = os.path.normpath(src) win_source_path = source_path link_path = os.path.normpath(dst) # Perform basic checks to make sure symlinking will succeed if os.path.lexists(link_path): raise AlreadyExistsError(f"Link path ({link_path}) already exists. Cannot create link.") if not os.path.exists(source_path): if os.path.isabs(source_path): # An absolute source path that does not exist will result in a broken link. raise SymlinkError( f"Source path ({source_path}) is absolute but does not exist. Resulting " f"link would be broken so not making link." ) else: # os.symlink can create a link when the given source path is relative to # the link path. Emulate this behavior and check to see if the source exists # relative to the link path ahead of link creation to prevent broken # links from being made. link_parent_dir = os.path.dirname(link_path) relative_path = os.path.join(link_parent_dir, source_path) if os.path.exists(relative_path): # In order to work on windows, the source path needs to be modified to be # relative because hardlink/junction dont resolve relative paths the same # way as os.symlink. This is ignored on other operating systems. win_source_path = relative_path else: raise SymlinkError( f"The source path ({source_path}) is not relative to the link path " f"({link_path}). Resulting link would be broken so not making link." ) # Create the symlink if not _windows_can_symlink(): _windows_create_link(win_source_path, link_path) else: os.symlink(source_path, link_path, target_is_directory=os.path.isdir(source_path)) def _windows_islink(path: str) -> bool: """Override os.islink to give correct answer for spack logic. For Non-Windows: a link can be determined with the os.path.islink method. Windows-only methods will return false for other operating systems. For Windows: spack considers symlinks, hard links, and junctions to all be links, so if any of those are True, return True. Args: path (str): path to check if it is a link. Returns: bool - whether the path is any kind link or not. """ return any([os.path.islink(path), _windows_is_junction(path), _windows_is_hardlink(path)]) def _windows_is_hardlink(path: str) -> bool: """Determines if a path is a windows hard link. This is accomplished by looking at the number of links using os.stat. A non-hard-linked file will have a st_nlink value of 1, whereas a hard link will have a value larger than 1. Note that both the original and hard-linked file will return True because they share the same inode. Args: path (str): Windows path to check for a hard link Returns: bool - Whether the path is a hard link or not. """ if sys.platform != "win32" or os.path.islink(path) or not os.path.exists(path): return False return os.stat(path).st_nlink > 1 def _windows_is_junction(path: str) -> bool: """Determines if a path is a windows junction. A junction can be determined using a bitwise AND operation between the file's attribute bitmask and the known junction bitmask (0x400). Args: path (str): A non-file path Returns: bool - whether the path is a junction or not. """ if sys.platform != "win32" or os.path.islink(path) or os.path.isfile(path): return False import ctypes.wintypes get_file_attributes = ctypes.windll.kernel32.GetFileAttributesW # type: ignore[attr-defined] get_file_attributes.argtypes = (ctypes.wintypes.LPWSTR,) get_file_attributes.restype = ctypes.wintypes.DWORD invalid_file_attributes = 0xFFFFFFFF reparse_point = 0x400 file_attr = get_file_attributes(str(path)) if file_attr == invalid_file_attributes: return False return file_attr & reparse_point > 0 @lang.memoized def _windows_can_symlink() -> bool: """ Determines if windows is able to make a symlink depending on the system configuration and the level of the user's permissions. """ if sys.platform != "win32": tty.warn("windows_can_symlink method can't be used on non-Windows OS.") return False tempdir = tempfile.mkdtemp() dpath = os.path.join(tempdir, "dpath") fpath = os.path.join(tempdir, "fpath.txt") dlink = os.path.join(tempdir, "dlink") flink = os.path.join(tempdir, "flink.txt") touchp(fpath) mkdirp(dpath) try: os.symlink(dpath, dlink) can_symlink_directories = os.path.islink(dlink) except OSError: can_symlink_directories = False try: os.symlink(fpath, flink) can_symlink_files = os.path.islink(flink) except OSError: can_symlink_files = False # Cleanup the test directory shutil.rmtree(tempdir) return can_symlink_directories and can_symlink_files def _windows_create_link(source: str, link: str): """ Attempts to create a Hard Link or Junction as an alternative to a symbolic link. This is called when symbolic links cannot be created. """ if sys.platform != "win32": raise SymlinkError("windows_create_link method can't be used on non-Windows OS.") elif os.path.isdir(source): _windows_create_junction(source=source, link=link) elif os.path.isfile(source): _windows_create_hard_link(path=source, link=link) else: raise SymlinkError( f"Cannot create link from {source}. It is neither a file nor a directory." ) def _windows_create_junction(source: str, link: str): """Duly verify that the path and link are eligible to create a junction, then create the junction. """ if sys.platform != "win32": raise SymlinkError("windows_create_junction method can't be used on non-Windows OS.") elif not os.path.exists(source): raise SymlinkError("Source path does not exist, cannot create a junction.") elif os.path.lexists(link): raise AlreadyExistsError("Link path already exists, cannot create a junction.") elif not os.path.isdir(source): raise SymlinkError("Source path is not a directory, cannot create a junction.") import subprocess cmd = ["cmd", "/C", "mklink", "/J", link, source] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() tty.debug(out.decode()) if proc.returncode != 0: err_str = err.decode() tty.error(err_str) raise SymlinkError("Make junction command returned a non-zero return code.", err_str) def _windows_create_hard_link(path: str, link: str): """Duly verify that the path and link are eligible to create a hard link, then create the hard link. """ if sys.platform != "win32": raise SymlinkError("windows_create_hard_link method can't be used on non-Windows OS.") elif not os.path.exists(path): raise SymlinkError(f"File path {path} does not exist. Cannot create hard link.") elif os.path.lexists(link): raise AlreadyExistsError(f"Link path ({link}) already exists. Cannot create hard link.") elif not os.path.isfile(path): raise SymlinkError(f"File path ({link}) is not a file. Cannot create hard link.") else: tty.debug(f"Creating hard link {link} pointing to {path}") CreateHardLink(link, path) def _windows_readlink(path: str, *, dir_fd=None): """Spack utility to override of os.readlink method to work cross platform""" if _windows_is_hardlink(path): return _windows_read_hard_link(path) elif _windows_is_junction(path): return _windows_read_junction(path) else: return sanitize_win_longpath(os.readlink(path, dir_fd=dir_fd)) def _windows_read_hard_link(link: str) -> str: """Find all of the files that point to the same inode as the link""" if sys.platform != "win32": raise SymlinkError("Can't read hard link on non-Windows OS.") link = os.path.abspath(link) fsutil_cmd = ["fsutil", "hardlink", "list", link] proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = proc.communicate() if proc.returncode != 0: raise SymlinkError(f"An error occurred while reading hard link: {err.decode()}") # fsutil response does not include the drive name, so append it back to each linked file. drive, link_tail = os.path.splitdrive(os.path.abspath(link)) links = set([os.path.join(drive, p) for p in out.decode().splitlines()]) links.remove(link) if len(links) == 1: return links.pop() elif len(links) > 1: # TODO: How best to handle the case where 3 or more paths point to a single inode? raise SymlinkError(f"Found multiple paths pointing to the same inode {links}") else: raise SymlinkError("Cannot determine hard link source path.") def _windows_read_junction(link: str): """Find the path that a junction points to.""" if sys.platform != "win32": raise SymlinkError("Can't read junction on non-Windows OS.") link = os.path.abspath(link) link_basename = os.path.basename(link) link_parent = os.path.dirname(link) fsutil_cmd = ["dir", "/a:l", link_parent] proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = proc.communicate() if proc.returncode != 0: raise SymlinkError(f"An error occurred while reading junction: {err.decode()}") matches = re.search(rf"\s+{link_basename} \[(.*)]", out.decode()) if matches: return matches.group(1) else: raise SymlinkError("Could not find junction path.") @system_path_filter def resolve_link_target_relative_to_the_link(link): """ os.path.isdir uses os.path.exists, which for links will check the existence of the link target. If the link target is relative to the link, we need to construct a pathname that is valid from our cwd (which may not be the same as the link's directory) """ target = readlink(link) if os.path.isabs(target): return target link_dir = os.path.dirname(os.path.abspath(link)) return os.path.join(link_dir, target) if sys.platform == "win32": symlink = _windows_symlink readlink = _windows_readlink islink = _windows_islink rename = _win_rename else: symlink = os.symlink readlink = os.readlink islink = os.path.islink rename = os.rename class SymlinkError(OSError): """Exception class for errors raised while creating symlinks, junctions and hard links """ class AlreadyExistsError(SymlinkError): """Link path already exists.""" ================================================ FILE: lib/spack/spack/llnl/util/lang.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections.abc import contextlib import fnmatch import functools import itertools import os import re import sys import traceback import types import typing import warnings from datetime import datetime, timedelta from typing import ( Any, Callable, Dict, Generic, Iterable, Iterator, List, Mapping, Optional, Tuple, TypeVar, Union, ) # Ignore emacs backups when listing modules ignore_modules = r"^\.#|~$" def index_by(objects, *funcs): """Create a hierarchy of dictionaries by splitting the supplied set of objects on unique values of the supplied functions. Values are used as keys. For example, suppose you have four objects with attributes that look like this:: a = Spec("boost %gcc target=skylake") b = Spec("mrnet %intel target=zen2") c = Spec("libelf %xlc target=skylake") d = Spec("libdwarf %intel target=zen2") list_of_specs = [a,b,c,d] index1 = index_by(list_of_specs, lambda s: str(s.target), lambda s: s.compiler) index2 = index_by(list_of_specs, lambda s: s.compiler) ``index1`` now has two levels of dicts, with lists at the leaves, like this:: { 'zen2' : { 'gcc' : [a], 'xlc' : [c] }, 'skylake' : { 'intel' : [b, d] } } And ``index2`` is a single level dictionary of lists that looks like this:: { 'gcc' : [a], 'intel' : [b,d], 'xlc' : [c] } If any elements in funcs is a string, it is treated as the name of an attribute, and acts like ``getattr(object, name)``. So shorthand for the above two indexes would be:: index1 = index_by(list_of_specs, 'arch', 'compiler') index2 = index_by(list_of_specs, 'compiler') You can also index by tuples by passing tuples:: index1 = index_by(list_of_specs, ('target', 'compiler')) Keys in the resulting dict will look like ``('gcc', 'skylake')``. """ if not funcs: return objects f = funcs[0] if isinstance(f, str): f = lambda x: getattr(x, funcs[0]) elif isinstance(f, tuple): f = lambda x: tuple(getattr(x, p, None) for p in funcs[0]) result = {} for o in objects: key = f(o) result.setdefault(key, []).append(o) for key, objects in result.items(): result[key] = index_by(objects, *funcs[1:]) return result def attr_setdefault(obj, name, value): """Like dict.setdefault, but for objects.""" if not hasattr(obj, name): setattr(obj, name, value) return getattr(obj, name) def memoized(func): """Decorator that caches the results of a function, storing them in an attribute of that function. Example:: @memoized def expensive_computation(x): # Some expensive computation return result """ return functools.lru_cache(maxsize=None)(func) def list_modules(directory, **kwargs): """Lists all of the modules, excluding ``__init__.py``, in a particular directory. Listed packages have no particular order.""" list_directories = kwargs.setdefault("directories", True) ignore = re.compile(ignore_modules) with os.scandir(directory) as it: for entry in it: if entry.name == "__init__.py" or entry.name == "__pycache__": continue if ( list_directories and entry.is_dir() and os.path.isfile(os.path.join(entry.path, "__init__.py")) ): yield entry.name elif entry.name.endswith(".py") and entry.is_file() and not ignore.search(entry.name): yield entry.name[:-3] # strip .py def decorator_with_or_without_args(decorator): """Allows a decorator to be used with or without arguments, e.g.:: # Calls the decorator function some args @decorator(with, arguments, and=kwargs) or:: # Calls the decorator function with zero arguments @decorator """ # See https://stackoverflow.com/questions/653368 for more on this @functools.wraps(decorator) def new_dec(*args, **kwargs): if len(args) == 1 and len(kwargs) == 0 and callable(args[0]): # actual decorated function return decorator(args[0]) else: # decorator arguments return lambda realf: decorator(realf, *args, **kwargs) return new_dec def key_ordering(cls): """Decorates a class with extra methods that implement rich comparison operations and ``__hash__``. The decorator assumes that the class implements a function called ``_cmp_key()``. The rich comparison operations will compare objects using this key, and the ``__hash__`` function will return the hash of this key. If a class already has ``__eq__``, ``__ne__``, ``__lt__``, ``__le__``, ``__gt__``, or ``__ge__`` defined, this decorator will overwrite them. Raises: TypeError: If the class does not have a ``_cmp_key`` method """ def setter(name, value): value.__name__ = name setattr(cls, name, value) if not hasattr(cls, "_cmp_key"): raise TypeError(f"'{cls.__name__}' doesn't define _cmp_key().") setter("__eq__", lambda s, o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key())) setter("__lt__", lambda s, o: o is not None and s._cmp_key() < o._cmp_key()) setter("__le__", lambda s, o: o is not None and s._cmp_key() <= o._cmp_key()) setter("__ne__", lambda s, o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key())) setter("__gt__", lambda s, o: o is None or s._cmp_key() > o._cmp_key()) setter("__ge__", lambda s, o: o is None or s._cmp_key() >= o._cmp_key()) setter("__hash__", lambda self: hash(self._cmp_key())) return cls #: sentinel for testing that iterators are done in lazy_lexicographic_ordering done = object() def tuplify(seq): """Helper for lazy_lexicographic_ordering().""" return tuple([(tuplify(x) if callable(x) else x) for x in seq()]) def lazy_eq(lseq, rseq): """Equality comparison for two lazily generated sequences. See ``lazy_lexicographic_ordering``. """ liter = lseq() # call generators riter = rseq() # zip_longest is implemented in native code, so use it for speed. # use zip_longest instead of zip because it allows us to tell # which iterator was longer. for left, right in itertools.zip_longest(liter, riter, fillvalue=done): if (left is done) or (right is done): return False # recursively enumerate any generators, otherwise compare equal = lazy_eq(left, right) if callable(left) else left == right if not equal: return False return True def lazy_lt(lseq, rseq): """Less-than comparison for two lazily generated sequences. See ``lazy_lexicographic_ordering``. """ liter = lseq() riter = rseq() for left, right in itertools.zip_longest(liter, riter, fillvalue=done): if (left is done) or (right is done): return left is done # left was shorter than right sequence = callable(left) equal = lazy_eq(left, right) if sequence else left == right if equal: continue if sequence: return lazy_lt(left, right) if left is None: return True if right is None: return False return left < right return False # if equal, return False @decorator_with_or_without_args def lazy_lexicographic_ordering(cls, set_hash=True): """Decorates a class with extra methods that implement rich comparison. This is a lazy version of the tuple comparison used frequently to implement comparison in Python. Given some objects with fields, you might use tuple keys to implement comparison, e.g.: .. code-block:: python class Widget: def _cmp_key(self): return (self.a, self.b, (self.c, self.d), self.e) def __eq__(self, other): return self._cmp_key() == other._cmp_key() def __lt__(self): return self._cmp_key() < other._cmp_key() # etc. Python would compare ``Widgets`` lexicographically based on their tuples. The issue there for simple comparators is that we have to build the tuples *and* we have to generate all the values in them up front. When implementing comparisons for large data structures, this can be costly. Lazy lexicographic comparison maps the tuple comparison shown above to generator functions. Instead of comparing based on pre-constructed tuple keys, users of this decorator can compare using elements from a generator. So, you'd write: .. code-block:: python @lazy_lexicographic_ordering class Widget: def _cmp_iter(self): yield a yield b def cd_fun(): yield c yield d yield cd_fun yield e # operators are added by decorator There are no tuples preconstructed, and the generator does not have to complete. Instead of tuples, we simply make functions that lazily yield what would've been in the tuple. The ``@lazy_lexicographic_ordering`` decorator handles the details of implementing comparison operators, and the ``Widget`` implementor only has to worry about writing ``_cmp_iter``, and making sure the elements in it are also comparable. In some cases, you may have a fast way to determine whether two objects are equal, e.g. the ``is`` function or an already-computed cryptographic hash. For this, you can implement your own ``_cmp_fast_eq`` function: .. code-block:: python @lazy_lexicographic_ordering class Widget: def _cmp_iter(self): yield a yield b def cd_fun(): yield c yield d yield cd_fun yield e def _cmp_fast_eq(self, other): return self is other or None ``_cmp_fast_eq`` should return: * ``True`` if ``self`` is equal to ``other``, * ``False`` if ``self`` is not equal to ``other``, and * ``None`` if it's not known whether they are equal, and the full comparison should be done. ``lazy_lexicographic_ordering`` uses ``_cmp_fast_eq`` to short-circuit the comparison if the answer can be determined quickly. If you do not implement it, it defaults to ``self is other or None``. Some things to note: * If a class already has ``__eq__``, ``__ne__``, ``__lt__``, ``__le__``, ``__gt__``, ``__ge__``, or ``__hash__`` defined, this decorator will overwrite them. * If ``set_hash`` is ``False``, this will not overwrite ``__hash__``. * This class uses Python 2 None-comparison semantics. If you yield None and it is compared to a non-None type, None will always be less than the other object. Raises: TypeError: If the class does not have a ``_cmp_iter`` method """ if not hasattr(cls, "_cmp_iter"): raise TypeError(f"'{cls.__name__}' doesn't define _cmp_iter().") # get an equal operation that allows us to short-circuit comparison # if it's not provided, default to `is` _cmp_fast_eq = getattr(cls, "_cmp_fast_eq", lambda x, y: x is y or None) # comparison operators are implemented in terms of lazy_eq and lazy_lt def eq(self, other): fast_eq = _cmp_fast_eq(self, other) if fast_eq is not None: return fast_eq return (other is not None) and lazy_eq(self._cmp_iter, other._cmp_iter) def lt(self, other): if _cmp_fast_eq(self, other) is True: return False return (other is not None) and lazy_lt(self._cmp_iter, other._cmp_iter) def gt(self, other): if _cmp_fast_eq(self, other) is True: return False return (other is None) or lazy_lt(other._cmp_iter, self._cmp_iter) def ne(self, other): return not (self == other) def le(self, other): return not (self > other) def ge(self, other): return not (self < other) def h(self): return hash(tuplify(self._cmp_iter)) def add_func_to_class(name, func): """Add a function to a class with a particular name.""" func.__name__ = name setattr(cls, name, func) add_func_to_class("__eq__", eq) add_func_to_class("__lt__", lt) add_func_to_class("__gt__", gt) add_func_to_class("__ne__", ne) add_func_to_class("__le__", le) add_func_to_class("__ge__", ge) if set_hash: add_func_to_class("__hash__", h) return cls K = TypeVar("K") V = TypeVar("V") @lazy_lexicographic_ordering class HashableMap(typing.MutableMapping[K, V]): """This is a hashable, comparable dictionary. Hash is performed on a tuple of the values in the dictionary.""" __slots__ = ("dict",) def __init__(self): self.dict: Dict[K, V] = {} def __getitem__(self, key: K) -> V: return self.dict[key] def __setitem__(self, key: K, value: V) -> None: self.dict[key] = value def __iter__(self) -> Iterator[K]: return iter(self.dict) def __len__(self) -> int: return len(self.dict) def __delitem__(self, key: K) -> None: del self.dict[key] def _cmp_iter(self): for _, v in sorted(self.dict.items()): yield v def match_predicate(*args): """Utility function for making string matching predicates. Each arg can be a: * regex * list or tuple of regexes * predicate that takes a string. This returns a predicate that is true if: * any arg regex matches * any regex in a list or tuple of regexes matches. * any predicate in args matches. """ def match(string): for arg in args: if isinstance(arg, str): if re.search(arg, string): return True elif isinstance(arg, list) or isinstance(arg, tuple): if any(re.search(i, string) for i in arg): return True elif callable(arg): if arg(string): return True else: raise ValueError( "args to match_predicate must be regex, list of regexes, or callable." ) return False return match def dedupe(sequence, key=None): """Yields a stable de-duplication of an hashable sequence by key Args: sequence: hashable sequence to be de-duplicated key: callable applied on values before uniqueness test; identity by default. Returns: stable de-duplication of the sequence Examples: Dedupe a list of integers:: [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3] [x for x in spack.llnl.util.lang.dedupe([1,-2,1,3,2], key=abs)] == [1, -2, 3] """ seen = set() for x in sequence: x_key = x if key is None else key(x) if x_key not in seen: yield x seen.add(x_key) def pretty_date(time: Union[datetime, int], now: Optional[datetime] = None) -> str: """Convert a datetime or timestamp to a pretty, relative date. Args: time: date to print prettily now: the date the pretty date is relative to (default is ``datetime.now()``) Returns: pretty string like "an hour ago", "Yesterday", "3 months ago", "just now", etc. Adapted from https://stackoverflow.com/questions/1551382. """ if now is None: now = datetime.now() if type(time) is int: diff = now - datetime.fromtimestamp(time) elif isinstance(time, datetime): diff = now - time else: raise ValueError("pretty_date requires a timestamp or datetime") second_diff = diff.seconds day_diff = diff.days if day_diff < 0: return "" if day_diff == 0: if second_diff < 10: return "just now" if second_diff < 60: return f"{second_diff} seconds ago" if second_diff < 120: return "a minute ago" if second_diff < 3600: return f"{second_diff // 60} minutes ago" if second_diff < 7200: return "an hour ago" if second_diff < 86400: return f"{second_diff // 3600} hours ago" if day_diff == 1: return "yesterday" if day_diff < 7: return f"{day_diff} days ago" if day_diff < 28: weeks = day_diff // 7 if weeks == 1: return "a week ago" else: return f"{day_diff // 7} weeks ago" if day_diff < 365: months = day_diff // 30 if months == 1: return "a month ago" elif months == 12: months -= 1 return f"{months} months ago" year_diff = day_diff // 365 if year_diff == 1: return "a year ago" return f"{year_diff} years ago" def pretty_string_to_date(date_str: str, now: Optional[datetime] = None) -> datetime: """Parses a string representing a date and returns a datetime object. Args: date_str: string representing a date. This string might be in different format (like ``YYYY``, ``YYYY-MM``, ``YYYY-MM-DD``, ``YYYY-MM-DD HH:MM``, ``YYYY-MM-DD HH:MM:SS``) or be a *pretty date* (like ``yesterday`` or ``two months ago``) Returns: datetime object corresponding to ``date_str`` """ pattern = {} now = now or datetime.now() # datetime formats pattern[re.compile(r"^\d{4}$")] = lambda x: datetime.strptime(x, "%Y") pattern[re.compile(r"^\d{4}-\d{2}$")] = lambda x: datetime.strptime(x, "%Y-%m") pattern[re.compile(r"^\d{4}-\d{2}-\d{2}$")] = lambda x: datetime.strptime(x, "%Y-%m-%d") pattern[re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}$")] = lambda x: datetime.strptime( x, "%Y-%m-%d %H:%M" ) pattern[re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$")] = lambda x: datetime.strptime( x, "%Y-%m-%d %H:%M:%S" ) pretty_regex = re.compile(r"(a|\d+)\s*(year|month|week|day|hour|minute|second)s?\s*ago") def _n_xxx_ago(x): how_many, time_period = pretty_regex.search(x).groups() how_many = 1 if how_many == "a" else int(how_many) # timedelta natively supports time periods up to 'weeks'. # To apply month or year we convert to 30 and 365 days if time_period == "month": how_many *= 30 time_period = "day" elif time_period == "year": how_many *= 365 time_period = "day" kwargs = {(time_period + "s"): how_many} return now - timedelta(**kwargs) pattern[pretty_regex] = _n_xxx_ago # yesterday callback = lambda x: now - timedelta(days=1) pattern[re.compile("^yesterday$")] = callback for regexp, parser in pattern.items(): if bool(regexp.match(date_str)): return parser(date_str) raise ValueError(f'date "{date_str}" does not match any valid format') def pretty_seconds_formatter(seconds): if seconds >= 1: multiplier, unit = 1, "s" elif seconds >= 1e-3: multiplier, unit = 1e3, "ms" elif seconds >= 1e-6: multiplier, unit = 1e6, "us" else: multiplier, unit = 1e9, "ns" return lambda s: "%.3f%s" % (multiplier * s, unit) def pretty_seconds(seconds): """Seconds to string with appropriate units Arguments: seconds (float): Number of seconds Returns: str: Time string with units """ return pretty_seconds_formatter(seconds)(seconds) def pretty_duration(seconds: float) -> str: """Format a duration in seconds as a compact human-readable string (e.g. "1h02m", "3m05s", "45s").""" s = int(seconds) if s < 60: return f"{s}s" m, s = divmod(s, 60) if m < 60: return f"{m}m{s:02d}s" h, m = divmod(m, 60) return f"{h}h{m:02d}m" class ObjectWrapper: """Base class that wraps an object. Derived classes can add new behavior while staying undercover. This class is modeled after the stackoverflow answer: * http://stackoverflow.com/a/1445289/771663 """ def __init__(self, wrapped_object): wrapped_cls = type(wrapped_object) wrapped_name = wrapped_cls.__name__ # If the wrapped object is already an ObjectWrapper, or a derived class # of it, adding type(self) in front of type(wrapped_object) # results in an inconsistent MRO. # # TODO: the implementation below doesn't account for the case where we # TODO: have different base classes of ObjectWrapper, say A and B, and # TODO: we want to wrap an instance of A with B. if type(self) not in wrapped_cls.__mro__: self.__class__ = type(wrapped_name, (type(self), wrapped_cls), {}) else: self.__class__ = type(wrapped_name, (wrapped_cls,), {}) self.__dict__ = wrapped_object.__dict__ class Singleton: """Wrapper for lazily initialized singleton objects.""" def __init__(self, factory: Callable[[], object]): """Create a new singleton to be inited with the factory function. Most factories will simply create the object to be initialized and return it. In some cases, e.g. when bootstrapping some global state, the singleton may need to be initialized incrementally. If the factory returns a generator instead of a regular object, the singleton will assign each result yielded by the generator to the singleton instance. This allows methods called by the factory in later stages to refer back to the singleton. Args: factory (function): function taking no arguments that creates the singleton instance. """ self.factory = factory self._instance = None @property def instance(self): if self._instance is None: try: instance = self.factory() if isinstance(instance, types.GeneratorType): # if it's a generator, assign every value for value in instance: self._instance = value else: # if not, just assign the result like a normal singleton self._instance = instance except AttributeError as e: # getattr will "absorb" an AttributeError that occurs # during the execution of the factory method: we'd like # to show that so wrap it in something that isn't absorbed raise SingletonInstantiationError( "AttrbuteError during creation of Singleton instance" ) from e return self._instance def __getattr__(self, name): # When unpickling Singleton objects, the 'instance' attribute may be # requested but not yet set. The final 'getattr' line here requires # 'instance'/'_instance' to be defined or it will enter an infinite # loop, so protect against that here. if name in ["_instance", "instance"]: raise AttributeError(f"cannot create {name}") return getattr(self.instance, name) def __getitem__(self, name): return self.instance[name] def __contains__(self, element): return element in self.instance def __call__(self, *args, **kwargs): return self.instance(*args, **kwargs) def __iter__(self): return iter(self.instance) def __str__(self): return str(self.instance) def __repr__(self): return repr(self.instance) class SingletonInstantiationError(Exception): """Error that indicates a singleton that cannot instantiate.""" def get_entry_points(*, group: str): """Wrapper for ``importlib.metadata.entry_points`` Args: group: entry points to select Returns: EntryPoints for ``group`` or empty list if unsupported """ try: import importlib.metadata # type: ignore # novermin except ImportError: return [] try: return importlib.metadata.entry_points(group=group) except TypeError: # Prior to Python 3.10, entry_points accepted no parameters and always # returned a dictionary of entry points, keyed by group. See # https://docs.python.org/3/library/importlib.metadata.html#entry-points return importlib.metadata.entry_points().get(group, []) def load_module_from_file(module_name, module_path): """Loads a python module from the path of the corresponding file. If the module is already in ``sys.modules`` it will be returned as is and not reloaded. Args: module_name (str): namespace where the python module will be loaded, e.g. ``foo.bar`` module_path (str): path of the python file containing the module Returns: A valid module object Raises: ImportError: when the module can't be loaded FileNotFoundError: when module_path doesn't exist """ import importlib.util if module_name in sys.modules: return sys.modules[module_name] # This recipe is adapted from https://stackoverflow.com/a/67692/771663 spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) # The module object needs to exist in sys.modules before the # loader executes the module code. # # See https://docs.python.org/3/reference/import.html#loading sys.modules[spec.name] = module try: spec.loader.exec_module(module) except BaseException: try: del sys.modules[spec.name] except KeyError: pass raise return module def uniq(sequence): """Remove strings of duplicate elements from a list. This works like the command-line ``uniq`` tool. It filters strings of duplicate elements in a list. Adjacent matching elements are merged into the first occurrence. For example:: uniq([1, 1, 1, 1, 2, 2, 2, 3, 3]) == [1, 2, 3] uniq([1, 1, 1, 1, 2, 2, 2, 1, 1]) == [1, 2, 1] """ if not sequence: return [] uniq_list = [sequence[0]] last = sequence[0] for element in sequence[1:]: if element != last: uniq_list.append(element) last = element return uniq_list def elide_list(line_list: List[str], max_num: int = 10) -> List[str]: """Takes a long list and limits it to a smaller number of elements, replacing intervening elements with ``"..."``. For example:: elide_list(["1", "2", "3", "4", "5", "6"], 4) gives:: ["1", "2", "3", "...", "6"] """ if len(line_list) > max_num: return [*line_list[: max_num - 1], "...", line_list[-1]] return line_list if sys.version_info >= (3, 9): PatternStr = re.Pattern[str] PatternBytes = re.Pattern[bytes] else: PatternStr = typing.Pattern[str] PatternBytes = typing.Pattern[bytes] def fnmatch_translate_multiple(named_patterns: Dict[str, str]) -> str: """Similar to ``fnmatch.translate``, but takes an ordered dictionary where keys are pattern names, and values are filename patterns. The output is a regex that matches any of the patterns in order, and named capture groups are used to identify which pattern matched.""" return "|".join(f"(?P<{n}>{fnmatch.translate(p)})" for n, p in named_patterns.items()) @contextlib.contextmanager def nullcontext(*args, **kwargs): """Empty context manager. TODO: replace with contextlib.nullcontext() if we ever require python 3.7. """ yield class UnhashableArguments(TypeError): """Raise when an @memoized function receives unhashable arg or kwarg values.""" T = TypeVar("T") def stable_partition( input_iterable: Iterable[T], predicate_fn: Callable[[T], bool] ) -> Tuple[List[T], List[T]]: """Partition the input iterable according to a custom predicate. Args: input_iterable: input iterable to be partitioned. predicate_fn: predicate function accepting an iterable item as argument. Return: Tuple of the list of elements evaluating to True, and list of elements evaluating to False. """ true_items: List[T] = [] false_items: List[T] = [] for item in input_iterable: if predicate_fn(item): true_items.append(item) else: false_items.append(item) return true_items, false_items def ensure_last(lst, *elements): """Performs a stable partition of lst, ensuring that ``elements`` occur at the end of ``lst`` in specified order. Mutates ``lst``. Raises ``ValueError`` if any ``elements`` are not already in ``lst``.""" for elt in elements: lst.append(lst.pop(lst.index(elt))) class Const: """Class level constant, raises when trying to set the attribute""" __slots__ = ["value"] def __init__(self, value): self.value = value def __get__(self, instance, owner): return self.value def __set__(self, instance, value): raise TypeError(f"Const value does not support assignment [value={self.value}]") class TypedMutableSequence(collections.abc.MutableSequence): """Base class that behaves like a list, just with a different type. Client code can inherit from this base class:: class Foo(TypedMutableSequence): pass and later perform checks based on types:: if isinstance(l, Foo): # do something """ def __init__(self, iterable): self.data = list(iterable) def __getitem__(self, item): return self.data[item] def __setitem__(self, key, value): self.data[key] = value def __delitem__(self, key): del self.data[key] def __len__(self): return len(self.data) def insert(self, index, item): self.data.insert(index, item) def __repr__(self): return repr(self.data) def __str__(self): return str(self.data) class GroupedExceptionHandler: """A generic mechanism to coalesce multiple exceptions and preserve tracebacks.""" def __init__(self): self.exceptions: List[Tuple[str, Exception, List[str]]] = [] def __bool__(self): """Whether any exceptions were handled.""" return bool(self.exceptions) def forward(self, context: str, base: type = BaseException) -> "GroupedExceptionForwarder": """Return a contextmanager which extracts tracebacks and prefixes a message.""" return GroupedExceptionForwarder(context, self, base) def _receive_forwarded(self, context: str, exc: Exception, tb: List[str]): self.exceptions.append((context, exc, tb)) def grouped_message(self, with_tracebacks: bool = True) -> str: """Print out an error message coalescing all the forwarded errors.""" each_exception_message = [ "\n\t{0} raised {1}: {2}\n{3}".format( context, exc.__class__.__name__, exc, f"\n{''.join(tb)}" if with_tracebacks else "" ) for context, exc, tb in self.exceptions ] return "due to the following failures:\n{0}".format("\n".join(each_exception_message)) class GroupedExceptionForwarder: """A contextmanager to capture exceptions and forward them to a GroupedExceptionHandler.""" def __init__(self, context: str, handler: GroupedExceptionHandler, base: type): self._context = context self._handler = handler self._base = base def __enter__(self): return None def __exit__(self, exc_type, exc_value, tb): if exc_value is not None: if not issubclass(exc_type, self._base): return False self._handler._receive_forwarded(self._context, exc_value, traceback.format_tb(tb)) # Suppress any exception from being re-raised: # https://docs.python.org/3/reference/datamodel.html#object.__exit__. return True ClassPropertyType = TypeVar("ClassPropertyType") class classproperty(Generic[ClassPropertyType]): """Non-data descriptor to evaluate a class-level property. The function that performs the evaluation is injected at creation time and takes an owner (i.e., the class that originated the instance). """ def __init__(self, callback: Callable[[Any], ClassPropertyType]) -> None: self.callback = callback self.__doc__ = callback.__doc__ def __get__(self, instance, owner) -> ClassPropertyType: return self.callback(owner) #: A type alias that represents either a classproperty descriptor or a constant value of the same #: type. This allows derived classes to override a computed class-level property with a constant #: value while retaining type compatibility. ClassProperty = Union[ClassPropertyType, classproperty[ClassPropertyType]] class DeprecatedProperty: """Data descriptor to error or warn when a deprecated property is accessed. Derived classes must define a factory method to return an adaptor for the deprecated property, if the descriptor is not set to error. """ __slots__ = ["name"] #: 0 - Nothing #: 1 - Warning #: 2 - Error error_lvl = 0 def __init__(self, name: str) -> None: self.name = name def __get__(self, instance, owner): if instance is None: return self if self.error_lvl == 1: warnings.warn( f"accessing the '{self.name}' property of '{instance}', which is deprecated" ) elif self.error_lvl == 2: raise AttributeError(f"cannot access the '{self.name}' attribute of '{instance}'") return self.factory(instance, owner) def __set__(self, instance, value): raise TypeError( f"the deprecated property '{self.name}' of '{instance}' does not support assignment" ) def factory(self, instance, owner): raise NotImplementedError("must be implemented by derived classes") KT = TypeVar("KT") VT = TypeVar("VT") class PriorityOrderedMapping(Mapping[KT, VT]): """Mapping that iterates over key according to an integer priority. If the priority is the same for two keys, insertion order is what matters. The priority is set when the key/value pair is added. If not set, the highest current priority is used. """ _data: Dict[KT, VT] _priorities: List[Tuple[int, KT]] def __init__(self) -> None: self._data = {} # Tuple of (priority, key) self._priorities = [] def __getitem__(self, key: KT) -> VT: return self._data[key] def __len__(self) -> int: return len(self._data) def __iter__(self): yield from (key for _, key in self._priorities) def __reversed__(self): yield from (key for _, key in reversed(self._priorities)) def reversed_keys(self): """Iterates over keys from the highest priority, to the lowest.""" return reversed(self) def reversed_values(self): """Iterates over values from the highest priority, to the lowest.""" yield from (self._data[key] for _, key in reversed(self._priorities)) def priority_values(self, priority: int): """Iterate over values of a given priority.""" if not any(p == priority for p, _ in self._priorities): raise KeyError(f"No such priority in PriorityOrderedMapping: {priority}") yield from (self._data[k] for p, k in self._priorities if p == priority) def _highest_priority(self) -> int: if not self._priorities: return 0 result, _ = self._priorities[-1] return result def add(self, key: KT, *, value: VT, priority: Optional[int] = None) -> None: """Adds a key/value pair to the mapping, with a specific priority. If the priority is None, then it is assumed to be the highest priority value currently in the container. Raises: ValueError: when the same priority is already in the mapping """ if priority is None: priority = self._highest_priority() if key in self._data: self.remove(key) self._priorities.append((priority, key)) # We rely on sort being stable self._priorities.sort(key=lambda x: x[0]) self._data[key] = value assert len(self._data) == len(self._priorities) def remove(self, key: KT) -> VT: """Removes a key from the mapping. Returns: The value associated with the key being removed Raises: KeyError: if the key is not in the mapping """ if key not in self._data: raise KeyError(f"cannot find {key}") popped_item = self._data.pop(key) self._priorities = [(p, k) for p, k in self._priorities if k != key] assert len(self._data) == len(self._priorities) return popped_item ================================================ FILE: lib/spack/spack/llnl/util/link_tree.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """LinkTree class for setting up trees of symbolic links.""" import filecmp import os import shutil from typing import Callable, Dict, List, Optional, Tuple import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty __all__ = ["LinkTree"] empty_file_name = ".spack-empty" def remove_link(src, dest): if not fs.islink(dest): raise ValueError("%s is not a link tree!" % dest) # remove if dest is a hardlink/symlink to src; this will only # be false if two packages are merged into a prefix and have a # conflicting file if filecmp.cmp(src, dest, shallow=True): os.remove(dest) class MergeConflict: """ The invariant here is that src_a and src_b are both mapped to dst: project(src_a) == project(src_b) == dst """ def __init__(self, dst, src_a=None, src_b=None): self.dst = dst self.src_a = src_a self.src_b = src_b def __repr__(self) -> str: return f"MergeConflict(dst={self.dst!r}, src_a={self.src_a!r}, src_b={self.src_b!r})" def _samefile(a: str, b: str): try: return os.path.samefile(a, b) except OSError: return False class SourceMergeVisitor(fs.BaseDirectoryVisitor): """ Visitor that produces actions: - An ordered list of directories to create in dst - A list of files to link in dst - A list of merge conflicts in dst/ """ def __init__( self, ignore: Optional[Callable[[str], bool]] = None, normalize_paths: bool = False ): self.ignore = ignore if ignore is not None else lambda f: False # On case-insensitive filesystems, normalize paths to detect duplications self.normalize_paths = normalize_paths # When mapping to /, we need to prepend the # bit to the relative path in the destination dir. self.projection: str = "" # Two files f and g conflict if they are not os.path.samefile(f, g) and they are both # projected to the same destination file. These conflicts are not necessarily fatal, and # can be resolved or ignored. For example /LICENSE or # //__init__.py conflicts can be ignored). self.file_conflicts: List[MergeConflict] = [] # When we have to create a dir where a file is, or a file where a dir is, we have fatal # errors, listed here. self.fatal_conflicts: List[MergeConflict] = [] # What directories we have to make; this is an ordered dict, so that we have a fast lookup # and can run mkdir in order. self.directories: Dict[str, Tuple[str, str]] = {} # If the visitor is configured to normalize paths, keep a map of # normalized path to: original path, root directory + relative path self._directories_normalized: Dict[str, Tuple[str, str, str]] = {} # Files to link. Maps dst_rel to (src_root, src_rel). This is an ordered dict, where files # are guaranteed to be grouped by src_root in the order they were visited. self.files: Dict[str, Tuple[str, str]] = {} # If the visitor is configured to normalize paths, keep a map of # normalized path to: original path, root directory + relative path self._files_normalized: Dict[str, Tuple[str, str, str]] = {} def _in_directories(self, proj_rel_path: str) -> bool: """ Check if a path is already in the directory list """ if self.normalize_paths: return proj_rel_path.lower() in self._directories_normalized else: return proj_rel_path in self.directories def _directory(self, proj_rel_path: str) -> Tuple[str, str, str]: """ Get the directory that is mapped to a path """ if self.normalize_paths: return self._directories_normalized[proj_rel_path.lower()] else: return (proj_rel_path, *self.directories[proj_rel_path]) def _del_directory(self, proj_rel_path: str): """ Remove a directory from the list of directories """ del self.directories[proj_rel_path] if self.normalize_paths: del self._directories_normalized[proj_rel_path.lower()] def _add_directory(self, proj_rel_path: str, root: str, rel_path: str): """ Add a directory to the list of directories. Also stores the normalized version for later lookups """ self.directories[proj_rel_path] = (root, rel_path) if self.normalize_paths: self._directories_normalized[proj_rel_path.lower()] = (proj_rel_path, root, rel_path) def _in_files(self, proj_rel_path: str) -> bool: """ Check if a path is already in the files list """ if self.normalize_paths: return proj_rel_path.lower() in self._files_normalized else: return proj_rel_path in self.files def _file(self, proj_rel_path: str) -> Tuple[str, str, str]: """ Get the file that is mapped to a path """ if self.normalize_paths: return self._files_normalized[proj_rel_path.lower()] else: return (proj_rel_path, *self.files[proj_rel_path]) def _del_file(self, proj_rel_path: str): """ Remove a file from the list of files """ del self.files[proj_rel_path] if self.normalize_paths: del self._files_normalized[proj_rel_path.lower()] def _add_file(self, proj_rel_path: str, root: str, rel_path: str): """ Add a file to the list of files Also stores the normalized version for later lookups """ self.files[proj_rel_path] = (root, rel_path) if self.normalize_paths: self._files_normalized[proj_rel_path.lower()] = (proj_rel_path, root, rel_path) def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool: """ Register a directory if dst / rel_path is not blocked by a file or ignored. """ proj_rel_path = os.path.join(self.projection, rel_path) if self.ignore(rel_path): # Don't recurse when dir is ignored. return False elif self._in_files(proj_rel_path): # A file-dir conflict is fatal except if they're the same file (symlinked dir). src_a = os.path.join(*self._file(proj_rel_path)) src_b = os.path.join(root, rel_path) if not _samefile(src_a, src_b): self.fatal_conflicts.append( MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b) ) return False # Remove the link in favor of the dir. existing_proj_rel_path, _, _ = self._file(proj_rel_path) self._del_file(existing_proj_rel_path) self._add_directory(proj_rel_path, root, rel_path) return True elif self._in_directories(proj_rel_path): # No new directory, carry on. return True else: # Register new directory. self._add_directory(proj_rel_path, root, rel_path) return True def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool: """ Replace symlinked dirs with actual directories when possible in low depths, otherwise handle it as a file (i.e. we link to the symlink). Transforming symlinks into dirs makes it more likely we can merge directories, e.g. when /lib -> /subdir/lib. We only do this when the symlink is pointing into a subdirectory from the symlink's directory, to avoid potential infinite recursion; and only at a constant level of nesting, to avoid potential exponential blowups in file duplication. """ if self.ignore(rel_path): return False # Only follow symlinked dirs in /**/**/* if depth > 1: handle_as_dir = False else: # Only follow symlinked dirs when pointing deeper src = os.path.join(root, rel_path) real_parent = os.path.realpath(os.path.dirname(src)) real_child = os.path.realpath(src) handle_as_dir = real_child.startswith(real_parent) if handle_as_dir: return self.before_visit_dir(root, rel_path, depth) self.visit_file(root, rel_path, depth, symlink=True) return False def visit_file(self, root: str, rel_path: str, depth: int, *, symlink: bool = False) -> None: proj_rel_path = os.path.join(self.projection, rel_path) if self.ignore(rel_path): pass elif self._in_directories(proj_rel_path): # Can't create a file where a dir is, unless they are the same file (symlinked dir), # in which case we simply drop the symlink in favor of the actual dir. src_a = os.path.join(*self._directory(proj_rel_path)) src_b = os.path.join(root, rel_path) if not symlink or not _samefile(src_a, src_b): self.fatal_conflicts.append( MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b) ) elif self._in_files(proj_rel_path): # When two files project to the same path, they conflict iff they are distinct. # If they are the same (i.e. one links to the other), register regular files rather # than symlinks. The reason is that in copy-type views, we need a copy of the actual # file, not the symlink. src_a = os.path.join(*self._file(proj_rel_path)) src_b = os.path.join(root, rel_path) if not _samefile(src_a, src_b): # Distinct files produce a conflict. self.file_conflicts.append( MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b) ) return if not symlink: # Remove the link in favor of the actual file. The del is necessary to maintain the # order of the files dict, which is grouped by root. existing_proj_rel_path, _, _ = self._file(proj_rel_path) self._del_file(existing_proj_rel_path) self._add_file(proj_rel_path, root, rel_path) else: # Otherwise register this file to be linked. self._add_file(proj_rel_path, root, rel_path) def visit_symlinked_file(self, root: str, rel_path: str, depth: int) -> None: # Treat symlinked files as ordinary files (without "dereferencing") self.visit_file(root, rel_path, depth, symlink=True) def set_projection(self, projection: str) -> None: self.projection = os.path.normpath(projection) # Todo, is this how to check in general for empty projection? if self.projection == ".": self.projection = "" return # If there is a projection, we'll also create the directories # it consists of, and check whether that's causing conflicts. path = "" for part in self.projection.split(os.sep): path = os.path.join(path, part) if not self._in_files(path): self._add_directory(path, "", path) else: # Can't create a dir where a file is. _, src_a_root, src_a_relpath = self._file(path) self.fatal_conflicts.append( MergeConflict( dst=path, src_a=os.path.join(src_a_root, src_a_relpath), src_b=os.path.join("", path), ) ) class DestinationMergeVisitor(fs.BaseDirectoryVisitor): """DestinationMergeVisitor takes a SourceMergeVisitor and: a. registers additional conflicts when merging to the destination prefix b. removes redundant mkdir operations when directories already exist in the destination prefix. This also makes sure that symlinked directories in the target prefix will never be merged with directories in the sources directories. """ def __init__(self, source_merge_visitor: SourceMergeVisitor): self.src = source_merge_visitor def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool: # If destination dir is a file in a src dir, add a conflict, # and don't traverse deeper if self.src._in_files(rel_path): _, src_a_root, src_a_relpath = self.src._file(rel_path) self.src.fatal_conflicts.append( MergeConflict( rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path) ) ) return False # If destination dir was also a src dir, remove the mkdir # action, and traverse deeper. if self.src._in_directories(rel_path): existing_proj_rel_path, _, _ = self.src._directory(rel_path) self.src._del_directory(existing_proj_rel_path) return True # If the destination dir does not appear in the src dir, # don't descend into it. return False def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool: """ Symlinked directories in the destination prefix should be seen as files; we should not accidentally merge source dir with a symlinked dest dir. """ self.visit_file(root, rel_path, depth) # Never descend into symlinked target dirs. return False def visit_file(self, root: str, rel_path: str, depth: int) -> None: # Can't merge a file if target already exists if self.src._in_directories(rel_path): _, src_a_root, src_a_relpath = self.src._directory(rel_path) self.src.fatal_conflicts.append( MergeConflict( rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path) ) ) elif self.src._in_files(rel_path): _, src_a_root, src_a_relpath = self.src._file(rel_path) self.src.fatal_conflicts.append( MergeConflict( rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path) ) ) def visit_symlinked_file(self, root: str, rel_path: str, depth: int) -> None: # Treat symlinked files as ordinary files (without "dereferencing") self.visit_file(root, rel_path, depth) class LinkTree: """Class to create trees of symbolic links from a source directory. LinkTree objects are constructed with a source root. Their methods allow you to create and delete trees of symbolic links back to the source tree in specific destination directories. Trees comprise symlinks only to files; directories are never symlinked to, to prevent the source directory from ever being modified. """ def __init__(self, source_root): if not os.path.exists(source_root): raise OSError("No such file or directory: '%s'", source_root) self._root = source_root def find_conflict(self, dest_root, ignore=None, ignore_file_conflicts=False): """Returns the first file in dest that conflicts with src""" ignore = ignore or (lambda x: False) conflicts = self.find_dir_conflicts(dest_root, ignore) if not ignore_file_conflicts: conflicts.extend( dst for src, dst in self.get_file_map(dest_root, ignore).items() if os.path.exists(dst) ) if conflicts: return conflicts[0] def find_dir_conflicts(self, dest_root, ignore): conflicts = [] kwargs = {"follow_nonexisting": False, "ignore": ignore} for src, dest in fs.traverse_tree(self._root, dest_root, **kwargs): if os.path.isdir(src): if os.path.exists(dest) and not os.path.isdir(dest): conflicts.append("File blocks directory: %s" % dest) elif os.path.exists(dest) and os.path.isdir(dest): conflicts.append("Directory blocks directory: %s" % dest) return conflicts def get_file_map(self, dest_root, ignore): merge_map = {} kwargs = {"follow_nonexisting": True, "ignore": ignore} for src, dest in fs.traverse_tree(self._root, dest_root, **kwargs): if not os.path.isdir(src): merge_map[src] = dest return merge_map def merge_directories(self, dest_root, ignore): for src, dest in fs.traverse_tree(self._root, dest_root, ignore=ignore): if os.path.isdir(src): if not os.path.exists(dest): fs.mkdirp(dest) continue if not os.path.isdir(dest): raise ValueError("File blocks directory: %s" % dest) # mark empty directories so they aren't removed on unmerge. if not os.listdir(dest): marker = os.path.join(dest, empty_file_name) fs.touch(marker) def unmerge_directories(self, dest_root, ignore): for src, dest in fs.traverse_tree(self._root, dest_root, ignore=ignore, order="post"): if os.path.isdir(src): if not os.path.exists(dest): continue elif not os.path.isdir(dest): raise ValueError("File blocks directory: %s" % dest) # remove directory if it is empty. if not os.listdir(dest): shutil.rmtree(dest, ignore_errors=True) # remove empty dir marker if present. marker = os.path.join(dest, empty_file_name) if os.path.exists(marker): os.remove(marker) def merge( self, dest_root, ignore_conflicts: bool = False, ignore: Optional[Callable[[str], bool]] = None, link: Callable = fs.symlink, relative: bool = False, ): """Link all files in src into dest, creating directories if necessary. Arguments: ignore_conflicts: if True, do not break when the target exists; return a list of files that could not be linked ignore: callable that returns True if a file is to be ignored in the merge (by default ignore nothing) link: function to create links with (defaults to ``spack.llnl.util.filesystem.symlink``) relative: create all symlinks relative to the target (default False) """ if ignore is None: ignore = lambda x: False conflict = self.find_conflict( dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts ) if conflict: raise SingleMergeConflictError(conflict) self.merge_directories(dest_root, ignore) existing = [] for src, dst in self.get_file_map(dest_root, ignore).items(): if os.path.exists(dst): existing.append(dst) elif relative: abs_src = os.path.abspath(src) dst_dir = os.path.dirname(os.path.abspath(dst)) rel = os.path.relpath(abs_src, dst_dir) link(rel, dst) else: link(src, dst) for c in existing: tty.warn("Could not merge: %s" % c) def unmerge(self, dest_root, ignore=None, remove_file=remove_link): """Unlink all files in dest that exist in src. Unlinks directories in dest if they are empty. """ if ignore is None: ignore = lambda x: False for src, dst in self.get_file_map(dest_root, ignore).items(): remove_file(src, dst) self.unmerge_directories(dest_root, ignore) class MergeConflictError(Exception): pass class ConflictingSpecsError(MergeConflictError): def __init__(self, spec_1, spec_2): super().__init__(spec_1, spec_2) class SingleMergeConflictError(MergeConflictError): def __init__(self, path): super().__init__("Package merge blocked by file: %s" % path) class MergeConflictSummary(MergeConflictError): def __init__(self, conflicts): """ A human-readable summary of file system view merge conflicts (showing only the first 3 issues.) """ msg = "{0} fatal error(s) when merging prefixes:".format(len(conflicts)) # show the first 3 merge conflicts. for conflict in conflicts[:3]: msg += "\n `{0}` and `{1}` both project to `{2}`".format( conflict.src_a, conflict.src_b, conflict.dst ) super().__init__(msg) ================================================ FILE: lib/spack/spack/llnl/util/lock.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import errno import os import socket import sys import time from datetime import datetime from types import TracebackType from typing import IO, Callable, Dict, Generator, Optional, Tuple, Type from spack.llnl.util import lang, tty from ..string import plural if sys.platform != "win32": import fcntl __all__ = [ "Lock", "LockDowngradeError", "LockUpgradeError", "LockTransaction", "WriteTransaction", "ReadTransaction", "LockError", "LockTimeoutError", "LockPermissionError", "LockROFileError", "CantCreateLockError", ] ExitFnType = Callable[ [Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]], Optional[bool], ] ReleaseFnType = Optional[Callable[[], Optional[bool]]] DevIno = Tuple[int, int] # (st_dev, st_ino) from os.stat_result def true_fn() -> bool: """A function that always returns True.""" return True class OpenFile: """Record for keeping track of open lockfiles (with reference counting).""" __slots__ = ("fh", "key", "refs") def __init__(self, fh: IO[bytes], key: DevIno): self.fh = fh self.key = key # (dev, ino) self.refs = 0 class OpenFileTracker: """Track open lockfiles by inode, to minimize the number of open file descriptors. ``fcntl`` locks are associated with an inode. If a process closes *any* file descriptor for an inode, all fcntl locks the process holds on that inode are released, even if other descriptors for the same inode are still open. To avoid accidentally dropping locks we keep at most one open file descriptor per inode and reference-count it. The descriptor is only closed when the reference count reaches zero (i.e. no ``Lock`` in this process still needs it). Descriptors are *not* released on unlock; they are kept alive across lock/unlock cycles so that the next lock operation can skip re-opening the file. ``Lock._ensure_valid_handle`` re-validates the on-disk inode before each lock operation and drops a stale descriptor when the file was deleted and replaced. """ def __init__(self): self._descriptors: Dict[DevIno, OpenFile] = {} def get_ref_for_inode(self, key: DevIno) -> Optional[OpenFile]: """Fast lookup: do we already have this inode open?""" return self._descriptors.get(key) def create_and_track(self, path: str) -> OpenFile: """Slow path: Open file, handle directory creation, track it.""" # Open the file and create it if it doesn't exist (incl. directories). try: try: fd = os.open(path, os.O_RDWR | os.O_CREAT) mode = "rb+" except PermissionError: fd = os.open(path, os.O_RDONLY) mode = "rb" except OSError as e: if e.errno != errno.ENOENT: raise # Directory missing, create and retry try: os.makedirs(os.path.dirname(path), exist_ok=True) fd = os.open(path, os.O_RDWR | os.O_CREAT) except OSError: raise CantCreateLockError(path) mode = "rb+" # Get file identifier (device, inode) for tracking. stat = os.fstat(fd) key = (stat.st_dev, stat.st_ino) # Did we open a file we already track, e.g. a symlink to existing tracker file. if key in self._descriptors: os.close(fd) existing = self._descriptors[key] existing.refs += 1 return existing # Track the new file. fh = os.fdopen(fd, mode) obj = OpenFile(fh, key) obj.refs += 1 self._descriptors[key] = obj return obj def release(self, open_file: OpenFile): """Decrement the reference count and close the file handle when it reaches zero.""" open_file.refs -= 1 if open_file.refs <= 0: if self._descriptors.get(open_file.key) is open_file: del self._descriptors[open_file.key] open_file.fh.close() def purge(self): """Close all tracked file descriptors and clear the cache.""" for open_file in self._descriptors.values(): open_file.fh.close() self._descriptors.clear() #: Open file descriptors for locks in this process. Used to prevent one process #: from opening the sam file many times for different byte range locks FILE_TRACKER = OpenFileTracker() def _attempts_str(wait_time, nattempts): # Don't print anything if we succeeded on the first try if nattempts <= 1: return "" attempts = plural(nattempts, "attempt") return " after {} and {}".format(lang.pretty_seconds(wait_time), attempts) class LockType: READ = 0 WRITE = 1 @staticmethod def to_str(tid): ret = "READ" if tid == LockType.WRITE: ret = "WRITE" return ret @staticmethod def to_module(tid): lock = fcntl.LOCK_SH if tid == LockType.WRITE: lock = fcntl.LOCK_EX return lock @staticmethod def is_valid(op: int) -> bool: return op == LockType.READ or op == LockType.WRITE class Lock: """This is an implementation of a filesystem lock using Python's lockf. In Python, ``lockf`` actually calls ``fcntl``, so this should work with any filesystem implementation that supports locking through the fcntl calls. This includes distributed filesystems like Lustre (when flock is enabled) and recent NFS versions. Note that this is for managing contention over resources *between* processes and not for managing contention between threads in a process: the functions of this object are not thread-safe. A process also must not maintain multiple locks on the same file (or, more specifically, on overlapping byte ranges in the same file). """ def __init__( self, path: str, *, start: int = 0, length: int = 0, default_timeout: Optional[float] = None, debug: bool = False, desc: str = "", ) -> None: """Construct a new lock on the file at ``path``. By default, the lock applies to the whole file. Optionally, caller can specify a byte range beginning ``start`` bytes from the start of the file and extending ``length`` bytes from there. This exposes a subset of fcntl locking functionality. It does not currently expose the ``whence`` parameter -- ``whence`` is always ``os.SEEK_SET`` and ``start`` is always evaluated from the beginning of the file. Args: path: path to the lock start: optional byte offset at which the lock starts length: optional number of bytes to lock default_timeout: seconds to wait for lock attempts, where None means to wait indefinitely debug: debug mode specific to locking desc: optional debug message lock description, which is helpful for distinguishing between different Spack locks. """ self.path = path self._reads = 0 self._writes = 0 self._file_ref: Optional[OpenFile] = None self._cached_key: Optional[DevIno] = None # byte range parameters self._start = start self._length = length # enable debug mode self.debug = debug # optional debug description self.desc = f" ({desc})" if desc else "" # If the user doesn't set a default timeout, or if they choose # None, 0, etc. then lock attempts will not time out (unless the # user sets a timeout for each attempt) self.default_timeout = default_timeout or None # PID and host of lock holder (only used in debug mode) self.pid: Optional[int] = None self.old_pid: Optional[int] = None self.host: Optional[str] = None self.old_host: Optional[str] = None def _ensure_valid_handle(self) -> IO[bytes]: """Return a valid file handle for the lock file, opening or re-opening as needed. On the happy path this costs a single ``os.stat`` syscall: if the inode on disk matches ``_cached_key``, the already-open file handle is returned immediately. If the inode changed (the lock file was deleted and replaced by another process), the stale reference is released and a fresh one is obtained. If the file does not exist yet it is created (along with any missing parent directories). """ try: # Check what is currently on disk. This is the only syscall in the happy path. stat_res = os.stat(self.path) current_key = (stat_res.st_dev, stat_res.st_ino) # Double-check that our cache corresponds the file on disk. if self._file_ref and not self._file_ref.fh.closed: if self._cached_key == current_key: return self._file_ref.fh # Stale path: file was deleted and replaced on disk. FILE_TRACKER.release(self._file_ref) self._file_ref = None # Get reference to the verified inode from the tracker if it exist, or a new one. existing_ref = FILE_TRACKER.get_ref_for_inode(current_key) if existing_ref: self._file_ref = existing_ref self._file_ref.refs += 1 else: # We don't have it tracked, so we need to open and track it ourselves. self._file_ref = FILE_TRACKER.create_and_track(self.path) except OSError as e: # Re-raise all errors except for "file not found". if e.errno != errno.ENOENT: raise # File was not found, so remove it from our cache. if self._file_ref: FILE_TRACKER.release(self._file_ref) self._file_ref = None self._file_ref = FILE_TRACKER.create_and_track(self.path) # Update our local cache of what we hold self._cached_key = self._file_ref.key return self._file_ref.fh @staticmethod def _poll_interval_generator( _wait_times: Optional[Tuple[float, float, float]] = None, ) -> Generator[float, None, None]: """This implements a backoff scheme for polling a contended resource by suggesting a succession of wait times between polls. It suggests a poll interval of .1s until 2 seconds have passed, then a poll interval of .2s until 10 seconds have passed, and finally (for all requests after 10s) suggests a poll interval of .5s. This doesn't actually track elapsed time, it estimates the waiting time as though the caller always waits for the full length of time suggested by this function. """ num_requests = 0 stage1, stage2, stage3 = _wait_times or (1e-1, 2e-1, 5e-1) wait_time = stage1 while True: if num_requests >= 60: # 40 * .2 = 8 wait_time = stage3 elif num_requests >= 20: # 20 * .1 = 2 wait_time = stage2 num_requests += 1 yield wait_time def __repr__(self) -> str: """Formal representation of the lock.""" rep = f"{self.__class__.__name__}(" for attr, value in self.__dict__.items(): rep += f"{attr}={value.__repr__()}, " return f"{rep.strip(', ')})" def __str__(self) -> str: """Readable string (with key fields) of the lock.""" location = f"{self.path}[{self._start}:{self._length}]" timeout = f"timeout={self.default_timeout}" activity = f"#reads={self._reads}, #writes={self._writes}" return f"({location}, {timeout}, {activity})" def __getstate__(self): """Don't include file handles or counts in pickled state.""" state = self.__dict__.copy() del state["_file_ref"] del state["_reads"] del state["_writes"] return state def __setstate__(self, state): self.__dict__.update(state) self._file_ref = None self._reads = 0 self._writes = 0 def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: """This takes a lock using POSIX locks (``fcntl.lockf``). The lock is implemented as a spin lock using a nonblocking call to ``lockf()``. If the lock times out, it raises a ``LockError``. If the lock is successfully acquired, the total wait time and the number of attempts is returned. """ assert LockType.is_valid(op) op_str = LockType.to_str(op) self._log_acquiring("{0} LOCK".format(op_str)) timeout = timeout or self.default_timeout fh = self._ensure_valid_handle() if LockType.to_module(op) == fcntl.LOCK_EX and fh.mode == "rb": # Attempt to upgrade to write lock w/a read-only file. # If the file were writable, we'd have opened it rb+ raise LockROFileError(self.path) self._log_debug( "{} locking [{}:{}]: timeout {}".format( op_str.lower(), self._start, self._length, lang.pretty_seconds(timeout or 0) ) ) start_time = time.monotonic() end_time = float("inf") if not timeout else start_time + timeout num_attempts = 1 poll_intervals = Lock._poll_interval_generator() while True: if self._poll_lock(op): return time.monotonic() - start_time, num_attempts if time.monotonic() >= end_time: break time.sleep(next(poll_intervals)) num_attempts += 1 raise LockTimeoutError(op, self.path, time.monotonic() - start_time, num_attempts) def _poll_lock(self, op: int) -> bool: """Attempt to acquire the lock in a non-blocking manner. Return whether the locking attempt succeeds """ assert self._file_ref is not None, "cannot poll a lock without the file being set" fh = self._file_ref.fh.fileno() module_op = LockType.to_module(op) try: # Try to get the lock (will raise if not available.) fcntl.lockf(fh, module_op | fcntl.LOCK_NB, self._length, self._start, os.SEEK_SET) # help for debugging distributed locking if self.debug: # All locks read the owner PID and host self._read_log_debug_data() self._log_debug( "{0} locked {1} [{2}:{3}] (owner={4})".format( LockType.to_str(op), self.path, self._start, self._length, self.pid ) ) # Exclusive locks write their PID/host if module_op == fcntl.LOCK_EX: self._write_log_debug_data() return True except OSError as e: # EAGAIN and EACCES == locked by another process (so try again) if e.errno not in (errno.EAGAIN, errno.EACCES): raise return False def _read_log_debug_data(self) -> None: """Read PID and host data out of the file if it is there.""" assert self._file_ref is not None, "cannot read debug log without the file being set" self.old_pid = self.pid self.old_host = self.host self._file_ref.fh.seek(0) line = self._file_ref.fh.read() if line: pid, host = line.decode("utf-8").strip().split(",") _, _, pid = pid.rpartition("=") _, _, self.host = host.rpartition("=") self.pid = int(pid) def _write_log_debug_data(self) -> None: """Write PID and host data to the file, recording old values.""" assert self._file_ref is not None, "cannot write debug log without the file being set" self.old_pid = self.pid self.old_host = self.host self.pid = os.getpid() self.host = socket.gethostname() # write pid, host to disk to sync over FS self._file_ref.fh.seek(0) self._file_ref.fh.write(f"pid={self.pid},host={self.host}".encode("utf-8")) self._file_ref.fh.truncate() self._file_ref.fh.flush() os.fsync(self._file_ref.fh.fileno()) def _unlock(self) -> None: """Releases a lock using POSIX locks (``fcntl.lockf``) Releases the lock regardless of mode. Note that read locks may be masquerading as write locks, but this removes either. """ assert self._file_ref is not None, "cannot unlock without the file being set" fcntl.lockf( self._file_ref.fh.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET ) self._reads = 0 self._writes = 0 def acquire_read(self, timeout: Optional[float] = None) -> bool: """Acquires a recursive, shared lock for reading. Read and write locks can be acquired and released in arbitrary order, but the POSIX lock is held until all local read and write locks are released. Returns True if it is the first acquire and actually acquires the POSIX lock, False if it is a nested transaction. """ timeout = timeout or self.default_timeout if self._reads == 0 and self._writes == 0: # can raise LockError. wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) self._reads += 1 # Log if acquired, which includes counts when verbose self._log_acquired("READ LOCK", wait_time, nattempts) return True else: # Increment the read count for nested lock tracking self._reaffirm_lock() self._reads += 1 return False def acquire_write(self, timeout: Optional[float] = None) -> bool: """Acquires a recursive, exclusive lock for writing. Read and write locks can be acquired and released in arbitrary order, but the POSIX lock is held until all local read and write locks are released. Returns True if it is the first acquire and actually acquires the POSIX lock, False if it is a nested transaction. """ timeout = timeout or self.default_timeout if self._writes == 0: # can raise LockError. wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._writes += 1 # Log if acquired, which includes counts when verbose self._log_acquired("WRITE LOCK", wait_time, nattempts) # return True only if we weren't nested in a read lock. # TODO: we may need to return two values: whether we got # the write lock, and whether this is acquiring a read OR # write lock for the first time. Now it returns the latter. return self._reads == 0 else: # Increment the write count for nested lock tracking self._reaffirm_lock() self._writes += 1 return False def _reaffirm_lock(self) -> None: """Fork-safety: always re-affirm the lock with one non-blocking attempt. In the same process, re-locking an already-held byte range succeeds instantly (POSIX). In a forked child that doesn't own the POSIX lock, the call fails immediately and we raise. Use WRITE if we hold an exclusive lock so we don't accidentally downgrade it.""" if self._writes > 0: op = LockType.WRITE elif self._reads > 0: op = LockType.READ else: return self._ensure_valid_handle() if not self._poll_lock(op): raise LockTimeoutError(op, self.path, time=0, attempts=1) def try_acquire_read(self) -> bool: """Non-blocking attempt to acquire a shared read lock. Returns True if the lock was acquired, False if it would block. """ if self._reads == 0 and self._writes == 0: self._ensure_valid_handle() if not self._poll_lock(LockType.READ): return False self._reads += 1 self._log_acquired("READ LOCK", 0, 1) return True else: self._reaffirm_lock() self._reads += 1 return True def try_acquire_write(self) -> bool: """Non-blocking attempt to acquire an exclusive write lock. Returns True if the lock was acquired, False if it would block. """ if self._writes == 0: fh = self._ensure_valid_handle() if LockType.to_module(LockType.WRITE) == fcntl.LOCK_EX and fh.mode == "rb": raise LockROFileError(self.path) if not self._poll_lock(LockType.WRITE): return False self._writes += 1 self._log_acquired("WRITE LOCK", 0, 1) return True else: self._reaffirm_lock() self._writes += 1 return True def is_write_locked(self) -> bool: """Returns ``True`` if the path is write locked, otherwise, ``False``""" try: self.acquire_read() # If we have a read lock then no other process has a write lock. self.release_read() except LockTimeoutError: # Another process is holding a write lock on the file return True return False def downgrade_write_to_read(self, timeout: Optional[float] = None) -> None: """Downgrade from an exclusive write lock to a shared read. Raises: LockDowngradeError: if this is an attempt at a nested transaction """ timeout = timeout or self.default_timeout if self._writes == 1 and self._reads == 0: self._log_downgrading() # can raise LockError. wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) self._reads = 1 self._writes = 0 self._log_downgraded(wait_time, nattempts) else: raise LockDowngradeError(self.path) def upgrade_read_to_write(self, timeout: Optional[float] = None) -> None: """Attempts to upgrade from a shared read lock to an exclusive write. Raises: LockUpgradeError: if this is an attempt at a nested transaction """ timeout = timeout or self.default_timeout if self._reads == 1 and self._writes == 0: self._log_upgrading() # can raise LockError. wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._reads = 0 self._writes = 1 self._log_upgraded(wait_time, nattempts) else: raise LockUpgradeError(self.path) def release_read(self, release_fn: ReleaseFnType = None) -> bool: """Releases a read lock. Arguments: release_fn: function to call *before* the last recursive lock (read or write) is released. If the last recursive lock will be released, then this will call release_fn and return its result (if provided), or return True (if release_fn was not provided). Otherwise, we are still nested inside some other lock, so do not call the release_fn and, return False. Does limited correctness checking: if a read lock is released when none are held, this will raise an assertion error. """ assert self._reads > 0 locktype = "READ LOCK" if self._reads == 1 and self._writes == 0: self._log_releasing(locktype) # we need to call release_fn before releasing the lock release_fn = release_fn or true_fn result = release_fn() self._unlock() # can raise LockError. self._reads = 0 self._log_released(locktype) return bool(result) else: self._reads -= 1 return False def release_write(self, release_fn: ReleaseFnType = None) -> bool: """Releases a write lock. Arguments: release_fn: function to call before the last recursive write is released. If the last recursive *write* lock will be released, then this will call release_fn and return its result (if provided), or return True (if release_fn was not provided). Otherwise, we are still nested inside some other write lock, so do not call the release_fn, and return False. Does limited correctness checking: if a read lock is released when none are held, this will raise an assertion error. """ assert self._writes > 0 release_fn = release_fn or true_fn locktype = "WRITE LOCK" if self._writes == 1: self._log_releasing(locktype) # we need to call release_fn before releasing the lock result = release_fn() if self._reads > 0: self._lock(LockType.READ) else: self._unlock() # can raise LockError. self._writes = 0 self._log_released(locktype) return bool(result) else: self._writes -= 1 return False def cleanup(self) -> None: if self._reads == 0 and self._writes == 0: os.unlink(self.path) else: raise LockError("Attempting to cleanup active lock.") def _get_counts_desc(self) -> str: return ( "(reads {0}, writes {1})".format(self._reads, self._writes) if tty.is_verbose() else "" ) def _log_acquired(self, locktype, wait_time, nattempts) -> None: attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() desc = "Acquired at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg(locktype, "{0}{1}".format(desc, attempts_part))) def _log_acquiring(self, locktype) -> None: self._log_debug(self._status_msg(locktype, "Acquiring"), level=3) def _log_debug(self, *args, **kwargs) -> None: """Output lock debug messages.""" kwargs["level"] = kwargs.get("level", 2) tty.debug(*args, **kwargs) def _log_downgraded(self, wait_time, nattempts) -> None: attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() desc = "Downgraded at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg("READ LOCK", "{0}{1}".format(desc, attempts_part))) def _log_downgrading(self) -> None: self._log_debug(self._status_msg("WRITE LOCK", "Downgrading"), level=3) def _log_released(self, locktype) -> None: now = datetime.now() desc = "Released at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg(locktype, desc)) def _log_releasing(self, locktype) -> None: self._log_debug(self._status_msg(locktype, "Releasing"), level=3) def _log_upgraded(self, wait_time, nattempts) -> None: attempts_part = _attempts_str(wait_time, nattempts) now = datetime.now() desc = "Upgraded at %s" % now.strftime("%H:%M:%S.%f") self._log_debug(self._status_msg("WRITE LOCK", "{0}{1}".format(desc, attempts_part))) def _log_upgrading(self) -> None: self._log_debug(self._status_msg("READ LOCK", "Upgrading"), level=3) def _status_msg(self, locktype: str, status: str) -> str: status_desc = "[{0}] {1}".format(status, self._get_counts_desc()) return "{0}{1.desc}: {1.path}[{1._start}:{1._length}] {2}".format( locktype, self, status_desc ) class LockTransaction: """Simple nested transaction context manager that uses a file lock. Arguments: lock: underlying lock for this transaction to be acquired on enter and released on exit acquire: function to be called after lock is acquired release: function to be called before release, with ``(exc_type, exc_value, traceback)`` timeout: number of seconds to set for the timeout when acquiring the lock (default no timeout) """ def __init__( self, lock: Lock, acquire: Optional[Callable[[], None]] = None, release: Optional[ExitFnType] = None, timeout: Optional[float] = None, ) -> None: self._lock = lock self._timeout = timeout self._acquire_fn = acquire self._release_fn = release def __enter__(self): if self._enter() and self._acquire_fn: return self._acquire_fn() def __exit__( self, exc_type: Optional[Type[BaseException]], exc_value: Optional[BaseException], traceback: Optional[TracebackType], ) -> bool: def release_fn(): if self._release_fn is not None: return self._release_fn(exc_type, exc_value, traceback) return bool(self._exit(release_fn)) def _enter(self) -> bool: raise NotImplementedError def _exit(self, release_fn: ReleaseFnType) -> bool: raise NotImplementedError class ReadTransaction(LockTransaction): """LockTransaction context manager that does a read and releases it.""" def _enter(self): return self._lock.acquire_read(self._timeout) def _exit(self, release_fn): return self._lock.release_read(release_fn) class WriteTransaction(LockTransaction): """LockTransaction context manager that does a write and releases it.""" def _enter(self): return self._lock.acquire_write(self._timeout) def _exit(self, release_fn): return self._lock.release_write(release_fn) class LockError(Exception): """Raised for any errors related to locks.""" class LockDowngradeError(LockError): """Raised when unable to downgrade from a write to a read lock.""" def __init__(self, path: str) -> None: msg = "Cannot downgrade lock from write to read on file: %s" % path super().__init__(msg) class LockTimeoutError(LockError): """Raised when an attempt to acquire a lock times out.""" def __init__(self, lock_type: int, path: str, time: float, attempts: int) -> None: lock_type_str = LockType.to_str(lock_type).lower() fmt = "Timed out waiting for a {} lock after {}.\n Made {} {} on file: {}" super().__init__( fmt.format( lock_type_str, lang.pretty_seconds(time), attempts, "attempt" if attempts == 1 else "attempts", path, ) ) class LockUpgradeError(LockError): """Raised when unable to upgrade from a read to a write lock.""" def __init__(self, path: str) -> None: msg = "Cannot upgrade lock from read to write on file: %s" % path super().__init__(msg) class LockPermissionError(LockError): """Raised when there are permission issues with a lock.""" class LockROFileError(LockPermissionError): """Tried to take an exclusive lock on a read-only file.""" def __init__(self, path: str) -> None: msg = "Can't take write lock on read-only file: %s" % path super().__init__(msg) class CantCreateLockError(LockPermissionError): """Attempt to create a lock in an unwritable location.""" def __init__(self, path: str) -> None: msg = "cannot create lock '%s': " % path msg += "file does not exist and location is not writable" super().__init__(msg) ================================================ FILE: lib/spack/spack/llnl/util/symlink.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import warnings import spack.error import spack.llnl.util.filesystem warnings.warn( "The `spack.llnl.util.symlink` module will be removed in Spack v1.1", category=spack.error.SpackAPIWarning, stacklevel=2, ) readlink = spack.llnl.util.filesystem.readlink islink = spack.llnl.util.filesystem.islink symlink = spack.llnl.util.filesystem.symlink ================================================ FILE: lib/spack/spack/llnl/util/tty/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import contextlib import io import os import shutil import sys import textwrap import traceback from datetime import datetime from types import TracebackType from typing import IO, Callable, Iterator, NoReturn, Optional, Type, Union from .color import cescape, clen, cprint, cwrite # Globals _debug = 0 _verbose = False _stacktrace = False _timestamp = False _msg_enabled = True _warn_enabled = True _error_enabled = True _output_filter: Callable[[str], str] = lambda s: s indent = " " def debug_level() -> int: return _debug def is_verbose() -> bool: return _verbose def is_debug(level: int = 1) -> bool: return _debug >= level def set_debug(level: int = 0) -> None: global _debug assert level >= 0, "Debug level must be a positive value" _debug = level def set_verbose(flag: bool) -> None: global _verbose _verbose = flag def set_timestamp(flag: bool) -> None: global _timestamp _timestamp = flag def set_msg_enabled(flag: bool) -> None: global _msg_enabled _msg_enabled = flag def set_warn_enabled(flag: bool) -> None: global _warn_enabled _warn_enabled = flag def set_error_enabled(flag: bool) -> None: global _error_enabled _error_enabled = flag def msg_enabled() -> bool: return _msg_enabled def warn_enabled() -> bool: return _warn_enabled def error_enabled() -> bool: return _error_enabled @contextlib.contextmanager def output_filter(filter_fn: Callable[[str], str]) -> Iterator[None]: """Context manager that applies a filter to all output.""" global _output_filter saved_filter = _output_filter try: _output_filter = filter_fn yield finally: _output_filter = saved_filter class SuppressOutput: """Class for disabling output in a scope using ``with`` keyword""" def __init__( self, msg_enabled: bool = True, warn_enabled: bool = True, error_enabled: bool = True ) -> None: self._msg_enabled_initial = _msg_enabled self._warn_enabled_initial = _warn_enabled self._error_enabled_initial = _error_enabled self._msg_enabled = msg_enabled self._warn_enabled = warn_enabled self._error_enabled = error_enabled def __enter__(self) -> None: set_msg_enabled(self._msg_enabled) set_warn_enabled(self._warn_enabled) set_error_enabled(self._error_enabled) def __exit__( self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> None: set_msg_enabled(self._msg_enabled_initial) set_warn_enabled(self._warn_enabled_initial) set_error_enabled(self._error_enabled_initial) def set_stacktrace(flag: bool) -> None: global _stacktrace _stacktrace = flag def process_stacktrace(countback: int) -> str: """Gives file and line frame ``countback`` frames from the bottom""" st = traceback.extract_stack() # Not all entries may be spack files, we have to remove those that aren't. file_list = [] for frame in st: # Check that the file is a spack file if frame[0].find(os.path.sep + "spack") >= 0: file_list.append(frame[0]) # We use commonprefix to find what the spack 'root' directory is. root_dir = os.path.commonprefix(file_list) root_len = len(root_dir) st_idx = len(st) - countback - 1 st_text = f"{st[st_idx][0][root_len:]}:{st[st_idx][1]:d} " return st_text def show_pid() -> bool: return is_debug(2) def get_timestamp(force: bool = False) -> str: """Get a string timestamp""" if _debug or _timestamp or force: # Note the inclusion of the PID is useful for parallel builds. pid = f", {os.getpid()}" if show_pid() else "" return f"[{datetime.now().strftime('%Y-%m-%d-%H:%M:%S.%f')}{pid}] " else: return "" def msg(message: Union[Exception, str], *args: str, newline: bool = True) -> None: """Print a message to the console.""" if not msg_enabled(): return if isinstance(message, Exception): message = f"{message.__class__.__name__}: {message}" else: message = str(message) st_text = "" if _stacktrace: st_text = process_stacktrace(2) nl = "\n" if newline else "" cwrite(f"@*b{{{st_text}==>}} {get_timestamp()}{cescape(_output_filter(message))}{nl}") for arg in args: print(indent + _output_filter(str(arg))) def info( message: Union[Exception, str], *args, format: str = "*b", stream: Optional[IO[str]] = None, wrap: bool = False, break_long_words: bool = False, countback: int = 3, ) -> None: """Print an informational message.""" if isinstance(message, Exception): message = f"{message.__class__.__name__}: {str(message)}" stream = stream or sys.stdout st_text = "" if _stacktrace: st_text = process_stacktrace(countback) cprint( "@%s{%s==>} %s%s" % (format, st_text, get_timestamp(), cescape(_output_filter(str(message)))), stream=stream, ) for arg in args: if wrap: lines = textwrap.wrap( _output_filter(str(arg)), initial_indent=indent, subsequent_indent=indent, break_long_words=break_long_words, ) for line in lines: stream.write(line + "\n") else: stream.write(indent + _output_filter(str(arg)) + "\n") stream.flush() def verbose(message, *args, format: str = "c", **kwargs) -> None: """Print a verbose message if the verbose flag is set.""" if _verbose: info(message, *args, format=format, **kwargs) def debug( message, *args, level: int = 1, format: str = "g", stream: Optional[IO[str]] = None, **kwargs ) -> None: """Print a debug message if the debug level is set.""" if is_debug(level): stream_arg = stream or sys.stderr info(message, *args, format=format, stream=stream_arg, **kwargs) def error(message, *args, format: str = "*r", stream: Optional[IO[str]] = None, **kwargs) -> None: """Print an error message.""" if not error_enabled(): return stream = stream or sys.stderr info(f"Error: {message}", *args, format=format, stream=stream, **kwargs) def warn(message, *args, format: str = "*Y", stream: Optional[IO[str]] = None, **kwargs) -> None: """Print a warning message.""" if not warn_enabled(): return stream = stream or sys.stderr info(f"Warning: {message}", *args, format=format, stream=stream, **kwargs) def die(message, *args, countback: int = 4, **kwargs) -> NoReturn: error(message, *args, countback=countback, **kwargs) sys.exit(1) def get_yes_or_no(prompt: str, default: Optional[bool] = None) -> Optional[bool]: if default is None: prompt += " [y/n] " elif default is True: prompt += " [Y/n] " elif default is False: prompt += " [y/N] " else: raise ValueError("default for get_yes_no() must be True, False, or None.") result = None while result is None: msg(prompt, newline=False) ans = input().lower() if not ans: result = default if result is None: print("Please enter yes or no.") else: if ans == "y" or ans == "yes": result = True elif ans == "n" or ans == "no": result = False return result def hline(label: Optional[str] = None, *, char: str = "-", max_width: int = 64) -> None: """Draw a labeled horizontal line. Args: char: char to draw the line with max_width: maximum width of the line """ cols = shutil.get_terminal_size().columns if not cols: cols = max_width else: cols -= 2 cols = min(max_width, cols) label = str(label) prefix = char * 2 + " " suffix = " " + (cols - len(prefix) - clen(label)) * char out = io.StringIO() out.write(prefix) out.write(label) out.write(suffix) print(out.getvalue()) ================================================ FILE: lib/spack/spack/llnl/util/tty/colify.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ Routines for printing columnar output. See ``colify()`` for more information. """ import io import os import shutil import sys from typing import IO, Any, List, Optional from spack.llnl.util.tty.color import cextra, clen class ColumnConfig: def __init__(self, cols: int) -> None: self.cols = cols self.line_length = 0 self.valid = True self.widths = [0] * cols # does not include ansi colors def __repr__(self) -> str: attrs = [(a, getattr(self, a)) for a in dir(self) if not a.startswith("__")] return f"" def config_variable_cols( elts: List[str], console_width: int, padding: int, cols: int = 0 ) -> ColumnConfig: """Variable-width column fitting algorithm. This function determines the most columns that can fit in the screen width. Unlike uniform fitting, where all columns take the width of the longest element in the list, each column takes the width of its own longest element. This packs elements more efficiently on screen. If cols is nonzero, force the table to use that many columns and just add minimal padding between the columns. """ if cols < 0: raise ValueError("cols must be non-negative.") # Get a bound on the most columns we could possibly have. # 'clen' ignores length of ansi color sequences. lengths = [clen(e) for e in elts] max_cols = max(1, console_width // (min(lengths) + padding)) max_cols = min(len(elts), max_cols) # Range of column counts to try. If forced, use the supplied value. col_range = [cols] if cols else range(1, max_cols + 1) # Determine the most columns possible for the console width. configs = [ColumnConfig(c) for c in col_range] for i, length in enumerate(lengths): for conf in configs: if conf.valid: rows = (len(elts) + conf.cols - 1) // conf.cols col = i // rows p = padding if col < (conf.cols - 1) else 0 if conf.widths[col] < (length + p): conf.line_length += length + p - conf.widths[col] conf.widths[col] = length + p conf.valid = conf.line_length < console_width try: # take the last valid config in the list (the one with most columns) config = next(conf for conf in reversed(configs) if conf.valid) except StopIteration: # If nothing was valid, the screen was too narrow -- use 1 col if cols was not # specified, otherwise, use the requested columns and overflow. config = configs[0] if cols: rows = (len(lengths) + cols - 1) // cols config.widths = [ max(length for i, length in enumerate(lengths) if i // rows == c) + (padding if c < cols - 1 else 0) for c in range(cols) ] # trim off any columns with nothing in them config.widths = [w for w in config.widths if w != 0] config.cols = len(config.widths) return config def config_uniform_cols( elts: List[str], console_width: int, padding: int, cols: int = 0 ) -> ColumnConfig: """Uniform-width column fitting algorithm. Determines the longest element in the list, and determines how many columns of that width will fit on screen. Returns a corresponding column config. """ if cols < 0: raise ValueError("cols must be non-negative.") # 'clen' ignores length of ansi color sequences. max_len = max(clen(e) for e in elts) + padding if cols == 0: cols = max(1, console_width // max_len) cols = min(len(elts), cols) config = ColumnConfig(cols) config.widths = [max_len] * cols return config def colify( elts: List[Any], *, cols: int = 0, output: Optional[IO] = None, indent: int = 0, padding: int = 2, tty: Optional[bool] = None, method: str = "variable", console_cols: Optional[int] = None, ): """Takes a list of elements as input and finds a good columnization of them, similar to how gnu ls does. This supports both uniform-width and variable-width (tighter) columns. If elts is not a list of strings, each element is first converted using :class:`str`. Keyword Arguments: output: A file object to write to. Default is ``sys.stdout`` indent: Optionally indent all columns by some number of spaces padding: Spaces between columns. Default is 2 width: Width of the output. Default is 80 if tty not detected cols: Force number of columns. Default is to size to terminal, or single-column if no tty tty: Whether to attempt to write to a tty. Default is to autodetect a tty. Set to False to force single-column output method: Method to use to fit columns. Options are variable or uniform. Variable-width columns are tighter, uniform columns are all the same width and fit less data on the screen console_cols: number of columns on this console (default: autodetect) """ if output is None: output = sys.stdout # elts needs to be an array of strings so we can count the elements elts = [str(elt) for elt in elts] if not elts: return (0, ()) # environment size is of the form "x" env_size = os.environ.get("COLIFY_SIZE") if env_size: try: console_cols = int(env_size.partition("x")[2]) tty = True except ValueError: pass # Use only one column if not a tty, unless cols specified explicitly if not cols and not tty: if tty is False or not output.isatty(): cols = 1 # Specify the number of character columns to use. if console_cols is None: console_cols = shutil.get_terminal_size().columns elif not isinstance(console_cols, int): raise ValueError("Number of columns must be an int") console_cols = max(1, console_cols - indent) # Choose a method. Variable-width columns vs uniform-width. if method == "variable": config = config_variable_cols(elts, console_cols, padding, cols) elif method == "uniform": config = config_uniform_cols(elts, console_cols, padding, cols) else: raise ValueError("method must be either 'variable' or 'uniform'") cols = config.cols rows = (len(elts) + cols - 1) // cols rows_last_col = len(elts) % rows for row in range(rows): output.write(" " * indent) for col in range(cols): elt = col * rows + row width = config.widths[col] + cextra(elts[elt]) if col < cols - 1: fmt = "%%-%ds" % width output.write(fmt % elts[elt]) else: # Don't pad the rightmost column (spaces can wrap on # small teriminals if one line is overlong) output.write(elts[elt]) output.write("\n") row += 1 if row == rows_last_col: cols -= 1 return (config.cols, tuple(config.widths)) def colify_table( table: List[List[Any]], *, output: Optional[IO] = None, indent: int = 0, padding: int = 2, console_cols: Optional[int] = None, ): """Version of ``colify()`` for data expressed in rows, (list of lists). Same as regular colify but: 1. This takes a list of lists, where each sub-list must be the same length, and each is interpreted as a row in a table. Regular colify displays a sequential list of values in columns. 2. Regular colify will always print with 1 column when the output is not a tty. This will always print with same dimensions of the table argument. """ if table is None: raise TypeError("Can't call colify_table on NoneType") elif not table or not table[0]: raise ValueError("Table is empty in colify_table!") columns = len(table[0]) def transpose(): for i in range(columns): for row in table: yield row[i] colify( transpose(), cols=columns, # this is always the number of cols in the table tty=True, # don't reduce to 1 column for non-tty output=output, indent=indent, padding=padding, console_cols=console_cols, ) def colified( elts: List[Any], *, cols: int = 0, indent: int = 0, padding: int = 2, tty: Optional[bool] = None, method: str = "variable", console_cols: Optional[int] = None, ): """Invokes the ``colify()`` function but returns the result as a string instead of writing it to an output string.""" sio = io.StringIO() colify( elts, cols=cols, output=sio, indent=indent, padding=padding, tty=tty, method=method, console_cols=console_cols, ) return sio.getvalue() ================================================ FILE: lib/spack/spack/llnl/util/tty/color.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This file implements an expression syntax, similar to ``printf``, for adding ANSI colors to text. See :func:`colorize`, :func:`cwrite`, and :func:`cprint` for routines that can generate colored output. :func:`colorize` will take a string and replace all color expressions with ANSI control codes. If the ``isatty`` keyword arg is set to False, then the color expressions will be converted to null strings, and the returned string will have no color. :func:`cwrite` and :func:`cprint` are equivalent to ``write()`` and ``print()`` calls in python, but they colorize their output. If the ``stream`` argument is not supplied, they write to ``sys.stdout``. Here are some example color expressions: ============== ============================================================ Expression Meaning ============== ============================================================ ``@r`` Turn on red coloring ``@R`` Turn on bright red coloring ``@*{foo}`` Bold foo, but don't change text color ``@_{bar}`` Underline bar, but don't change text color ``@*b`` Turn on bold, blue text ``@_B`` Turn on bright blue text with an underline ``@.`` Revert to plain formatting ``@*g{green}`` Print out 'green' in bold, green text, then reset to plain. ``@*ggreen@.`` Print out 'green' in bold, green text, then reset to plain. ============== ============================================================ The syntax consists of: ========== ===================================================== color-expr ``'@' [style] color-code '{' text '}' | '@.' | '@@'`` style ``'*' | '_'`` color-code ``[krgybmcwKRGYBMCW]`` text ``.*`` ========== ===================================================== ``@`` indicates the start of a color expression. It can be followed by an optional ``*`` or ``_`` that indicates whether the font should be bold or underlined. If ``*`` or ``_`` is not provided, the text will be plain. Then an optional color code is supplied. This can be ``[krgybmcw]`` or ``[KRGYBMCW]``, where the letters map to ``black(k)``, ``red(r)``, ``green(g)``, ``yellow(y)``, ``blue(b)``, ``magenta(m)``, ``cyan(c)``, and ``white(w)``. Lowercase letters denote normal ANSI colors and capital letters denote bright ANSI colors. Finally, the color expression can be followed by text enclosed in ``{}``. If braces are present, only the text in braces is colored. If the braces are NOT present, then just the control codes to enable the color will be output. The console can be reset later to plain text with ``@.``. To output an ``@``, use ``@@``. To output a ``}`` inside braces, use ``}}``. """ import io import os import re import sys import textwrap from contextlib import contextmanager from typing import IO, Iterator, List, NamedTuple, Optional, Tuple, Union class ColorParseError(Exception): """Raised when a color format fails to parse.""" def __init__(self, message: str) -> None: super().__init__(message) # Text styles for ansi codes styles = {"*": "1", "_": "4", None: "0"} # bold # underline # plain # Dim and bright ansi colors colors = { "k": 30, "K": 90, # black "r": 31, "R": 91, # red "g": 32, "G": 92, # green "y": 33, "Y": 93, # yellow "b": 34, "B": 94, # blue "m": 35, "M": 95, # magenta "c": 36, "C": 96, # cyan "w": 37, "W": 97, } # white # Regex to be used for color formatting COLOR_RE = re.compile(r"@(?:(@)|(\.)|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)") # Mapping from color arguments to values for tty.set_color color_when_values = {"always": True, "auto": None, "never": False} def _color_when_value(when: Union[str, bool, None]) -> Optional[bool]: """Raise a ValueError for an invalid color setting. Valid values are 'always', 'never', and 'auto', or equivalently, True, False, and None. """ if isinstance(when, bool) or when is None: return when elif when not in color_when_values: raise ValueError(f"Invalid color setting: {when}") return color_when_values[when] def _color_from_environ() -> Optional[bool]: try: return _color_when_value(os.environ.get("SPACK_COLOR", "auto")) except ValueError: return None #: When `None` colorize when stdout is tty, when `True` or `False` always or never colorize resp. _force_color = _color_from_environ() def try_enable_terminal_color_on_windows() -> None: """Turns coloring in Windows terminal by enabling VTP in Windows consoles (CMD/PWSH/CONHOST) Method based on the link below https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#example-of-enabling-virtual-terminal-processing Note: No-op on non windows platforms """ if sys.platform == "win32": import ctypes import msvcrt from ctypes import wintypes try: ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 DISABLE_NEWLINE_AUTO_RETURN = 0x0008 kernel32 = ctypes.WinDLL("kernel32") def _err_check(result, func, args): if not result: raise ctypes.WinError(ctypes.get_last_error()) return args kernel32.GetConsoleMode.errcheck = _err_check kernel32.GetConsoleMode.argtypes = ( wintypes.HANDLE, # hConsoleHandle, i.e. GetStdHandle output type ctypes.POINTER(wintypes.DWORD), # result of GetConsoleHandle ) kernel32.SetConsoleMode.errcheck = _err_check kernel32.SetConsoleMode.argtypes = ( wintypes.HANDLE, # hConsoleHandle, i.e. GetStdHandle output type wintypes.DWORD, # result of GetConsoleHandle ) # Use conout$ here to handle a redirectired stdout/get active console associated # with spack with open(r"\\.\CONOUT$", "w", encoding="utf-8") as conout: # Link above would use kernel32.GetStdHandle(-11) however this would not handle # a redirected stdout appropriately, so we always refer to the current CONSOLE out # which is defined as conout$ on Windows. # linked example is follow more or less to the letter beyond this point con_handle = msvcrt.get_osfhandle(conout.fileno()) dw_orig_mode = wintypes.DWORD() kernel32.GetConsoleMode(con_handle, ctypes.byref(dw_orig_mode)) dw_new_mode_request = ( ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN ) dw_new_mode = dw_new_mode_request | dw_orig_mode.value kernel32.SetConsoleMode(con_handle, wintypes.DWORD(dw_new_mode)) except OSError: # We failed to enable color support for associated console # report and move on but spack will no longer attempt to # color global _force_color _force_color = False def get_color_when(stdout=None) -> bool: """Return whether commands should print color or not.""" if _force_color is not None: return _force_color if stdout is None: stdout = sys.stdout return stdout.isatty() def set_color_when(when: Union[str, bool, None]) -> None: """Set when color should be applied. Options are: * True or ``"always"``: always print color * False or ``"never"``: never print color * None or ``"auto"``: only print color if sys.stdout is a tty. """ global _force_color _force_color = _color_when_value(when) @contextmanager def color_when(value: Union[str, bool, None]) -> Iterator[None]: """Context manager to temporarily use a particular color setting.""" old_value = _force_color set_color_when(value) yield set_color_when(old_value) _ConvertibleToStr = Union[str, int, bool, None] def _escape(s: _ConvertibleToStr, color: bool, enclose: bool, zsh: bool) -> str: """Returns a TTY escape sequence for a color""" if not color: return "" elif zsh: return f"\033[0;{s}m" result = f"\033[{s}m" if enclose: result = rf"\[{result}\]" return result def colorize( string: str, color: Optional[bool] = None, enclose: bool = False, zsh: bool = False ) -> str: """Replace all color expressions in a string with ANSI control codes. Args: string: The string to replace color: If False, output will be plain text without control codes, for output to non-console devices (default: automatically choose color or not) enclose: If True, enclose ansi color sequences with square brackets to prevent misestimation of terminal width. zsh: If True, use zsh ansi codes instead of bash ones (for variables like PS1) """ if color is None: color = get_color_when() def match_to_ansi(match) -> str: """Convert a match object generated by ``COLOR_RE`` into an ansi color code. This can be used as a handler in ``re.sub``. """ escaped_at, dot, style, color_code, text = match.groups() if escaped_at: return "@" elif dot: return _escape(0, color, enclose, zsh) elif not (style or color_code): raise ColorParseError( f"Incomplete color format: '{match.group(0)}' in '{match.string}'" ) color_number = colors.get(color_code, "") semi = ";" if color_number else "" ansi_code = _escape(f"{styles[style]}{semi}{color_number}", color, enclose, zsh) if text: # must be here, not in the final return: top-level @@ is already handled by # the regex, and its @-results could form new @@ pairs. text = text.replace("@@", "@") return f"{ansi_code}{text}{_escape(0, color, enclose, zsh)}" else: return ansi_code return COLOR_RE.sub(match_to_ansi, string).replace("}}", "}") #: matches a standard ANSI color code ANSI_CODE_RE = re.compile(r"\033[^m]*m") def csub(string: str) -> str: """Return the string with ANSI color sequences removed.""" return ANSI_CODE_RE.sub("", string) class ColorMapping(NamedTuple): color: str #: color string colors: List[str] #: ANSI color codes in the color string, in order offsets: List[Tuple[int, int]] #: map indices in plain string to offsets in color string def plain_to_color(self, index: int) -> int: """Convert plain string index to color index.""" offset = 0 for i, off in self.offsets: if i > index: break offset = off return index + offset def cmapping(string: str) -> ColorMapping: """Return a mapping for translating indices in a plain string to indices in colored text. The returned dictionary maps indices in the plain string to the offset of the corresponding indices in the colored string. """ colors = [] offsets = [] color_offset = 0 for m in ANSI_CODE_RE.finditer(string): start, end = m.start(), m.end() start_offset = color_offset color_offset += end - start offsets.append((start - start_offset, color_offset)) colors.append(m.group()) return ColorMapping(string, colors, offsets) def cwrap( string: str, *, initial_indent: str = "", subsequent_indent: str = "", **kwargs ) -> List[str]: """Wrapper around ``textwrap.wrap()`` that handles ANSI color codes.""" plain = csub(string) lines = textwrap.wrap( plain, initial_indent=initial_indent, subsequent_indent=subsequent_indent, **kwargs ) # do nothing if string has no ANSI codes if plain == string: return lines # otherwise add colors back to lines after wrapping plain text cmap = cmapping(string) clines = [] start = 0 for i, line in enumerate(lines): # scan to find the actual start, skipping any whitespace from a prior line break # can assume this b/c textwrap only collapses whitespace at line breaks while start < len(plain) and plain[start].isspace(): start += 1 # map the start and end positions in the plain string to the color string cstart = cmap.plain_to_color(start) # rewind to include any color codes before cstart while cstart and string[cstart - 1] == "m": cstart = string.rfind("\033", 0, cstart - 1) indent = initial_indent if i == 0 else subsequent_indent end = start + len(line) - len(indent) cend = cmap.plain_to_color(end) # append the color line to the result clines.append(indent + string[cstart:cend]) start = end return clines def clen(string: str) -> int: """Return the length of a string, excluding ansi color sequences.""" return len(csub(string)) def cextra(string: str) -> int: """Length of extra color characters in a string""" return len("".join(re.findall(r"\033[^m]*m", string))) def cwrite(string: str, stream: Optional[IO[str]] = None, color: Optional[bool] = None) -> None: """Replace all color expressions in string with ANSI control codes and write the result to the stream. If color is False, this will write plain text with no color. If True, then it will always write colored output. If not supplied, then it will be set based on stream.isatty(). """ stream = sys.stdout if stream is None else stream if color is None: color = get_color_when() stream.write(colorize(string, color=color)) def cprint(string: str, stream: Optional[IO[str]] = None, color: Optional[bool] = None) -> None: """Same as cwrite, but writes a trailing newline to the stream.""" cwrite(string + "\n", stream, color) def cescape(string: str) -> str: """Escapes special characters needed for color codes. Replaces the following symbols with their equivalent literal forms: ===== ====== ``@`` ``@@`` ``}`` ``}}`` ===== ====== Parameters: string (str): the string to escape Returns: (str): the string with color codes escaped """ return string.replace("@", "@@").replace("}", "}}") class ColorStream: def __init__(self, stream: io.IOBase, color: Optional[bool] = None) -> None: self._stream = stream self._color = color def write(self, string: str, *, raw: bool = False) -> None: raw_write = getattr(self._stream, "write") color = self._color if self._color is None: if raw: color = True else: color = get_color_when() raw_write(colorize(string, color=color)) ================================================ FILE: lib/spack/spack/llnl/util/tty/log.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Utility classes for logging the output of blocks of code.""" import atexit import ctypes import errno import io import multiprocessing import os import re import select import signal import sys import threading import traceback from contextlib import contextmanager from multiprocessing.connection import Connection from threading import Thread from typing import IO, Callable, Optional, Tuple import spack.llnl.util.tty as tty try: import termios except ImportError: termios = None # type: ignore[assignment] esc, bell, lbracket, bslash, newline = r"\x1b", r"\x07", r"\[", r"\\", r"\n" # Ansi Control Sequence Introducers (CSI) are a well-defined format # Standard ECMA-48: Control Functions for Character-Imaging I/O Devices, section 5.4 # https://www.ecma-international.org/wp-content/uploads/ECMA-48_5th_edition_june_1991.pdf csi_pre = f"{esc}{lbracket}" csi_param, csi_inter, csi_post = r"[0-?]", r"[ -/]", r"[@-~]" ansi_csi = f"{csi_pre}{csi_param}*{csi_inter}*{csi_post}" # General ansi escape sequences have well-defined prefixes, # but content and suffixes are less reliable. # Conservatively assume they end with either "\" or "", # with no intervening ""/"" keys or newlines esc_pre = f"{esc}[@-_]" esc_content = f"[^{esc}{bell}{newline}]" esc_post = f"(?:{esc}{bslash}|{bell})" ansi_esc = f"{esc_pre}{esc_content}*{esc_post}" # Use this to strip escape sequences _escape = re.compile(f"{ansi_csi}|{ansi_esc}") # control characters for enabling/disabling echo # # We use control characters to ensure that echo enable/disable are inline # with the other output. We always follow these with a newline to ensure # one per line the following newline is ignored in output. xon, xoff = "\x11\n", "\x13\n" control = re.compile("(\x11\n|\x13\n)") @contextmanager def ignore_signal(signum): """Context manager to temporarily ignore a signal.""" old_handler = signal.signal(signum, signal.SIG_IGN) try: yield finally: signal.signal(signum, old_handler) def _is_background_tty(stdin: IO[str]) -> bool: """True if the stream is a tty and calling process is in the background.""" return stdin.isatty() and os.getpgrp() != os.tcgetpgrp(stdin.fileno()) def _strip(line: str) -> str: """Strip color and control characters from a line.""" return _escape.sub("", line) class preserve_terminal_settings: """Context manager to preserve terminal settings on a stream. Stores terminal settings before the context and ensures they are restored after. Ensures that things like echo and canonical line mode are not left disabled if terminal settings in the context are not properly restored. """ def __init__(self, stdin: Optional[IO[str]]) -> None: """Create a context manager that preserves terminal settings on a stream. Args: stream: keyboard input stream, typically sys.stdin """ self.stdin = stdin def _restore_default_terminal_settings(self) -> None: """Restore the original input configuration on ``self.stdin``.""" # Can be called in foreground or background. When called in the background, tcsetattr # triggers SIGTTOU, which we must ignore, or the process will be stopped. assert self.stdin is not None and self.old_cfg is not None and termios is not None with ignore_signal(signal.SIGTTOU): termios.tcsetattr(self.stdin, termios.TCSANOW, self.old_cfg) def __enter__(self) -> "preserve_terminal_settings": """Store terminal settings.""" self.old_cfg = None # Ignore all this if the input stream is not a tty. if not self.stdin or not self.stdin.isatty() or not termios: return self # save old termios settings to restore later self.old_cfg = termios.tcgetattr(self.stdin) # add an atexit handler to ensure the terminal is restored atexit.register(self._restore_default_terminal_settings) return self def __exit__(self, exc_type, exception, traceback): """If termios was available, restore old settings.""" if self.old_cfg: self._restore_default_terminal_settings() atexit.unregister(self._restore_default_terminal_settings) class keyboard_input(preserve_terminal_settings): """Context manager to disable line editing and echoing. Use this with ``sys.stdin`` for keyboard input, e.g.:: with keyboard_input(sys.stdin) as kb: while True: kb.check_fg_bg() r, w, x = select.select([sys.stdin], [], []) # ... do something with keypresses ... The ``keyboard_input`` context manager disables canonical (line-based) input and echoing, so that keypresses are available on the stream immediately, and they are not printed to the terminal. Typically, standard input is line-buffered, which means keypresses won't be sent until the user hits return. In this mode, a user can hit, e.g., ``v``, and it will be read on the other end of the pipe immediately but not printed. The handler takes care to ensure that terminal changes only take effect when the calling process is in the foreground. If the process is backgrounded, canonical mode and echo are re-enabled. They are disabled again when the calling process comes back to the foreground. This context manager works through a single signal handler for ``SIGTSTP``, along with a poolling routine called ``check_fg_bg()``. Here are the relevant states, transitions, and POSIX signals:: [Running] -------- Ctrl-Z sends SIGTSTP ------------. [ in FG ] <------- fg sends SIGCONT --------------. | ^ | | | fg (no signal) | | | | v [Running] <------- bg sends SIGCONT ---------- [Stopped] [ in BG ] [ in BG ] We handle all transitions except for ``SIGTSTP`` generated by Ctrl-Z by periodically calling ``check_fg_bg()``. This routine notices if we are in the background with canonical mode or echo disabled, or if we are in the foreground without canonical disabled and echo enabled, and it fixes the terminal settings in response. ``check_fg_bg()`` works *except* for when the process is stopped with ``SIGTSTP``. We cannot rely on a periodic timer in this case, as it may not rrun before the process stops. We therefore restore terminal settings in the ``SIGTSTP`` handler. Additional notes: * We mostly use polling here instead of a SIGARLM timer or a thread. This is to avoid the complexities of many interrupts, which seem to make system calls (like I/O) unreliable in older Python versions (2.6 and 2.7). See these issues for details: 1. https://www.python.org/dev/peps/pep-0475/ 2. https://bugs.python.org/issue8354 There are essentially too many ways for asynchronous signals to go wrong if we also have to support older Python versions, so we opt not to use them. * ``SIGSTOP`` can stop a process (in the foreground or background), but it can't be caught. Because of this, we can't fix any terminal settings on ``SIGSTOP``, and the terminal will be left with ``ICANON`` and ``ECHO`` disabled until it is resumes execution. * Technically, a process *could* be sent ``SIGTSTP`` while running in the foreground, without the shell backgrounding that process. This doesn't happen in practice, and we assume that ``SIGTSTP`` always means that defaults should be restored. * We rely on ``termios`` support. Without it, or if the stream isn't a TTY, ``keyboard_input`` has no effect. """ def __init__(self, stdin: Optional[IO[str]]) -> None: """Create a context manager that will enable keyboard input on stream. Args: stdin: text io wrapper of stdin (keyboard input) Note that stdin can be None, in which case ``keyboard_input`` will do nothing. """ super().__init__(stdin) def _is_background(self) -> bool: """True iff calling process is in the background.""" assert self.stdin is not None, "stdin should be available" return _is_background_tty(self.stdin) def _get_canon_echo_flags(self) -> Tuple[bool, bool]: """Get current termios canonical and echo settings.""" assert termios is not None and self.stdin is not None cfg = termios.tcgetattr(self.stdin) return (bool(cfg[3] & termios.ICANON), bool(cfg[3] & termios.ECHO)) def _enable_keyboard_input(self) -> None: """Disable canonical input and echoing on ``self.stdin``.""" # "enable" input by disabling canonical mode and echo assert termios is not None and self.stdin is not None new_cfg = termios.tcgetattr(self.stdin) new_cfg[3] &= ~termios.ICANON new_cfg[3] &= ~termios.ECHO # Apply new settings for terminal with ignore_signal(signal.SIGTTOU): termios.tcsetattr(self.stdin, termios.TCSANOW, new_cfg) def _tstp_handler(self, signum, frame): self._restore_default_terminal_settings() os.kill(os.getpid(), signal.SIGSTOP) def check_fg_bg(self) -> None: # old_cfg is set up in __enter__ and indicates that we have # termios and a valid stream. if not self.old_cfg: return # query terminal flags and fg/bg status flags = self._get_canon_echo_flags() bg = self._is_background() # restore sanity if flags are amiss -- see diagram in class docs if not bg and any(flags): # fg, but input not enabled self._enable_keyboard_input() elif bg and not all(flags): # bg, but input enabled self._restore_default_terminal_settings() def __enter__(self) -> "keyboard_input": """Enable immediate keypress input, while this process is foreground. If the stream is not a TTY or the system doesn't support termios, do nothing. """ super().__enter__() self.old_handlers = {} # Ignore all this if the input stream is not a tty. if not self.stdin or not self.stdin.isatty(): return self if termios: # Install a signal handler to disable/enable keyboard input # when the process moves between foreground and background. self.old_handlers[signal.SIGTSTP] = signal.signal(signal.SIGTSTP, self._tstp_handler) # enable keyboard input initially (if foreground) if not self._is_background(): self._enable_keyboard_input() return self def __exit__(self, exc_type, exception, traceback): """If termios was available, restore old settings.""" super().__exit__(exc_type, exception, traceback) # restore SIGSTP and SIGCONT handlers if self.old_handlers: for signum, old_handler in self.old_handlers.items(): signal.signal(signum, old_handler) class Unbuffered: """Wrapper for Python streams that forces them to be unbuffered. This is implemented by forcing a flush after each write. """ def __init__(self, stream): self.stream = stream def write(self, data): self.stream.write(data) self.stream.flush() def writelines(self, datas): self.stream.writelines(datas) self.stream.flush() def __getattr__(self, attr): return getattr(self.stream, attr) def log_output(*args, **kwargs): """Context manager that logs its output to a file. In the simplest case, the usage looks like this:: with log_output('logfile.txt'): # do things ... output will be logged Any output from the with block will be redirected to ``logfile.txt``. If you also want the output to be echoed to ``stdout``, use the ``echo`` parameter:: with log_output('logfile.txt', echo=True): # do things ... output will be logged and printed out The following is available on Unix only. No-op on Windows. And, if you just want to echo *some* stuff from the parent, use ``force_echo``:: with log_output('logfile.txt', echo=False) as logger: # do things ... output will be logged with logger.force_echo(): # things here will be echoed *and* logged See individual log classes for more information. This method is actually a factory serving a per platform (unix vs windows) log_output class """ if sys.platform == "win32": return winlog(*args, **kwargs) else: return nixlog(*args, **kwargs) class nixlog: """ Under the hood, we spawn a daemon and set up a pipe between this process and the daemon. The daemon writes our output to both the file and to stdout (if echoing). The parent process can communicate with the daemon to tell it when and when not to echo; this is what force_echo does. You can also enable/disable echoing by typing ``v``. We use OS-level file descriptors to do the redirection, which redirects output for subprocesses and system calls. """ def __init__( self, filename: str, echo=False, debug=0, buffer=False, env=None, filter_fn=None, append=False, ): """Create a new output log context manager. Args: filename (str): path to file where output should be logged echo (bool): whether to echo output in addition to logging it debug (int): positive to enable tty debug mode during logging buffer (bool): pass buffer=True to skip unbuffering output; note this doesn't set up any *new* buffering filter_fn (callable, optional): Callable[str] -> str to filter each line of output append (bool): whether to append to file ('a' mode) The filename will be opened and closed entirely within ``__enter__`` and ``__exit__``. By default, we unbuffer sys.stdout and sys.stderr because the logger will include output from executed programs and from python calls. If stdout and stderr are buffered, their output won't be printed in the right place w.r.t. output from commands. Logger daemon is not started until ``__enter__()``. """ self.filename = filename self.echo = echo self.debug = debug self.buffer = buffer self.filter_fn = filter_fn self.append = append self._active = False # used to prevent re-entry def __enter__(self): if self._active: raise RuntimeError("Can't re-enter the same log_output!") # record parent color settings before redirecting. We do this # because color output depends on whether the *original* stdout # is a TTY. New stdout won't be a TTY so we force colorization. self._saved_color = tty.color._force_color forced_color = tty.color.get_color_when() # also record parent debug settings -- in case the logger is # forcing debug output. self._saved_debug = tty._debug # Pipe for redirecting output to logger read_fd, self.write_fd = multiprocessing.Pipe(duplex=False) # Pipe for communication back from the daemon # Currently only used to save echo value between uses self.parent_pipe, child_pipe = multiprocessing.Pipe(duplex=False) stdin_fd = None stdout_fd = None try: # need to pass this b/c multiprocessing closes stdin in child. try: if sys.stdin.isatty(): stdin_fd = Connection(os.dup(sys.stdin.fileno())) except BaseException: # just don't forward input if this fails pass # If our process has redirected stdout after the forkserver was started, we need to # make the forked processes use the new file descriptors. if multiprocessing.get_start_method() == "forkserver": stdout_fd = Connection(os.dup(sys.stdout.fileno())) self.process = multiprocessing.Process( target=_writer_daemon, args=( stdin_fd, stdout_fd, read_fd, self.write_fd, self.echo, self.filename, self.append, child_pipe, self.filter_fn, ), ) self.process.daemon = True # must set before start() self.process.start() finally: if stdin_fd: stdin_fd.close() if stdout_fd: stdout_fd.close() read_fd.close() # Flush immediately before redirecting so that anything buffered # goes to the original stream sys.stdout.flush() sys.stderr.flush() # Now do the actual output redirection. # We use OS-level file descriptors, as this # redirects output for subprocesses and system calls. self._redirected_fds = {} # sys.stdout and sys.stderr may have been replaced with file objects under pytest, so # redirect their file descriptors in addition to the original fds 1 and 2. fds = {sys.stdout.fileno(), sys.stderr.fileno(), 1, 2} for fd in fds: self._redirected_fds[fd] = os.dup(fd) os.dup2(self.write_fd.fileno(), fd) self.write_fd.close() # Unbuffer stdout and stderr at the Python level if not self.buffer: sys.stdout = Unbuffered(sys.stdout) sys.stderr = Unbuffered(sys.stderr) # Force color and debug settings now that we have redirected. tty.color.set_color_when(forced_color) tty._debug = self.debug # track whether we're currently inside this log_output self._active = True # return this log_output object so that the user can do things # like temporarily echo some output. return self def __exit__(self, exc_type, exc_val, exc_tb): # Flush any buffered output to the logger daemon. sys.stdout.flush() sys.stderr.flush() # restore previous output settings using the OS-level way for fd, saved_fd in self._redirected_fds.items(): os.dup2(saved_fd, fd) os.close(saved_fd) # recover and store echo settings from the child before it dies try: self.echo = self.parent_pipe.recv() except EOFError: # This may occur if some exception prematurely terminates the # _writer_daemon. An exception will have already been generated. pass # now that the write pipe is closed (in this __exit__, when we restore # stdout with dup2), the logger daemon process loop will terminate. We # wait for that here. self.process.join() # restore old color and debug settings tty.color._force_color = self._saved_color tty._debug = self._saved_debug self._active = False # safe to enter again @contextmanager def force_echo(self): """Context manager to force local echo, even if echo is off.""" if not self._active: raise RuntimeError("Can't call force_echo() outside log_output region!") # This uses the xon/xoff to highlight regions to be echoed in the # output. We us these control characters rather than, say, a # separate pipe, because they're in-band and assured to appear # exactly before and after the text we want to echo. sys.stdout.write(xon) sys.stdout.flush() try: yield finally: sys.stdout.write(xoff) sys.stdout.flush() class StreamWrapper: """Wrapper class to handle redirection of io streams""" def __init__(self, sys_attr): self.sys_attr = sys_attr self.saved_stream = None if sys.platform.startswith("win32"): if hasattr(sys, "gettotalrefcount"): # debug build libc = ctypes.CDLL("ucrtbased") else: libc = ctypes.CDLL("api-ms-win-crt-stdio-l1-1-0") kernel32 = ctypes.WinDLL("kernel32") # https://docs.microsoft.com/en-us/windows/console/getstdhandle if self.sys_attr == "stdout": STD_HANDLE = -11 elif self.sys_attr == "stderr": STD_HANDLE = -12 else: raise KeyError(self.sys_attr) c_stdout = kernel32.GetStdHandle(STD_HANDLE) self.libc = libc self.c_stream = c_stdout else: self.libc = ctypes.CDLL(None) self.c_stream = ctypes.c_void_p.in_dll(self.libc, self.sys_attr) self.sys_stream = getattr(sys, self.sys_attr) self.orig_stream_fd = self.sys_stream.fileno() # Save a copy of the original stdout fd in saved_stream self.saved_stream = os.dup(self.orig_stream_fd) def redirect_stream(self, to_fd): """Redirect stdout to the given file descriptor.""" # Flush the C-level buffer stream if sys.platform.startswith("win32"): self.libc.fflush(None) else: self.libc.fflush(self.c_stream) # Flush and close sys_stream - also closes the file descriptor (fd) sys_stream = getattr(sys, self.sys_attr) sys_stream.flush() sys_stream.close() # Make orig_stream_fd point to the same file as to_fd os.dup2(to_fd, self.orig_stream_fd) # Set sys_stream to a new stream that points to the redirected fd new_buffer = open(self.orig_stream_fd, "wb") new_stream = io.TextIOWrapper(new_buffer) setattr(sys, self.sys_attr, new_stream) self.sys_stream = getattr(sys, self.sys_attr) def flush(self): if sys.platform.startswith("win32"): self.libc.fflush(None) else: self.libc.fflush(self.c_stream) self.sys_stream.flush() def close(self): """Redirect back to the original system stream, and close stream""" try: if self.saved_stream is not None: self.redirect_stream(self.saved_stream) finally: if self.saved_stream is not None: os.close(self.saved_stream) class winlog: """ Similar to nixlog, with underlying functionality ported to support Windows. Does not support the use of ``v`` toggling as nixlog does. """ def __init__( self, filename: str, echo=False, debug=0, buffer=False, filter_fn=None, append=False ): self.debug = debug self.echo = echo self.logfile = filename self.stdout = StreamWrapper("stdout") self.stderr = StreamWrapper("stderr") self._active = False self.old_stdout = sys.stdout self.old_stderr = sys.stderr self.append = append def __enter__(self): if self._active: raise RuntimeError("Can't re-enter the same log_output!") # Open both write and reading on logfile write_mode = "ab+" if self.append else "wb+" self.writer = open(self.logfile, mode=write_mode) self.reader = open(self.logfile, mode="rb+") # Dup stdout so we can still write to it after redirection self.echo_writer = open(os.dup(sys.stdout.fileno()), "w", encoding=sys.stdout.encoding) # Redirect stdout and stderr to write to logfile self.stderr.redirect_stream(self.writer.fileno()) self.stdout.redirect_stream(self.writer.fileno()) self._kill = threading.Event() def background_reader(reader, echo_writer, _kill): # for each line printed to logfile, read it # if echo: write line to user try: while True: is_killed = _kill.wait(0.1) # Flush buffered build output to file # stdout/err fds refer to log file self.stderr.flush() self.stdout.flush() line = reader.readline() if self.echo and line: echo_writer.write("{0}".format(line.decode())) echo_writer.flush() if is_killed: break finally: reader.close() self._active = True self._thread = Thread( target=background_reader, args=(self.reader, self.echo_writer, self._kill) ) self._thread.start() return self def __exit__(self, exc_type, exc_val, exc_tb): self.writer.close() self.echo_writer.flush() self.stdout.flush() self.stderr.flush() self._kill.set() self._thread.join() self.stdout.close() self.stderr.close() self._active = False @contextmanager def force_echo(self): """Context manager to force local echo, even if echo is off.""" if not self._active: raise RuntimeError("Can't call force_echo() outside log_output region!") yield def _writer_daemon( stdin_fd: Optional[Connection], stdout_fd: Optional[Connection], read_fd: Connection, write_fd: Connection, echo: bool, log_filename: str, append: bool, control_fd: Connection, filter_fn: Optional[Callable[[str], str]], ) -> None: """Daemon used by ``log_output`` to write to a log file and to ``stdout``. The daemon receives output from the parent process and writes it both to a log and, optionally, to ``stdout``. The relationship looks like this:: Terminal | | +-------------------------+ | | Parent Process | +--------> | with log_output(): | | stdin | ... | | +-------------------------+ | ^ | write_fd (parent's redirected stdout) | | control | | | pipe | | | v read_fd | +-------------------------+ stdout | | Writer daemon |------------> +--------> | read from read_fd | log_file stdin | write to out and log |------------> +-------------------------+ Within the ``log_output`` handler, the parent's output is redirected to a pipe from which the daemon reads. The daemon writes each line from the pipe to a log file and (optionally) to ``stdout``. The user can hit ``v`` to toggle output on ``stdout``. In addition to the input and output file descriptors, the daemon interacts with the parent via ``control_pipe``. It reports whether ``stdout`` was enabled or disabled when it finished. Arguments: stdin_fd: optional input from the terminal read_fd: pipe for reading from parent's redirected stdout echo: initial echo setting -- controlled by user and preserved across multiple writer daemons log_filename: filename where output should be logged append: whether to append to the file or overwrite it control_pipe: multiprocessing pipe on which to send control information to the parent filter_fn: optional function to filter each line of output """ # This process depends on closing all instances of write_pipe to terminate the reading loop write_fd.close() # 1. Use line buffering (3rd param = 1) since Python 3 has a bug # that prevents unbuffered text I/O. [needs citation] # 2. Enforce a UTF-8 interpretation of build process output with errors replaced by '?'. # The downside is that the log file will not contain the exact output of the build process. # 3. closefd=False because Connection has "ownership" read_file = os.fdopen( read_fd.fileno(), "r", 1, encoding="utf-8", errors="replace", closefd=False ) if stdin_fd: stdin_file = os.fdopen(stdin_fd.fileno(), closefd=False) else: stdin_file = None if stdout_fd: os.dup2(stdout_fd.fileno(), sys.stdout.fileno()) stdout_fd.close() # list of streams to select from istreams = [read_file, stdin_file] if stdin_file else [read_file] force_echo = False # parent can force echo for certain output log_file = open(log_filename, mode="a" if append else "w", encoding="utf-8") try: with keyboard_input(stdin_file) as kb: while True: # fix the terminal settings if we recently came to # the foreground kb.check_fg_bg() # wait for input from any stream. use a coarse timeout to # allow other checks while we wait for input rlist, _, _ = select.select(istreams, [], [], 0.1) # Allow user to toggle echo with 'v' key. # Currently ignores other chars. # only read stdin if we're in the foreground if stdin_file and stdin_file in rlist and not _is_background_tty(stdin_file): # it's possible to be backgrounded between the above # check and the read, so we ignore SIGTTIN here. with ignore_signal(signal.SIGTTIN): try: if stdin_file.read(1) == "v": echo = not echo except OSError as e: # If SIGTTIN is ignored, the system gives EIO # to let the caller know the read failed b/c it # was in the bg. Ignore that too. if e.errno != errno.EIO: raise if read_file in rlist: line_count = 0 try: while line_count < 100: # Handle output from the calling process. line = read_file.readline() if not line: return line_count += 1 # find control characters and strip them. clean_line, num_controls = control.subn("", line) # Echo to stdout if requested or forced. if echo or force_echo: output_line = clean_line if filter_fn: output_line = filter_fn(clean_line) enc = sys.stdout.encoding if enc != "utf-8": # On Python 3.6 and 3.7-3.14 with non-{utf-8,C} locale stdout # may not be able to handle utf-8 output. We do an inefficient # dance of re-encoding with errors replaced, so stdout.write # does not raise. output_line = output_line.encode(enc, "replace").decode(enc) sys.stdout.write(output_line) # Stripped output to log file. log_file.write(_strip(clean_line)) if num_controls > 0: controls = control.findall(line) if xon in controls: force_echo = True if xoff in controls: force_echo = False if not _input_available(read_file): break finally: if line_count > 0: if echo or force_echo: sys.stdout.flush() log_file.flush() except BaseException: tty.error("Exception occurred in writer daemon!") traceback.print_exc() finally: log_file.close() read_fd.close() if stdin_fd: stdin_fd.close() # send echo value back to the parent so it can be preserved. control_fd.send(echo) def _input_available(f): return f in select.select([f], [], [], 0)[0] ================================================ FILE: lib/spack/spack/main.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This is the implementation of the Spack command line executable. In a normal Spack installation, this is invoked from the bin/spack script after the system path is set up. """ import argparse import gc import inspect import multiprocessing import operator import os import pstats import re import shlex import signal import sys import tempfile import textwrap import traceback import warnings from contextlib import contextmanager from typing import Any, List, Optional, Set, Tuple import spack.vendor.archspec.cpu import spack import spack.cmd import spack.config import spack.environment import spack.environment as ev import spack.environment.environment import spack.error import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.llnl.util.tty.colify import spack.llnl.util.tty.color as color import spack.paths import spack.platforms import spack.solver.asp import spack.spec import spack.util.environment import spack.util.lock from .enums import ConfigScopePriority #: names of profile statistics stat_names = pstats.Stats.sort_arg_dict_default #: help levels in order of detail (i.e., number of commands shown) levels = ["short", "long"] #: intro text for help at different levels intro_by_level = {"short": "Common spack commands:", "long": "Commands:"} #: control top-level spack options shown in basic vs. advanced help options_by_level = {"short": ["e", "h", "k", "V", "color"], "long": "all"} #: Longer text for each section, to show in help section_descriptions = { "query": "query packages", "build": "build, install, and test packages", "environment": "environment", "config": "configuration", "packaging": "create packages", "admin": "administration", "developer": "spack development", } #: preferential command order for some sections (e.g., build pipeline is #: in execution order, not alphabetical) section_order = { "basic": ["list", "info", "find"], "build": [ "fetch", "stage", "patch", "configure", "build", "restage", "install", "uninstall", "clean", ], "packaging": ["create", "edit"], } #: Properties that commands are required to set. required_command_properties = ["level", "section", "description"] spack_ld_library_path = os.environ.get("LD_LIBRARY_PATH", "") def add_all_commands(parser): """Add all spack subcommands to the parser.""" for cmd in spack.cmd.all_commands(): parser.add_command(cmd) def index_commands(): """create an index of commands by section for this help level""" index = {} for command in spack.cmd.all_commands(): cmd_module = spack.cmd.get_module(command) # make sure command modules have required properties for p in required_command_properties: prop = getattr(cmd_module, p, None) if not prop: tty.die("Command doesn't define a property '%s': %s" % (p, command)) # add commands to lists for their level and higher levels for level in reversed(levels): level_sections = index.setdefault(level, {}) commands = level_sections.setdefault(cmd_module.section, []) commands.append(command) if level == cmd_module.level: break return index class SpackHelpFormatter(argparse.RawTextHelpFormatter): def _format_actions_usage(self, actions, groups): """Formatter with more concise usage strings.""" usage = super()._format_actions_usage(actions, groups) # Eliminate any occurrence of two or more consecutive spaces usage = re.sub(r"[ ]{2,}", " ", usage) # compress single-character flags that are not mutually exclusive # at the beginning of the usage string chars = "".join(re.findall(r"\[-(.)\]", usage)) usage = re.sub(r"\[-.\] ?", "", usage) if chars: usage = "[-%s] %s" % (chars, usage) return usage.strip() def start_section(self, heading): return super().start_section(color.colorize(f"@*B{{{heading}}}")) def _format_usage(self, usage, actions, groups, prefix=None): # if no optionals or positionals are available, usage is just prog if usage is None and not actions: return super()._format_usage(usage, actions, groups, prefix) # add color *after* argparse aligns the text, so as not to interfere result = super()._format_usage(usage, actions, groups, prefix) escaped = color.cescape(result) escaped = escaped.replace(self._prog, f"@.@*C{{{self._prog}}}@c") return color.colorize(f"@B{escaped}@.") def add_argument(self, action): if action.help is not argparse.SUPPRESS: # find all invocations get_invocation = self._format_action_invocation invocation_lengths = [color.clen(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append( color.clen(get_invocation(subaction)) + self._current_indent ) # update the maximum item length action_length = max(invocation_lengths) self._action_max_length = max(self._action_max_length, action_length) # add the item to the list self._add_item(self._format_action, [action]) def _format_action(self, action): # this is where argparse aligns the help text next to each option help_position = min(self._action_max_length + 2, self._max_help_position) result = super()._format_action(action) # add color *after* argparse aligns the text, so we don't interfere with lengths if len(result) <= help_position: header, rest = result, "" elif result[help_position - 1] == " ": header, rest = result[:help_position], result[help_position:] else: first_newline = result.index("\n") header, rest = result[:first_newline], result[first_newline:] return color.colorize(f"@c{{{color.cescape(header)}}}{color.cescape(rest)}") def add_arguments(self, actions): actions = sorted(actions, key=operator.attrgetter("option_strings")) super().add_arguments(actions) class SpackArgumentParser(argparse.ArgumentParser): def format_help_sections(self, level): """Format help on sections for a particular verbosity level. Args: level (str): ``"short"`` or ``"long"`` (more commands shown for long) """ if level not in levels: raise ValueError("level must be one of: %s" % levels) # lazily add all commands to the parser when needed. add_all_commands(self) # Print help on subcommands in neatly formatted sections. formatter = self._get_formatter() # Create a list of subcommand actions. Argparse internals are nasty! # Note: you can only call _get_subactions() once. Even nastier! if not hasattr(self, "actions"): self.actions = self._subparsers._actions[-1]._get_subactions() # make a set of commands not yet added. remaining = set(spack.cmd.all_commands()) def add_group(group): formatter.start_section(group.title) formatter.add_text(group.description) formatter.add_arguments(group._group_actions) formatter.end_section() def add_subcommand_group(title, commands): """Add informational help group for a specific subcommand set.""" cmd_set = set(c for c in commands) # make a dict of commands of interest cmds = dict((a.dest, a) for a in self.actions if a.dest in cmd_set) # add commands to a group in order, and add the group group = argparse._ArgumentGroup(self, title=title) for name in commands: group._add_action(cmds[name]) if name in remaining: remaining.remove(name) add_group(group) # select only the options for the particular level we're showing. show_options = options_by_level[level] options = [ opt for group in self._action_groups for opt in group._group_actions if group.title not in ["positional arguments"] ] opts = {opt.option_strings[0].strip("-"): opt for opt in options} actions = [o for o in opts.values()] if show_options != "all": actions = [opts[letter] for letter in show_options] # custom, more concise usage for top level help_options = actions + [self._positionals._group_actions[-1]] formatter.add_usage(self.usage, help_options, self._mutually_exclusive_groups) # description formatter.add_text(self.description) # start subcommands formatter.add_text(color.colorize(f"@*C{{{intro_by_level[level]}}}")) # add argument groups based on metadata in commands index = index_commands() sections = index[level] for section in sorted(sections): if section == "help": continue # Cover help in the epilog. group_description = section_descriptions.get(section, section) to_display = sections[section] commands = [] # add commands whose order we care about first. if section in section_order: commands.extend(cmd for cmd in section_order[section] if cmd in to_display) # add rest in alphabetical order. commands.extend(cmd for cmd in sorted(sections[section]) if cmd not in commands) # add the group to the parser add_subcommand_group(group_description, commands) # start subcommands formatter.add_text(color.colorize("@*C{Options:}")) # optionals and user-defined groups for group in sorted( self._action_groups, key=lambda g: (g.title == "help", g.title != "general", g.title) ): if group.title == "positional arguments": continue # handled by subcommand help above filtered_actions = [action for action in group._group_actions if action in actions] if not filtered_actions: continue formatter.start_section(group.title) formatter.add_text(group.description) formatter.add_arguments(filtered_actions) formatter.end_section() # epilog help_section = textwrap.dedent( """\ @*C{More help}: @c{spack help --all} list all commands and options @c{spack help } help on a specific command @c{spack help --spec} help on the package specification syntax @c{spack docs} open https://spack.rtfd.io/ in a browser """ ) formatter.add_text(color.colorize(help_section)) # determine help from format above return formatter.format_help() def add_subparsers(self, **kwargs): """Ensure that sensible defaults are propagated to subparsers""" kwargs.setdefault("metavar", "SUBCOMMAND") # From Python 3.7 we can require a subparser, earlier versions # of argparse will error because required=True is unknown if sys.version_info[:2] > (3, 6): kwargs.setdefault("required", True) sp = super().add_subparsers(**kwargs) # This monkey patching is needed for Python 3.6, which supports # having a required subparser but don't expose the API used above if sys.version_info[:2] == (3, 6): sp.required = True old_add_parser = sp.add_parser def add_parser(name, **kwargs): kwargs.setdefault("formatter_class", SpackHelpFormatter) return old_add_parser(name, **kwargs) sp.add_parser = add_parser return sp def add_command(self, cmd_name): """Add one subcommand to this parser.""" # lazily initialize any subparsers if not hasattr(self, "subparsers"): # remove the dummy "command" argument. if self._actions[-1].dest == "command": self._remove_action(self._actions[-1]) self.subparsers = self.add_subparsers(metavar="COMMAND", dest="command") if cmd_name not in self.subparsers._name_parser_map: # each command module implements a parser() function, to which we # pass its subparser for setup. module = spack.cmd.get_module(cmd_name) # build a list of aliases alias_list = [] aliases = spack.config.get("config:aliases") if aliases: alias_list = [k for k, v in aliases.items() if shlex.split(v)[0] == cmd_name] subparser = self.subparsers.add_parser( cmd_name, aliases=alias_list, help=module.description, description=module.description, ) module.setup_parser(subparser) # return the callable function for the command return spack.cmd.get_command(cmd_name) def format_help(self, level="short"): if self.prog == "spack": # use format_help_sections for the main spack parser, but not # for subparsers return self.format_help_sections(level) else: # in subparsers, self.prog is, e.g., 'spack install' return super().format_help() def _check_value(self, action, value): # converted value must be one of the choices (if specified) if action.choices is not None and value not in action.choices: cols = spack.llnl.util.tty.colify.colified(sorted(action.choices), indent=4, tty=True) msg = "invalid choice: %r choose from:\n%s" % (value, cols) raise argparse.ArgumentError(action, msg) def make_argument_parser(**kwargs): """Create an basic argument parser without any subcommands added.""" parser = SpackArgumentParser( prog="spack", formatter_class=SpackHelpFormatter, add_help=False, description=( "A flexible package manager that supports multiple versions,\n" "configurations, platforms, and compilers." ), **kwargs, ) general = parser.add_argument_group("general") general.add_argument( "--color", action="store", default=None, choices=("always", "never", "auto"), help="when to colorize output (default: auto)", ) general.add_argument( "-v", "--verbose", action="store_true", help="print additional output during builds" ) general.add_argument( "-k", "--insecure", action="store_true", help="do not check ssl certificates when downloading", ) general.add_argument( "-b", "--bootstrap", action="store_true", help="use bootstrap config, store, and externals" ) general.add_argument( "-V", "--version", action="store_true", help="show version number and exit" ) general.add_argument( "-h", "--help", dest="help", action="store_const", const="short", default=None, help="show this help message and exit", ) general.add_argument( "-H", "--all-help", dest="help", action="store_const", const="long", default=None, help="show help for all commands (same as `spack help --all`)", ) config = parser.add_argument_group("configuration and environments") config.add_argument( "-c", "--config", default=None, action="append", dest="config_vars", help="add one or more custom, one-off config settings", ) config.add_argument( "-C", "--config-scope", dest="config_scopes", action="append", metavar="DIR|ENV", help="add directory or environment as read-only config scope", ) envs = config # parser.add_argument_group("environments") env_mutex = envs.add_mutually_exclusive_group() env_mutex.add_argument( "-e", "--env", dest="env", metavar="ENV", action="store", help="run with an environment" ) env_mutex.add_argument( "-D", "--env-dir", dest="env_dir", metavar="DIR", action="store", help="run with environment in directory (ignore managed envs)", ) env_mutex.add_argument( "-E", "--no-env", dest="no_env", action="store_true", help="run without any environments activated (see spack env)", ) envs.add_argument( "--use-env-repo", action="store_true", help="when in an environment, use its package repository", ) debug = parser.add_argument_group("debug") debug.add_argument( "-d", "--debug", action="count", default=0, help="write out debug messages\n\n(more d's for more verbosity: -d, -dd, -ddd, etc.)", ) debug.add_argument( "-t", "--backtrace", action="store_true", default="SPACK_BACKTRACE" in os.environ, help="always show backtraces for exceptions", ) debug.add_argument("--pdb", action="store_true", help=argparse.SUPPRESS) debug.add_argument("--timestamp", action="store_true", help="add a timestamp to tty output") debug.add_argument( "-m", "--mock", action="store_true", help="use mock packages instead of real ones" ) debug.add_argument( "--print-shell-vars", action="store", help="print info needed by setup-env.*sh" ) debug.add_argument( "--stacktrace", action="store_true", default="SPACK_STACKTRACE" in os.environ, help="add stacktraces to all printed statements", ) locks = general lock_mutex = locks.add_mutually_exclusive_group() lock_mutex.add_argument( "-l", "--enable-locks", action="store_true", dest="locks", default=None, help="use filesystem locking (default)", ) lock_mutex.add_argument( "-L", "--disable-locks", action="store_false", dest="locks", help="do not use filesystem locking (unsafe)", ) debug.add_argument( "-p", "--profile", action="store_true", dest="spack_profile", help=argparse.SUPPRESS ) debug.add_argument("--profile-file", default=None, help=argparse.SUPPRESS) debug.add_argument("--sorted-profile", default=None, metavar="STAT", help=argparse.SUPPRESS) debug.add_argument("--lines", default=20, action="store", help=argparse.SUPPRESS) return parser def showwarning(message, category, filename, lineno, file=None, line=None): """Redirects messages to tty.warn.""" if category is spack.error.SpackAPIWarning: tty.warn(f"{filename}:{lineno}: {message}") else: tty.warn(message) def setup_main_options(args): """Configure spack globals based on the basic options.""" # Set up environment based on args. tty.set_verbose(args.verbose) tty.set_debug(args.debug) tty.set_stacktrace(args.stacktrace) # debug must be set first so that it can even affect behavior of # errors raised by spack.config. if args.debug or args.backtrace: spack.error.debug = True spack.error.SHOW_BACKTRACE = True if args.debug: spack.config.set("config:debug", True, scope="command_line") spack.util.environment.TRACING_ENABLED = True if args.timestamp: tty.set_timestamp(True) # override lock configuration if passed on command line if args.locks is not None: if args.locks is False: spack.util.lock.check_lock_safety(spack.paths.prefix) spack.config.set("config:locks", args.locks, scope="command_line") if args.mock: import spack.util.spack_yaml as syaml key = syaml.syaml_str("repos") key.override = True spack.config.CONFIG.scopes["command_line"].sections["repos"] = syaml.syaml_dict( [(key, [spack.paths.mock_packages_path])] ) # If the user asked for it, don't check ssl certs. if args.insecure: tty.warn("You asked for --insecure. Will NOT check SSL certificates.") spack.config.set("config:verify_ssl", False, scope="command_line") # Use the spack config command to handle parsing the config strings for config_var in args.config_vars or []: spack.config.add(fullpath=config_var, scope="command_line") # On Windows10 console handling for ASCI/VT100 sequences is not # on by default. Turn on before we try to write to console # with color color.try_enable_terminal_color_on_windows() # when to use color (takes always, auto, or never) if args.color is not None: color.set_color_when(args.color) def allows_unknown_args(command): """Implements really simple argument injection for unknown arguments. Commands may add an optional argument called "unknown args" to indicate they can handle unknown args, and we'll pass the unknown args in. """ info = dict(inspect.getmembers(command)) varnames = info["__code__"].co_varnames argcount = info["__code__"].co_argcount return argcount == 3 and varnames[2] == "unknown_args" def _invoke_command(command, parser, args, unknown_args): """Run a spack command *without* setting spack global options.""" if allows_unknown_args(command): return_val = command(parser, args, unknown_args) else: if unknown_args: tty.die("unrecognized arguments: %s" % " ".join(unknown_args)) return_val = command(parser, args) # Allow commands to return and error code if they want return 0 if return_val is None else return_val class SpackCommand: """Callable object that invokes a Spack command (for testing). Example usage:: install = SpackCommand("install") install("-v", "mpich") Use this to invoke Spack commands directly from Python and check their output.""" def __init__(self, command_name: str) -> None: """Create a new SpackCommand that invokes ``command_name`` when called. Args: command_name: name of the command to invoke """ self.parser = make_argument_parser() self.command_name = command_name #: Return code of the last command invocation self.returncode: Any = None #: Error raised during the last command invocation, if any self.error: Optional[BaseException] = None #: Binary output captured from the last command invocation self.binary_output = b"" #: Decoded output captured from the last command invocation self.output = "" def __call__(self, *argv: str, capture: bool = True, fail_on_error: bool = True) -> str: """Invoke this SpackCommand. Returns the combined stdout/stderr. Args: argv: command line arguments. Keyword Args: capture: Capture output from the command fail_on_error: Don't raise an exception on error On return, if ``fail_on_error`` is False, return value of command is set in ``returncode`` property, and the error is set in the ``error`` property. Otherwise, raise an error.""" self.returncode = None self.error = None self.binary_output = b"" self.output = "" try: with self.capture_output(enable=capture): command = self.parser.add_command(self.command_name) args, unknown = self.parser.parse_known_args([self.command_name, *argv]) setup_main_options(args) self.returncode = _invoke_command(command, self.parser, args, unknown) except SystemExit as e: # When the command calls sys.exit instead of returning an exit code self.error = e self.returncode = e.code except BaseException as e: # For other exceptions, raise the original exception if fail_on_error is True self.error = e if fail_on_error: raise finally: self.output = self.binary_output.decode("utf-8", errors="replace") if fail_on_error and self.returncode not in (0, None): raise SpackCommandError(self.returncode, self.output) from self.error return self.output @contextmanager def capture_output(self, enable: bool = True): """Captures stdout and stderr from the current process and all subprocesses. This uses a temporary file and os.dup2 to redirect file descriptors.""" if not enable: yield self return with tempfile.TemporaryFile(mode="w+b") as tmp_file: # sys.stdout and sys.stderr may have been replaced with file objects under pytest, so # redirect their file descriptors in addition to the original fds 1 and 2. fds: Set[int] = {sys.stdout.fileno(), sys.stderr.fileno(), 1, 2} saved_fds = {fd: os.dup(fd) for fd in fds} sys.stdout.flush() sys.stderr.flush() for fd in fds: os.dup2(tmp_file.fileno(), fd) try: yield self finally: sys.stdout.flush() sys.stderr.flush() for fd, saved_fd in saved_fds.items(): os.dup2(saved_fd, fd) os.close(saved_fd) tmp_file.seek(0) self.binary_output = tmp_file.read() def _profile_wrapper(command, main_args, parser, args, unknown_args): import cProfile try: nlines = int(main_args.lines) except ValueError: if main_args.lines != "all": tty.die("Invalid number for --lines: %s" % main_args.lines) nlines = -1 # allow comma-separated list of fields sortby = ["time"] if main_args.sorted_profile: sortby = main_args.sorted_profile.split(",") for stat in sortby: if stat not in stat_names: tty.die("Invalid sort field: %s" % stat) try: # make a profiler and run the code. pr = cProfile.Profile() pr.enable() return _invoke_command(command, parser, args, unknown_args) finally: pr.disable() if main_args.profile_file: pr.dump_stats(main_args.profile_file) # print out profile stats. stats = pstats.Stats(pr, stream=sys.stderr) stats.sort_stats(*sortby) stats.print_stats(nlines) @spack.llnl.util.lang.memoized def _compatible_sys_types(): """Return a list of all the platform-os-target tuples compatible with the current host. """ host_platform = spack.platforms.host() host_os = str(host_platform.default_operating_system()) host_target = spack.vendor.archspec.cpu.host() compatible_targets = [host_target] + host_target.ancestors compatible_archs = [ str(spack.spec.ArchSpec((str(host_platform), host_os, str(target)))) for target in compatible_targets ] return compatible_archs def print_setup_info(*info): """Print basic information needed by setup-env.[c]sh. Args: info (list): list of things to print: comma-separated list of ``"csh"``, ``"sh"``, or ``"modules"`` This is in ``main.py`` to make it fast; the setup scripts need to invoke spack in login scripts, and it needs to be quick. """ from spack.modules.common import root_path shell = "csh" if "csh" in info else "sh" def shell_set(var, value): if shell == "sh": print("%s='%s'" % (var, value)) elif shell == "csh": print("set %s = '%s'" % (var, value)) else: tty.die("shell must be sh or csh") # print sys type shell_set("_sp_sys_type", str(spack.spec.ArchSpec.default_arch())) shell_set("_sp_compatible_sys_types", ":".join(_compatible_sys_types())) # print roots for all module systems module_to_roots = {"tcl": list(), "lmod": list()} for name in module_to_roots.keys(): path = root_path(name, "default") module_to_roots[name].append(path) other_spack_instances = spack.config.get("upstreams") or {} for install_properties in other_spack_instances.values(): upstream_module_roots = install_properties.get("modules", {}) upstream_module_roots = dict( (k, v) for k, v in upstream_module_roots.items() if k in module_to_roots ) for module_type, root in upstream_module_roots.items(): module_to_roots[module_type].append(root) for name, paths in module_to_roots.items(): # Environment setup prepends paths, so the order is reversed here to # preserve the intended priority: the modules of the local Spack # instance are the highest-precedence. roots_val = ":".join(reversed(paths)) shell_set("_sp_%s_roots" % name, roots_val) def restore_macos_dyld_vars(): """ Spack mutates ``DYLD_*`` variables in ``spack load`` and ``spack env activate``. Unlike Linux, macOS SIP clears these variables in new processes, meaning that ``os.environ["DYLD_*"]`` in our Python process is not the same as the user's shell. Therefore, we store the user's ``DYLD_*`` variables in ``SPACK_DYLD_*`` and restore them here. """ if not sys.platform == "darwin": return for dyld_var in ("DYLD_LIBRARY_PATH", "DYLD_FALLBACK_LIBRARY_PATH"): stored_var_name = f"SPACK_{dyld_var}" if stored_var_name in os.environ: os.environ[dyld_var] = os.environ[stored_var_name] def resolve_alias(cmd_name: str, cmd: List[str]) -> Tuple[str, List[str]]: """Resolves aliases in the given command. Args: cmd_name: command name. cmd: command line arguments. Returns: new command name and arguments. """ all_commands = spack.cmd.all_commands() aliases = spack.config.get("config:aliases") if aliases: for key, value in aliases.items(): if " " in key: tty.warn( f"Alias '{key}' (mapping to '{value}') contains a space" ", which is not supported." ) if key in all_commands: tty.warn( f"Alias '{key}' (mapping to '{value}') attempts to override built-in command." ) if cmd_name not in all_commands: alias = None if aliases: alias = aliases.get(cmd_name) if alias is not None: alias_parts = shlex.split(alias) cmd_name = alias_parts[0] cmd = alias_parts + cmd[1:] return cmd_name, cmd # sentinel scope marker for environments passed on the command line _ENV = object() def add_command_line_scopes( cfg: spack.config.Configuration, command_line_scopes: List[str] ) -> None: """Add additional scopes from the ``--config-scope`` argument, either envs or dirs. Args: cfg: configuration instance command_line_scopes: list of configuration scope paths Raises: spack.error.ConfigError: if the path is an invalid configuration scope """ for i, path in enumerate(command_line_scopes): name = f"cmd_scope_{i}" scope = ev.environment_path_scope(name, path) if scope is None: if os.path.isdir(path): # directory with config files cfg.push_scope( spack.config.DirectoryConfigScope(name, path, writable=False), priority=ConfigScopePriority.CUSTOM, ) continue else: raise spack.error.ConfigError(f"Invalid configuration scope: {path}") cfg.push_scope(scope, priority=ConfigScopePriority.CUSTOM) def _main(argv=None): """Logic for the main entry point for the Spack command. ``main()`` calls ``_main()`` and catches any errors that emerge. ``_main()`` handles: 1. Parsing arguments; 2. Setting up configuration; and 3. Finding and executing a Spack command. Args: argv (list or None): command line arguments, NOT including the executable name. If None, parses from ``sys.argv``. """ # ------------------------------------------------------------------------ # main() is tricky to get right, so be careful where you put things. # # Things in this first part of `main()` should *not* require any # configuration. This doesn't include much -- setting up the parser, # restoring some key environment variables, very simple CLI options, etc. # ------------------------------------------------------------------------ warnings.showwarning = showwarning # Create a parser with a simple positional argument first. We'll # lazily load the subcommand(s) we need later. This allows us to # avoid loading all the modules from spack.cmd when we don't need # them, which reduces startup latency. parser = make_argument_parser() parser.add_argument("command", nargs=argparse.REMAINDER) args = parser.parse_args(argv) # Just print help and exit if run with no arguments at all no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0) if no_args: parser.print_help() return 1 # version is special as it does not require a command or loading and additional infrastructure if args.version: print(spack.get_version()) return 0 # ------------------------------------------------------------------------ # This part of the `main()` sets up Spack's configuration. # # We set command line options (like --debug), then command line config # scopes, then environment configuration here. # ------------------------------------------------------------------------ # Make spack load / env activate work on macOS restore_macos_dyld_vars() # store any error that occurred loading an env env_format_error = None env = None # try to find an active environment here, so that we can activate it later if not args.no_env: try: env = spack.cmd.find_environment(args) except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError) as e: # print the context but delay this exception so that commands like # `spack config edit` can still work with a bad environment. e.print_context() env_format_error = e def add_environment_scope(): if env_format_error: # Allow command to continue without env in case it is `spack config edit` # All other cases will raise in `finish_parse_and_run` spack.environment.environment._active_environment_error = env_format_error return # do not call activate here, as it has a lot of expensive function calls to deal # with mutation of spack.config.CONFIG -- but we are still building the config. env.manifest.prepare_config_scope() spack.environment.environment._active_environment = env # add the environment if env: add_environment_scope() # Push scopes from the command line last if args.config_scopes: add_command_line_scopes(spack.config.CONFIG, args.config_scopes) spack.config.CONFIG.push_scope( spack.config.InternalConfigScope("command_line"), priority=ConfigScopePriority.COMMAND_LINE ) setup_main_options(args) # ------------------------------------------------------------------------ # Things that require configuration should go below here # ------------------------------------------------------------------------ if args.print_shell_vars: print_setup_info(*args.print_shell_vars.split(",")) return 0 # -h and -H are special as they do not require a command, but # all the other options do nothing without a command. if args.help: sys.stdout.write(parser.format_help(level=args.help)) return 0 # At this point we've considered all the options to spack itself, so we # need a command or we're done. if not args.command: parser.print_help() return 1 # Try to load the particular command the caller asked for. cmd_name = args.command[0] cmd_name, args.command = resolve_alias(cmd_name, args.command) # set up a bootstrap context, if asked. # bootstrap context needs to include parsing the command, b/c things # like `ConstraintAction` and `ConfigSetAction` happen at parse time. bootstrap_context = spack.llnl.util.lang.nullcontext() if args.bootstrap: import spack.bootstrap as bootstrap # avoid circular imports bootstrap_context = bootstrap.ensure_bootstrap_configuration() with bootstrap_context: return finish_parse_and_run(parser, cmd_name, args, env_format_error) def finish_parse_and_run(parser, cmd_name, main_args, env_format_error): """Finish parsing after we know the command to run.""" # add the found command to the parser and re-run then re-parse command = parser.add_command(cmd_name) args, unknown = parser.parse_known_args(main_args.command) # we need to inherit verbose since the install command checks for it args.verbose = main_args.verbose args.lines = main_args.lines # Now that we know what command this is and what its args are, determine # whether we can continue with a bad environment and raise if not. if env_format_error: subcommand = getattr(args, "config_command", None) if (cmd_name, subcommand) != ("config", "edit"): raise env_format_error # many operations will fail without a working directory. spack.paths.set_working_dir() # now we can actually execute the command. if main_args.spack_profile or main_args.sorted_profile or main_args.profile_file: new_args = [sys.executable, "-m", "cProfile"] if main_args.sorted_profile: new_args.extend(["-s", main_args.sorted_profile]) if main_args.profile_file: new_args.extend(["-o", main_args.profile_file]) new_args.append(spack.paths.spack_script) skip_next = False for arg in sys.argv[1:]: if skip_next: skip_next = False continue if arg in ("--sorted-profile", "--profile-file", "--lines"): skip_next = True continue if arg.startswith(("--sorted-profile=", "--profile-file=", "--lines=")): continue if arg in ("--profile", "-p"): continue new_args.append(arg) formatted_args = " ".join(shlex.quote(a) for a in new_args) tty.warn( "The --profile flag is deprecated and will be removed in Spack v1.3. " f"Use `{formatted_args}` instead." ) _profile_wrapper(command, main_args, parser, args, unknown) elif main_args.pdb: new_args = [sys.executable, "-m", "pdb", spack.paths.spack_script] new_args.extend(arg for arg in sys.argv[1:] if arg != "--pdb") formatted_args = " ".join(shlex.quote(arg) for arg in new_args) tty.warn( "The --pdb flag is deprecated and will be removed in Spack v1.3. " f"Use `{formatted_args}` instead." ) import pdb pdb.runctx("_invoke_command(command, parser, args, unknown)", globals(), locals()) return 0 else: return _invoke_command(command, parser, args, unknown) def main(argv=None): """This is the entry point for the Spack command. ``main()`` itself is just an error handler -- it handles errors for everything in Spack that makes it to the top level. The logic is all in ``_main()``. Args: argv (list or None): command line arguments, NOT including the executable name. If None, parses from sys.argv. """ # When using the forkserver start method, preload the following modules to improve startup # time of child processes. multiprocessing.set_forkserver_preload(["spack.main", "spack.package", "spack.new_installer"]) try: g0, g1, g2 = gc.get_threshold() gc.set_threshold(50 * g0, g1, g2) return _main(argv) except spack.solver.asp.OutputDoesNotSatisfyInputError as e: _handle_solver_bug(e) return 1 except spack.error.SpackError as e: tty.debug(e) e.die() # gracefully die on any SpackErrors except KeyboardInterrupt: if spack.config.get("config:debug") or spack.error.SHOW_BACKTRACE: raise sys.stderr.write("\n") tty.error("Keyboard interrupt.") return signal.SIGINT.value except SystemExit as e: if spack.config.get("config:debug") or spack.error.SHOW_BACKTRACE: traceback.print_exc() return e.code except Exception as e: if spack.config.get("config:debug") or spack.error.SHOW_BACKTRACE: raise tty.error(e) return 3 def _handle_solver_bug( e: spack.solver.asp.OutputDoesNotSatisfyInputError, out=sys.stderr, root=None ) -> None: # when the solver outputs specs that do not satisfy the input and spack is used as a command # line tool, we dump the incorrect output specs to json so users can upload them in bug reports wrong_output = [(input, output) for input, output in e.input_to_output if output is not None] no_output = [input for input, output in e.input_to_output if output is None] if no_output: tty.error( "internal solver error: the following specs were not solved:\n - " + "\n - ".join(str(s) for s in no_output), stream=out, ) if wrong_output: msg = "internal solver error: the following specs were concretized, but do not satisfy " msg += "the input:\n" for in_spec, out_spec in wrong_output: msg += f" - input: {in_spec}\n" msg += f" output: {out_spec.long_spec}\n" msg += "\n Please report a bug at https://github.com/spack/spack/issues" # try to write the input/output specs to a temporary directory for bug reports try: tmpdir = tempfile.mkdtemp(prefix="spack-asp-", dir=root) files = [] for i, (input, output) in enumerate(wrong_output, start=1): in_file = os.path.join(tmpdir, f"input-{i}.json") out_file = os.path.join(tmpdir, f"output-{i}.json") files.append(in_file) files.append(out_file) with open(in_file, "w", encoding="utf-8") as f: input.to_json(f) with open(out_file, "w", encoding="utf-8") as f: output.to_json(f) msg += " and attach the following files:\n - " + "\n - ".join(files) except Exception: msg += "." tty.error(msg, stream=out) class SpackCommandError(Exception): """Raised when SpackCommand execution fails, replacing SystemExit.""" def __init__(self, code, output): self.code = code self.output = output super().__init__(f"Spack command failed with exit code {code}") ================================================ FILE: lib/spack/spack/mirrors/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/mirrors/layout.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os from typing import TYPE_CHECKING, Optional import spack.fetch_strategy import spack.llnl.url import spack.oci.image import spack.repo from spack.error import MirrorError from spack.llnl.util.filesystem import mkdirp, symlink if TYPE_CHECKING: import spack.spec class MirrorLayout: """A ``MirrorLayout`` object describes the relative path of a mirror entry.""" def __init__(self, path: str) -> None: self.path = path def __iter__(self): """Yield all paths including aliases where the resource can be found.""" yield self.path def make_alias(self, root: str) -> None: """Make the entry ``root / self.path`` available under a human readable alias""" pass class DefaultLayout(MirrorLayout): def __init__(self, alias_path: str, digest_path: Optional[str] = None) -> None: # When we have a digest, it is used as the primary storage location. If not, then we use # the human-readable alias. In case of mirrors of a VCS checkout, we currently do not have # a digest, that's why an alias is required and a digest optional. super().__init__(path=digest_path or alias_path) self.alias = alias_path self.digest_path = digest_path def make_alias(self, root: str) -> None: """Symlink a human readable path in our mirror to the actual storage location.""" # We already use the human-readable path as the main storage location. if not self.digest_path: return alias, digest = os.path.join(root, self.alias), os.path.join(root, self.digest_path) alias_dir = os.path.dirname(alias) relative_dst = os.path.relpath(digest, start=alias_dir) mkdirp(alias_dir) tmp = f"{alias}.tmp" symlink(relative_dst, tmp) try: os.rename(tmp, alias) except OSError: # Clean up the temporary if possible try: os.unlink(tmp) except OSError: pass raise def __iter__(self): if self.digest_path: yield self.digest_path yield self.alias class OCILayout(MirrorLayout): """Follow the OCI Image Layout Specification to archive blobs where paths are of the form ``blobs//``""" def __init__(self, digest: spack.oci.image.Digest) -> None: super().__init__(os.path.join("blobs", digest.algorithm, digest.digest)) def _determine_extension(fetcher): if isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy): if fetcher.expand_archive: # If we fetch with a URLFetchStrategy, use URL's archive type ext = spack.llnl.url.determine_url_file_extension(fetcher.url) if ext: # Remove any leading dots ext = ext.lstrip(".") else: msg = """\ Unable to parse extension from {0}. If this URL is for a tarball but does not include the file extension in the name, you can explicitly declare it with the following syntax: version('1.2.3', 'hash', extension='tar.gz') If this URL is for a download like a .jar or .whl that does not need to be expanded, or an uncompressed installation script, you can tell Spack not to expand it with the following syntax: version('1.2.3', 'hash', expand=False) """ raise MirrorError(msg.format(fetcher.url)) else: # If the archive shouldn't be expanded, don't check extension. ext = None else: # Otherwise we'll make a .tar.gz ourselves ext = "tar.gz" return ext def default_mirror_layout( fetcher: "spack.fetch_strategy.FetchStrategy", per_package_ref: str, spec: Optional["spack.spec.Spec"] = None, ) -> MirrorLayout: """Returns a ``MirrorReference`` object which keeps track of the relative storage path of the resource associated with the specified ``fetcher``.""" ext = None if spec: pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) versions = pkg_cls.versions.get(spec.version, {}) ext = versions.get("extension", None) # If the spec does not explicitly specify an extension (the default case), # then try to determine it automatically. An extension can only be # specified for the primary source of the package (e.g. the source code # identified in the 'version' declaration). Resources/patches don't have # an option to specify an extension, so it must be inferred for those. ext = ext or _determine_extension(fetcher) if ext: per_package_ref += ".%s" % ext global_ref = fetcher.mirror_id() if global_ref: global_ref = os.path.join("_source-cache", global_ref) if global_ref and ext: global_ref += ".%s" % ext return DefaultLayout(per_package_ref, global_ref) ================================================ FILE: lib/spack/spack/mirrors/mirror.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import operator import os import urllib.parse from typing import Any, Dict, List, Mapping, Optional, Tuple, Union import spack.config import spack.llnl.util.tty as tty import spack.util.path import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.util.url as url_util from spack.error import MirrorError from spack.oci.image import is_oci_url #: What schemes do we support supported_url_schemes = ("file", "http", "https", "sftp", "ftp", "s3", "gs", "oci", "oci+http") #: The layout version spack can current install SUPPORTED_LAYOUT_VERSIONS = (3, 2) def _url_or_path_to_url(url_or_path: str) -> str: """For simplicity we allow mirror URLs in config files to be local, relative paths. This helper function takes care of distinguishing between URLs and paths, and canonicalizes paths before transforming them into file:// URLs.""" # Is it a supported URL already? Then don't do path-related canonicalization. parsed = urllib.parse.urlparse(url_or_path) if parsed.scheme in supported_url_schemes: return url_or_path # Otherwise we interpret it as path, and we should promote it to file:// URL. return url_util.path_to_file_url(spack.util.path.canonicalize_path(url_or_path)) class Mirror: """Represents a named location for storing source tarballs and binary packages. Mirrors have a fetch_url that indicate where and how artifacts are fetched from them, and a push_url that indicate where and how artifacts are pushed to them. These two URLs are usually the same. """ def __init__(self, data: Union[str, dict], name: Optional[str] = None): self._data = data self._name = name @staticmethod def from_yaml(stream, name=None): return Mirror(syaml.load(stream), name) @staticmethod def from_json(stream, name=None): try: return Mirror(sjson.load(stream), name) except Exception as e: raise sjson.SpackJSONError("error parsing JSON mirror:", str(e)) from e @staticmethod def from_local_path(path: str): return Mirror(url_util.path_to_file_url(path)) @staticmethod def from_url(url: str): """Create an anonymous mirror by URL. This method validates the URL.""" if urllib.parse.urlparse(url).scheme not in supported_url_schemes: raise ValueError( f'"{url}" is not a valid mirror URL. ' f"Scheme must be one of {supported_url_schemes}." ) return Mirror(url) def __eq__(self, other): if not isinstance(other, Mirror): return NotImplemented return self._data == other._data and self._name == other._name def __str__(self): return f"{self._name}: {self.push_url} {self.fetch_url}" def __repr__(self): return f"Mirror(name={self._name!r}, data={self._data!r})" def to_json(self, stream=None): return sjson.dump(self.to_dict(), stream) def to_yaml(self, stream=None): return syaml.dump(self.to_dict(), stream) def to_dict(self): return self._data def display(self, max_len=0): fetch, push = self.fetch_url, self.push_url # don't print the same URL twice url = fetch if fetch == push else f"fetch: {fetch} push: {push}" source = "s" if self.source else " " binary = "b" if self.binary else " " print(f"{self.name: <{max_len}} [{source}{binary}] {url}") @property def name(self): return self._name or "" @property def binary(self): return isinstance(self._data, str) or self._data.get("binary", True) @property def source(self): return isinstance(self._data, str) or self._data.get("source", True) @property def signed(self) -> bool: # TODO: OCI support signing # Only checking for fetch, push is handled by OCI implementation if is_oci_url(self.fetch_url): return False return isinstance(self._data, str) or self._data.get("signed", True) @property def autopush(self) -> bool: if isinstance(self._data, str): return False return self._data.get("autopush", False) @property def fetch_url(self): """Get the valid, canonicalized fetch URL""" return self.get_url("fetch") @property def push_url(self): """Get the valid, canonicalized fetch URL""" return self.get_url("push") @property def fetch_view(self): """Get the valid, canonicalized fetch URL""" return self.get_view("fetch") @property def push_view(self): """Get the valid, canonicalized fetch URL""" return self.get_view("push") def ensure_mirror_usable(self, direction: str = "push"): access_pair = self._get_value("access_pair", direction) access_token_variable = self._get_value("access_token_variable", direction) errors = [] # Verify that the credentials that are variables expand if access_pair and isinstance(access_pair, dict): if "id_variable" in access_pair and access_pair["id_variable"] not in os.environ: errors.append(f"id_variable {access_pair['id_variable']} not set in environment") if "secret_variable" in access_pair: if access_pair["secret_variable"] not in os.environ: errors.append( f"environment variable `{access_pair['secret_variable']}` " "(secret_variable) not set" ) if access_token_variable: if access_token_variable not in os.environ: errors.append( f"environment variable `{access_pair['access_token_variable']}` " "(access_token_variable) not set" ) if errors: msg = f"invalid {direction} configuration for mirror {self.name}: " msg += "\n ".join(errors) raise MirrorError(msg) @property def supported_layout_versions(self) -> List[int]: """List all of the supported layouts a mirror can fetch from""" # Only check the fetch configuration, the push configuration is whatever the latest # mirror version is which should support all configurable features. # All configured mirrors support the latest version supported_versions = [SUPPORTED_LAYOUT_VERSIONS[0]] has_view = self.fetch_view is not None # Check if the mirror supports older layout versions # OCI - Only return the newest version, the layout version is a dummy version since OCI # has its own layout. # Views - Only versions >=3 support the views feature if not is_oci_url(self.fetch_url) and not has_view: supported_versions.extend(SUPPORTED_LAYOUT_VERSIONS[1:]) return supported_versions def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool): # Only allow one to exist in the config if "access_token" in current_data and "access_token_variable" in new_data: current_data.pop("access_token") elif "access_token_variable" in current_data and "access_token" in new_data: current_data.pop("access_token_variable") # If updating to a new access_pair that is the deprecated list, warn warn_deprecated_access_pair = False if "access_pair" in new_data: warn_deprecated_access_pair = isinstance(new_data["access_pair"], list) # If the not updating the current access_pair, and it is the deprecated list, warn elif "access_pair" in current_data: warn_deprecated_access_pair = isinstance(current_data["access_pair"], list) if warn_deprecated_access_pair: tty.warn( f"in mirror {self.name}: support for plain text secrets in config files " "(access_pair: [id, secret]) is deprecated and will be removed in a future Spack " "version. Use environment variables instead (access_pair: " "{id: ..., secret_variable: ...})" ) keys = [ "url", "access_pair", "access_token", "access_token_variable", "profile", "endpoint_url", ] if top_level: keys += ["binary", "source", "signed", "autopush"] changed = False for key in keys: if key in new_data and current_data.get(key) != new_data[key]: current_data[key] = new_data[key] changed = True return changed def update(self, data: dict, direction: Optional[str] = None) -> bool: """Modify the mirror with the given data. This takes care of expanding trivial mirror definitions by URL to something more rich with a dict if necessary Args: data (dict): The data to update the mirror with. direction (str): The direction to update the mirror in (fetch or push or None for top-level update) Returns: bool: True if the mirror was updated, False otherwise.""" # Modify the top-level entry when no direction is given. if not data: return False # If we only update a URL, there's typically no need to expand things to a dict. set_url = data["url"] if len(data) == 1 and "url" in data else None if direction is None: # First deal with the case where the current top-level entry is just a string. if isinstance(self._data, str): # Can we replace that string with something new? if set_url: if self._data == set_url: return False self._data = set_url return True # Otherwise promote to a dict self._data = {"url": self._data} # And update the dictionary accordingly. return self._update_connection_dict(self._data, data, top_level=True) # Otherwise, update the fetch / push entry; turn top-level # url string into a dict if necessary. if isinstance(self._data, str): self._data = {"url": self._data} # Create a new fetch / push entry if necessary if direction not in self._data: # Keep config minimal if we're just setting the URL. if set_url: self._data[direction] = set_url return True self._data[direction] = {} entry = self._data[direction] # Keep the entry simple if we're just swapping out the URL. if isinstance(entry, str): if set_url: if entry == set_url: return False self._data[direction] = set_url return True # Otherwise promote to a dict self._data[direction] = {"url": entry} return self._update_connection_dict(self._data[direction], data, top_level=False) def _get_value(self, attribute: str, direction: str): """Returns the most specific value for a given attribute (either push/fetch or global)""" if direction not in ("fetch", "push"): raise ValueError(f"direction must be either 'fetch' or 'push', not {direction}") if isinstance(self._data, str): return None # Either a string (url) or a dictionary, we care about the dict here. value = self._data.get(direction, {}) # Return top-level entry if only a URL was set. if isinstance(value, str) or attribute not in value: return self._data.get(attribute) return value[attribute] def get_url(self, direction: str) -> str: if direction not in ("fetch", "push"): raise ValueError(f"direction must be either 'fetch' or 'push', not {direction}") # Whole mirror config is just a url. if isinstance(self._data, str): return _url_or_path_to_url(self._data) # Default value url = self._data.get("url") # Override it with a direction-specific value if direction in self._data: # Either a url as string or a dict with url key info = self._data[direction] if isinstance(info, str): url = info elif "url" in info: url = info["url"] if not url: raise ValueError(f"Mirror {self.name} has no URL configured") return _url_or_path_to_url(url) def get_view(self, direction: str): return self._get_value("view", direction) def get_credentials(self, direction: str) -> Dict[str, Any]: """Get the mirror credentials from the mirror config Args: direction: fetch or push mirror config Returns: Dictionary from credential type string to value Credential Type Map: * ``access_token``: ``str`` * ``access_pair``: ``Tuple[str, str]`` * ``profile``: ``str`` """ creddict: Dict[str, Any] = {} access_token = self.get_access_token(direction) if access_token: creddict["access_token"] = access_token access_pair = self.get_access_pair(direction) if access_pair: creddict.update({"access_pair": access_pair}) profile = self.get_profile(direction) if profile: creddict["profile"] = profile return creddict def get_access_token(self, direction: str) -> Optional[str]: tok = self._get_value("access_token_variable", direction) if tok: return os.environ.get(tok) else: return self._get_value("access_token", direction) return None def get_access_pair(self, direction: str) -> Optional[Tuple[str, str]]: pair = self._get_value("access_pair", direction) if isinstance(pair, (tuple, list)) and len(pair) == 2: return (pair[0], pair[1]) if all(pair) else None elif isinstance(pair, dict): id_ = os.environ.get(pair["id_variable"]) if "id_variable" in pair else pair["id"] secret = os.environ.get(pair["secret_variable"]) return (id_, secret) if id_ and secret else None else: return None def get_profile(self, direction: str) -> Optional[str]: return self._get_value("profile", direction) def get_endpoint_url(self, direction: str) -> Optional[str]: return self._get_value("endpoint_url", direction) class MirrorCollection(Mapping[str, Mirror]): """A mapping of mirror names to mirrors.""" def __init__( self, mirrors=None, scope=None, binary: Optional[bool] = None, source: Optional[bool] = None, autopush: Optional[bool] = None, ): """Initialize a mirror collection. Args: mirrors: A name-to-mirror mapping to initialize the collection with. scope: The scope to use when looking up mirrors from the config. binary: If True, only include binary mirrors. If False, omit binary mirrors. If None, do not filter on binary mirrors. source: If True, only include source mirrors. If False, omit source mirrors. If None, do not filter on source mirrors. autopush: If True, only include mirrors that have autopush enabled. If False, omit mirrors that have autopush enabled. If None, do not filter on autopush.""" mirrors_data = ( mirrors.items() if mirrors is not None else spack.config.CONFIG.get_config("mirrors", scope=scope).items() ) mirrors = (Mirror(data=mirror, name=name) for name, mirror in mirrors_data) def _filter(m: Mirror): if source is not None and m.source != source: return False if binary is not None and m.binary != binary: return False if autopush is not None and m.autopush != autopush: return False return True self._mirrors = {m.name: m for m in mirrors if _filter(m)} def __eq__(self, other): return self._mirrors == other._mirrors def to_json(self, stream=None): return sjson.dump(self.to_dict(True), stream) def to_yaml(self, stream=None): return syaml.dump(self.to_dict(True), stream) # TODO: this isn't called anywhere @staticmethod def from_yaml(stream, name=None): data = syaml.load(stream) return MirrorCollection(data) @staticmethod def from_json(stream, name=None): try: d = sjson.load(stream) return MirrorCollection(d) except Exception as e: raise sjson.SpackJSONError("error parsing JSON mirror collection:", str(e)) from e def to_dict(self, recursive=False): return syaml.syaml_dict( sorted( ((k, (v.to_dict() if recursive else v)) for (k, v) in self._mirrors.items()), key=operator.itemgetter(0), ) ) @staticmethod def from_dict(d): return MirrorCollection(d) def __getitem__(self, item): return self._mirrors[item] def display(self): max_len = max(len(mirror.name) for mirror in self._mirrors.values()) for mirror in self._mirrors.values(): mirror.display(max_len) def lookup(self, name_or_url): """Looks up and returns a Mirror. If this MirrorCollection contains a named Mirror under the name [name_or_url], then that mirror is returned. Otherwise, [name_or_url] is assumed to be a mirror URL, and an anonymous mirror with the given URL is returned. """ result = self.get(name_or_url) if result is None: result = Mirror(fetch=name_or_url) return result def __iter__(self): return iter(self._mirrors) def __len__(self): return len(self._mirrors) ================================================ FILE: lib/spack/spack/mirrors/utils.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import traceback from collections import Counter import spack.caches import spack.config import spack.llnl.util.tty as tty import spack.repo import spack.spec import spack.util.spack_yaml as syaml import spack.version from spack.error import MirrorError from spack.llnl.util.filesystem import mkdirp from spack.mirrors.mirror import Mirror, MirrorCollection from spack.package import InstallError def get_all_versions(specs): """Given a set of initial specs, return a new set of specs that includes each version of each package in the original set. Note that if any spec in the original set specifies properties other than version, this information will be omitted in the new set; for example; the new set of specs will not include variant settings. """ version_specs = [] for spec in specs: pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) # Skip any package that has no known versions. if not pkg_cls.versions: tty.msg("No safe (checksummed) versions for package %s" % pkg_cls.name) continue for version in pkg_cls.versions: version_spec = spack.spec.Spec(pkg_cls.name) version_spec.versions = spack.version.VersionList([version]) version_specs.append(version_spec) return version_specs def get_matching_versions(specs, num_versions=1): """Get a spec for EACH known version matching any spec in the list. For concrete specs, this retrieves the concrete version and, if more than one version per spec is requested, retrieves the latest versions of the package. """ matching = [] for spec in specs: pkg = spec.package # Skip any package that has no known versions. if not pkg.versions: tty.msg("No safe (checksummed) versions for package %s" % pkg.name) continue pkg_versions = num_versions version_order = list(reversed(sorted(pkg.versions))) matching_spec = [] if spec.concrete: matching_spec.append(spec) pkg_versions -= 1 if spec.version in version_order: version_order.remove(spec.version) for v in version_order: # Generate no more than num_versions versions for each spec. if pkg_versions < 1: break # Generate only versions that satisfy the spec. if spec.concrete or v.intersects(spec.versions): s = spack.spec.Spec(pkg.name) s.versions = spack.version.VersionList([v]) s.variants = spec.variants.copy() # This is needed to avoid hanging references during the # concretization phase s.variants.spec = s matching_spec.append(s) pkg_versions -= 1 if not matching_spec: tty.warn("No known version matches spec: %s" % spec) matching.extend(matching_spec) return matching def get_mirror_cache(path, skip_unstable_versions=False): """Returns a mirror cache, starting from the path where a mirror ought to be created. Args: path (str): path to create a mirror directory hierarchy in. skip_unstable_versions: if true, this skips adding resources when they do not have a stable archive checksum (as determined by ``fetch_strategy.stable_target``). Returns: spack.caches.MirrorCache: mirror cache object for the given path. """ # Get the absolute path of the root before we start jumping around. if not os.path.isdir(path): try: mkdirp(path) except OSError as e: raise MirrorError("Cannot create directory '%s':" % path, str(e)) mirror_cache = spack.caches.MirrorCache(path, skip_unstable_versions=skip_unstable_versions) return mirror_cache def add(mirror: Mirror, scope=None): """Add a named mirror in the given scope""" mirrors = spack.config.get("mirrors", scope=scope) if not mirrors: mirrors = syaml.syaml_dict() if mirror.name in mirrors: tty.die("Mirror with name {} already exists.".format(mirror.name)) items = [(n, u) for n, u in mirrors.items()] items.insert(0, (mirror.name, mirror.to_dict())) mirrors = syaml.syaml_dict(items) spack.config.set("mirrors", mirrors, scope=scope) def remove(name, scope): """Remove the named mirror in the given scope""" mirrors = spack.config.get("mirrors", scope=scope) if not mirrors: mirrors = syaml.syaml_dict() removed = mirrors.pop(name, False) spack.config.set("mirrors", mirrors, scope=scope) return bool(removed) class MirrorStatsForOneSpec: def __init__(self, spec): self.present = Counter() self.new = Counter() self.errors = Counter() self.spec = spec self.added_resources = set() self.existing_resources = set() def finalize(self): if self.spec: if self.added_resources: self.new[self.spec] = len(self.added_resources) if self.existing_resources: self.present[self.spec] = len(self.existing_resources) self.added_resources = set() self.existing_resources = set() def already_existed(self, resource): # If an error occurred after caching a subset of a spec's # resources, a secondary attempt may consider them already added if resource not in self.added_resources: self.existing_resources.add(resource) def added(self, resource): self.added_resources.add(resource) def error(self): if self.spec: self.errors[self.spec] += 1 class MirrorStatsForAllSpecs: def __init__(self): # Counter is used to easily merge mirror stats for one spec into mirror stats for all specs self.present = Counter() self.new = Counter() self.errors = Counter() def merge(self, ext_mirror_stat: MirrorStatsForOneSpec): # For the sake of parallelism we need a way to reduce/merge different # MirrorStats objects. self.present.update(ext_mirror_stat.present) self.new.update(ext_mirror_stat.new) self.errors.update(ext_mirror_stat.errors) def stats(self): # Convert dictionary to list present_list = list(self.present.keys()) new_list = list(self.new.keys()) errors_list = list(self.errors.keys()) return present_list, new_list, errors_list def create_mirror_from_package_object( pkg_obj, mirror_cache: "spack.caches.MirrorCache", mirror_stats: MirrorStatsForOneSpec ) -> bool: """Add a single package object to a mirror. The package object is only required to have an associated spec with a concrete version. Args: pkg_obj (spack.package_base.PackageBase): package object with to be added. mirror_cache: mirror where to add the spec. mirror_stats: statistics on the current mirror Return: True if the spec was added successfully, False otherwise """ tty.msg("Adding package {} to mirror".format(pkg_obj.spec.format("{name}{@version}"))) # Skip placeholder packages try: pkg_obj.fetcher except InstallError: return False max_retries = 3 for num_retries in range(max_retries): try: # Includes patches and resources with pkg_obj.stage as pkg_stage: pkg_stage.cache_mirror(mirror_cache, mirror_stats) break except Exception as e: pkg_obj.stage.destroy() if num_retries + 1 == max_retries: if spack.config.get("config:debug"): traceback.print_exc() else: tty.warn( "Error while fetching %s" % pkg_obj.spec.format("{name}{@version}"), str(e) ) mirror_stats.error() return False return True def require_mirror_name(mirror_name): """Find a mirror by name and raise if it does not exist""" mirror = MirrorCollection().get(mirror_name) if not mirror: raise ValueError(f'no mirror named "{mirror_name}"') return mirror ================================================ FILE: lib/spack/spack/mixins.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module contains additional behavior that can be attached to any given package.""" import os from typing import Optional import spack.llnl.util.filesystem import spack.phase_callbacks def filter_compiler_wrappers( *files: str, after: str = "install", relative_root: Optional[str] = None, ignore_absent: bool = True, backup: bool = False, recursive: bool = False, **kwargs, # for compatibility with package api v2.0 ) -> None: """Registers a phase callback (e.g. post-install) to look for references to Spack's compiler wrappers in the given files and replace them with the underlying compilers. Example usage:: class MyPackage(Package): filter_compiler_wrappers("mpicc", "mpicxx", relative_root="bin") This is useful for packages that register the path to the compiler they are built with to be used later at runtime. Spack's compiler wrappers cannot be used at runtime, as they require Spack's build environment to be set up. Using this function, the compiler wrappers are replaced with the actual compilers, so that the package works correctly at runtime. Args: *files: files to be filtered relative to the search root (install prefix by default). after: specifies after which phase the files should be filtered (defaults to ``"install"``). relative_root: path relative to install prefix where to start searching for the files to be filtered. If not set the install prefix will be used as the search root. It is *highly recommended* to set this, as searching recursively from the installation prefix can be very slow. ignore_absent: if present, will be forwarded to :func:`~spack.llnl.util.filesystem.filter_file` backup: if present, will be forwarded to :func:`~spack.llnl.util.filesystem.filter_file` recursive: if present, will be forwarded to :func:`~spack.llnl.util.filesystem.find` """ def _filter_compiler_wrappers_impl(pkg_or_builder): pkg = getattr(pkg_or_builder, "pkg", pkg_or_builder) # Compute the absolute path of the search root root = os.path.join(pkg.prefix, relative_root) if relative_root else pkg.prefix # Compute the absolute path of the files to be filtered and remove links from the list. abs_files = spack.llnl.util.filesystem.find(root, files, recursive=recursive) abs_files = [x for x in abs_files if not os.path.islink(x)] x = spack.llnl.util.filesystem.FileFilter(*abs_files) compiler_vars = [] if "c" in pkg.spec: compiler_vars.append(("CC", pkg.spec["c"].package.cc)) if "cxx" in pkg.spec: compiler_vars.append(("CXX", pkg.spec["cxx"].package.cxx)) if "fortran" in pkg.spec: compiler_vars.append(("FC", pkg.spec["fortran"].package.fortran)) compiler_vars.append(("F77", pkg.spec["fortran"].package.fortran)) # Some paths to the compiler wrappers might be substrings of the others. # For example: # CC=/path/to/spack/lib/spack/env/cc (realpath to the wrapper) # FC=/path/to/spack/lib/spack/env/cce/ftn # Therefore, we perform the filtering in the reversed sorted order of the substituted # strings. If, however, the strings are identical (e.g. both CC and FC are set using # realpath), the filtering is done according to the order in compiler_vars. To achieve # that, we populate the following array with tuples of three elements: path to the wrapper, # negated index of the variable in compiler_vars, path to the real compiler. This way, the # reversed sorted order of the resulting array is the order of replacements that we need. replacements = [] for idx, (env_var, compiler_path) in enumerate(compiler_vars): if env_var in os.environ and compiler_path is not None: # filter spack wrapper and links to spack wrapper in case # build system runs realpath wrapper = os.environ[env_var] for wrapper_path in (wrapper, os.path.realpath(wrapper)): replacements.append((wrapper_path, -idx, compiler_path)) for wrapper_path, _, compiler_path in sorted(replacements, reverse=True): x.filter( wrapper_path, compiler_path, ignore_absent=ignore_absent, backup=backup, string=True, ) # Remove this linking flag if present (it turns RPATH into RUNPATH) for compiler_lang in ("c", "cxx", "fortran"): if compiler_lang not in pkg.spec: continue compiler_pkg = pkg.spec[compiler_lang].package x.filter( f"{compiler_pkg.linker_arg}--enable-new-dtags", "", ignore_absent=ignore_absent, backup=backup, string=True, ) # NAG compiler is usually mixed with GCC, which has a different # prefix for linker arguments. if pkg.compiler.name == "nag": x.filter( "-Wl,--enable-new-dtags", "", ignore_absent=ignore_absent, backup=backup, string=True, ) spack.phase_callbacks.run_after(after)(_filter_compiler_wrappers_impl) ================================================ FILE: lib/spack/spack/modules/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This package contains code for creating environment modules, which can include Tcl non-hierarchical modules, Lua hierarchical modules, and others. """ import os from typing import Dict, Type import spack.llnl.util.tty as tty import spack.repo import spack.spec import spack.store from . import common from .common import BaseModuleFileWriter, disable_modules from .lmod import LmodModulefileWriter from .tcl import TclModulefileWriter __all__ = ["TclModulefileWriter", "LmodModulefileWriter", "disable_modules"] module_types: Dict[str, Type[BaseModuleFileWriter]] = { "tcl": TclModulefileWriter, "lmod": LmodModulefileWriter, } def get_module( module_type, spec: spack.spec.Spec, get_full_path, module_set_name="default", required=True ): """Retrieve the module file for a given spec and module type. Retrieve the module file for the given spec if it is available. If the module is not available, this will raise an exception unless the module is excluded or if the spec is installed upstream. Args: module_type: the type of module we want to retrieve (e.g. lmod) spec: refers to the installed package that we want to retrieve a module for required: if the module is required but excluded, this function will print a debug message. If a module is missing but not excluded, then an exception is raised (regardless of whether it is required) get_full_path: if ``True``, this returns the full path to the module. Otherwise, this returns the module name. module_set_name: the named module configuration set from modules.yaml for which to retrieve the module. Returns: The module name or path. May return ``None`` if the module is not available. """ try: upstream = spec.installed_upstream except spack.repo.UnknownPackageError: upstream, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash()) if upstream: module = common.upstream_module_index.upstream_module(spec, module_type) if not module: return None if get_full_path: return module.path else: return module.use_name else: writer = module_types[module_type](spec, module_set_name) if not os.path.isfile(writer.layout.filename): fmt_str = "{name}{@version}{/hash:7}" if not writer.conf.excluded: raise common.ModuleNotFoundError( "The module for package {} should be at {}, but it does not exist".format( spec.format(fmt_str), writer.layout.filename ) ) elif required: tty.debug( "The module configuration has excluded {}: omitting it".format( spec.format(fmt_str) ) ) else: return None if get_full_path: return writer.layout.filename else: return writer.layout.use_name ================================================ FILE: lib/spack/spack/modules/common.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Here we consolidate the logic for creating an abstract description of the information that module systems need. This information maps **a single spec** to: * a unique module filename * the module file content and is divided among four classes: * a configuration class that provides a convenient interface to query details about the configuration for the spec under consideration. * a layout class that provides the information associated with module file names and directories * a context class that provides the dictionary used by the template engine to generate the module file * a writer that collects and uses the information above to either write or remove the module file Each of the four classes needs to be sub-classed when implementing a new module type. """ import collections import contextlib import copy import datetime import inspect import os import re import string from typing import List, Optional import spack.vendor.jinja2 import spack.build_environment import spack.config import spack.deptypes as dt import spack.environment import spack.error import spack.llnl.util.filesystem import spack.llnl.util.tty as tty import spack.paths import spack.projections as proj import spack.schema import spack.schema.environment import spack.spec import spack.store import spack.tengine as tengine import spack.user_environment import spack.util.environment import spack.util.file_permissions as fp import spack.util.path import spack.util.spack_yaml as syaml from spack.context import Context from spack.llnl.util.lang import Singleton, dedupe, memoized #: config section for this file def configuration(module_set_name): config_path = f"modules:{module_set_name}" return spack.config.get(config_path, {}) #: Valid tokens for naming scheme and env variable names _valid_tokens = ( "name", "version", "compiler", "compiler.name", "compiler.version", "architecture", # tokens from old-style format strings "package", "compilername", "compilerver", ) _FORMAT_STRING_RE = re.compile(r"({[^}]*})") def _format_env_var_name(spec, var_name_fmt): """Format the variable name, but uppercase any formatted fields.""" fmt_parts = _FORMAT_STRING_RE.split(var_name_fmt) return "".join( spec.format(part).upper() if _FORMAT_STRING_RE.match(part) else part for part in fmt_parts ) def _check_tokens_are_valid(format_string, message): """Checks that the tokens used in the format string are valid in the context of module file and environment variable naming. Args: format_string (str): string containing the format to be checked. This is supposed to be a 'template' for ``Spec.format`` message (str): first sentence of the error message in case invalid tokens are found """ named_tokens = re.findall(r"{(\w*)}", format_string) invalid_tokens = [x for x in named_tokens if x.lower() not in _valid_tokens] if invalid_tokens: raise RuntimeError( f"{message} [{', '.join(invalid_tokens)}]. " f"Did you check your 'modules.yaml' configuration?" ) def update_dictionary_extending_lists(target, update): """Updates a dictionary, but extends lists instead of overriding them. Args: target: dictionary to be updated update: update to be applied """ for key in update: value = target.get(key, None) if isinstance(value, list): target[key].extend(update[key]) elif isinstance(value, dict): update_dictionary_extending_lists(target[key], update[key]) else: target[key] = update[key] def dependencies(spec: spack.spec.Spec, request: str = "all") -> List[spack.spec.Spec]: """Returns the list of dependent specs for a given spec. Args: spec: spec to be analyzed request: one of ``"none"``, ``"run"``, ``"direct"``, ``"all"`` Returns: list of requested dependencies """ if request == "none": return [] elif request == "run": return spec.dependencies(deptype=dt.RUN) elif request == "direct": return spec.dependencies(deptype=dt.RUN | dt.LINK) elif request == "all": return list(spec.traverse(order="topo", deptype=dt.LINK | dt.RUN, root=False)) raise ValueError(f'request "{request}" is not one of "none", "direct", "run", "all"') def merge_config_rules(configuration, spec): """Parses the module specific part of a configuration and returns a dictionary containing the actions to be performed on the spec passed as an argument. Args: configuration: module specific configuration (e.g. entries under the top-level 'tcl' key) spec: spec for which we need to generate a module file Returns: dict: actions to be taken on the spec passed as an argument """ # The keyword 'all' is always evaluated first, all the others are # evaluated in order of appearance in the module file spec_configuration = copy.deepcopy(configuration.get("all", {})) for constraint, action in configuration.items(): if spec.satisfies(constraint): if hasattr(constraint, "override") and constraint.override: spec_configuration = {} update_dictionary_extending_lists(spec_configuration, copy.deepcopy(action)) # Transform keywords for dependencies or prerequisites into a list of spec # Which modulefiles we want to autoload autoload_strategy = spec_configuration.get("autoload", "direct") spec_configuration["autoload"] = dependencies(spec, autoload_strategy) # Which instead we want to mark as prerequisites prerequisite_strategy = spec_configuration.get("prerequisites", "none") spec_configuration["prerequisites"] = dependencies(spec, prerequisite_strategy) # Attach options that are spec-independent to the spec-specific # configuration # Hash length in module files hash_length = configuration.get("hash_length", 7) spec_configuration["hash_length"] = hash_length verbose = configuration.get("verbose", False) spec_configuration["verbose"] = verbose # module defaults per-package defaults = configuration.get("defaults", []) spec_configuration["defaults"] = defaults return spec_configuration def root_path(name, module_set_name): """Returns the root folder for module file installation. Args: name: name of the module system to be used (``"tcl"`` or ``"lmod"``) module_set_name: name of the set of module configs to use Returns: root folder for module file installation """ defaults = {"lmod": "$spack/share/spack/lmod", "tcl": "$spack/share/spack/modules"} # Root folders where the various module files should be written roots = spack.config.get(f"modules:{module_set_name}:roots", {}) # Merge config values into the defaults so we prefer configured values roots = spack.schema.merge_yaml(defaults, roots) path = roots.get(name, os.path.join(spack.paths.share_path, name)) return spack.util.path.canonicalize_path(path) def generate_module_index(root, modules, overwrite=False): index_path = os.path.join(root, "module-index.yaml") if overwrite or not os.path.exists(index_path): entries = syaml.syaml_dict() else: with open(index_path, encoding="utf-8") as index_file: yaml_content = syaml.load(index_file) entries = yaml_content["module_index"] for m in modules: entry = {"path": m.layout.filename, "use_name": m.layout.use_name} entries[m.spec.dag_hash()] = entry index = {"module_index": entries} spack.llnl.util.filesystem.mkdirp(root) with open(index_path, "w", encoding="utf-8") as index_file: syaml.dump(index, default_flow_style=False, stream=index_file) def _generate_upstream_module_index(): module_indices = read_module_indices() return UpstreamModuleIndex(spack.store.STORE.db, module_indices) upstream_module_index = Singleton(_generate_upstream_module_index) ModuleIndexEntry = collections.namedtuple("ModuleIndexEntry", ["path", "use_name"]) def read_module_index(root): index_path = os.path.join(root, "module-index.yaml") if not os.path.exists(index_path): return {} with open(index_path, encoding="utf-8") as index_file: return _read_module_index(index_file) def _read_module_index(str_or_file): """Read in the mapping of spec hash to module location/name. For a given Spack installation there is assumed to be (at most) one such mapping per module type.""" yaml_content = syaml.load(str_or_file) index = {} yaml_index = yaml_content["module_index"] for dag_hash, module_properties in yaml_index.items(): index[dag_hash] = ModuleIndexEntry( module_properties["path"], module_properties["use_name"] ) return index def read_module_indices(): other_spack_instances = spack.config.get("upstreams") or {} module_indices = [] for install_properties in other_spack_instances.values(): module_type_to_index = {} module_type_to_root = install_properties.get("modules", {}) for module_type, root in module_type_to_root.items(): module_type_to_index[module_type] = read_module_index(root) module_indices.append(module_type_to_index) return module_indices class UpstreamModuleIndex: """This is responsible for taking the individual module indices of all upstream Spack installations and locating the module for a given spec based on which upstream install it is located in.""" def __init__(self, local_db, module_indices): self.local_db = local_db self.upstream_dbs = local_db.upstream_dbs self.module_indices = module_indices def upstream_module(self, spec, module_type): db_for_spec = self.local_db.db_for_spec_hash(spec.dag_hash()) if db_for_spec in self.upstream_dbs: db_index = self.upstream_dbs.index(db_for_spec) elif db_for_spec: raise spack.error.SpackError(f"Unexpected: {spec} is installed locally") else: raise spack.error.SpackError(f"Unexpected: no install DB found for {spec}") module_index = self.module_indices[db_index] module_type_index = module_index.get(module_type, {}) if not module_type_index: tty.debug( f"No {module_type} modules associated with the Spack instance " f"where {spec} is installed" ) return None if spec.dag_hash() in module_type_index: return module_type_index[spec.dag_hash()] else: tty.debug(f"No module is available for upstream package {spec}") return None class BaseConfiguration: """Manipulates the information needed to generate a module file to make querying easier. It needs to be sub-classed for specific module types. """ default_projections = {"all": "{name}/{version}-{compiler.name}-{compiler.version}"} def __init__(self, spec: spack.spec.Spec, module_set_name: str, explicit: bool) -> None: # Spec for which we want to generate a module file self.spec = spec self.name = module_set_name self.explicit = explicit # Dictionary of configuration options that should be applied to the spec self.conf = merge_config_rules(self.module.configuration(self.name), self.spec) @property def module(self): return inspect.getmodule(self) @property def projections(self): """Projection from specs to module names""" # backwards compatibility for naming_scheme key conf = self.module.configuration(self.name) if "naming_scheme" in conf: default = {"all": conf["naming_scheme"]} else: default = self.default_projections projections = conf.get("projections", default) # Ensure the named tokens we are expanding are allowed, see # issue #2884 for reference msg = "some tokens cannot be part of the module naming scheme" for projection in projections.values(): _check_tokens_are_valid(projection, message=msg) return projections @property def template(self): """Returns the name of the template to use for the module file or None if not specified in the configuration. """ return self.conf.get("template", None) @property def defaults(self): """Returns the specs configured as defaults or [].""" return self.conf.get("defaults", []) @property def env(self): """List of environment modifications that should be done in the module. """ return spack.schema.environment.parse(self.conf.get("environment", {})) @property def suffixes(self): """List of suffixes that should be appended to the module file name. """ suffixes = [] for constraint, suffix in self.conf.get("suffixes", {}).items(): if constraint in self.spec: suffixes.append(suffix) suffixes = list(dedupe(suffixes)) # For hidden modules we can always add a fixed length hash as suffix, since it guards # against file name clashes, and the module is not exposed to the user anyways. if self.hidden: suffixes.append(self.spec.dag_hash(length=7)) elif self.hash: suffixes.append(self.hash) return suffixes @property def hash(self): """Hash tag for the module or None""" hash_length = self.conf.get("hash_length", 7) if hash_length != 0: return self.spec.dag_hash(length=hash_length) return None @property def conflicts(self): """Conflicts for this module file""" return self.conf.get("conflict", []) @property def excluded(self): """Returns True if the module has been excluded, False otherwise.""" # A few variables for convenience of writing the method spec = self.spec conf = self.module.configuration(self.name) # Compute the list of matching include / exclude rules, and whether excluded as implicit include_matches = [x for x in conf.get("include", []) if spec.satisfies(x)] exclude_matches = [x for x in conf.get("exclude", []) if spec.satisfies(x)] excluded_as_implicit = not self.explicit and conf.get("exclude_implicits", False) def debug_info(line_header, match_list): if match_list: tty.debug(f"\t{line_header} : {spec.cshort_spec}") for rule in match_list: tty.debug(f"\t\tmatches rule: {rule}") debug_info("INCLUDE", include_matches) debug_info("EXCLUDE", exclude_matches) if excluded_as_implicit: tty.debug(f"\tEXCLUDED_AS_IMPLICIT : {spec.cshort_spec}") return not include_matches and (exclude_matches or excluded_as_implicit) @property def hidden(self): """Returns True if the module has been hidden, False otherwise.""" conf = self.module.configuration(self.name) hidden_as_implicit = not self.explicit and conf.get("hide_implicits", False) if hidden_as_implicit: tty.debug(f"\tHIDDEN_AS_IMPLICIT : {self.spec.cshort_spec}") return hidden_as_implicit @property def context(self): return self.conf.get("context", {}) @property def specs_to_load(self): """List of specs that should be loaded in the module file.""" return self._create_list_for("autoload") @property def literals_to_load(self): """List of literal modules to be loaded.""" return self.conf.get("load", []) @property def specs_to_prereq(self): """List of specs that should be prerequisite of the module file.""" return self._create_list_for("prerequisites") @property def exclude_env_vars(self): """List of variables that should be left unmodified.""" filter_subsection = self.conf.get("filter", {}) return filter_subsection.get("exclude_env_vars", {}) def _create_list_for(self, what): include = [] for item in self.conf[what]: if not self.module.make_configuration(item, self.name).excluded: include.append(item) return include @property def verbose(self): """Returns True if the module file needs to be verbose, False otherwise """ return self.conf.get("verbose") class BaseFileLayout: """Provides information on the layout of module files. Needs to be sub-classed for specific module types. """ #: This needs to be redefined extension: Optional[str] = None def __init__(self, configuration): self.conf = configuration @property def spec(self): """Spec under consideration""" return self.conf.spec def dirname(self): """Root folder for module files of this type.""" module_system = str(self.conf.module.__name__).split(".")[-1] return root_path(module_system, self.conf.name) @property def use_name(self): """Returns the 'use' name of the module i.e. the name you have to type to console to use it. This implementation fits the needs of most non-hierarchical layouts. """ projection = proj.get_projection(self.conf.projections, self.spec) if not projection: projection = self.conf.default_projections["all"] name = self.spec.format_path(projection) # Not everybody is working on linux... parts = name.split("/") name = os.path.join(*parts) # Add optional suffixes based on constraints path_elements = [name] path_elements.extend(map(self.spec.format, self.conf.suffixes)) return "-".join(path_elements) @property def filename(self): """Name of the module file for the current spec.""" # Just the name of the file filename = self.use_name if self.extension: filename = f"{self.use_name}.{self.extension}" # Architecture sub-folder arch_folder_conf = spack.config.get("modules:%s:arch_folder" % self.conf.name, True) if arch_folder_conf: # include an arch specific folder between root and filename arch_folder = str(self.spec.architecture) filename = os.path.join(arch_folder, filename) # Return the absolute path return os.path.join(self.dirname(), filename) class BaseContext(tengine.Context): """Provides the base context needed for template rendering. This class needs to be sub-classed for specific module types. The following attributes need to be implemented: - fields """ def __init__(self, configuration): self.conf = configuration @tengine.context_property def spec(self): return self.conf.spec @tengine.context_property def tags(self): if not hasattr(self.spec.package, "tags"): return [] return self.spec.package.tags @tengine.context_property def timestamp(self): return datetime.datetime.now() @tengine.context_property def category(self): return getattr(self.spec, "category", "spack") @tengine.context_property def short_description(self): # If we have a valid docstring return the first paragraph. docstring = type(self.spec.package).__doc__ if docstring: value = docstring.split("\n\n")[0] # Transform tabs and friends into spaces value = re.sub(r"\s+", " ", value) # Turn double quotes into single quotes (double quotes are needed # to start and end strings) value = re.sub(r'"', "'", value) return value # Otherwise the short description is just the package + version return self.spec.format("{name} {@version}") @tengine.context_property def long_description(self): # long description is the docstring with reduced whitespace. if self.spec.package.__doc__: return re.sub(r"\s+", " ", self.spec.package.__doc__) return None @tengine.context_property def configure_options(self): pkg = self.spec.package # If the spec is external Spack doesn't know its configure options if self.spec.external: msg = "unknown, software installed outside of Spack" return msg if os.path.exists(pkg.install_configure_args_path): with open(pkg.install_configure_args_path, encoding="utf-8") as args_file: return spack.util.path.padding_filter(args_file.read()) # Returning a false-like value makes the default templates skip # the configure option section return None def modification_needs_formatting(self, modification): """Returns True if environment modification entry needs to be formatted.""" return ( not isinstance(modification, (spack.util.environment.SetEnv)) or not modification.raw ) @tengine.context_property @memoized def environment_modifications(self): """List of environment modifications to be processed.""" # Modifications guessed by inspecting the spec prefix prefix_inspections = syaml.syaml_dict() spack.schema.merge_yaml( prefix_inspections, spack.config.get("modules:prefix_inspections", {}) ) spack.schema.merge_yaml( prefix_inspections, spack.config.get(f"modules:{self.conf.name}:prefix_inspections", {}), ) use_view = spack.config.get(f"modules:{self.conf.name}:use_view", False) assert isinstance(use_view, (bool, str)) if use_view: env = spack.environment.active_environment() if not env: raise spack.environment.SpackEnvironmentViewError( "Module generation with views requires active environment" ) view_name = spack.environment.default_view_name if use_view is True else use_view if not env.has_view(view_name): raise spack.environment.SpackEnvironmentViewError( f"View {view_name} not found in environment {env.name} when generating modules" ) view = env.views[view_name] else: view = None env = spack.util.environment.inspect_path( self.spec.prefix, prefix_inspections, exclude=spack.util.environment.is_system_path ) # Let the extendee/dependency modify their extensions/dependencies # The only thing we care about is `setup_dependent_run_environment`, but # for that to work, globals have to be set on the package modules, and the # whole chain of setup_dependent_package has to be followed from leaf to spec. # So: just run it here, but don't collect env mods. spack.build_environment.SetupContext( self.spec, context=Context.RUN ).set_all_package_py_globals() # Then run setup_dependent_run_environment before setup_run_environment. for dep in self.spec.dependencies(deptype=("link", "run")): dep.package.setup_dependent_run_environment(env, self.spec) self.spec.package.setup_run_environment(env) # Project the environment variables from prefix to view if needed if view and self.spec in view: spack.user_environment.project_env_mods( *self.spec.traverse(deptype=dt.LINK | dt.RUN), view=view, env=env ) # Modifications required from modules.yaml env.extend(self.conf.env) # List of variables that are excluded in modules.yaml exclude = self.conf.exclude_env_vars # We may have tokens to substitute in environment commands for x in env: # Ensure all the tokens are valid in this context msg = "some tokens cannot be expanded in an environment variable name" _check_tokens_are_valid(x.name, message=msg) x.name = _format_env_var_name(self.spec, x.name) if self.modification_needs_formatting(x): try: # Not every command has a value x.value = self.spec.format(x.value) except AttributeError: pass x.name = str(x.name).replace("-", "_") return [(type(x).__name__, x) for x in env if x.name not in exclude] @tengine.context_property def has_manpath_modifications(self): """True if MANPATH environment variable is modified.""" for modification_type, cmd in self.environment_modifications: if not isinstance( cmd, (spack.util.environment.PrependPath, spack.util.environment.AppendPath) ): continue if cmd.name == "MANPATH": return True else: return False @tengine.context_property def conflicts(self): """List of conflicts for the module file.""" fmts = [] projection = proj.get_projection(self.conf.projections, self.spec) for item in self.conf.conflicts: self._verify_conflict_naming_consistency_or_raise(item, projection) item = self.spec.format(item) fmts.append(item) return fmts def _verify_conflict_naming_consistency_or_raise(self, item, projection): f = string.Formatter() errors = [] if len([x for x in f.parse(item)]) > 1: for naming_dir, conflict_dir in zip(projection.split("/"), item.split("/")): if naming_dir != conflict_dir: errors.extend( [ f"spec={self.spec.cshort_spec}", f"conflict_scheme={item}", f"naming_scheme={projection}", ] ) if errors: raise ModulesError( message="conflict scheme does not match naming scheme", long_message="\n ".join(errors), ) @tengine.context_property def autoload(self): """List of modules that needs to be loaded automatically.""" # From 'autoload' configuration option specs = self._create_module_list_of("specs_to_load") # From 'load' configuration option literals = self.conf.literals_to_load return specs + literals def _create_module_list_of(self, what): m = self.conf.module name = self.conf.name return [m.make_layout(x, name).use_name for x in getattr(self.conf, what)] @tengine.context_property def verbose(self): """Verbosity level.""" return self.conf.verbose class BaseModuleFileWriter: default_template: str hide_cmd_format: str modulerc_header: List[str] def __init__( self, spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None ) -> None: self.spec = spec m = self.module # Create the triplet of configuration/layout/context self.conf = m.make_configuration(spec, module_set_name, explicit) self.layout = m.make_layout(spec, module_set_name, explicit) self.context = m.make_context(spec, module_set_name, explicit) # Check if a default template has been defined, # throw if not found try: self.default_template except AttributeError: msg = "'{0}' object has no attribute 'default_template'\n" msg += "Did you forget to define it in the class?" name = type(self).__name__ raise DefaultTemplateNotDefined(msg.format(name)) # Check if format for module hide command has been defined, # throw if not found try: self.hide_cmd_format except AttributeError: msg = "'{0}' object has no attribute 'hide_cmd_format'\n" msg += "Did you forget to define it in the class?" name = type(self).__name__ raise HideCmdFormatNotDefined(msg.format(name)) # Check if modulerc header content has been defined, # throw if not found try: self.modulerc_header except AttributeError: msg = "'{0}' object has no attribute 'modulerc_header'\n" msg += "Did you forget to define it in the class?" name = type(self).__name__ raise ModulercHeaderNotDefined(msg.format(name)) @property def module(self): return inspect.getmodule(self) def _get_template(self): """Gets the template that will be rendered for this spec.""" # Get templates and put them in the order of importance: # 1. template specified in "modules.yaml" # 2. template specified in a package directly # 3. default template (must be defined, check in __init__) module_system_name = str(self.module.__name__).split(".")[-1] package_attribute = f"{module_system_name}_template" choices = [ self.conf.template, getattr(self.spec.package, package_attribute, None), self.default_template, # This is always defined at this point ] # Filter out false-ish values choices = list(filter(lambda x: bool(x), choices)) # ... and return the first match return choices.pop(0) def write(self, overwrite=False): """Writes the module file. Args: overwrite (bool): if True it is fine to overwrite an already existing file. If False the operation is skipped an we print a warning to the user. """ # Return immediately if the module is excluded if self.conf.excluded: msg = "\tNOT WRITING: {0} [EXCLUDED]" tty.debug(msg.format(self.spec.cshort_spec)) return # Print a warning in case I am accidentally overwriting # a module file that is already there (name clash) if not overwrite and os.path.exists(self.layout.filename): message = "Module file {0.filename} exists and will not be overwritten" tty.warn(message.format(self.layout)) return # If we are here it means it's ok to write the module file msg = "\tWRITE: {0} [{1}]" tty.debug(msg.format(self.spec.cshort_spec, self.layout.filename)) # If the directory where the module should reside does not exist # create it module_dir = os.path.dirname(self.layout.filename) if not os.path.exists(module_dir): spack.llnl.util.filesystem.mkdirp(module_dir) # Get the template for the module template_name = self._get_template() try: env = tengine.make_environment() template = env.get_template(template_name) except spack.vendor.jinja2.TemplateNotFound: # If the template was not found raise an exception with a little # more information msg = "template '{0}' was not found for '{1}'" name = type(self).__name__ msg = msg.format(template_name, name) raise ModulesTemplateNotFoundError(msg) # Construct the context following the usual hierarchy of updates: # 1. start with the default context from the module writer class # 2. update with package specific context # 3. update with 'modules.yaml' specific context context = self.context.to_dict() # Attribute from package module_name = str(self.module.__name__).split(".")[-1] attr_name = f"{module_name}_context" pkg_update = getattr(self.spec.package, attr_name, {}) context.update(pkg_update) # Context key in modules.yaml conf_update = self.conf.context context.update(conf_update) # Render the template text = template.render(context) # Write it to file with open(self.layout.filename, "w", encoding="utf-8") as f: f.write(text) # Set the file permissions of the module to match that of the package if os.path.exists(self.layout.filename): fp.set_permissions_by_spec(self.layout.filename, self.spec) # Symlink defaults if needed self.update_module_defaults() # record module hiddenness if implicit self.update_module_hiddenness() def update_module_defaults(self): if any(self.spec.satisfies(default) for default in self.conf.defaults): # This spec matches a default, it needs to be symlinked to default # Symlink to a tmp location first and move, so that existing # symlinks do not cause an error. default_path = os.path.join(os.path.dirname(self.layout.filename), "default") default_tmp = os.path.join(os.path.dirname(self.layout.filename), ".tmp_spack_default") os.symlink(self.layout.filename, default_tmp) os.rename(default_tmp, default_path) def update_module_hiddenness(self, remove=False): """Update modulerc file corresponding to module to add or remove command that hides module depending on its hidden state. Args: remove (bool): if True, hiddenness information for module is removed from modulerc. """ modulerc_path = self.layout.modulerc hide_module_cmd = self.hide_cmd_format % self.layout.use_name hidden = self.conf.hidden and not remove modulerc_exists = os.path.exists(modulerc_path) updated = False if modulerc_exists: # retrieve modulerc content with open(modulerc_path, encoding="utf-8") as f: content = f.readlines() content = "".join(content).split("\n") # remove last empty item if any if len(content[-1]) == 0: del content[-1] already_hidden = hide_module_cmd in content # remove hide command if module not hidden if already_hidden and not hidden: content.remove(hide_module_cmd) updated = True # add hide command if module is hidden elif not already_hidden and hidden: if len(content) == 0: content = self.modulerc_header.copy() content.append(hide_module_cmd) updated = True else: content = self.modulerc_header.copy() if hidden: content.append(hide_module_cmd) updated = True # no modulerc file change if no content update if updated: is_empty = content == self.modulerc_header or len(content) == 0 # remove existing modulerc if empty if modulerc_exists and is_empty: os.remove(modulerc_path) # create or update modulerc elif content != self.modulerc_header: # ensure file ends with a newline character content.append("") with open(modulerc_path, "w", encoding="utf-8") as f: f.write("\n".join(content)) def remove(self): """Deletes the module file.""" mod_file = self.layout.filename if os.path.exists(mod_file): try: os.remove(mod_file) # Remove the module file self.remove_module_defaults() # Remove default targeting module file self.update_module_hiddenness(remove=True) # Remove hide cmd in modulerc os.removedirs( os.path.dirname(mod_file) ) # Remove all the empty directories from the leaf up except OSError: # removedirs throws OSError on first non-empty directory found pass def remove_module_defaults(self): if not any(self.spec.satisfies(default) for default in self.conf.defaults): return # This spec matches a default, symlink needs to be removed as we remove the module # file it targets. default_symlink = os.path.join(os.path.dirname(self.layout.filename), "default") try: os.unlink(default_symlink) except OSError: pass @contextlib.contextmanager def disable_modules(): """Disable the generation of modulefiles within the context manager.""" data = {"modules:": {"default": {"enable": []}}} disable_scope = spack.config.InternalConfigScope("disable_modules", data=data) with spack.config.override(disable_scope): yield class ModulesError(spack.error.SpackError): """Base error for modules.""" class ModuleNotFoundError(ModulesError): """Raised when a module cannot be found for a spec""" class DefaultTemplateNotDefined(AttributeError, ModulesError): """Raised if the attribute ``default_template`` has not been specified in the derived classes. """ class HideCmdFormatNotDefined(AttributeError, ModulesError): """Raised if the attribute ``hide_cmd_format`` has not been specified in the derived classes. """ class ModulercHeaderNotDefined(AttributeError, ModulesError): """Raised if the attribute ``modulerc_header`` has not been specified in the derived classes. """ class ModulesTemplateNotFoundError(ModulesError, RuntimeError): """Raised if the template for a module file was not found.""" ================================================ FILE: lib/spack/spack/modules/lmod.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import itertools import os import pathlib import warnings from typing import Dict, List, Optional, Tuple import spack.compilers.config import spack.config import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.lang as lang import spack.spec import spack.tengine as tengine import spack.util.environment from spack.aliases import BUILTIN_TO_LEGACY_COMPILER from .common import BaseConfiguration, BaseContext, BaseFileLayout, BaseModuleFileWriter #: lmod specific part of the configuration def configuration(module_set_name: str) -> dict: return spack.config.get(f"modules:{module_set_name}:lmod", {}) # Caches the configuration {spec_hash: configuration} configuration_registry: Dict[Tuple[str, str, bool], BaseConfiguration] = {} def make_configuration( spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None ) -> BaseConfiguration: """Returns the lmod configuration for spec""" explicit = bool(spec._installed_explicitly()) if explicit is None else explicit key = (spec.dag_hash(), module_set_name, explicit) try: return configuration_registry[key] except KeyError: return configuration_registry.setdefault( key, LmodConfiguration(spec, module_set_name, explicit) ) def make_layout( spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None ) -> BaseFileLayout: """Returns the layout information for spec""" return LmodFileLayout(make_configuration(spec, module_set_name, explicit)) def make_context( spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None ) -> BaseContext: """Returns the context information for spec""" return LmodContext(make_configuration(spec, module_set_name, explicit)) def guess_core_compilers(name, store=False) -> List[spack.spec.Spec]: """Guesses the list of core compilers installed in the system. Args: store (bool): if True writes the core compilers to the modules.yaml configuration file Returns: List of found core compilers """ core_compilers = [] for compiler in spack.compilers.config.all_compilers(init_config=False): try: cc_dir = pathlib.Path(compiler.package.cc).parent is_system_compiler = str(cc_dir) in spack.util.environment.SYSTEM_DIRS if is_system_compiler: core_compilers.append(compiler) except (KeyError, TypeError, AttributeError): continue if store and core_compilers: # If we asked to store core compilers, update the entry # in the default modify scope (i.e. within the directory hierarchy # of Spack itself) modules_cfg = spack.config.get( "modules:" + name, {}, scope=spack.config.default_modify_scope() ) modules_cfg.setdefault("lmod", {})["core_compilers"] = [str(x) for x in core_compilers] spack.config.set("modules:" + name, modules_cfg, scope=spack.config.default_modify_scope()) return core_compilers class LmodConfiguration(BaseConfiguration): """Configuration class for lmod module files.""" default_projections = {"all": "{name}/{version}"} compiler: Optional[spack.spec.Spec] def __init__(self, spec: spack.spec.Spec, module_set_name: str, explicit: bool) -> None: super().__init__(spec, module_set_name, explicit) candidates = collections.defaultdict(list) language_virtuals = ("c", "cxx", "fortran") for node in spec.traverse(deptype=("link", "run")): for language in language_virtuals: candidates[language].extend(node.dependencies(virtuals=(language,))) self.compiler = None for language in language_virtuals: if candidates[language]: self.compiler = candidates[language][0] if len(set(candidates[language])) > 1: warnings.warn( f"{spec.short_spec} uses more than one compiler, and might not fit the " f"LMod hierarchy. Using {self.compiler.short_spec} as the LMod compiler." ) break @property def core_compilers(self) -> List[spack.spec.Spec]: """Returns the list of "Core" compilers Raises: CoreCompilersNotFoundError: if the key was not specified in the configuration file or the sequence is empty """ compilers = [] for c in configuration(self.name).get("core_compilers", []): compilers.extend(spack.spec.Spec(f"%{c}").dependencies()) if not compilers: compilers = guess_core_compilers(self.name, store=True) if not compilers: msg = 'the key "core_compilers" must be set in modules.yaml' raise CoreCompilersNotFoundError(msg) return compilers @property def core_specs(self): """Returns the list of "Core" specs""" return configuration(self.name).get("core_specs", []) @property def filter_hierarchy_specs(self): """Returns the dict of specs with modified hierarchies""" return configuration(self.name).get("filter_hierarchy_specs", {}) @property @lang.memoized def hierarchy_tokens(self): """Returns the list of tokens that are part of the modulefile hierarchy. ``compiler`` is always present. """ tokens = configuration(self.name).get("hierarchy", []) # Append 'compiler' which is always implied tokens.append("compiler") # Deduplicate tokens in case duplicates have been coded tokens = list(lang.dedupe(tokens)) return tokens @property @lang.memoized def requires(self): """Returns a dictionary mapping all the requirements of this spec to the actual provider. The ``compiler`` key is always present among the requirements. """ # If it's a core_spec, lie and say it requires a core compiler if any(self.spec.satisfies(core_spec) for core_spec in self.core_specs): return {"compiler": self.core_compilers[0]} hierarchy_filter_list = [] for spec, filter_list in self.filter_hierarchy_specs.items(): if self.spec.satisfies(spec): hierarchy_filter_list = filter_list break # Keep track of the requirements that this package has in terms # of virtual packages that participate in the hierarchical structure requirements = {"compiler": self.compiler or self.core_compilers[0]} # For each dependency in the hierarchy for x in self.hierarchy_tokens: # Skip anything filtered for this spec if x in hierarchy_filter_list: continue # If I depend on it if x in self.spec and not (self.spec.name == x or self.spec.package.provides(x)): requirements[x] = self.spec[x] # record the actual provider return requirements @property def provides(self): """Returns a dictionary mapping all the services provided by this spec to the spec itself. """ provides = {} # Treat the 'compiler' case in a special way, as compilers are not # virtual dependencies in spack # If it is in the list of supported compilers family -> compiler if self.spec.name in spack.compilers.config.supported_compilers(): provides["compiler"] = spack.spec.Spec(self.spec.format("{name}{@versions}")) elif self.spec.name in BUILTIN_TO_LEGACY_COMPILER: # If it is the package for a supported compiler, but of a different name cname = BUILTIN_TO_LEGACY_COMPILER[self.spec.name] provides["compiler"] = spack.spec.Spec(cname, self.spec.versions) # All the other tokens in the hierarchy must be virtual dependencies for x in self.hierarchy_tokens: if self.spec.name == x or self.spec.package.provides(x): provides[x] = self.spec return provides @property def available(self): """Returns a dictionary of the services that are currently available. """ available = {} # What is available is what I require plus what I provide. # 'compiler' is the only key that may be overridden. available.update(self.requires) available.update(self.provides) return available @property @lang.memoized def missing(self): """Returns the list of tokens that are not available.""" return [x for x in self.hierarchy_tokens if x not in self.available] @property def hidden(self): # Never hide a module that opens a hierarchy if any( self.spec.name == x or self.spec.package.provides(x) for x in self.hierarchy_tokens ): return False return super().hidden class LmodFileLayout(BaseFileLayout): """File layout for lmod module files.""" #: file extension of lua module files extension = "lua" @property def arch_dirname(self): """Returns the root folder for THIS architecture""" # Architecture sub-folder arch_folder_conf = spack.config.get("modules:%s:arch_folder" % self.conf.name, True) if arch_folder_conf: # include an arch specific folder between root and filename arch_folder = "-".join( [str(self.spec.platform), str(self.spec.os), str(self.spec.target.family)] ) return os.path.join(self.dirname(), arch_folder) return self.dirname() @property def filename(self): """Returns the filename for the current module file""" # Get the list of requirements and build an **ordered** # list of the path parts requires = self.conf.requires hierarchy = self.conf.hierarchy_tokens path_parts = lambda x: self.token_to_path(x, requires[x]) parts = [path_parts(x) for x in hierarchy if x in requires] # My relative path if just a join of all the parts hierarchy_name = os.path.join(*parts) # Compute the absolute path return os.path.join( self.arch_dirname, # root for lmod files on this architecture hierarchy_name, # relative path f"{self.use_name}.{self.extension}", # file name ) @property def modulerc(self): """Returns the modulerc file associated with current module file""" return os.path.join(os.path.dirname(self.filename), f".modulerc.{self.extension}") def token_to_path(self, name, value): """Transforms a hierarchy token into the corresponding path part. Args: name (str): name of the service in the hierarchy value: actual provider of the service Returns: str: part of the path associated with the service """ # General format for the path part def path_part_fmt(token): return fs.polite_path([f"{token.name}", f"{token.version}"]) # If we are dealing with a core compiler, return 'Core' core_compilers = self.conf.core_compilers if name == "compiler" and any(spack.spec.Spec(value).satisfies(c) for c in core_compilers): return "Core" # Spec does not have a hash, as we are not allowed to # use different flavors of the same compiler if name == "compiler": return path_part_fmt(token=value) # In case the hierarchy token refers to a virtual provider # we need to append a hash to the version to distinguish # among flavors of the same library (e.g. openblas~openmp vs. # openblas+openmp) return f"{path_part_fmt(token=value)}-{value.dag_hash(length=7)}" @property def available_path_parts(self): """List of path parts that are currently available. Needed to construct the file name. """ # List of available services available = self.conf.available # List of services that are part of the hierarchy hierarchy = self.conf.hierarchy_tokens # Tokenize each part that is both in the hierarchy and available return [self.token_to_path(x, available[x]) for x in hierarchy if x in available] @property @lang.memoized def unlocked_paths(self): """Returns a dictionary mapping conditions to a list of unlocked paths. The paths that are unconditionally unlocked are under the key 'None'. The other keys represent the list of services you need loaded to unlock the corresponding paths. """ unlocked = collections.defaultdict(list) # Get the list of services we require and we provide requires_key = list(self.conf.requires) provides_key = list(self.conf.provides) # A compiler is always required. To avoid duplication pop the # 'compiler' item from required if we also **provide** one if "compiler" in provides_key: requires_key.remove("compiler") # Compute the unique combinations of the services we provide combinations = [] for ii in range(len(provides_key)): combinations += itertools.combinations(provides_key, ii + 1) # Attach the services required to each combination to_be_processed = [x + tuple(requires_key) for x in combinations] # Compute the paths that are unconditionally added # and append them to the dictionary (key = None) available_combination = [] for item in to_be_processed: hierarchy = self.conf.hierarchy_tokens available = self.conf.available ac = [x for x in hierarchy if x in item] available_combination.append(tuple(ac)) parts = [self.token_to_path(x, available[x]) for x in ac] unlocked[None].append(tuple([self.arch_dirname] + parts)) # Deduplicate the list unlocked[None] = list(lang.dedupe(unlocked[None])) # Compute the combination of missing requirements: this will lead to # paths that are unlocked conditionally missing = self.conf.missing missing_combinations = [] for ii in range(len(missing)): missing_combinations += itertools.combinations(missing, ii + 1) # Attach the services required to each combination for m in missing_combinations: to_be_processed = [m + x for x in available_combination] for item in to_be_processed: hierarchy = self.conf.hierarchy_tokens available = self.conf.available token2path = lambda x: self.token_to_path(x, available[x]) parts = [] for x in hierarchy: if x not in item: continue value = token2path(x) if x in available else x parts.append(value) unlocked[m].append(tuple([self.arch_dirname] + parts)) # Deduplicate the list unlocked[m] = list(lang.dedupe(unlocked[m])) return unlocked class LmodContext(BaseContext): """Context class for lmod module files.""" @tengine.context_property def has_modulepath_modifications(self): """True if this module modifies MODULEPATH, False otherwise.""" return bool(self.conf.provides) @tengine.context_property def has_conditional_modifications(self): """True if this module modifies MODULEPATH conditionally to the presence of other services in the environment, False otherwise. """ # In general we have conditional modifications if we have modifications # and we are not providing **only** a compiler provides = self.conf.provides provide_compiler_only = "compiler" in provides and len(provides) == 1 has_modifications = self.has_modulepath_modifications return has_modifications and not provide_compiler_only @tengine.context_property def name_part(self): """Name of this provider.""" return self.spec.name @tengine.context_property def version_part(self): """Version of this provider.""" s = self.spec return "-".join([str(s.version), s.dag_hash(length=7)]) @tengine.context_property def provides(self): """Returns the dictionary of provided services.""" return self.conf.provides @tengine.context_property def missing(self): """Returns a list of missing services.""" return self.conf.missing @tengine.context_property @lang.memoized def unlocked_paths(self): """Returns the list of paths that are unlocked unconditionally.""" layout = make_layout(self.spec, self.conf.name) return [os.path.join(*parts) for parts in layout.unlocked_paths[None]] @tengine.context_property def conditionally_unlocked_paths(self): """Returns the list of paths that are unlocked conditionally. Each item in the list is a tuple with the structure (condition, path). """ layout = make_layout(self.spec, self.conf.name) value = [] conditional_paths = layout.unlocked_paths conditional_paths.pop(None) for services_needed, list_of_path_parts in conditional_paths.items(): condition = " and ".join([x + "_name" for x in services_needed]) for parts in list_of_path_parts: def manipulate_path(token): if token in self.conf.hierarchy_tokens: return "{0}_name, {0}_version".format(token) return '"' + token + '"' path = ", ".join([manipulate_path(x) for x in parts]) value.append((condition, path)) return value class LmodModulefileWriter(BaseModuleFileWriter): """Writer class for lmod module files.""" default_template = "modules/modulefile.lua" modulerc_header = [] hide_cmd_format = 'hide_version("%s")' class CoreCompilersNotFoundError(spack.error.SpackError, KeyError): """Error raised if the key ``core_compilers`` has not been specified in the configuration file. """ ================================================ FILE: lib/spack/spack/modules/tcl.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module implements the classes necessary to generate Tcl non-hierarchical modules. """ import os from typing import Dict, Optional, Tuple import spack.config import spack.spec import spack.tengine as tengine from .common import BaseConfiguration, BaseContext, BaseFileLayout, BaseModuleFileWriter #: Tcl specific part of the configuration def configuration(module_set_name: str) -> dict: return spack.config.get(f"modules:{module_set_name}:tcl", {}) # Caches the configuration {spec_hash: configuration} configuration_registry: Dict[Tuple[str, str, bool], BaseConfiguration] = {} def make_configuration( spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None ) -> BaseConfiguration: """Returns the tcl configuration for spec""" explicit = bool(spec._installed_explicitly()) if explicit is None else explicit key = (spec.dag_hash(), module_set_name, explicit) try: return configuration_registry[key] except KeyError: return configuration_registry.setdefault( key, TclConfiguration(spec, module_set_name, explicit) ) def make_layout( spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None ) -> BaseFileLayout: """Returns the layout information for spec""" return TclFileLayout(make_configuration(spec, module_set_name, explicit)) def make_context( spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None ) -> BaseContext: """Returns the context information for spec""" return TclContext(make_configuration(spec, module_set_name, explicit)) class TclConfiguration(BaseConfiguration): """Configuration class for tcl module files.""" class TclFileLayout(BaseFileLayout): """File layout for tcl module files.""" @property def modulerc(self): """Returns the modulerc file associated with current module file""" return os.path.join(os.path.dirname(self.filename), ".modulerc") class TclContext(BaseContext): """Context class for tcl module files.""" @tengine.context_property def prerequisites(self): """List of modules that needs to be loaded automatically.""" return self._create_module_list_of("specs_to_prereq") class TclModulefileWriter(BaseModuleFileWriter): """Writer class for tcl module files.""" default_template = "modules/modulefile.tcl" modulerc_header = ["#%Module4.7"] hide_cmd_format = "module-hide --soft --hidden-loaded %s" ================================================ FILE: lib/spack/spack/multimethod.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module contains utilities for using multi-methods in spack. You can think of multi-methods like overloaded methods -- they're methods with the same name, and we need to select a version of the method based on some criteria. e.g., for overloaded methods, you would select a version of the method to call based on the types of its arguments. In spack, multi-methods are used to ease the life of package authors. They allow methods like install() (or other methods called by install()) to declare multiple versions to be called when the package is instantiated with different specs. e.g., if the package is built with OpenMPI on x86_64,, you might want to call a different install method than if it was built for mpich2 on BlueGene/Q. Likewise, you might want to do a different type of install for different versions of the package. Multi-methods provide a simple decorator-based syntax for this that avoids overly complicated rat nests of if statements. Obviously, depending on the scenario, regular old conditionals might be clearer, so package authors should use their judgement. """ import functools from contextlib import contextmanager from typing import Optional, Union import spack.directives_meta import spack.error import spack.spec class MultiMethodMeta(type): """This allows us to track the class's dict during instantiation.""" #: saved dictionary of attrs on the class being constructed _locals = None @classmethod def __prepare__(cls, name, bases, **kwargs): """Save the dictionary that will be used for the class namespace.""" MultiMethodMeta._locals = dict() return MultiMethodMeta._locals def __init__(cls, name, bases, attr_dict): """Clear out the cached locals dict once the class is built.""" MultiMethodMeta._locals = None super(MultiMethodMeta, cls).__init__(name, bases, attr_dict) class SpecMultiMethod: """This implements a multi-method for Spack specs. Packages are instantiated with a particular spec, and you may want to execute different versions of methods based on what the spec looks like. For example, you might want to call a different version of install() for one platform than you call on another. The SpecMultiMethod class implements a callable object that handles method dispatch. When it is called, it looks through registered methods and their associated specs, and it tries to find one that matches the package's spec. If it finds one (and only one), it will call that method. This is intended for use with decorators (see below). The decorator (see docs below) creates SpecMultiMethods and registers method versions with them. To register a method, you can do something like this:: mm = SpecMultiMethod() mm.register("^chaos_5_x86_64_ib", some_method) The object registered needs to be a Spec or some string that will parse to be a valid spec. When the ``mm`` is actually called, it selects a version of the method to call based on the ``sys_type`` of the object it is called on. See the docs for decorators below for more details. """ def __init__(self, default=None): self.method_list = [] self.default = default if default: functools.update_wrapper(self, default) def register(self, spec, method): """Register a version of a method for a particular spec.""" self.method_list.append((spec, method)) if not hasattr(self, "__name__"): functools.update_wrapper(self, method) else: assert self.__name__ == method.__name__ def __get__(self, obj, objtype): """This makes __call__ support instance methods.""" # Method_list is a list of tuples (constraint, method) # Here we are going to assume that we have at least one # element in the list. The first registered function # will be the one 'wrapped'. wrapped_method = self.method_list[0][1] # Call functools.wraps manually to get all the attributes # we need to be disguised as the wrapped_method func = functools.wraps(wrapped_method)(functools.partial(self.__call__, obj)) return func def _get_method_by_spec(self, spec): """Find the method of this SpecMultiMethod object that satisfies the given spec, if one exists """ for condition, method in self.method_list: if spec.satisfies(condition): return method return self.default or None def __call__(self, package_or_builder_self, *args, **kwargs): """Find the first method with a spec that matches the package's spec. If none is found, call the default or if there is none, then raise a NoSuchMethodError. """ spec_method = self._get_method_by_spec(package_or_builder_self.spec) if spec_method: return spec_method(package_or_builder_self, *args, **kwargs) # Unwrap the MRO of `package_self by hand. Note that we can't # use `super()` here, because using `super()` recursively # requires us to know the class of `package_self`, as well as # its superclasses for successive calls. We don't have that # information within `SpecMultiMethod`, because it is not # associated with the package class. for cls in package_or_builder_self.__class__.__mro__[1:]: superself = cls.__dict__.get(self.__name__, None) if isinstance(superself, SpecMultiMethod): # Check parent multimethod for method for spec. superself_method = superself._get_method_by_spec(package_or_builder_self.spec) if superself_method: return superself_method(package_or_builder_self, *args, **kwargs) elif superself: return superself(package_or_builder_self, *args, **kwargs) raise NoSuchMethodError( type(package_or_builder_self), self.__name__, package_or_builder_self.spec, [m[0] for m in self.method_list], ) class when: """This is a multi-purpose class, which can be used 1. As a context manager to **group directives together** that share the same ``when=`` argument. 2. As a **decorator** for defining multi-methods (multiple methods with the same name are defined, but the version that is called depends on the condition of the package's spec) As a **context manager** it groups directives together. It allows you to write:: with when("+nvptx"): conflicts("@:6", msg="NVPTX only supported from gcc 7") conflicts("languages=ada") conflicts("languages=brig") instead of the more repetitive:: conflicts("@:6", when="+nvptx", msg="NVPTX only supported from gcc 7") conflicts("languages=ada", when="+nvptx") conflicts("languages=brig", when="+nvptx") This context manager is composable both with nested ``when`` contexts and with other ``when=`` arguments in directives. For example:: with when("+foo"): with when("+bar"): depends_on("dependency", when="+baz") is equilavent to:: depends_on("dependency", when="+foo +bar +baz") As a **decorator**, it allows packages to declare multiple versions of methods like ``install()`` that depend on the package's spec. For example:: class SomePackage(Package): ... def install(self, spec: Spec, prefix: Prefix): # Do default install @when("target=x86_64:") def install(self, spec: Spec, prefix: Prefix): # This will be executed instead of the default install if # the package's target is in the x86_64 family. @when("target=aarch64:") def install(self, spec: Spec, prefix: Prefix): # This will be executed if the package's target is in # the aarch64 family This allows each package to have a default version of ``install()`` AND specialized versions for particular platforms. The version that is called depends on the architecture of the instantiated package. Note that this works for methods other than install, as well. So, if you only have part of the install that is platform specific, you could do this: .. code-block:: python class SomePackage(Package): ... # virtual dependence on MPI. # could resolve to mpich, mpich2, OpenMPI depends_on("mpi") def setup(self): # do nothing in the default case pass @when("^openmpi") def setup(self): # do something special when this is built with OpenMPI for its MPI implementations. pass def install(self, prefix): # Do common install stuff self.setup() # Do more common install stuff Note that the default version of decorated methods must *always* come first. Otherwise it will override all of the decorated versions. This is a limitation of the Python language. """ spec: Optional[spack.spec.Spec] def __init__(self, condition: Union[str, bool]): """Can be used both as a decorator, for multimethods, or as a context manager to group ``when=`` arguments together. Args: condition (str): condition to be met """ self.when = condition def __call__(self, method): assert MultiMethodMeta._locals is not None, ( "cannot use multimethod, missing MultiMethodMeta metaclass?" ) # Create a multimethod with this name if there is not one already original_method = MultiMethodMeta._locals.get(method.__name__) if not isinstance(original_method, SpecMultiMethod): original_method = SpecMultiMethod(original_method) if self.when is True: original_method.register(spack.spec.EMPTY_SPEC, method) elif self.when is not False: original_method.register(spack.directives_meta.get_spec(self.when), method) return original_method def __enter__(self): # TODO: support when=False. if isinstance(self.when, str): spack.directives_meta.DirectiveMeta.push_when_constraint(self.when) def __exit__(self, exc_type, exc_val, exc_tb): if isinstance(self.when, str): spack.directives_meta.DirectiveMeta.pop_when_constraint() @contextmanager def default_args(**kwargs): """Context manager to override the default arguments of directives. Example:: with default_args(type=("build", "run")): depends_on("py-foo") depends_on("py-bar") depends_on("py-baz") Notice that unlike then :func:`when` context manager, this one is *not* composable, as it merely overrides the default argument values for the duration of the context. For example:: with default_args(when="+foo"): depends_on("pkg-a") depends_on("pkg-b", when="+bar") is equivalent to:: depends_on("pkg-a", when="+foo") depends_on("pkg-b", when="+bar") """ spack.directives_meta.DirectiveMeta.push_default_args(kwargs) yield spack.directives_meta.DirectiveMeta.pop_default_args() class MultiMethodError(spack.error.SpackError): """Superclass for multimethod dispatch errors""" def __init__(self, message): super().__init__(message) class NoSuchMethodError(spack.error.SpackError): """Raised when we can't find a version of a multi-method.""" def __init__(self, cls, method_name, spec, possible_specs): super().__init__( "Package %s does not support %s called with %s. Options are: %s" % (cls.__name__, method_name, spec, ", ".join(str(s) for s in possible_specs)) ) ================================================ FILE: lib/spack/spack/new_installer.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """New installer that will ultimately replace installer.py. It features an event loop, non-blocking I/O, and a POSIX jobserver to limit concurrency. It also has a more advanced terminal UI. It's mostly self-contained to avoid interfering with the rest of Spack too much while it's being developed and tested. The installer consists of a UI process that manages multiple build processes and handles updates to the database. It detects or creates a jobserver, and then kicks off an event loop in which it runs through a build queue, always running at least one build. Concurrent builds run as jobserver tokens are obtained. This means only one -j flag is needed to control concurrency. The UI process has two modes: an overview mode where it shows the status of all builds, and a mode where it follows the logs of a specific build. It listens to keyboard input to switch between modes. The build process does an ordinary install, but also spawns a "tee" thread that forwards its build output to both a log file and the UI process (if the UI process has requested it). This thread also runs an event loop to listen for control messages from the UI process (to enable/disable echoing of logs), and for output from the build process.""" import codecs import fcntl import glob import io import json import multiprocessing import os import re import selectors import shlex import shutil import signal import sys import tempfile import termios import threading import time import traceback import tty import warnings from gzip import GzipFile from multiprocessing import Pipe, Process from multiprocessing.connection import Connection from typing import ( TYPE_CHECKING, Callable, Dict, FrozenSet, Generator, List, NamedTuple, Optional, Set, Tuple, Union, ) from spack.vendor.typing_extensions import Literal import spack.binary_distribution import spack.build_environment import spack.builder import spack.config import spack.database import spack.deptypes as dt import spack.error import spack.hooks import spack.llnl.util.filesystem as fs import spack.llnl.util.tty import spack.llnl.util.tty.color import spack.paths import spack.report import spack.spec import spack.stage import spack.store import spack.subprocess_context import spack.traverse import spack.url_buildcache import spack.util.environment import spack.util.lock from spack.installer import _do_fake_install, dump_packages from spack.llnl.util.lang import pretty_duration from spack.llnl.util.tty.log import _is_background_tty, ignore_signal from spack.util.path import padding_filter, padding_filter_bytes if TYPE_CHECKING: import spack.package_base #: Type for specifying installation source modes InstallPolicy = Literal["auto", "cache_only", "source_only"] #: How often to update a spinner in seconds SPINNER_INTERVAL = 0.1 #: How often to wake up in headless mode to check for background->foreground transition (seconds) HEADLESS_WAKE_INTERVAL = 1.0 #: How long to display finished packages before graying them out CLEANUP_TIMEOUT = 2.0 #: How often to flush completed builds to the database DATABASE_WRITE_INTERVAL = 5.0 #: Size of the output buffer for child processes OUTPUT_BUFFER_SIZE = 32768 #: Suffix for temporary backup during overwrite install OVERWRITE_BACKUP_SUFFIX = ".old" #: Suffix for temporary cleanup during failed install OVERWRITE_GARBAGE_SUFFIX = ".garbage" #: Exit code used by the child process to signal that the build was stopped at a phase boundary EXIT_STOPPED_AT_PHASE = 3 class DatabaseAction: """Base class for objects that need to be persisted to the database.""" __slots__ = ("spec", "prefix_lock") spec: "spack.spec.Spec" prefix_lock: Optional[spack.util.lock.Lock] def save_to_db(self, db: spack.database.Database) -> None: ... def release_lock(self) -> None: if self.prefix_lock is not None: try: self.prefix_lock.release_write() except Exception: pass self.prefix_lock = None class MarkExplicitAction(DatabaseAction): """Action to mark an already installed spec as explicitly installed. Similar to ChildInfo, but used when no build process was needed.""" __slots__ = () def __init__(self, spec: "spack.spec.Spec") -> None: self.spec = spec self.prefix_lock = None def save_to_db(self, db: spack.database.Database) -> None: db._mark(self.spec, "explicit", True) class ChildInfo(DatabaseAction): """Information about a child process.""" __slots__ = ("proc", "output_r_conn", "state_r_conn", "control_w_conn", "explicit", "log_path") def __init__( self, proc: Process, spec: spack.spec.Spec, output_r_conn: Connection, state_r_conn: Connection, control_w_conn: Connection, log_path: str, explicit: bool = False, ) -> None: self.proc = proc self.spec = spec self.output_r_conn = output_r_conn self.state_r_conn = state_r_conn self.control_w_conn = control_w_conn self.log_path = log_path self.explicit = explicit self.prefix_lock: Optional[spack.util.lock.Lock] = None def save_to_db(self, db: spack.database.Database) -> None: return db._add(self.spec, explicit=self.explicit) def cleanup(self, selector: selectors.BaseSelector) -> None: """Unregister and close file descriptors, and join the child process.""" try: selector.unregister(self.output_r_conn.fileno()) except KeyError: pass try: selector.unregister(self.state_r_conn.fileno()) except KeyError: pass try: selector.unregister(self.proc.sentinel) except (KeyError, ValueError): pass self.output_r_conn.close() self.state_r_conn.close() self.control_w_conn.close() self.proc.join() def send_state(state: str, state_pipe: io.TextIOWrapper) -> None: """Send a state update message.""" json.dump({"state": state}, state_pipe, separators=(",", ":")) state_pipe.write("\n") def send_progress(current: int, total: int, state_pipe: io.TextIOWrapper) -> None: """Send a progress update message.""" json.dump({"progress": current, "total": total}, state_pipe, separators=(",", ":")) state_pipe.write("\n") def send_installed_from_binary_cache(state_pipe: io.TextIOWrapper) -> None: """Send a notification that the package was installed from binary cache.""" json.dump({"installed_from_binary_cache": True}, state_pipe, separators=(",", ":")) state_pipe.write("\n") def tee(control_r: int, log_r: int, log_path: str, parent_w: int) -> None: """Forward log_r to file_w and parent_w (if echoing is enabled). Echoing is enabled and disabled by reading from control_r.""" echo_on = False selector = selectors.DefaultSelector() selector.register(log_r, selectors.EVENT_READ) selector.register(control_r, selectors.EVENT_READ) try: with open(log_path, "wb") as log_file, open(parent_w, "wb", closefd=False) as parent: while True: for key, _ in selector.select(): if key.fd == log_r: data = os.read(log_r, OUTPUT_BUFFER_SIZE) if not data: # EOF: exit the thread return log_file.write(data) log_file.flush() if echo_on: parent.write(data) parent.flush() elif key.fd == control_r: control_data = os.read(control_r, 1) if not control_data: return else: echo_on = control_data == b"1" except OSError: # do not raise pass finally: os.close(log_r) class Tee: """Emulates ./build 2>&1 | tee build.log. The output is sent both to a log file and the parent process (if echoing is enabled). The control_fd is used to enable/disable echoing.""" def __init__(self, control: Connection, parent: Connection, log_path: str) -> None: self.control = control self.parent = parent # sys.stdout and sys.stderr may have been replaced with file objects under pytest, so # redirect their file descriptors in addition to the original fds 1 and 2. fds = {sys.stdout.fileno(), sys.stderr.fileno(), 1, 2} self.saved_fds = {fd: os.dup(fd) for fd in fds} #: The path of the log file self.log_path = log_path r, w = os.pipe() self.tee_thread = threading.Thread( target=tee, args=(self.control.fileno(), r, self.log_path, self.parent.fileno()), daemon=True, ) self.tee_thread.start() for fd in fds: os.dup2(w, fd) os.close(w) def close(self) -> None: # Closing stdout and stderr should close the last reference to the write end of the pipe, # causing the tee thread to wake up, flush the last data, and exit. We restore stdout and # stderr, because between sys.exit and the actual process exit buffers may be flushed, and # can cause exit code 120 (witnessed under pytest+coverage on macOS). sys.stdout.flush() sys.stderr.flush() for fd, saved_fd in self.saved_fds.items(): os.dup2(saved_fd, fd) os.close(saved_fd) self.tee_thread.join() # Only then close the other fds. self.control.close() self.parent.close() def install_from_buildcache( mirrors: List[spack.url_buildcache.MirrorMetadata], spec: spack.spec.Spec, unsigned: Optional[bool], state_stream: io.TextIOWrapper, ) -> bool: send_state("fetching from build cache", state_stream) try: tarball_stage = spack.binary_distribution.download_tarball( spec.build_spec, unsigned, mirrors ) except spack.binary_distribution.NoConfiguredBinaryMirrors: return False if tarball_stage is None: return False send_state("relocating", state_stream) spack.binary_distribution.extract_tarball(spec, tarball_stage, force=False) if spec.spliced: # overwrite old metadata with new spack.store.STORE.layout.write_spec(spec, spack.store.STORE.layout.spec_file_path(spec)) # now a block of curious things follow that should be fixed. pkg = spec.package if hasattr(pkg, "_post_buildcache_install_hook"): pkg._post_buildcache_install_hook() pkg.installed_from_binary_cache = True # inform also the parent that this package was installed from binary cache. send_installed_from_binary_cache(state_stream) return True class GlobalState: """Global state needed in a build subprocess. This is similar to spack.subprocess_context, but excludes the Spack environment, which is slow to serialize and should not be needed during the build.""" __slots__ = ("store", "config", "monkey_patches", "spack_working_dir", "repo_cache") def __init__(self): if multiprocessing.get_start_method() == "fork": return self.config = spack.config.CONFIG.ensure_unwrapped() self.store = spack.store.STORE self.monkey_patches = spack.subprocess_context.TestPatches.create() self.spack_working_dir = spack.paths.spack_working_dir def restore(self): if multiprocessing.get_start_method() == "fork": # In the forking case we must erase SSL contexts. from spack.oci import opener from spack.util import web from spack.util.s3 import s3_client_cache web.urlopen._instance = None opener.urlopen._instance = None s3_client_cache.clear() return spack.store.STORE = self.store spack.config.CONFIG = self.config self.monkey_patches.restore() spack.paths.spack_working_dir = self.spack_working_dir class PrefixPivoter: """Manages the installation prefix of a build.""" def __init__(self, prefix: str, keep_prefix: bool = False) -> None: """Initialize the prefix pivoter. Args: prefix: The installation prefix path keep_prefix: Whether to keep a failed installation prefix """ self.prefix = prefix #: Whether to keep a failed installation prefix self.keep_prefix = keep_prefix #: Temporary location for the original prefix self.tmp_prefix: Optional[str] = None self.parent = os.path.dirname(prefix) def __enter__(self) -> "PrefixPivoter": """Enter the context: move existing prefix to temporary location if needed.""" if not self._lexists(self.prefix): return self # Move the existing prefix to a temporary location so the build starts fresh self.tmp_prefix = self._mkdtemp( dir=self.parent, prefix=".", suffix=OVERWRITE_BACKUP_SUFFIX ) self._rename(self.prefix, self.tmp_prefix) return self def __exit__( self, exc_type: Optional[type], exc_val: Optional[BaseException], exc_tb: Optional[object] ) -> None: """Exit the context: cleanup on success, restore on failure.""" if exc_type is None: # Success: remove the backup if self.tmp_prefix is not None: self._rmtree_ignore_errors(self.tmp_prefix) return # Failure handling: if self.keep_prefix: # Leave the failed prefix in place, discard the backup if self.tmp_prefix is not None: self._rmtree_ignore_errors(self.tmp_prefix) elif self.tmp_prefix is not None: # There was a pre-existing prefix: pivot back to it and discard the failed build garbage = self._mkdtemp(dir=self.parent, prefix=".", suffix=OVERWRITE_GARBAGE_SUFFIX) try: self._rename(self.prefix, garbage) has_failed_prefix = True except FileNotFoundError: # build never created the prefix dir has_failed_prefix = False self._rename(self.tmp_prefix, self.prefix) if has_failed_prefix: self._rmtree_ignore_errors(garbage) elif self._lexists(self.prefix): # No backup, just remove the failed installation garbage = self._mkdtemp(dir=self.parent, prefix=".", suffix=OVERWRITE_GARBAGE_SUFFIX) self._rename(self.prefix, garbage) self._rmtree_ignore_errors(garbage) def _lexists(self, path: str) -> bool: return os.path.lexists(path) def _rename(self, src: str, dst: str) -> None: os.rename(src, dst) def _mkdtemp(self, dir: str, prefix: str, suffix: str) -> str: return tempfile.mkdtemp(dir=dir, prefix=prefix, suffix=suffix) def _rmtree_ignore_errors(self, path: str) -> None: shutil.rmtree(path, ignore_errors=True) def worker_function( spec: spack.spec.Spec, explicit: bool, mirrors: List[spack.url_buildcache.MirrorMetadata], unsigned: Optional[bool], install_policy: InstallPolicy, dirty: bool, keep_stage: bool, restage: bool, keep_prefix: bool, skip_patch: bool, fake: bool, install_source: bool, run_tests: bool, state: Connection, parent: Connection, echo_control: Connection, makeflags: str, js1: Optional[Connection], js2: Optional[Connection], log_path: str, global_state: GlobalState, stop_before: Optional[str] = None, stop_at: Optional[str] = None, ): """ Function run in the build child process. Installs the specified spec, sending state updates and build output back to the parent process. Args: spec: Spec to install explicit: Whether the spec was explicitly requested by the user mirrors: List of buildcache mirrors to try unsigned: Whether to allow unsigned buildcache entries install_policy: ``"auto"``, ``"cache_only"``, or ``"source_only"`` dirty: Whether to preserve user environment in the build environment keep_stage: Whether to keep the build stage after installation restage: Whether to restage the source before building keep_prefix: Whether to keep a failed installation prefix skip_patch: Whether to skip the patch phase run_tests: Whether to run install-time tests for this package state: Connection to send state updates to parent: Connection to send build output to echo_control: Connection to receive echo control messages from makeflags: MAKEFLAGS to set, so that the build process uses the POSIX jobserver js1: Connection for old style jobserver read fd (if any). Unused, just to inherit fd. js2: Connection for old style jobserver write fd (if any). Unused, just to inherit fd. log_path: Path to the log file to write build output to global_state: Global state to restore """ # TODO: don't start a build for external packages if spec.external: return global_state.restore() # Isolate the process group to shield against Ctrl+C and enable safe killpg() cleanup. In # constrast to setsid(), this keeps a neat process group hierarchy for utils like pstree. os.setpgid(0, 0) # Reset SIGTSTP to default in case the parent had a custom handler. signal.signal(signal.SIGTSTP, signal.SIG_DFL) def handle_sigterm(signum, frame): # This SIGTERM handler forwards the signal to child processes (cmake, make, etc). We wait # for all child processes to exit before raising KeyboardInterrupt. This ensures all # __exit__ and finally blocks run after the child processes have stopped, meaning that we # get to clean up the prefix without risking that the child process writes to it # afterwards. signal.signal(signal.SIGTERM, signal.SIG_IGN) os.killpg(0, signal.SIGTERM) try: while True: os.waitpid(-1, 0) except ChildProcessError: pass raise KeyboardInterrupt("Installation interrupted") signal.signal(signal.SIGTERM, handle_sigterm) os.environ["MAKEFLAGS"] = makeflags # Force line buffering for Python's textio wrappers of stdout/stderr. We're not going to print # much ourselves, but what we print should appear before output from `make` and other build # tools. sys.stdout = os.fdopen( sys.stdout.fileno(), "w", buffering=1, encoding=sys.stdout.encoding, closefd=False ) sys.stderr = os.fdopen( sys.stderr.fileno(), "w", buffering=1, encoding=sys.stderr.encoding, closefd=False ) # Detach stdin from the terminal like `./build < /dev/null`. This would not be necessary if we # used os.setsid() instead of os.setpgid(), but that would "break" pstree output. devnull_fd = os.open(os.devnull, os.O_RDONLY) os.dup2(devnull_fd, 0) os.close(devnull_fd) sys.stdin = open(os.devnull, "r", encoding=sys.stdin.encoding) # Start the tee thread to forward output to the log file and parent process. tee = Tee(echo_control, parent, log_path) # Use closedfd=false because of the connection objects. Use line buffering. state_stream = os.fdopen(state.fileno(), "w", buffering=1, closefd=False) exit_code = 0 try: with PrefixPivoter(spec.prefix, keep_prefix): _install( spec, explicit, mirrors, unsigned, install_policy, dirty, keep_stage, restage, skip_patch, fake, install_source, state_stream, log_path, spack.store.STORE, run_tests, stop_before, stop_at, ) except spack.error.StopPhase: exit_code = EXIT_STOPPED_AT_PHASE except BaseException: traceback.print_exc() # log the traceback to the log file exit_code = 1 finally: tee.close() state_stream.close() if exit_code == 0: # Try to install the compressed log file if not os.path.lexists(spec.package.install_log_path): try: with open(log_path, "rb") as f, open(spec.package.install_log_path, "wb") as g: # Use GzipFile directly so we can omit filename / mtime in header gzip_file = GzipFile( filename="", mode="wb", compresslevel=6, mtime=0, fileobj=g ) shutil.copyfileobj(f, gzip_file) gzip_file.close() except Exception: pass # don't fail the build just because log compression failed # Remove the uncompressed log file from the stage dir on successful install. if not keep_stage: try: os.unlink(log_path) except OSError: pass sys.exit(exit_code) def _archive_build_metadata(pkg: "spack.package_base.PackageBase") -> None: """Copy build metadata from stage to install prefix .spack directory. Mirrors what the old installer's log() function does in the parent process. Only called after a successful source build (not for binary cache installs). Errors are suppressed to avoid failing the build over metadata archiving.""" try: if os.path.lexists(pkg.env_mods_path): shutil.copy2(pkg.env_mods_path, pkg.install_env_path) except OSError as e: spack.llnl.util.tty.debug(e) try: if os.path.lexists(pkg.configure_args_path): shutil.copy2(pkg.configure_args_path, pkg.install_configure_args_path) except OSError as e: spack.llnl.util.tty.debug(e) # Archive install-phase test log if present try: pkg.archive_install_test_log() except Exception as e: spack.llnl.util.tty.debug(e) # Archive package-specific files matched by archive_files glob patterns try: with fs.working_dir(pkg.stage.path): target_dir = os.path.join( spack.store.STORE.layout.metadata_path(pkg.spec), "archived-files" ) errors = io.StringIO() for glob_expr in spack.builder.create(pkg).archive_files: abs_expr = os.path.realpath(glob_expr) if os.path.realpath(pkg.stage.path) not in abs_expr: errors.write(f"[OUTSIDE SOURCE PATH]: {glob_expr}\n") continue if os.path.isabs(glob_expr): glob_expr = os.path.relpath(glob_expr, pkg.stage.path) for f in glob.glob(glob_expr): try: target = os.path.join(target_dir, f) fs.mkdirp(os.path.dirname(target)) fs.install(f, target) except Exception as e: spack.llnl.util.tty.debug(e) errors.write(f"[FAILED TO ARCHIVE]: {f}") if errors.getvalue(): error_file = os.path.join(target_dir, "errors.txt") fs.mkdirp(target_dir) with open(error_file, "w", encoding="utf-8") as err: err.write(errors.getvalue()) spack.llnl.util.tty.warn( f"Errors occurred when archiving files.\n\tSee: {error_file}" ) except Exception as e: spack.llnl.util.tty.debug(e) try: packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec) dump_packages(pkg.spec, packages_dir) except Exception as e: spack.llnl.util.tty.debug(e) try: spack.store.STORE.layout.write_host_environment(pkg.spec) except Exception as e: spack.llnl.util.tty.debug(e) def _install( spec: spack.spec.Spec, explicit: bool, mirrors: List[spack.url_buildcache.MirrorMetadata], unsigned: Optional[bool], install_policy: InstallPolicy, dirty: bool, keep_stage: bool, restage: bool, skip_patch: bool, fake: bool, install_source: bool, state_stream: io.TextIOWrapper, log_path: str, store: spack.store.Store = spack.store.STORE, run_tests: bool = False, stop_before: Optional[str] = None, stop_at: Optional[str] = None, ) -> None: """Install a spec from build cache or source.""" # Create the stage and log file before starting the tee thread. pkg = spec.package pkg.run_tests = run_tests if fake: store.layout.create_install_directory(spec) _do_fake_install(pkg) spack.hooks.post_install(spec, explicit) return # Try to install from buildcache, unless user asked for source only if install_policy != "source_only": if install_from_buildcache(mirrors, spec, unsigned, state_stream): spack.hooks.post_install(spec, explicit) return elif install_policy == "cache_only": # Binary required but not available send_state("no binary available", state_stream) raise spack.error.InstallError(f"No binary available for {spec}") unmodified_env = os.environ.copy() env_mods = spack.build_environment.setup_package(pkg, dirty=dirty) store.layout.create_install_directory(spec) stage = pkg.stage stage.keep = keep_stage # Then try a source build. with stage: if restage: stage.destroy() stage.create() # Write build environment and env-mods to stage spack.util.environment.dump_environment(pkg.env_path) with open(pkg.env_mods_path, "w", encoding="utf-8") as f: f.write(env_mods.shell_modifications(explicit=True, env=unmodified_env)) # Try to snapshot configure/cmake args before phases run for attr in ("configure_args", "cmake_args"): try: args = getattr(pkg, attr)() with open(pkg.configure_args_path, "w", encoding="utf-8") as f: f.write(" ".join(shlex.quote(a) for a in args)) break except Exception: pass # For develop packages or non-develop packages with --keep-stage there may be a # pre-existing symlink at pkg.log_path which would cause the new symlink to fail. # Try removing it if it exists. try: os.unlink(pkg.log_path) except OSError: pass os.symlink(log_path, pkg.log_path) send_state("staging", state_stream) if not skip_patch: pkg.do_patch() else: pkg.do_stage() os.chdir(stage.source_path) if install_source and os.path.isdir(stage.source_path): src_target = os.path.join(spec.prefix, "share", spec.name, "src") fs.install_tree(stage.source_path, src_target) spack.hooks.pre_install(spec) builder = spack.builder.create(pkg) if stop_before is not None and stop_before not in builder.phases: raise spack.error.InstallError(f"'{stop_before}' is not a valid phase for {pkg.name}") if stop_at is not None and stop_at not in builder.phases: raise spack.error.InstallError(f"'{stop_at}' is not a valid phase for {pkg.name}") for phase in builder: if stop_before is not None and phase.name == stop_before: send_state(f"stopped before {stop_before}", state_stream) raise spack.error.StopPhase(f"Stopping before '{stop_before}'") send_state(phase.name, state_stream) spack.llnl.util.tty.msg(f"{pkg.name}: Executing phase: '{phase.name}'") # Run the install phase with debug output enabled. old_debug = spack.llnl.util.tty.debug_level() spack.llnl.util.tty.set_debug(1) try: phase.execute() finally: spack.llnl.util.tty.set_debug(old_debug) if stop_at is not None and phase.name == stop_at: send_state(f"stopped after {stop_at}", state_stream) raise spack.error.StopPhase(f"Stopping at '{stop_at}'") _archive_build_metadata(pkg) spack.hooks.post_install(spec, explicit) class JobServer: """Attach to an existing POSIX jobserver or create a FIFO-based one.""" def __init__(self, num_jobs: int) -> None: #: Keep track of how many tokens Spack itself has acquired, which is used to release them. self.tokens_acquired = 0 #: The number of jobs to run concurrently. This translates to `num_jobs - 1` tokens in the #: jobserver. self.num_jobs = num_jobs #: The target number of jobs to run concurrently, which may differ from num_jobs if the #: user has requested a decrease in parallelism, but we haven't consumed enough tokens to #: reflect that yet. This value is used in the UI. The invariant is that self.target_jobs #: can only be modified if self.created is True. self.target_jobs = num_jobs self.fifo_path: Optional[str] = None self.created = False self._setup() # Ensure that Executable()(...) in build processes ultimately inherit jobserver fds. os.set_inheritable(self.r, True) os.set_inheritable(self.w, True) # r_conn and w_conn are used to make build processes inherit the jobserver fds if needed. # Connection objects close the fd as they are garbage collected, so store them. self.r_conn = Connection(self.r) self.w_conn = Connection(self.w) def _setup(self) -> None: fifo_config = get_jobserver_config() if type(fifo_config) is str: # FIFO-based jobserver. Try to open the FIFO. open_attempt = open_existing_jobserver_fifo(fifo_config) if open_attempt: self.r, self.w = open_attempt self.fifo_path = fifo_config return elif type(fifo_config) is tuple: # Old style pipe-based jobserver. Validate the fds before using them. r, w = fifo_config if fcntl.fcntl(r, fcntl.F_GETFD) != -1 and fcntl.fcntl(w, fcntl.F_GETFD) != -1: self.r, self.w = r, w return # No existing jobserver we can connect to: create a FIFO-based one. self.r, self.w, self.fifo_path = create_jobserver_fifo(self.num_jobs) self.created = True def makeflags(self, gmake: Optional[spack.spec.Spec]) -> str: """Return the MAKEFLAGS for a build process, depending on its gmake build dependency.""" if self.fifo_path and (not gmake or gmake.satisfies("@4.4:")): return f" -j{self.num_jobs} --jobserver-auth=fifo:{self.fifo_path}" elif not gmake or gmake.satisfies("@4.0:"): return f" -j{self.num_jobs} --jobserver-auth={self.r},{self.w}" else: return f" -j{self.num_jobs} --jobserver-fds={self.r},{self.w}" def has_target_parallelism(self) -> bool: return self.num_jobs == self.target_jobs def increase_parallelism(self) -> None: """Add one token to the jobserver to increase parallelism; this should always work.""" if not self.created: return os.write(self.w, b"+") self.target_jobs += 1 self.num_jobs += 1 def decrease_parallelism(self) -> None: """Request an eventual concurrency decrease by 1.""" if not self.created or self.target_jobs <= 1: return self.target_jobs -= 1 self.maybe_discard_tokens() def maybe_discard_tokens(self) -> None: """Try to get reduce parallelism by discarding tokens.""" to_discard = self.num_jobs - self.target_jobs if to_discard <= 0: return try: # The read may return zero or just fewer bytes than requested; we'll try again later. self.num_jobs -= len(os.read(self.r, to_discard)) except BlockingIOError: pass def acquire(self, jobs: int) -> int: """Try and acquire at most 'jobs' tokens from the jobserver. Returns the number of tokens actually acquired (may be less than requested, or zero).""" try: num_acquired = len(os.read(self.r, jobs)) self.tokens_acquired += num_acquired return num_acquired except BlockingIOError: return 0 def release(self) -> None: """Release a token back to the jobserver.""" # The last job to quit has an implicit token, so don't release if we have none. if self.tokens_acquired == 0: return self.tokens_acquired -= 1 if self.target_jobs < self.num_jobs: # If a decrease in parallelism is requested, discard a token instead of releasing it. self.num_jobs -= 1 else: os.write(self.w, b"+") def close(self) -> None: if self.created and self.num_jobs > 1: if self.tokens_acquired != 0: # It's a non-fatal internal error to close the jobserver with acquired tokens. warnings.warn("Spack failed to release jobserver tokens", stacklevel=2) else: # Verify that all build processes released the tokens they acquired. total = self.num_jobs - 1 drained = self.acquire(total) if drained != total: n = total - drained warnings.warn( f"{n} jobserver {'token was' if n == 1 else 'tokens were'} not released " "by the build processes. This can indicate that the build ran with " "limited parallelism.", stacklevel=2, ) self.r_conn.close() self.w_conn.close() # Remove the FIFO if we created it. if self.created and self.fifo_path: try: os.unlink(self.fifo_path) except OSError: pass try: os.rmdir(os.path.dirname(self.fifo_path)) except OSError: pass def start_build( spec: spack.spec.Spec, explicit: bool, mirrors: List[spack.url_buildcache.MirrorMetadata], unsigned: Optional[bool], install_policy: InstallPolicy, dirty: bool, keep_stage: bool, restage: bool, keep_prefix: bool, skip_patch: bool, fake: bool, install_source: bool, run_tests: bool, jobserver: JobServer, stop_before: Optional[str] = None, stop_at: Optional[str] = None, ) -> ChildInfo: """Start a new build.""" # Create pipes for the child's output, state reporting, and control. state_r_conn, state_w_conn = Pipe(duplex=False) output_r_conn, output_w_conn = Pipe(duplex=False) control_r_conn, control_w_conn = Pipe(duplex=False) # Obtain the MAKEFLAGS to be set in the child process, and determine whether it's necessary # for the child process to inherit our jobserver fds. gmake = next(iter(spec.dependencies("gmake")), None) makeflags = jobserver.makeflags(gmake) fifo = "--jobserver-auth=fifo:" in makeflags # TODO: remove once external specs do not create a build process if spec.external: log_path = os.devnull else: log_fd, log_path = tempfile.mkstemp( prefix=f"spack-stage-{spec.name}-{spec.version}-{spec.dag_hash()}-", suffix=".log", dir=spack.stage.get_stage_root(), ) os.close(log_fd) # child will open it proc = Process( target=worker_function, args=( spec, explicit, mirrors, unsigned, install_policy, dirty, keep_stage, restage, keep_prefix, skip_patch, fake, install_source, run_tests, state_w_conn, output_w_conn, control_r_conn, makeflags, None if fifo else jobserver.r_conn, None if fifo else jobserver.w_conn, log_path, GlobalState(), stop_before, stop_at, ), ) proc.start() # The parent process does not need the write ends of the main pipes or the read end of control. state_w_conn.close() output_w_conn.close() control_r_conn.close() # Set the read ends to non-blocking: in principle redundant with epoll/kqueue, but safer. os.set_blocking(output_r_conn.fileno(), False) os.set_blocking(state_r_conn.fileno(), False) return ChildInfo(proc, spec, output_r_conn, state_r_conn, control_w_conn, log_path, explicit) def get_jobserver_config(makeflags: Optional[str] = None) -> Optional[Union[str, Tuple[int, int]]]: """Parse MAKEFLAGS for jobserver. Either it's a FIFO or (r, w) pair of file descriptors. Args: makeflags: MAKEFLAGS string to parse. If None, reads from os.environ. """ makeflags = os.environ.get("MAKEFLAGS", "") if makeflags is None else makeflags if not makeflags: return None # We can have the following flags: # --jobserver-fds=R,W (before GNU make 4.2) # --jobserver-auth=fifo:PATH or --jobserver-auth=R,W (after GNU make 4.2) # In case of multiple, the last one wins. matches = re.findall(r" --jobserver-[^=]+=([^ ]+)", makeflags) if not matches: return None last_match: str = matches[-1] assert isinstance(last_match, str) if last_match.startswith("fifo:"): return last_match[5:] parts = last_match.split(",", 1) if len(parts) != 2: return None try: return int(parts[0]), int(parts[1]) except ValueError: return None def create_jobserver_fifo(num_jobs: int) -> Tuple[int, int, str]: """Create a new jobserver FIFO with the specified number of job tokens.""" tmpdir = tempfile.mkdtemp() fifo_path = os.path.join(tmpdir, "jobserver_fifo") try: os.mkfifo(fifo_path, 0o600) read_fd = os.open(fifo_path, os.O_RDONLY | os.O_NONBLOCK) write_fd = os.open(fifo_path, os.O_WRONLY) # write num_jobs - 1 tokens, because the first job is implicit os.write(write_fd, b"+" * (num_jobs - 1)) return read_fd, write_fd, fifo_path except Exception: try: os.unlink(fifo_path) except OSError as e: spack.llnl.util.tty.debug(f"Failed to remove POSIX jobserver FIFO: {e}", level=3) pass try: os.rmdir(tmpdir) except OSError as e: spack.llnl.util.tty.debug(f"Failed to remove POSIX jobserver FIFO dir: {e}", level=3) pass raise def open_existing_jobserver_fifo(fifo_path: str) -> Optional[Tuple[int, int]]: """Open an existing jobserver FIFO for reading and writing.""" try: read_fd = os.open(fifo_path, os.O_RDONLY | os.O_NONBLOCK) write_fd = os.open(fifo_path, os.O_WRONLY) return read_fd, write_fd except OSError: return None class FdInfo: """Information about a file descriptor mapping.""" __slots__ = ("pid", "name") def __init__(self, pid: int, name: str) -> None: self.pid = pid self.name = name class BuildInfo: """Information about a package being built.""" __slots__ = ( "state", "explicit", "version", "hash", "name", "external", "prefix", "finished_time", "start_time", "duration", "progress_percent", "control_w_conn", "log_path", "log_summary", ) def __init__( self, spec: spack.spec.Spec, explicit: bool, control_w_conn: Optional[Connection], log_path: Optional[str] = None, start_time: float = 0.0, ) -> None: self.state: str = "starting" self.explicit: bool = explicit self.version: str = str(spec.version) self.hash: str = spec.dag_hash(7) self.name: str = spec.name self.external: bool = spec.external self.prefix: str = spec.prefix self.finished_time: Optional[float] = None self.start_time: float = start_time self.duration: Optional[float] = None self.progress_percent: Optional[int] = None self.control_w_conn = control_w_conn self.log_path: Optional[str] = log_path self.log_summary: Optional[str] = None class BuildStatus: """Tracks the build status display for terminal output.""" def __init__( self, total: int, stdout: io.TextIOWrapper = sys.stdout, # type: ignore[assignment] get_terminal_size: Callable[[], os.terminal_size] = os.get_terminal_size, get_time: Callable[[], float] = time.monotonic, is_tty: Optional[bool] = None, color: Optional[bool] = None, verbose: bool = False, filter_padding: bool = False, ) -> None: #: Ordered dict of build ID -> info self.total = total self.completed = 0 self.builds: Dict[str, BuildInfo] = {} self.finished_builds: List[BuildInfo] = [] self.spinner_chars = ["|", "/", "-", "\\"] self.spinner_index = 0 self.dirty = True # Start dirty to draw initial state self.active_area_rows = 0 self.total_lines = 0 self.next_spinner_update = 0.0 self.next_update = 0.0 self.overview_mode = True # Whether to draw the package overview self.tracked_build_id = "" # identifier of the package whose logs we follow self.search_term = "" self.search_mode = False self.log_ends_with_newline = True self.actual_jobs: int = 0 self.target_jobs: int = 0 self.stdout = stdout self.get_terminal_size = get_terminal_size self.terminal_size = os.terminal_size((0, 0)) self.terminal_size_changed: bool = True self.get_time = get_time self.is_tty = is_tty if is_tty is not None else stdout.isatty() if color is not None: self.color = color else: self.color = spack.llnl.util.tty.color.get_color_when(stdout) #: Verbose mode only applies to non-TTY where we want to track a single build log. self.verbose = verbose and not self.is_tty self.filter_padding = filter_padding #: When True, suppress all terminal output (process is in background). #: Controlling code is responsible for modifying this variable based on process state self.headless = False def on_resize(self) -> None: """Refresh cached terminal size and trigger a redraw.""" self.terminal_size_changed = True self.dirty = True def add_build( self, spec: spack.spec.Spec, explicit: bool, control_w_conn: Optional[Connection] = None, log_path: Optional[str] = None, ) -> None: """Add a new build to the display and mark the display as dirty.""" build_info = BuildInfo(spec, explicit, control_w_conn, log_path, int(self.get_time())) self.builds[spec.dag_hash()] = build_info self.dirty = True # Track the new build's logs when we're not already following another build. This applies # only in non-TTY verbose mode. if self.verbose and not self.tracked_build_id and control_w_conn is not None: self.tracked_build_id = spec.dag_hash() try: os.write(control_w_conn.fileno(), b"1") except OSError: pass def toggle(self) -> None: """Toggle between overview mode and following a specific build.""" if self.overview_mode: self.next() else: if not self.log_ends_with_newline: self.stdout.buffer.write(b"\n") self.log_ends_with_newline = True self.active_area_rows = 0 self.search_term = "" self.search_mode = False self.overview_mode = True self.dirty = True try: conn = self.builds[self.tracked_build_id].control_w_conn if conn is not None: os.write(conn.fileno(), b"0") except (KeyError, OSError): pass self.tracked_build_id = "" def search_input(self, input: str) -> None: """Handle keyboard input when in search mode""" if input in ("\r", "\n"): self.log_ends_with_newline = False self.next(1) elif input == "\x1b": # Escape self.search_mode = False self.search_term = "" self.dirty = True elif input in ("\x7f", "\b"): # Backspace self.search_term = self.search_term[:-1] self.dirty = True elif input.isprintable(): self.search_term += input self.dirty = True def enter_search(self) -> None: self.search_mode = True self.dirty = True def _is_displayed(self, build: BuildInfo) -> bool: """Returns true if the build matches the search term, or when no search term is set.""" # When not in search mode, the search_term is "", which always evaluates to True below return self.search_term in build.name or build.hash.startswith(self.search_term) def _get_next(self, direction: int) -> Optional[str]: """Returns the next or previous unfinished build ID matching the search term, or None if none found. Direction should be 1 for next, -1 for previous.""" matching = [ build_id for build_id, build in self.builds.items() if (build.finished_time is None or build.state == "failed") and self._is_displayed(build) ] if not matching: return None try: idx = matching.index(self.tracked_build_id) except ValueError: return matching[0] if direction == 1 else matching[-1] return matching[(idx + direction) % len(matching)] def next(self, direction: int = 1) -> None: """Follow the logs of the next build in the list.""" new_build_id = self._get_next(direction) if not new_build_id or self.tracked_build_id == new_build_id: return new_build = self.builds[new_build_id] if self.overview_mode: self.overview_mode = False # Stop following the previous and start following the new build. if self.tracked_build_id: try: conn = self.builds[self.tracked_build_id].control_w_conn if conn is not None: os.write(conn.fileno(), b"0") except (KeyError, OSError): pass self.tracked_build_id = new_build_id version_str = ( f"\033[0;36m@{new_build.version}\033[0m" if self.color else f"@{new_build.version}" ) prefix = "" if self.log_ends_with_newline else "\n" if new_build.state == "failed": # For failed builds, show the stored log summary instead of following live logs. self.stdout.write(f"{prefix}==> Log summary of {new_build.name}{version_str}\n") self.log_ends_with_newline = True if new_build.log_summary: self.stdout.write(new_build.log_summary) if new_build.log_path: if not new_build.log_summary: self.stdout.write("No errors parsed from log, see full log: ") else: self.stdout.write("Full log: ") self.stdout.write(f"{new_build.log_path}\n") self.stdout.flush() else: # Tell the user we're following new logs, and instruct the child to start sending. self.stdout.write(f"{prefix}==> Following logs of {new_build.name}{version_str}\n") self.log_ends_with_newline = True self.stdout.flush() try: conn = new_build.control_w_conn if conn is not None: os.write(conn.fileno(), b"1") except (KeyError, OSError): pass def set_jobs(self, actual: int, target: int) -> None: """Set the actual and target number of jobs to run concurrently.""" if actual == self.actual_jobs and target == self.target_jobs: return self.actual_jobs = actual self.target_jobs = target self.dirty = True def update_state(self, build_id: str, state: str) -> None: """Update the state of a package and mark the display as dirty.""" build_info = self.builds[build_id] build_info.state = state build_info.progress_percent = None if state in ("finished", "failed"): self.completed += 1 now = self.get_time() build_info.duration = now - build_info.start_time build_info.finished_time = now + CLEANUP_TIMEOUT # Stop tracking the finished build's logs. if build_id == self.tracked_build_id: if not self.overview_mode: self.toggle() if self.verbose: self.tracked_build_id = "" self.dirty = True # For non-TTY output, print state changes immediately if not self.is_tty and not self.headless: line = "".join( self._generate_line_components(build_info, static=True, now=self.get_time()) ) self.stdout.write(line + "\n") self.stdout.flush() def parse_log_summary(self, build_id: str) -> None: """Parse the build log for errors/warnings and store the summary.""" build_info = self.builds[build_id] if not build_info.log_path or not os.path.exists(build_info.log_path): return buf = io.StringIO() spack.build_environment.write_log_summary( buf, f"{build_info.name}@{build_info.version} build", build_info.log_path ) summary = buf.getvalue() if summary: build_info.log_summary = summary def update_progress(self, build_id: str, current: int, total: int) -> None: """Update the progress of a package and mark the display as dirty.""" percent = int((current / total) * 100) build_info = self.builds[build_id] if build_info.progress_percent != percent: build_info.progress_percent = percent self.dirty = True def update(self, finalize: bool = False) -> None: """Redraw the interactive display.""" if self.headless or not self.is_tty or not self.overview_mode: return now = self.get_time() # Avoid excessive redraws if not finalize and now < self.next_update: return # Only update the spinner if there are still running packages if now >= self.next_spinner_update and any( pkg.finished_time is None for pkg in self.builds.values() ): self.spinner_index = (self.spinner_index + 1) % len(self.spinner_chars) self.dirty = True self.next_spinner_update = now + SPINNER_INTERVAL for build_id in list(self.builds): build_info = self.builds[build_id] if build_info.state == "failed" or build_info.finished_time is None: continue if finalize or now >= build_info.finished_time: self.finished_builds.append(build_info) del self.builds[build_id] self.dirty = True if not self.dirty and not finalize: return # Build the overview output in a buffer and print all at once to avoid flickering. buffer = io.StringIO() # Move cursor up to the start of the display area assuming the same terminal width. If the # terminal resized, lines may have wrapped, and we should've moved up further. We do not # try to track that (would require keeping track of each line's width). if self.active_area_rows > 0: buffer.write(f"\033[{self.active_area_rows}A\r") if self.terminal_size_changed: self.terminal_size = self.get_terminal_size() self.terminal_size_changed = False # After resize, active_area_rows is invalidated due to possible line wrapping. Set to # 0 to force newlines instead of cursor movement. self.active_area_rows = 0 max_width, max_height = self.terminal_size # First flush the finished builds. These are "persisted" in terminal history. if self.finished_builds: for build in self.finished_builds: self._render_build(build, buffer, now=now) self._println(buffer, force_newline=True) # should scroll the terminal self.finished_builds.clear() # Finished builds can span multiple lines, overlapping our "active area", invalidating # active_area_rows. Set to 0 to force newlines instead of cursor movement. self.active_area_rows = 0 # Then a header followed by the active builds. This is the "mutable" part of the display. self.total_lines = 0 if not finalize: if self.color: bold = "\033[1m" reset = "\033[0m" cyan = "\033[36m" else: bold = reset = cyan = "" if self.actual_jobs != self.target_jobs: jobs_str = f"{self.actual_jobs}=>{self.target_jobs}" else: jobs_str = str(self.target_jobs) long_header_len = len( f"Progress: {self.completed}/{self.total} +/-: {jobs_str} jobs" " /: filter v: logs n/p: next/prev" ) if long_header_len < max_width: self._println( buffer, f"{bold}Progress:{reset} {self.completed}/{self.total}" f" {cyan}+{reset}/{cyan}-{reset}: " f"{jobs_str} jobs" f" {cyan}/{reset}: filter {cyan}v{reset}: logs" f" {cyan}n{reset}/{cyan}p{reset}: next/prev", ) else: self._println(buffer, f"{bold}Progress:{reset} {self.completed}/{self.total}") displayed_builds = ( [b for b in self.builds.values() if self._is_displayed(b)] if self.search_term else self.builds.values() ) len_builds = len(displayed_builds) # Truncate if we have more builds than fit on the screen. In that case we have to reserve # an additional line for the "N more..." message. truncate_at = max_height - 3 if len_builds + 2 > max_height else len_builds for i, build in enumerate(displayed_builds, 1): if i > truncate_at: self._println(buffer, f"{len_builds - i + 1} more...") break self._render_build(build, buffer, max_width, now=now) self._println(buffer) if self.search_mode: buffer.write(f"filter> {self.search_term}\033[K") # Clear any remaining lines from previous display buffer.write("\033[0J") # Print everything at once to avoid flickering self.stdout.write(buffer.getvalue()) self.stdout.flush() # Update the number of lines drawn for next time. It reflects the number of active builds. self.active_area_rows = self.total_lines self.dirty = False # Schedule next UI update self.next_update = now + SPINNER_INTERVAL / 2 def _println(self, buffer: io.StringIO, line: str = "", force_newline: bool = False) -> None: """Print a line to the buffer, handling line clearing and cursor movement.""" self.total_lines += 1 if line: buffer.write(line) if self.total_lines > self.active_area_rows or force_newline: buffer.write("\033[0m\033[K\n") # reset, clear to EOL, newline else: buffer.write("\033[0m\033[K\033[1B\r") # reset, clear to EOL, move to next line def print_logs(self, build_id: str, data: bytes) -> None: if self.headless: return # Discard logs we are not following. Generally this should not happen as we tell the child # to only send logs when we are following it. It could maybe happen while transitioning # between builds. if build_id != self.tracked_build_id: return if self.filter_padding: data = padding_filter_bytes(data) self.stdout.buffer.write(data) self.stdout.flush() self.log_ends_with_newline = data.endswith(b"\n") def _render_build( self, build_info: BuildInfo, buffer: io.StringIO, max_width: int = 0, now: float = 0.0 ) -> None: """Print a single build line to the buffer, truncating to max_width (if > 0).""" line_width = 0 for component in self._generate_line_components(build_info, now=now): # ANSI escape sequence(s), does not contribute to width if not component.startswith("\033") and max_width > 0: line_width += len(component) if line_width > max_width: break buffer.write(component) def _generate_line_components( self, build_info: BuildInfo, static: bool = False, now: float = 0.0 ) -> Generator[str, None, None]: """Yield formatted line components for a package. Escape sequences are yielded as separate strings so they do not contribute to the line width.""" if build_info.external: indicator = "[e]" elif build_info.state == "finished": indicator = "[+]" elif build_info.state == "failed": indicator = "[x]" elif static: indicator = "[ ]" else: indicator = f"[{self.spinner_chars[self.spinner_index]}]" if self.color: if build_info.state == "failed": yield "\033[31m" # red elif build_info.state == "finished": yield "\033[32m" # green yield indicator if self.color: yield "\033[0m" # reset yield " " if self.color: yield "\033[0;90m" # dark gray yield build_info.hash if self.color: yield "\033[0m" # reset yield " " # Package name in bold white if explicit, default otherwise if build_info.explicit: if self.color: yield "\033[1;37m" # bold white yield build_info.name if self.color: yield "\033[0m" # reset else: yield build_info.name if self.color: yield "\033[0;36m" # cyan yield f"@{build_info.version}" if self.color: yield "\033[0m" # reset # progress or state if build_info.progress_percent is not None: yield " fetching" yield f": {build_info.progress_percent}%" elif build_info.state == "finished": prefix = build_info.prefix yield f" {padding_filter(prefix) if self.filter_padding else prefix}" elif build_info.state == "failed": yield " failed" if build_info.log_path: yield f": {build_info.log_path}" else: yield f" {build_info.state}" # Duration elapsed = ( build_info.duration if build_info.duration is not None else (now - build_info.start_time) ) if elapsed > 0: if self.color: yield "\033[0;90m" # dark gray yield f" ({pretty_duration(elapsed)})" if self.color: yield "\033[0m" Nodes = Dict[str, spack.spec.Spec] Edges = Dict[str, Set[str]] class BuildGraph: """Represents the dependency graph for package installation.""" def __init__( self, specs: List[spack.spec.Spec], root_policy: InstallPolicy, dependencies_policy: InstallPolicy, include_build_deps: bool, install_package: bool, install_deps: bool, database: spack.database.Database, overwrite_set: Optional[Set[str]] = None, tests: Union[bool, List[str], Set[str]] = False, explicit_set: Optional[Set[str]] = None, ): """Construct a build graph from the given specs. This includes only packages that need to be installed. Installed packages are pruned from the graph, and build dependencies are only included when necessary.""" self.roots = {s.dag_hash() for s in specs} self.nodes = {s.dag_hash(): s for s in specs} self.parent_to_child: Dict[str, Set[str]] = {} self.child_to_parent: Dict[str, Set[str]] = {} overwrite_set = overwrite_set or set() explicit_set = explicit_set or set() self.pruned: Set[str] = set() stack: List[Tuple[spack.spec.Spec, InstallPolicy]] = [ (s, root_policy) for s in self.nodes.values() ] with database.read_transaction(): # Set the install prefix for each spec based on the db record or store layout for s in spack.traverse.traverse_nodes(specs): _, record = database.query_by_spec_hash(s.dag_hash()) if record and record.path: s.set_prefix(record.path) else: s.set_prefix(spack.store.STORE.layout.path_for_spec(s)) # Build the graph and determine which specs to prune while stack: spec, install_policy = stack.pop() key = spec.dag_hash() _, record = database.query_by_spec_hash(key) # Conditionally include build dependencies. Don't prune installed specs # that need to be marked explicit so they flow through the DB write path. if record and record.installed and key not in overwrite_set: # Installed spec only needs link/run deps traversed. dependencies = spec.dependencies(deptype=dt.LINK | dt.RUN) # If it needs to be marked explicit, keep it in the graph (don't prune). if not (key in explicit_set and not record.explicit): self.pruned.add(key) elif install_policy == "cache_only" and not include_build_deps: dependencies = spec.dependencies(deptype=dt.LINK | dt.RUN) else: deptype = dt.BUILD | dt.LINK | dt.RUN if tests is True or (tests and spec.name in tests): deptype |= dt.TEST dependencies = spec.dependencies(deptype=deptype) self.parent_to_child[key] = {d.dag_hash() for d in dependencies} # Enqueue new dependencies for d in dependencies: if d.dag_hash() in self.nodes: continue self.nodes[d.dag_hash()] = d stack.append((d, dependencies_policy)) # Construct reverse lookup from child to parent for parent, children in self.parent_to_child.items(): for child in children: if child in self.child_to_parent: self.child_to_parent[child].add(parent) else: self.child_to_parent[child] = {parent} # If we're not installing the package itself, mark root specs for pruning too if not install_package: self.pruned.update(s.dag_hash() for s in specs) # Prune specs from the build graph. Their parents become parents of their children and # their children become children of their parents. for key in self.pruned: for parent in self.child_to_parent.get(key, ()): self.parent_to_child[parent].remove(key) self.parent_to_child[parent].update(self.parent_to_child.get(key, ())) for child in self.parent_to_child.get(key, ()): self.child_to_parent[child].remove(key) self.child_to_parent[child].update(self.child_to_parent.get(key, ())) self.parent_to_child.pop(key, None) self.child_to_parent.pop(key, None) self.nodes.pop(key, None) # Check that all prefixes to be created are unique. prefixes = [s.prefix for s in self.nodes.values() if not s.external] if len(prefixes) != len(set(prefixes)): raise spack.error.InstallError( "Install prefix collision: " + ", ".join(p for p in prefixes if prefixes.count(p) > 1) ) # If we're not installing dependencies, verify that all remaining nodes in the build graph # after pruning are roots. If there are any non-root nodes, it means there are uninstalled # dependencies that we're not supposed to install. if not install_deps: non_root_spec = next((v for k, v in self.nodes.items() if k not in self.roots), None) if non_root_spec is not None: raise spack.error.InstallError( f"Failed to install in package only mode: dependency {non_root_spec} is not " "installed" ) def enqueue_parents(self, dag_hash: str, pending_builds: List[str]) -> None: """After a spec is installed, remove it from the graph and enqueue any parents that are now ready to install. Args: dag_hash: The dag_hash of the spec that was just installed pending_builds: List to append parent specs that are ready to build """ # Remove node and edges from the node in the build graph self.parent_to_child.pop(dag_hash, None) self.nodes.pop(dag_hash, None) parents = self.child_to_parent.pop(dag_hash, None) if not parents: return # Enqueue any parents and remove edges to the installed child for parent in parents: children = self.parent_to_child[parent] children.remove(dag_hash) if not children: pending_builds.append(parent) class ScheduleResult(NamedTuple): """Return value of :func:`schedule_builds`.""" #: True if any pending builds were blocked on locks held by other processes. blocked: bool #: ``(dag_hash, lock)`` pairs where the write lock is held and the caller must start the build #: and eventually release the lock. to_start: List[Tuple[str, spack.util.lock.Lock]] #: ``(dag_hash, spec, lock)`` triples found already installed by another process; the read lock #: is held and the caller must add it to retained_read_locks. newly_installed: List[Tuple[str, spack.spec.Spec, spack.util.lock.Lock]] #: Actions to mark already installed specs explicit in the DB. to_mark_explicit: List[MarkExplicitAction] def schedule_builds( pending: List[str], build_graph: BuildGraph, db: spack.database.Database, prefix_locker: spack.database.SpecLocker, overwrite: Set[str], overwrite_time: float, capacity: int, needs_jobserver_token: bool, jobserver: JobServer, explicit: Set[str], ) -> ScheduleResult: """Try to schedule as many pending builds as possible. For each pending spec, attempts to acquire a non-blocking per-spec write lock. If the write lock times out, a read lock is tried as a fallback: a successful read lock means the first process finished and downgraded its write lock. If the DB confirms the spec is installed, it is captured as newly_installed; if the DB says it is not installed, the concurrent process was likely killed mid-build, and the spec is retried next iteration. Under both the DB read lock and the prefix lock, checks whether another process has already installed the spec. If so, captures it as newly_installed (caller enqueues parents) and keeps a read lock on the prefix to prevent concurrent uninstall. Otherwise, acquires a jobserver token if needed and adds the (dag_hash, lock) pair to to_start (caller launches the build). Args: pending: List of dag hashes pending installation; modified in-place. build_graph: The build dependency graph; used for node lookup and parent enqueueing. db: Package database; used for read lock and installed-status queries. prefix_locker: Per-spec write locker. overwrite: Set of dag hashes to overwrite even if already installed. overwrite_time: Timestamp (from time.time()) at which the overwrite install was requested. A spec in ``overwrite`` whose DB installation_time >= overwrite_time was installed by a concurrent process after our request started and should be treated as done. capacity: Maximum number of new builds to add to to_start in this call. needs_jobserver_token: True if a jobserver token is required for the first new build. jobserver: Jobserver for acquiring tokens. explicit: Set of dag hashes to mark explicit in the DB if found already installed. Returns: A :class:`ScheduleResult` with ``blocked``, ``to_start``, and ``newly_installed`` fields; see :class:`ScheduleResult` for field semantics. """ to_start: List[Tuple[str, spack.util.lock.Lock]] = [] newly_installed: List[Tuple[str, spack.spec.Spec, spack.util.lock.Lock]] = [] to_mark_explicit: List[MarkExplicitAction] = [] blocked = True # Acquire the DB read lock non-blocking; hold it throughout the loop so the in-memory snapshot # stays consistent while we acquire per-spec prefix locks. if not db.lock.try_acquire_read(): return ScheduleResult(blocked, to_start, newly_installed, to_mark_explicit) try: db._read() # refresh in-memory snapshot under the read lock idx = 0 while capacity and idx < len(pending): dag_hash = pending[idx] spec = build_graph.nodes[dag_hash] lock = prefix_locker.lock(spec) if lock.try_acquire_write(): blocked = False have_write = True elif lock.try_acquire_read(): have_write = False else: idx += 1 continue # Check installed status under the DB read lock and prefix lock. upstream, record = db.query_by_spec_hash(dag_hash) # If the spec is already installed, treat it as done regardless of lock type. # A spec in the overwrite set is also treated as done if another process installed it # after our overwrite request was created (installation_time >= overwrite_time). if ( record and record.installed and (dag_hash not in overwrite or record.installation_time >= overwrite_time) ): if have_write: lock.downgrade_write_to_read() # keep the read lock (either downgraded or already a read lock) del pending[idx] newly_installed.append((dag_hash, spec, lock)) # It's already installed, but needs to be marked as explicitly installed in the DB. if dag_hash in explicit and not record.explicit: to_mark_explicit.append(MarkExplicitAction(spec)) build_graph.enqueue_parents(dag_hash, pending) continue if not have_write: # If have to install but only got a read lock, try it in next iteration of the # event loop. lock.release_read() idx += 1 continue # Write lock acquired: proceed with scheduling. # Don't schedule builds for specs from upstream databases. if upstream and record and not record.installed: lock.release_write() raise spack.error.InstallError( f"Cannot install {spec}: it is uninstalled in an upstream database." ) # Defensively assert prefix invariants if not spec.external: if ( dag_hash in overwrite and record and record.installed and record.path != spec.prefix ): # Cannot do an overwrite install to a different prefix. lock.release_write() raise spack.error.InstallError( f"Prefix mismatch in overwrite of {spec}: expected {record.path}, " f"got {spec.prefix}" ) elif dag_hash not in overwrite and spec.prefix in db._installed_prefixes: # Prevent install prefix collision with other specs. lock.release_write() raise spack.error.InstallError( f"Cannot install {spec}: prefix {spec.prefix} already exists" ) # Acquire a jobserver token if needed. The first (implicit) job needs no token. if needs_jobserver_token and not jobserver.acquire(1): lock.release_write() break # no tokens available right now; stop scheduling del pending[idx] to_start.append((dag_hash, lock)) capacity -= 1 needs_jobserver_token = True # all subsequent jobs need a token finally: db.lock.release_read() return ScheduleResult(blocked, to_start, newly_installed, to_mark_explicit) def _node_to_roots(roots: List[spack.spec.Spec]) -> Dict[str, FrozenSet[str]]: """Map each node in a graph to the set of root node DAG hashes that can reach it. Args: roots: List of root specs. Returns: A dictionary mapping each node's dag_hash to a frozenset of root dag_hashes. """ node_to_roots: Dict[str, FrozenSet[str]] = { s.dag_hash(): frozenset([s.dag_hash()]) for s in roots } for edge in spack.traverse.traverse_edges( roots, order="topo", cover="edges", root=False, key=spack.traverse.by_dag_hash ): parent_roots = node_to_roots[edge.parent.dag_hash()] child_hash = edge.spec.dag_hash() existing = node_to_roots.get(child_hash) if existing is None: node_to_roots[child_hash] = parent_roots # keep a reference if no mutation is needed elif not parent_roots.issubset(existing): node_to_roots[child_hash] = existing | parent_roots return node_to_roots class ReportData: """Data collected for reports during installation.""" def __init__(self, roots: List[spack.spec.Spec]): self.roots = roots self.build_records: Dict[str, spack.report.InstallRecord] = {} def start_record(self, spec: spack.spec.Spec) -> None: """Begin an InstallRecord for a spec that is about to be built.""" if spec.external: return record = spack.report.InstallRecord(spec) record.start() self.build_records[spec.dag_hash()] = record def finish_record(self, spec: spack.spec.Spec, exitcode: int) -> None: """Mark the InstallRecord for a spec as succeeded or failed.""" record = self.build_records.get(spec.dag_hash()) if record is None or spec.external: return if exitcode == 0: record.succeed() else: record.fail( spack.error.InstallError( f"Installation of {spec.name} failed; see log for details" ) ) def finalize( self, reports: Dict[str, spack.report.RequestRecord], build_graph: BuildGraph ) -> None: """Finalize InstallRecords and append them to RequestRecords after all builds finish. Args: reports: Map of root dag_hash to RequestRecord to append to. build_graph: The build graph containing all nodes and their states. """ node_to_roots = _node_to_roots(self.roots) for spec in spack.traverse.traverse_nodes(self.roots): h = spec.dag_hash() if h in self.build_records: record = self.build_records[h] else: record = spack.report.InstallRecord(spec) if spec.external: msg = "Spec is external" elif h in build_graph.pruned: msg = "Spec was not scheduled for installation" elif h in build_graph.nodes: msg = "Dependencies failed to install" else: # If not installed or failed (build_records), not statically pruned ahead of # time (build_graph.pruned), and also not scheduled (build_graph.nodes), it # means it was in pending_builds or running_builds but never started/finished. # This branch is followed on KeyboardInterrupt and --fail-fast. msg = "Installation was interrupted" record.skip(msg=msg) for root_hash in node_to_roots[h]: reports[root_hash].append_record(record) class NullReportData(ReportData): """No-op drop-in for ReportData when no reporter is configured. Avoids creating InstallRecords and reading log files on every completed build.""" def __init__(self) -> None: pass def start_record(self, spec: spack.spec.Spec) -> None: pass def finish_record(self, spec: spack.spec.Spec, exitcode: int) -> None: pass def finalize( self, reports: Dict[str, spack.report.RequestRecord], build_graph: "BuildGraph" ) -> None: pass class TerminalState: """Manages terminal settings, stdin selector registration, and suspend/resume signals. Installs a SIGTSTP handler that restores the terminal before suspending and re-applies it on resume. After waking up it checks whether the process is in the foreground or background and enables or suppresses interactive output accordingly. Optional ``on_suspend`` / ``on_resume`` hooks are called just before the process suspends and just after it wakes, allowing callers to pause and resume child processes.""" def __init__( self, selector: selectors.BaseSelector, build_status: BuildStatus, on_suspend: Optional[Callable[[], None]] = None, on_resume: Optional[Callable[[], None]] = None, ) -> None: self.selector = selector self.build_status = build_status self.on_suspend = on_suspend self.on_resume = on_resume self.old_stdin_settings = termios.tcgetattr(sys.stdin) self.sigwinch_r = -1 self.sigwinch_w = -1 def setup(self) -> None: """Set cbreak mode, register stdin and signal pipes in the selector.""" # SIGWINCH self-pipe (stdout must be a tty too) if sys.stdout.isatty(): self.sigwinch_r, self.sigwinch_w = os.pipe() os.set_blocking(self.sigwinch_r, False) os.set_blocking(self.sigwinch_w, False) self.selector.register(self.sigwinch_r, selectors.EVENT_READ, "sigwinch") self.old_sigwinch = signal.signal(signal.SIGWINCH, self._handle_sigwinch) else: self.old_sigwinch = None self.old_sigtstp = signal.signal(signal.SIGTSTP, self._handle_sigtstp) # Start correctly depending on whether we're foregrounded or backgrounded self.build_status.headless = True if not _is_background_tty(sys.stdin): self.enter_foreground() def teardown(self) -> None: """Restore terminal settings and signal handlers, close pipes.""" with ignore_signal(signal.SIGTTOU): termios.tcsetattr(sys.stdin, termios.TCSADRAIN, self.old_stdin_settings) for sig, old in ((signal.SIGTSTP, self.old_sigtstp), (signal.SIGWINCH, self.old_sigwinch)): if old is not None: try: signal.signal(sig, old) except Exception as e: spack.llnl.util.tty.debug(f"Failed to restore signal handler for {sig}: {e}") if sys.stdin.fileno() in self.selector.get_map(): self.selector.unregister(sys.stdin.fileno()) for fd in (self.sigwinch_r, self.sigwinch_w): if fd < 0: continue if fd in self.selector.get_map(): self.selector.unregister(fd) try: os.close(fd) except Exception as e: spack.llnl.util.tty.debug(f"Failed to close sigwinch pipe {fd}: {e}") def _handle_sigtstp(self, signum: int, frame: object) -> None: """Restore terminal before suspending, then re-install handler after resume.""" # Reset so the first redraw after resume doesn't overwrite the shell's # prompt / "$ fg" line. self.build_status.active_area_rows = 0 # Restore terminal so the user's shell works normally while we're stopped. with ignore_signal(signal.SIGTTOU): termios.tcsetattr(sys.stdin, termios.TCSANOW, self.old_stdin_settings) # Force headless mode before suspending so that enter_foreground() doesn't # exit early when we resume, ensuring terminal settings are re-applied. self.build_status.headless = True # Actually suspend: reset to default handler then re-send SIGTSTP. if self.on_suspend is not None: self.on_suspend() signal.signal(signal.SIGTSTP, signal.SIG_DFL) os.kill(os.getpid(), signal.SIGTSTP) # Execution resumes here after SIGCONT. Re-install our handler. signal.signal(signal.SIGTSTP, self._handle_sigtstp) if self.on_resume is not None: self.on_resume() self.handle_continue() def _handle_sigwinch(self, signum: int, frame: object) -> None: try: os.write(self.sigwinch_w, b"\x00") except OSError: pass def enter_foreground(self) -> None: """Restore interactive terminal mode.""" if not self.build_status.headless: return # We save old settings right before applying cbreak. # If we started in the background, bash may have had the terminal in its own # readline (raw) mode when __init__ ran. Waiting until we are foregrounded # ensures we capture the shell's exported 'sane' configuration for this job. self.old_stdin_settings = termios.tcgetattr(sys.stdin) with ignore_signal(signal.SIGTTOU): tty.setcbreak(sys.stdin.fileno()) if sys.stdin.fileno() not in self.selector.get_map(): self.selector.register(sys.stdin.fileno(), selectors.EVENT_READ, "stdin") self.build_status.headless = False self.build_status.dirty = True def enter_background(self) -> None: """Suppress output and stop reading stdin to avoid SIGTTIN/SIGTTOU.""" if sys.stdin.fileno() in self.selector.get_map(): self.selector.unregister(sys.stdin.fileno()) self.build_status.headless = True def handle_continue(self) -> None: """Detect whether the process is in the foreground or background and adjust accordingly.""" if _is_background_tty(sys.stdin): self.enter_background() else: self.enter_foreground() def _signal_children(running_builds: Dict[int, ChildInfo], sig: signal.Signals) -> None: """Send a signal to the process group of each running build.""" for child in running_builds.values(): try: pid = child.proc.pid if pid is not None: os.killpg(pid, sig) except OSError: pass class StdinReader: """Helper class to do non-blocking, incremental decoding of stdin, stripping ANSI escape sequences. The input is the backing file descriptor for stdin (instead of the TextIOWrapper) to avoid double buffering issues: the event loop triggers when the fd is ready to read, and if we do a partial read from the TextIOWrapper, it will likely drain the fd and buffer the remainder internally, which the event loop is not aware of, and user input doesn't come through.""" def __init__(self, fd: int) -> None: self.fd = fd #: Handle multi-byte UTF-8 characters self.decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") #: For stripping out arrow and navigation keys self.ansi_escape_re = re.compile(r"\x1b\[[0-9;]*[A-Za-z~]") def read(self) -> str: try: chars = self.decoder.decode(os.read(self.fd, 1024)) return self.ansi_escape_re.sub("", chars) except OSError: return "" class PackageInstaller: explicit: Set[str] def __init__( self, packages: List["spack.package_base.PackageBase"], *, dirty: bool = False, explicit: Union[Set[str], bool] = False, overwrite: Optional[Union[List[str], Set[str]]] = None, fail_fast: bool = False, fake: bool = False, include_build_deps: bool = False, install_deps: bool = True, install_package: bool = True, install_source: bool = False, keep_prefix: bool = False, keep_stage: bool = False, restage: bool = True, skip_patch: bool = False, stop_at: Optional[str] = None, stop_before: Optional[str] = None, tests: Union[bool, List[str], Set[str]] = False, unsigned: Optional[bool] = None, verbose: bool = False, concurrent_packages: Optional[int] = None, root_policy: InstallPolicy = "auto", dependencies_policy: InstallPolicy = "auto", create_reports: bool = False, ) -> None: assert install_package or install_deps, "Must install package, dependencies or both" self.install_source = install_source self.stop_at = stop_at self.stop_before = stop_before self.tests: Union[bool, List[str], Set[str]] = tests self.db = spack.store.STORE.db specs = [pkg.spec for pkg in packages] self.root_policy: InstallPolicy = root_policy self.dependencies_policy: InstallPolicy = dependencies_policy self.include_build_deps = include_build_deps #: Set of DAG hashes to overwrite (if already installed) self.overwrite: Set[str] = set(overwrite) if overwrite else set() #: Time at which the overwrite install was requested; used to detect concurrent overwrites. self.overwrite_time: float = time.time() self.keep_prefix = keep_prefix self.fail_fast = fail_fast # Buffer for incoming, partially received state data from child processes self.state_buffers: Dict[int, str] = {} if explicit is True: self.explicit = {spec.dag_hash() for spec in specs} elif explicit is False: self.explicit = set() else: self.explicit = explicit # Build the dependency graph self.build_graph = BuildGraph( specs, root_policy, dependencies_policy, include_build_deps, install_package, install_deps, self.db, self.overwrite, tests, self.explicit, ) #: check what specs we could fetch from binaries (checks against cache, not remotely) try: spack.binary_distribution.BINARY_INDEX.update() except spack.binary_distribution.FetchCacheError: pass self.binary_cache_for_spec = { s.dag_hash(): spack.binary_distribution.BINARY_INDEX.find_by_hash(s.dag_hash()) for s in self.build_graph.nodes.values() } self.unsigned = unsigned self.dirty = dirty self.fake = fake self.restage = restage self.keep_stage = keep_stage self.skip_patch = skip_patch #: queue of packages ready to install (no children) self.pending_builds = [ parent for parent, children in self.build_graph.parent_to_child.items() if not children ] self.verbose = verbose self.running_builds: Dict[int, ChildInfo] = {} self.log_paths: Dict[str, str] = {} self.build_status = BuildStatus( len(self.build_graph.nodes), verbose=verbose, filter_padding=spack.store.STORE.has_padding(), ) self.jobs = spack.config.determine_number_of_jobs(parallel=True) self.build_status.actual_jobs = self.jobs self.build_status.target_jobs = self.jobs if concurrent_packages is None: concurrent_packages_config = spack.config.get("config:concurrent_packages", 0) # The value 0 in config means no limit (other than self.jobs) if concurrent_packages_config == 0: self.capacity = sys.maxsize else: self.capacity = concurrent_packages_config else: self.capacity = concurrent_packages # The reports property is what the old installer has and used as public interface. if create_reports: self.reports = {spec.dag_hash(): spack.report.RequestRecord(spec) for spec in specs} self.report_data = ReportData(specs) else: self.reports = {} self.report_data = NullReportData() self.next_database_write = 0.0 def install(self) -> None: self._installer() def _installer(self) -> None: jobserver = JobServer(self.jobs) selector = selectors.DefaultSelector() # Set up terminal handling (cbreak, signals, stdin registration) terminal: Optional[TerminalState] = None stdin_reader: Optional[StdinReader] = None if sys.stdin.isatty(): stdin_reader = StdinReader(sys.stdin.fileno()) terminal = TerminalState( selector, self.build_status, on_suspend=lambda: _signal_children(self.running_builds, signal.SIGSTOP), on_resume=lambda: _signal_children(self.running_builds, signal.SIGCONT), ) terminal.setup() # Finished builds that have not yet been written to the database. database_actions: List[DatabaseAction] = [] # Prefix read locks retained after DB flush (downgraded from write locks in _save_to_db). retained_read_locks: List[spack.util.lock.Lock] = [] failures: List[spack.spec.Spec] = [] finished_pids: List[int] = [] try: # Try to schedule builds immediately. The first job does not require a token. blocked = self._schedule_builds( selector, jobserver, retained_read_locks, database_actions ) while self.pending_builds or self.running_builds or database_actions: # Monitor the jobserver when we have pending builds, capacity, and at least one # spec is not locked by another process. Also listen if the target parallelism is # reduced. wake_on_jobserver = ( self.pending_builds and self.capacity and not blocked or not jobserver.has_target_parallelism() ) if wake_on_jobserver and jobserver.r not in selector.get_map(): selector.register(jobserver.r, selectors.EVENT_READ, "jobserver") elif not wake_on_jobserver and jobserver.r in selector.get_map(): selector.unregister(jobserver.r) stdin_ready = False if self.build_status.headless: # no UI to update, but check background to foreground transition periodically timeout = HEADLESS_WAKE_INTERVAL elif self.build_status.is_tty: timeout = SPINNER_INTERVAL else: # when not in interactive mode, wake least often (no spinner/terminal updates) timeout = DATABASE_WRITE_INTERVAL events = selector.select(timeout=timeout) finished_pids.clear() # The transition "suspended to foreground/background" is handled in the signal # handler, but there's no SIGCONT event in the transition of background to # foreground, so we conditionally poll for that here (headless case). In the # headless case the event loop only fires once per second, so this is cheap enough. if terminal and self.build_status.headless and not _is_background_tty(sys.stdin): terminal.enter_foreground() for key, _ in events: data = key.data if isinstance(data, FdInfo): # Child output (logs and state updates) child_info = self.running_builds[data.pid] if data.name == "output": self._handle_child_logs(key.fd, child_info, selector) elif data.name == "state": self._handle_child_state(key.fd, child_info, selector) elif data.name == "sentinel": finished_pids.append(data.pid) elif data == "stdin": stdin_ready = True elif data == "sigwinch": assert terminal is not None os.read(terminal.sigwinch_r, 64) # drain the pipe self.build_status.on_resize() elif data == "jobserver" and not jobserver.has_target_parallelism(): jobserver.maybe_discard_tokens() self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs) if finished_pids: self._handle_finished_builds( finished_pids, selector, jobserver, database_actions, failures ) if failures and self.fail_fast: # Terminate other builds to actually fail fast. We continue in the event loop # waiting for child processes to finish, which may take a little while. for child in self.running_builds.values(): child.proc.terminate() self.pending_builds.clear() if stdin_ready and stdin_reader is not None: for char in stdin_reader.read(): overview = self.build_status.overview_mode if overview and self.build_status.search_mode: self.build_status.search_input(char) elif overview and char == "/": self.build_status.enter_search() elif char == "v" or char in ("q", "\x1b") and not overview: self.build_status.toggle() elif char == "n": self.build_status.next(1) elif char == "p" or char == "N": self.build_status.next(-1) elif char == "+": jobserver.increase_parallelism() self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs) elif char == "-": jobserver.decrease_parallelism() self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs) # Insert into the database if we have any finished builds, and either the delay # interval has passed, or we're done with all builds. The database save is not # guaranteed; it fails if another process holds the lock. We'll try again next # iteration of the event loop in that case. if ( database_actions and ( time.monotonic() >= self.next_database_write or not (self.pending_builds or self.running_builds) ) and self._save_to_db(database_actions, retained_read_locks) ): database_actions.clear() # Try to schedule more builds, acquiring per-spec locks and jobserver tokens. if self.capacity and self.pending_builds: blocked = self._schedule_builds( selector, jobserver, retained_read_locks, database_actions ) # Finally update the UI self.build_status.update() finally: # First ensure that the user's terminal state is restored. if terminal is not None: terminal.teardown() # Flush any not-yet-written successful builds to the DB; save the exception on error # to be re-raised after best-effort cleanup. db_exc = None if database_actions: try: with self.db.write_transaction(): for action in database_actions: action.save_to_db(self.db) except Exception as e: db_exc = e # Send SIGTERM to running builds; this is a no-op in the successful case. for child in self.running_builds.values(): try: child.proc.terminate() except Exception: pass # Release our jobserver token for each terminated build and then join. for child in self.running_builds.values(): try: jobserver.release() child.proc.join(timeout=30) if child.proc.is_alive(): child.proc.kill() child.proc.join() except Exception: pass # Release all held locks best-effort, so that one failure does not prevent the others # from being released. for child in self.running_builds.values(): child.release_lock() for lock in retained_read_locks: try: lock.release_read() except Exception: pass for action in database_actions: action.release_lock() try: self.build_status.overview_mode = True self.build_status.update(finalize=True) selector.close() jobserver.close() except Exception: pass # Re-raise the DB exception if any. if db_exc is not None: raise db_exc try: self.report_data.finalize(self.reports, build_graph=self.build_graph) except Exception as e: spack.llnl.util.tty.debug(f"[{__name__}]: Failed to finalize reports: {e}]") if failures: for s in failures: build_info = self.build_status.builds[s.dag_hash()] if build_info and build_info.log_summary: sys.stderr.write(build_info.log_summary) lines = [f"{s}: {self.log_paths[s.dag_hash()]}" for s in failures] raise spack.error.InstallError( "The following packages failed to install:\n" + "\n".join(lines) ) def _handle_finished_builds( self, finished_pids: List[int], selector: selectors.BaseSelector, jobserver: JobServer, database_actions: List[DatabaseAction], failures: List[spack.spec.Spec], ) -> None: """Handle builds that finished since the last event loop iteration.""" current_time = time.monotonic() for pid in finished_pids: build = self.running_builds.pop(pid) self.capacity += 1 jobserver.release() self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs) self._drain_child_output(build, selector) self._drain_child_state(build, selector) build.cleanup(selector) exitcode = build.proc.exitcode assert exitcode is not None, "Finished build should have exit code set" self.report_data.finish_record(build.spec, exitcode) if exitcode == 0: # Add successful builds for database insertion (after a short delay) database_actions.append(build) self.build_graph.enqueue_parents(build.spec.dag_hash(), self.pending_builds) self.next_database_write = current_time + DATABASE_WRITE_INTERVAL self.build_status.update_state(build.spec.dag_hash(), "finished") elif exitcode == EXIT_STOPPED_AT_PHASE: # Partial build: neither failure nor success. Should not be persisted in # the database, but also not treated as a failure in the UI. Just release # locks and move on. build.release_lock() elif not self.fail_fast or not failures: # In fail-fast mode, only record the first failure. Subsequent failures may # be a consequence of us terminating other builds, and should not be # reported as failures in the UI. failures.append(build.spec) self.build_status.update_state(build.spec.dag_hash(), "failed") self.build_status.parse_log_summary(build.spec.dag_hash()) def _save_to_db( self, database_actions: List[DatabaseAction], retained_read_locks: List[spack.util.lock.Lock], ) -> bool: if not self.db.lock.try_acquire_write(): return False try: self.db._read() for action in database_actions: action.save_to_db(self.db) finally: self.db.lock.release_write(self.db._write) # DB has been written and flushed; downgrade per-spec prefix write locks to read locks so # other processes can see the specs are installed, while preventing concurrent uninstalls. for action in database_actions: if action.prefix_lock is not None: try: action.prefix_lock.downgrade_write_to_read() retained_read_locks.append(action.prefix_lock) except Exception: action.prefix_lock.release_write() raise finally: action.prefix_lock = None return True def _schedule_builds( self, selector: selectors.BaseSelector, jobserver: JobServer, retained_read_locks: List[spack.util.lock.Lock], database_actions: List[DatabaseAction], ) -> bool: """Try to schedule as many pending builds as possible. Delegates to the module-level schedule_builds() function and then performs the side-effects that require the selector and running-build state: updating build_status for specs that were found already installed, and launching new builds via _start(). Preconditions: self.capacity > 0 and self.pending_builds is not empty. Returns True if we had capacity to schedule, but were blocked by locks held by other processes. In that case we should not monitor the jobserver for new tokens, since we'd end up in a busy wait loop until the locks are released. """ result = schedule_builds( pending=self.pending_builds, build_graph=self.build_graph, db=self.db, prefix_locker=spack.store.STORE.prefix_locker, overwrite=self.overwrite, overwrite_time=self.overwrite_time, capacity=self.capacity, needs_jobserver_token=bool(self.running_builds), jobserver=jobserver, explicit=self.explicit, ) blocked = result.blocked database_actions.extend(result.to_mark_explicit) # Specs installed by another process. for dag_hash, spec, lock in result.newly_installed: retained_read_locks.append(lock) explicit = dag_hash in self.explicit self.build_status.add_build(spec, explicit=explicit) self.build_status.update_state(dag_hash, "finished") # Specs we can start building ourselves. for dag_hash, lock in result.to_start: self._start(selector, jobserver, dag_hash, lock) return blocked def _start( self, selector: selectors.BaseSelector, jobserver: JobServer, dag_hash: str, prefix_lock: spack.util.lock.Lock, ) -> None: self.capacity -= 1 explicit = dag_hash in self.explicit spec = self.build_graph.nodes[dag_hash] is_develop = spec.is_develop tests = self.tests run_tests = tests is True or bool(tests and spec.name in tests) is_root = dag_hash in self.build_graph.roots child_info = start_build( spec, explicit=explicit, mirrors=self.binary_cache_for_spec[dag_hash], unsigned=self.unsigned, install_policy=self.root_policy if is_root else self.dependencies_policy, dirty=self.dirty, # keep_stage/restage logic taken from installer.py keep_stage=self.keep_stage or is_develop, restage=self.restage and not is_develop, keep_prefix=self.keep_prefix, skip_patch=self.skip_patch, fake=self.fake, install_source=self.install_source, run_tests=run_tests, jobserver=jobserver, stop_before=self.stop_before if is_root else None, stop_at=self.stop_at if is_root else None, ) self.log_paths[dag_hash] = child_info.log_path child_info.prefix_lock = prefix_lock pid = child_info.proc.pid assert type(pid) is int self.running_builds[pid] = child_info selector.register( child_info.output_r_conn.fileno(), selectors.EVENT_READ, FdInfo(pid, "output") ) selector.register( child_info.state_r_conn.fileno(), selectors.EVENT_READ, FdInfo(pid, "state") ) selector.register(child_info.proc.sentinel, selectors.EVENT_READ, FdInfo(pid, "sentinel")) self.build_status.add_build( child_info.spec, explicit=explicit, control_w_conn=child_info.control_w_conn, log_path=child_info.log_path, ) self.report_data.start_record(spec) def _handle_child_logs( self, r_fd: int, child_info: ChildInfo, selector: selectors.BaseSelector ) -> None: """Handle reading output logs from a child process pipe.""" try: # There might be more data than OUTPUT_BUFFER_SIZE, but we will read that in the next # iteration of the event loop to keep things responsive. data = os.read(r_fd, OUTPUT_BUFFER_SIZE) except BlockingIOError: return except OSError: data = None if not data: # EOF or error try: selector.unregister(r_fd) except KeyError: pass return self.build_status.print_logs(child_info.spec.dag_hash(), data) def _drain_child_output(self, child_info: ChildInfo, selector: selectors.BaseSelector) -> None: """Read and print any remaining output from a finished child's pipe.""" r_fd = child_info.output_r_conn.fileno() while r_fd in selector.get_map(): self._handle_child_logs(r_fd, child_info, selector) def _drain_child_state(self, child_info: ChildInfo, selector: selectors.BaseSelector) -> None: """Read and process any remaining state messages from a finished child's pipe.""" r_fd = child_info.state_r_conn.fileno() while r_fd in selector.get_map(): self._handle_child_state(r_fd, child_info, selector) def _handle_child_state( self, r_fd: int, child_info: ChildInfo, selector: selectors.BaseSelector ) -> None: """Handle reading state updates from a child process pipe.""" try: # There might be more data than OUTPUT_BUFFER_SIZE, but we will read that in the next # iteration of the event loop to keep things responsive. data = os.read(r_fd, OUTPUT_BUFFER_SIZE) except BlockingIOError: return except OSError: data = None if not data: # EOF or error try: selector.unregister(r_fd) except KeyError: pass self.state_buffers.pop(r_fd, None) return # Append new data to the buffer for this fd and process it buffer = self.state_buffers.get(r_fd, "") + data.decode(errors="replace") lines = buffer.split("\n") # The last element of split() will be a partial line or an empty string. # We store it back in the buffer for the next read. self.state_buffers[r_fd] = lines.pop() for line in lines: if not line: continue try: message = json.loads(line) except json.JSONDecodeError: continue if "state" in message: self.build_status.update_state(child_info.spec.dag_hash(), message["state"]) elif "progress" in message and "total" in message: self.build_status.update_progress( child_info.spec.dag_hash(), message["progress"], message["total"] ) elif "installed_from_binary_cache" in message: child_info.spec.package.installed_from_binary_cache = True ================================================ FILE: lib/spack/spack/oci/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/oci/image.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import urllib.parse from typing import Optional, Union # notice: Docker is more strict (no uppercase allowed). We parse image names *with* uppercase # and normalize, so: example.com/Organization/Name -> example.com/organization/name. Tags are # case sensitive though. alphanumeric_with_uppercase = r"[a-zA-Z0-9]+" separator = r"(?:[._]|__|[-]+)" localhost = r"localhost" domainNameComponent = r"(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])" optionalPort = r"(?::[0-9]+)?" tag = r"[\w][\w.-]{0,127}" digestPat = r"[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][0-9a-fA-F]{32,}" ipv6address = r"\[(?:[a-fA-F0-9:]+)\]" # domain name domainName = rf"{domainNameComponent}(?:\.{domainNameComponent})*" host = rf"(?:{domainName}|{ipv6address})" domainAndPort = rf"{host}{optionalPort}" # image name pathComponent = rf"{alphanumeric_with_uppercase}(?:{separator}{alphanumeric_with_uppercase})*" remoteName = rf"{pathComponent}(?:\/{pathComponent})*" namePat = rf"(?:{domainAndPort}\/)?{remoteName}" # Regex for a full image reference, with 3 groups: name, tag, digest referencePat = re.compile(rf"^({namePat})(?::({tag}))?(?:@({digestPat}))?$") # Regex for splitting the name into domain and path components anchoredNameRegexp = re.compile(rf"^(?:({domainAndPort})\/)?({remoteName})$") def ensure_sha256_checksum(oci_blob: str): """Validate that the reference is of the format sha256: Return the checksum if valid, raise ValueError otherwise.""" if ":" not in oci_blob: raise ValueError(f"Invalid OCI blob format: {oci_blob}") alg, checksum = oci_blob.split(":", 1) if alg != "sha256": raise ValueError(f"Unsupported OCI blob checksum algorithm: {alg}") if len(checksum) != 64: raise ValueError(f"Invalid OCI blob checksum length: {len(checksum)}") return checksum def is_oci_url(url: str) -> bool: """Check if the URL is an OCI URL.""" return url.startswith("oci://") or url.startswith("oci+http://") class Digest: """Represents a digest in the format :. Currently only supports sha256 digests.""" __slots__ = ["algorithm", "digest"] def __init__(self, *, algorithm: str, digest: str) -> None: self.algorithm = algorithm self.digest = digest def __eq__(self, __value: object) -> bool: if not isinstance(__value, Digest): return NotImplemented return self.algorithm == __value.algorithm and self.digest == __value.digest @classmethod def from_string(cls, string: str) -> "Digest": return cls(algorithm="sha256", digest=ensure_sha256_checksum(string)) @classmethod def from_sha256(cls, digest: str) -> "Digest": return cls(algorithm="sha256", digest=digest) def __str__(self) -> str: return f"{self.algorithm}:{self.digest}" class ImageReference: """A parsed image of the form domain/name:tag[@digest]. The digest is optional, and domain and tag are automatically filled out with defaults when parsed from string.""" __slots__ = ["scheme", "domain", "name", "tag", "digest"] def __init__( self, *, domain: str, name: str, tag: str = "latest", digest: Optional[Digest] = None, scheme: str = "https", ): self.scheme = scheme self.domain = domain self.name = name self.tag = tag self.digest = digest @classmethod def from_string(cls, string: str, *, scheme: str = "https") -> "ImageReference": match = referencePat.match(string) if not match: raise ValueError(f"Invalid image reference: {string}") image, tag, digest = match.groups() assert isinstance(image, str) assert isinstance(tag, (str, type(None))) assert isinstance(digest, (str, type(None))) match = anchoredNameRegexp.match(image) # This can never happen, since the regex is implied # by the regex above. It's just here to make mypy happy. assert match, f"Invalid image reference: {string}" domain, name = match.groups() assert isinstance(domain, (str, type(None))) assert isinstance(name, str) # Fill out defaults like docker would do... # Based on github.com/distribution/distribution: allow short names like "ubuntu" # and "user/repo" to be interpreted as "library/ubuntu" and "user/repo:latest # Not sure if Spack should follow Docker, but it's what people expect... if not domain: domain = "index.docker.io" name = f"library/{name}" elif ( "." not in domain and ":" not in domain and domain != "localhost" and domain == domain.lower() ): name = f"{domain}/{name}" domain = "index.docker.io" # Lowercase the image name. This is enforced by Docker, although the OCI spec isn't clear? # We do this anyways, cause for example in Github Actions the / # part can have uppercase, and may be interpolated when specifying the relevant OCI image. name = name.lower() if not tag: tag = "latest" # sha256 is currently the only algorithm that # we implement, even though the spec allows for more if isinstance(digest, str): digest = Digest.from_string(digest) return cls(domain=domain, name=name, tag=tag, digest=digest, scheme=scheme) @classmethod def from_url(cls, url: str) -> "ImageReference": """Parse an OCI URL into an ImageReference, either oci:// or oci+http://.""" if url.startswith("oci://"): img = url[6:] scheme = "https" elif url.startswith("oci+http://"): img = url[11:] scheme = "http" else: raise ValueError(f"Invalid OCI URL: {url}") return cls.from_string(img, scheme=scheme) def manifest_url(self) -> str: digest_or_tag = self.digest or self.tag return f"{self.scheme}://{self.domain}/v2/{self.name}/manifests/{digest_or_tag}" def blob_url(self, digest: Union[str, Digest]) -> str: if isinstance(digest, str): digest = Digest.from_string(digest) return f"{self.scheme}://{self.domain}/v2/{self.name}/blobs/{digest}" def with_digest(self, digest: Union[str, Digest]) -> "ImageReference": if isinstance(digest, str): digest = Digest.from_string(digest) return ImageReference( domain=self.domain, name=self.name, tag=self.tag, digest=digest, scheme=self.scheme ) def with_tag(self, tag: str) -> "ImageReference": return ImageReference( domain=self.domain, name=self.name, tag=tag, digest=self.digest, scheme=self.scheme ) def uploads_url(self, digest: Optional[Digest] = None) -> str: url = f"{self.scheme}://{self.domain}/v2/{self.name}/blobs/uploads/" if digest: url += f"?digest={digest}" return url def tags_url(self) -> str: return f"{self.scheme}://{self.domain}/v2/{self.name}/tags/list" def endpoint(self, path: str = "") -> str: return urllib.parse.urljoin(f"{self.scheme}://{self.domain}/v2/", path) def __str__(self) -> str: s = f"{self.domain}/{self.name}" if self.tag: s += f":{self.tag}" if self.digest: s += f"@{self.digest}" return s def __eq__(self, __value: object) -> bool: if not isinstance(__value, ImageReference): return NotImplemented return ( self.domain == __value.domain and self.name == __value.name and self.tag == __value.tag and self.digest == __value.digest and self.scheme == __value.scheme ) def ensure_valid_tag(tag: str) -> str: """Ensure a tag is valid for an OCI registry.""" sanitized = re.sub(r"[^\w.-]", "_", tag) if len(sanitized) > 128: return sanitized[:64] + sanitized[-64:] return sanitized def default_config(architecture: str, os: str): return { "architecture": architecture, "os": os, "rootfs": {"type": "layers", "diff_ids": []}, "config": {"Env": []}, } def default_manifest(): return { "mediaType": "application/vnd.oci.image.manifest.v1+json", "schemaVersion": 2, "config": {"mediaType": "application/vnd.oci.image.config.v1+json"}, "layers": [], } ================================================ FILE: lib/spack/spack/oci/oci.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import hashlib import json import os import urllib.error import urllib.parse from typing import List, NamedTuple, Tuple from urllib.request import Request import spack.fetch_strategy import spack.llnl.util.tty as tty import spack.mirrors.layout import spack.mirrors.mirror import spack.oci.opener import spack.stage import spack.util.url from .image import Digest, ImageReference class Blob(NamedTuple): compressed_digest: Digest uncompressed_digest: Digest size: int def with_query_param(url: str, param: str, value: str) -> str: """Add a query parameter to a URL Args: url: The URL to add the parameter to. param: The parameter name. value: The parameter value. Returns: The URL with the parameter added. """ parsed = urllib.parse.urlparse(url) query = urllib.parse.parse_qs(parsed.query) if param in query: query[param].append(value) else: query[param] = [value] return urllib.parse.urlunparse( parsed._replace(query=urllib.parse.urlencode(query, doseq=True)) ) def list_tags(ref: ImageReference, _urlopen: spack.oci.opener.MaybeOpen = None) -> List[str]: """Retrieves the list of tags associated with an image, handling pagination.""" _urlopen = _urlopen or spack.oci.opener.urlopen tags = set() fetch_url = ref.tags_url() while True: # Fetch tags request = Request(url=fetch_url) with _urlopen(request) as response: spack.oci.opener.ensure_status(request, response, 200) tags.update(json.load(response)["tags"]) # Check for pagination link_header = response.headers["Link"] if link_header is None: break tty.debug(f"OCI tag pagination: {link_header}") rel_next_value = spack.util.url.parse_link_rel_next(link_header) if rel_next_value is None: break rel_next = urllib.parse.urlparse(rel_next_value) if rel_next.scheme not in ("https", ""): break fetch_url = ref.endpoint(rel_next_value) return sorted(tags) def upload_blob( ref: ImageReference, file: str, digest: Digest, force: bool = False, small_file_size: int = 0, _urlopen: spack.oci.opener.MaybeOpen = None, ) -> bool: """Uploads a blob to an OCI registry We only do monolithic uploads, even though it's very simple to do chunked. Observed problems with chunked uploads: (1) it's slow, many sequential requests, (2) some registries set an *unknown* max chunk size, and the spec doesn't say how to obtain it Args: ref: The image reference. file: The file to upload. digest: The digest of the file. force: Whether to force upload the blob, even if it already exists. small_file_size: For files at most this size, attempt to do a single POST request instead of POST + PUT. Some registries do no support single requests, and others do not specify what size they support in single POST. For now this feature is disabled by default (0KB) Returns: True if the blob was uploaded, False if it already existed. """ _urlopen = _urlopen or spack.oci.opener.urlopen # Test if the blob already exists, if so, early exit. if not force and blob_exists(ref, digest, _urlopen): return False with open(file, "rb") as f: file_size = os.fstat(f.fileno()).st_size # For small blobs, do a single POST request. # The spec says that registries MAY support this if file_size <= small_file_size: request = Request( url=ref.uploads_url(digest), method="POST", data=f, headers={ "Content-Type": "application/octet-stream", "Content-Length": str(file_size), }, ) else: request = Request( url=ref.uploads_url(), method="POST", headers={"Content-Length": "0"} ) with _urlopen(request) as response: # Created the blob in one go. if response.status == 201: return True # Otherwise, do another PUT request. spack.oci.opener.ensure_status(request, response, 202) assert "Location" in response.headers # Can be absolute or relative, joining handles both upload_url = with_query_param( ref.endpoint(response.headers["Location"]), "digest", str(digest) ) f.seek(0) request = Request( url=upload_url, method="PUT", data=f, headers={"Content-Type": "application/octet-stream", "Content-Length": str(file_size)}, ) with _urlopen(request) as response: spack.oci.opener.ensure_status(request, response, 201) return True def upload_manifest( ref: ImageReference, manifest: dict, tag: bool = True, _urlopen: spack.oci.opener.MaybeOpen = None, ): """Uploads a manifest/index to a registry Args: ref: The image reference. manifest: The manifest or index. tag: When true, use the tag, otherwise use the digest, this is relevant for multi-arch images, where the tag is an index, referencing the manifests by digest. Returns: The digest and size of the uploaded manifest. """ _urlopen = _urlopen or spack.oci.opener.urlopen data = json.dumps(manifest, separators=(",", ":")).encode() digest = Digest.from_sha256(hashlib.sha256(data).hexdigest()) size = len(data) if not tag: ref = ref.with_digest(digest) request = Request( url=ref.manifest_url(), method="PUT", data=data, headers={"Content-Type": manifest["mediaType"]}, ) with _urlopen(request) as response: spack.oci.opener.ensure_status(request, response, 201) return digest, size def image_from_mirror(mirror: spack.mirrors.mirror.Mirror) -> ImageReference: """Given an OCI based mirror, extract the URL and image name from it""" return ImageReference.from_url(mirror.push_url) def blob_exists( ref: ImageReference, digest: Digest, _urlopen: spack.oci.opener.MaybeOpen = None ) -> bool: """Checks if a blob exists in an OCI registry""" try: _urlopen = _urlopen or spack.oci.opener.urlopen with _urlopen(Request(url=ref.blob_url(digest), method="HEAD")) as response: return response.status == 200 except urllib.error.HTTPError as e: if e.getcode() == 404: return False raise def copy_missing_layers( src: ImageReference, dst: ImageReference, architecture: str, _urlopen: spack.oci.opener.MaybeOpen = None, ) -> Tuple[dict, dict]: """Copy image layers from src to dst for given architecture. Args: src: The source image reference. dst: The destination image reference. architecture: The architecture (when referencing an index) Returns: Tuple of manifest and config of the base image. """ _urlopen = _urlopen or spack.oci.opener.urlopen manifest, config = get_manifest_and_config(src, architecture, _urlopen=_urlopen) # Get layer digests digests = [Digest.from_string(layer["digest"]) for layer in manifest["layers"]] # Filter digests that are don't exist in the registry missing_digests = [ digest for digest in digests if not blob_exists(dst, digest, _urlopen=_urlopen) ] if not missing_digests: return manifest, config # Pull missing blobs, push them to the registry with spack.stage.StageComposite.from_iterable( make_stage(url=src.blob_url(digest), digest=digest, _urlopen=_urlopen) for digest in missing_digests ) as stages: stages.fetch() stages.check() stages.cache_local() for stage, digest in zip(stages, missing_digests): # No need to check existence again, force=True. upload_blob( dst, file=stage.save_filename, force=True, digest=digest, _urlopen=_urlopen ) return manifest, config #: OCI manifest content types (including docker type) manifest_content_type = [ "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json", ] #: OCI index content types (including docker type) index_content_type = [ "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json", ] #: All OCI manifest / index content types all_content_type = manifest_content_type + index_content_type def get_manifest_and_config( ref: ImageReference, architecture="amd64", recurse=3, _urlopen: spack.oci.opener.MaybeOpen = None, ) -> Tuple[dict, dict]: """Recursively fetch manifest and config for a given image reference with a given architecture. Args: ref: The image reference. architecture: The architecture (when referencing an index) recurse: How many levels of index to recurse into. Returns: A tuple of (manifest, config)""" _urlopen = _urlopen or spack.oci.opener.urlopen # Get manifest with _urlopen( Request(url=ref.manifest_url(), headers={"Accept": ", ".join(all_content_type)}) ) as response: # Recurse when we find an index if response.headers["Content-Type"] in index_content_type: if recurse == 0: raise Exception("Maximum recursion depth reached while fetching OCI manifest") index = json.load(response) manifest_meta = next( manifest for manifest in index["manifests"] if manifest["platform"]["architecture"] == architecture ) return get_manifest_and_config( ref.with_digest(manifest_meta["digest"]), architecture=architecture, recurse=recurse - 1, _urlopen=_urlopen, ) # Otherwise, require a manifest if response.headers["Content-Type"] not in manifest_content_type: raise Exception(f"Unknown content type {response.headers['Content-Type']}") manifest = json.load(response) # Download, verify and cache config file config_digest = Digest.from_string(manifest["config"]["digest"]) with make_stage(ref.blob_url(config_digest), config_digest, _urlopen=_urlopen) as stage: stage.fetch() stage.check() stage.cache_local() with open(stage.save_filename, "rb") as f: config = json.load(f) return manifest, config #: Same as upload_manifest, but with retry wrapper upload_manifest_with_retry = spack.oci.opener.default_retry(upload_manifest) #: Same as upload_blob, but with retry wrapper upload_blob_with_retry = spack.oci.opener.default_retry(upload_blob) #: Same as get_manifest_and_config, but with retry wrapper get_manifest_and_config_with_retry = spack.oci.opener.default_retry(get_manifest_and_config) #: Same as copy_missing_layers, but with retry wrapper copy_missing_layers_with_retry = spack.oci.opener.default_retry(copy_missing_layers) def make_stage( url: str, digest: Digest, keep: bool = False, _urlopen: spack.oci.opener.MaybeOpen = None ) -> spack.stage.Stage: _urlopen = _urlopen or spack.oci.opener.urlopen fetch_strategy = spack.fetch_strategy.OCIRegistryFetchStrategy( url=url, checksum=digest.digest, _urlopen=_urlopen ) # Use blobs// as the cache path, which follows # the OCI Image Layout Specification. What's missing though, # is the `oci-layout` and `index.json` files, which are # required by the spec. return spack.stage.Stage( fetch_strategy, mirror_paths=spack.mirrors.layout.OCILayout(digest), name=digest.digest, keep=keep, ) ================================================ FILE: lib/spack/spack/oci/opener.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """All the logic for OCI fetching and authentication""" import base64 import json import re import urllib.error import urllib.parse import urllib.request from enum import Enum, auto from http.client import HTTPResponse from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Tuple from urllib.request import Request import spack.config import spack.llnl.util.lang import spack.mirrors.mirror import spack.tokenize import spack.util.web from .image import ImageReference def _urlopen(): opener = create_opener() def dispatch_open(fullurl, data=None, timeout=None): timeout = timeout or spack.config.get("config:connect_timeout", 10) return opener.open(fullurl, data, timeout) return dispatch_open OpenType = Callable[..., HTTPResponse] MaybeOpen = Optional[OpenType] #: Opener that automatically uses OCI authentication based on mirror config urlopen: OpenType = spack.llnl.util.lang.Singleton(_urlopen) SP = r" " OWS = r"[ \t]*" BWS = OWS HTAB = r"\t" VCHAR = r"\x21-\x7E" tchar = r"[!#$%&'*+\-.^_`|~0-9A-Za-z]" token = rf"{tchar}+" obs_text = r"\x80-\xFF" qdtext = rf"[{HTAB}{SP}\x21\x23-\x5B\x5D-\x7E{obs_text}]" quoted_pair = rf"\\([{HTAB}{SP}{VCHAR}{obs_text}])" quoted_string = rf'"(?:({qdtext}*)|{quoted_pair})*"' class WwwAuthenticateTokens(spack.tokenize.TokenBase): AUTH_PARAM = rf"({token}){BWS}={BWS}({token}|{quoted_string})" # TOKEN68 = r"([A-Za-z0-9\-._~+/]+=*)" # todo... support this? TOKEN = rf"{tchar}+" EQUALS = rf"{BWS}={BWS}" COMMA = rf"{OWS},{OWS}" SPACE = r" +" EOF = r"$" ANY = r"." WWW_AUTHENTICATE_TOKENIZER = spack.tokenize.Tokenizer(WwwAuthenticateTokens) class State(Enum): CHALLENGE = auto() AUTH_PARAM_LIST_START = auto() AUTH_PARAM = auto() NEXT_IN_LIST = auto() AUTH_PARAM_OR_SCHEME = auto() class Challenge: __slots__ = ["scheme", "params"] def __init__( self, scheme: Optional[str] = None, params: Optional[List[Tuple[str, str]]] = None ) -> None: self.scheme = scheme or "" self.params = params or [] def __repr__(self) -> str: return f"Challenge({self.scheme}, {self.params})" def __eq__(self, other: object) -> bool: return ( isinstance(other, Challenge) and self.scheme == other.scheme and self.params == other.params ) def matches_scheme(self, scheme: str) -> bool: """Checks whether the challenge matches the given scheme, case-insensitive.""" return self.scheme == scheme.lower() def get_param(self, key: str) -> Optional[str]: """Get the value of an auth param by key, or None if not found.""" return next((v for k, v in self.params if k == key.lower()), None) def parse_www_authenticate(input: str): """Very basic parsing of www-authenticate parsing (RFC7235 section 4.1) Notice: this omits token68 support.""" # auth-scheme = token # auth-param = token BWS "=" BWS ( token / quoted-string ) # challenge = auth-scheme [ 1*SP ( token68 / #auth-param ) ] # WWW-Authenticate = 1#challenge challenges: List[Challenge] = [] _unquote = re.compile(quoted_pair).sub unquote = lambda s: _unquote(r"\1", s[1:-1]) mode: State = State.CHALLENGE tokens = WWW_AUTHENTICATE_TOKENIZER.tokenize(input) current_challenge = Challenge() def extract_auth_param(input: str) -> Tuple[str, str]: key, value = input.split("=", 1) key = key.rstrip().lower() value = value.lstrip() if value.startswith('"'): value = unquote(value) return key, value while True: token: spack.tokenize.Token = next(tokens) if mode == State.CHALLENGE: if token.kind == WwwAuthenticateTokens.EOF: raise ValueError(token) elif token.kind == WwwAuthenticateTokens.TOKEN: current_challenge.scheme = token.value.lower() mode = State.AUTH_PARAM_LIST_START else: raise ValueError(token) elif mode == State.AUTH_PARAM_LIST_START: if token.kind == WwwAuthenticateTokens.EOF: challenges.append(current_challenge) break elif token.kind == WwwAuthenticateTokens.COMMA: # Challenge without param list, followed by another challenge. challenges.append(current_challenge) current_challenge = Challenge() mode = State.CHALLENGE elif token.kind == WwwAuthenticateTokens.SPACE: # A space means it must be followed by param list mode = State.AUTH_PARAM else: raise ValueError(token) elif mode == State.AUTH_PARAM: if token.kind == WwwAuthenticateTokens.EOF: raise ValueError(token) elif token.kind == WwwAuthenticateTokens.AUTH_PARAM: key, value = extract_auth_param(token.value) current_challenge.params.append((key, value)) mode = State.NEXT_IN_LIST else: raise ValueError(token) elif mode == State.NEXT_IN_LIST: if token.kind == WwwAuthenticateTokens.EOF: challenges.append(current_challenge) break elif token.kind == WwwAuthenticateTokens.COMMA: mode = State.AUTH_PARAM_OR_SCHEME else: raise ValueError(token) elif mode == State.AUTH_PARAM_OR_SCHEME: if token.kind == WwwAuthenticateTokens.EOF: raise ValueError(token) elif token.kind == WwwAuthenticateTokens.TOKEN: challenges.append(current_challenge) current_challenge = Challenge(token.value.lower()) mode = State.AUTH_PARAM_LIST_START elif token.kind == WwwAuthenticateTokens.AUTH_PARAM: key, value = extract_auth_param(token.value) current_challenge.params.append((key, value)) mode = State.NEXT_IN_LIST return challenges class RealmServiceScope(NamedTuple): realm: str service: str scope: str class UsernamePassword(NamedTuple): username: str password: str @property def basic_auth_header(self) -> str: encoded = base64.b64encode(f"{self.username}:{self.password}".encode("utf-8")).decode( "utf-8" ) return f"Basic {encoded}" def _get_bearer_challenge(challenges: List[Challenge]) -> Optional[RealmServiceScope]: """Return the realm/service/scope for a Bearer auth challenge, or None if not found.""" challenge = next((c for c in challenges if c.matches_scheme("Bearer")), None) if challenge is None: return None # Get realm / service / scope from challenge realm = challenge.get_param("realm") service = challenge.get_param("service") scope = challenge.get_param("scope") if realm is None or service is None or scope is None: return None return RealmServiceScope(realm, service, scope) def _get_basic_challenge(challenges: List[Challenge]) -> Optional[str]: """Return the realm for a Basic auth challenge, or None if not found.""" challenge = next((c for c in challenges if c.matches_scheme("Basic")), None) if challenge is None: return None return challenge.get_param("realm") class OCIAuthHandler(urllib.request.BaseHandler): def __init__(self, credentials_provider: Callable[[str], Optional[UsernamePassword]]): """ Args: credentials_provider: A function that takes a domain and may return a UsernamePassword. """ self.credentials_provider = credentials_provider # Cached authorization headers for a given domain. self.cached_auth_headers: Dict[str, str] = {} def https_request(self, req: Request): # Eagerly add the bearer token to the request if no # auth header is set yet, to avoid 401s in multiple # requests to the same registry. # Use has_header, not .headers, since there are two # types of headers (redirected and unredirected) if req.has_header("Authorization"): return req parsed = urllib.parse.urlparse(req.full_url) auth_header = self.cached_auth_headers.get(parsed.netloc) if not auth_header: return req req.add_unredirected_header("Authorization", auth_header) return req def _try_bearer_challenge( self, challenges: List[Challenge], credentials: Optional[UsernamePassword], timeout: Optional[float], ) -> Optional[str]: # Check whether a Bearer challenge is present in the WWW-Authenticate header challenge = _get_bearer_challenge(challenges) if not challenge: return None # Get the token from the auth handler query = urllib.parse.urlencode( {"service": challenge.service, "scope": challenge.scope, "client_id": "spack"} ) parsed = urllib.parse.urlparse(challenge.realm)._replace( query=query, fragment="", params="" ) # Don't send credentials over insecure transport. if parsed.scheme != "https": raise ValueError(f"Cannot login over insecure {parsed.scheme} connection") request = Request(urllib.parse.urlunparse(parsed), method="GET") if credentials is not None: request.add_unredirected_header("Authorization", credentials.basic_auth_header) # Do a GET request. response = self.parent.open(request, timeout=timeout) try: response_json = json.load(response) token = response_json.get("token") if token is None: token = response_json.get("access_token") assert type(token) is str except Exception as e: raise ValueError(f"Malformed token response from {challenge.realm}") from e return f"Bearer {token}" def _try_basic_challenge( self, challenges: List[Challenge], credentials: UsernamePassword ) -> Optional[str]: # Check whether a Basic challenge is present in the WWW-Authenticate header # A realm is required for Basic auth, although we don't use it here. Leave this as a # validation step. realm = _get_basic_challenge(challenges) if not realm: return None return credentials.basic_auth_header def http_error_401(self, req: Request, fp, code, msg, headers): # Login failed, avoid infinite recursion where we go back and # forth between auth server and registry if hasattr(req, "login_attempted"): raise spack.util.web.DetailedHTTPError( req, code, f"Failed to login: {msg}", headers, fp ) # On 401 Unauthorized, parse the WWW-Authenticate header # to determine what authentication is required if "WWW-Authenticate" not in headers: raise spack.util.web.DetailedHTTPError( req, code, "Cannot login to registry, missing WWW-Authenticate header", headers, fp ) www_auth_str = headers["WWW-Authenticate"] try: challenges = parse_www_authenticate(www_auth_str) except ValueError as e: raise spack.util.web.DetailedHTTPError( req, code, f"Cannot login to registry, malformed WWW-Authenticate header: {www_auth_str}", headers, fp, ) from e registry = urllib.parse.urlparse(req.get_full_url()).netloc credentials = self.credentials_provider(registry) # First try Bearer, then Basic try: auth_header = self._try_bearer_challenge(challenges, credentials, req.timeout) if not auth_header and credentials: auth_header = self._try_basic_challenge(challenges, credentials) except Exception as e: raise spack.util.web.DetailedHTTPError( req, code, f"Cannot login to registry: {e}", headers, fp ) from e if not auth_header: raise spack.util.web.DetailedHTTPError( req, code, f"Cannot login to registry, unsupported authentication scheme: {www_auth_str}", headers, fp, ) self.cached_auth_headers[registry] = auth_header # Add the authorization header to the request req.add_unredirected_header("Authorization", auth_header) setattr(req, "login_attempted", True) return self.parent.open(req, timeout=req.timeout) def credentials_from_mirrors( domain: str, *, mirrors: Optional[Iterable[spack.mirrors.mirror.Mirror]] = None ) -> Optional[UsernamePassword]: """Filter out OCI registry credentials from a list of mirrors.""" mirrors = mirrors or spack.mirrors.mirror.MirrorCollection().values() for mirror in mirrors: # Prefer push credentials over fetch. Unlikely that those are different # but our config format allows it. for direction in ("push", "fetch"): pair = mirror.get_credentials(direction).get("access_pair") if not pair: continue url = mirror.get_url(direction) try: parsed = ImageReference.from_url(url) except ValueError: continue if parsed.domain == domain: return UsernamePassword(*pair) return None def create_opener(): """Create an opener that can handle OCI authentication.""" opener = urllib.request.OpenerDirector() for handler in [ urllib.request.ProxyHandler(), urllib.request.UnknownHandler(), urllib.request.HTTPHandler(), spack.util.web.SpackHTTPSHandler(context=spack.util.web.ssl_create_default_context()), spack.util.web.SpackHTTPDefaultErrorHandler(), urllib.request.HTTPRedirectHandler(), urllib.request.HTTPErrorProcessor(), OCIAuthHandler(credentials_from_mirrors), ]: opener.add_handler(handler) return opener def ensure_status(request: urllib.request.Request, response: HTTPResponse, status: int): """Raise an error if the response status is not the expected one.""" if response.status == status: return raise spack.util.web.DetailedHTTPError( request, response.status, response.reason, response.info(), None ) default_retry = spack.util.web.retry_on_transient_error ================================================ FILE: lib/spack/spack/operating_systems/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from ._operating_system import OperatingSystem from .freebsd import FreeBSDOs from .linux_distro import LinuxDistro from .mac_os import MacOs from .windows_os import WindowsOs __all__ = ["OperatingSystem", "LinuxDistro", "MacOs", "WindowsOs", "FreeBSDOs"] #: List of all the Operating Systems known to Spack operating_systems = [LinuxDistro, MacOs, WindowsOs, FreeBSDOs] ================================================ FILE: lib/spack/spack/operating_systems/_operating_system.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.llnl.util.lang @spack.llnl.util.lang.lazy_lexicographic_ordering class OperatingSystem: """Base class for all the Operating Systems. On a multiple architecture machine, the architecture spec field can be set to build a package against any target and operating system that is present on the platform. On Cray platforms or any other architecture that has different front and back end environments, the operating system will determine the method of compiler detection. There are two different types of compiler detection: 1. Through the $PATH env variable (front-end detection) 2. Through the module system. (back-end detection) Depending on which operating system is specified, the compiler will be detected using one of those methods. For platforms such as linux and darwin, the operating system is autodetected. """ def __init__(self, name, version): self.name = name.replace("-", "_") self.version = str(version).replace("-", "_") def __str__(self): return "%s%s" % (self.name, self.version) def __repr__(self): return self.__str__() def _cmp_iter(self): yield self.name yield self.version def to_dict(self): return {"name": self.name, "version": self.version} ================================================ FILE: lib/spack/spack/operating_systems/freebsd.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform as py_platform from spack.version import Version from ._operating_system import OperatingSystem class FreeBSDOs(OperatingSystem): def __init__(self): release = py_platform.release().split("-", 1)[0] super().__init__("freebsd", Version(release)) ================================================ FILE: lib/spack/spack/operating_systems/linux_distro.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform as py_platform import re from subprocess import check_output from spack.version import StandardVersion from ._operating_system import OperatingSystem def kernel_version() -> StandardVersion: """Return the host's kernel version as a :class:`~spack.version.StandardVersion` object.""" # Strip '+' characters just in case we're running a # version built from git/etc return StandardVersion.from_string(re.sub(r"\+", r"", py_platform.release())) class LinuxDistro(OperatingSystem): """This class will represent the autodetected operating system for a Linux System. Since there are many different flavors of Linux, this class will attempt to encompass them all through autodetection using the python module platform and the method platform.dist() """ def __init__(self): try: # This will throw an error if imported on a non-Linux platform. from spack.vendor import distro distname, version = distro.id(), distro.version() except ImportError: distname, version = "unknown", "" # Grabs major version from tuple on redhat; on other platforms # grab the first legal identifier in the version field. On # debian you get things like 'wheezy/sid'; sid means unstable. # We just record 'wheezy' and don't get quite so detailed. version = re.split(r"[^\w-]", version) if "ubuntu" in distname: version = ".".join(version[0:2]) # openSUSE Tumbleweed is a rolling release which can change # more than once in a week, so set version to tumbleweed$GLIBVERS elif "opensuse-tumbleweed" in distname or "opensusetumbleweed" in distname: distname = "opensuse" output = check_output(["ldd", "--version"]).decode() libcvers = re.findall(r"ldd \(GNU libc\) (.*)", output) if len(libcvers) == 1: version = "tumbleweed" + libcvers[0] else: version = "tumbleweed" + version[0] else: version = version[0] super().__init__(distname, version) ================================================ FILE: lib/spack/spack/operating_systems/mac_os.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import platform as py_platform import re import spack.llnl.util.lang from spack.util.executable import Executable from spack.version import StandardVersion, Version from ._operating_system import OperatingSystem @spack.llnl.util.lang.memoized def macos_version() -> StandardVersion: """Get the current macOS version as a version object. This has three mechanisms for determining the macOS version, which is used for spack identification (the ``os`` in the spec's ``arch``) and indirectly for setting the value of ``MACOSX_DEPLOYMENT_TARGET``, which affects the ``minos`` value of the ``LC_BUILD_VERSION`` macho header. Mixing ``minos`` values can lead to lots of linker warnings, and using a consistent version (pinned to the major OS version) allows distribution across clients that might be slightly behind. The version determination is made with three mechanisms in decreasing priority: 1. The ``MACOSX_DEPLOYMENT_TARGET`` variable overrides the actual operating system version, just like the value can be used to build for older macOS targets on newer systems. Spack currently will truncate this value when building packages, but at least the major version will be the same. 2. The system ``sw_vers`` command reports the actual operating system version. 3. The Python ``platform.mac_ver`` function is a fallback if the operating system identification fails, because some Python versions and/or installations report the OS on which Python was *built* rather than the one on which it is running. """ env_ver = os.environ.get("MACOSX_DEPLOYMENT_TARGET", None) if env_ver: return StandardVersion.from_string(env_ver) try: output = Executable("sw_vers")(output=str, fail_on_error=False) except Exception: # FileNotFoundError, or spack.util.executable.ProcessError pass else: match = re.search(r"ProductVersion:\s*([0-9.]+)", output) if match: return StandardVersion.from_string(match.group(1)) # Fall back to python-reported version, which can be inaccurate around # macOS 11 (e.g. showing 10.16 for macOS 12) return StandardVersion.from_string(py_platform.mac_ver()[0]) @spack.llnl.util.lang.memoized def macos_cltools_version(): """Find the last installed version of the CommandLineTools. The CLT version might only affect the build if it's selected as the macOS SDK path. """ pkgutil = Executable("pkgutil") output = pkgutil( "--pkg-info=com.apple.pkg.cltools_executables", output=str, fail_on_error=False ) match = re.search(r"version:\s*([0-9.]+)", output) if match: return Version(match.group(1)) # No CLTools installed by package manager: try Xcode output = pkgutil("--pkg-info=com.apple.pkg.Xcode", output=str, fail_on_error=False) match = re.search(r"version:\s*([0-9.]+)", output) if match: return Version(match.group(1)) return None @spack.llnl.util.lang.memoized def macos_sdk_path(): """Return path to the active macOS SDK.""" xcrun = Executable("xcrun") return xcrun("--show-sdk-path", output=str).rstrip() def macos_sdk_version(): """Return the version of the active macOS SDK. The SDK version usually corresponds to the installed Xcode version and can affect how some packages (especially those that use the GUI) can fail. This information should somehow be embedded into the future "compilers are dependencies" feature. The macOS deployment target cannot be greater than the SDK version, but usually it can be at least a few versions less. """ xcrun = Executable("xcrun") return Version(xcrun("--show-sdk-version", output=str).rstrip()) class MacOs(OperatingSystem): """This class represents the macOS operating system. This will be auto detected using the python platform.mac_ver. The macOS platform will be represented using the major version operating system name, i.e el capitan, yosemite...etc. """ def __init__(self): """Autodetects the mac version from a dictionary. If the mac version is too old or too new for Spack to recognize, will use a generic "macos" version string until Spack is updated. """ mac_releases = { "10.0": "cheetah", "10.1": "puma", "10.2": "jaguar", "10.3": "panther", "10.4": "tiger", "10.5": "leopard", "10.6": "snowleopard", "10.7": "lion", "10.8": "mountainlion", "10.9": "mavericks", "10.10": "yosemite", "10.11": "elcapitan", "10.12": "sierra", "10.13": "highsierra", "10.14": "mojave", "10.15": "catalina", "10.16": "bigsur", "11": "bigsur", "12": "monterey", "13": "ventura", "14": "sonoma", "15": "sequoia", "26": "tahoe", } version = macos_version() # Big Sur versions go 11.0, 11.0.1, 11.1 (vs. prior versions that # only used the minor component) part = 1 if version >= Version("11") else 2 mac_ver = str(version.up_to(part)) name = mac_releases.get(mac_ver, "macos") super().__init__(name, mac_ver) def __str__(self): return self.name ================================================ FILE: lib/spack/spack/operating_systems/windows_os.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import glob import os import pathlib import platform import subprocess from spack.error import SpackError from spack.llnl.util import tty from spack.util import windows_registry as winreg from spack.version import Version from ._operating_system import OperatingSystem def windows_version(): """Windows version as a Version object""" # include the build number as this provides important information # for low lever packages and components like the SDK and WDK # The build number is the version component that would otherwise # be the patch version in semantic versioning, i.e. z of x.y.z return Version(platform.version()) class WindowsOs(OperatingSystem): """This class represents the Windows operating system. This will be auto detected using the python platform.win32_ver() once we have a python setup that runs natively. The Windows platform will be represented using the major version operating system number, e.g. 10. """ def __init__(self): plat_ver = windows_version() if plat_ver < Version("10"): raise SpackError("Spack is not supported on Windows versions older than 10") super().__init__("windows{}".format(plat_ver), plat_ver) def __str__(self): return self.name @property def vs_install_paths(self): vs_install_paths = [] root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") if root: try: extra_args = {"encoding": "mbcs", "errors": "strict"} paths = subprocess.check_output( # type: ignore[call-overload] # novermin [ os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), "-prerelease", "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath", "-products", "*", ], **extra_args, ).strip() vs_install_paths = paths.split("\n") except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): pass return vs_install_paths @property def msvc_paths(self): return [os.path.join(path, "VC", "Tools", "MSVC") for path in self.vs_install_paths] @property def oneapi_root(self): root = os.environ.get("ONEAPI_ROOT", "") or os.path.join( os.environ.get("ProgramFiles(x86)", ""), "Intel", "oneAPI" ) if os.path.exists(root): return root @property def compiler_search_paths(self): # First Strategy: Find MSVC directories using vswhere _compiler_search_paths = [] for p in self.msvc_paths: _compiler_search_paths.extend(glob.glob(os.path.join(p, "*", "bin", "Hostx64", "x64"))) oneapi_root = self.oneapi_root if oneapi_root: _compiler_search_paths.extend( glob.glob(os.path.join(oneapi_root, "compiler", "**", "bin"), recursive=True) ) # Second strategy: Find MSVC via the registry def try_query_registry(retry=False): winreg_report_error = lambda e: tty.debug( 'Windows registry query on "SOFTWARE\\WOW6432Node\\Microsoft"' f"under HKEY_LOCAL_MACHINE: {str(e)}" ) try: # Registry interactions are subject to race conditions, etc and can generally # be flakey, do this in a catch block to prevent reg issues from interfering # with compiler detection msft = winreg.WindowsRegistryView( "SOFTWARE\\WOW6432Node\\Microsoft", winreg.HKEY.HKEY_LOCAL_MACHINE ) return msft.find_subkeys(r"VisualStudio_.*", recursive=False) except OSError as e: # OSErrors propagated into caller by Spack's registry module are expected # and indicate a known issue with the registry query # i.e. user does not have permissions or the key/value # doesn't exist winreg_report_error(e) return [] except winreg.InvalidRegistryOperation as e: # Other errors raised by the Spack's reg module indicate # an unexpected error type, and are handled specifically # as the underlying cause is difficult/impossible to determine # without manually exploring the registry # These errors can also be spurious (race conditions) # and may resolve on re-execution of the query # or are permanent (specific types of permission issues) # but the registry raises the same exception for all types of # atypical errors if retry: winreg_report_error(e) return [] vs_entries = try_query_registry() if not vs_entries: # Occasional spurious race conditions can arise when reading the MS reg # typically these race conditions resolve immediately and we can safely # retry the reg query without waiting # Note: Winreg does not support locking vs_entries = try_query_registry(retry=True) vs_paths = [] def clean_vs_path(path): path = path.split(",")[0].lstrip("@") return str((pathlib.Path(path).parent / "..\\..").resolve()) for entry in vs_entries: try: val = entry.get_subkey("Capabilities").get_value("ApplicationDescription").value vs_paths.append(clean_vs_path(val)) except FileNotFoundError as e: if hasattr(e, "winerror") and e.winerror == 2: pass else: raise _compiler_search_paths.extend(vs_paths) return _compiler_search_paths ================================================ FILE: lib/spack/spack/package.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import warnings from os import chdir, environ, getcwd, makedirs, mkdir, remove, removedirs from shutil import move, rmtree # import most common types used in packages from typing import Dict, Iterable, List, Optional, Tuple from spack.vendor.macholib.MachO import LC_ID_DYLIB, MachO import spack.builder import spack.llnl.util.tty as _tty from spack.archspec import microarchitecture_flags, microarchitecture_flags_from_target from spack.build_environment import ( MakeExecutable, ModuleChangePropagator, get_cmake_prefix_path, get_effective_jobs, shared_library_suffix, static_library_suffix, ) from spack.builder import ( BaseBuilder, Builder, BuilderWithDefaults, GenericBuilder, Package, apply_macos_rpath_fixups, execute_install_time_tests, register_builder, ) from spack.compilers.config import find_compilers from spack.compilers.libraries import CompilerPropertyDetector, compiler_spec from spack.config import determine_number_of_jobs from spack.deptypes import ALL_TYPES as all_deptypes from spack.directives import ( build_system, can_splice, conditional, conflicts, depends_on, extends, license, maintainers, patch, provides, redistribute, requires, resource, variant, version, ) from spack.error import ( CompilerError, InstallError, NoHeadersError, NoLibrariesError, SpackAPIWarning, SpackError, ) from spack.hooks.sbang import filter_shebang, sbang_install_path, sbang_shebang_line from spack.install_test import ( SkipTest, cache_extra_test_sources, check_outputs, find_required_file, get_escaped_text_output, install_test_root, test_part, ) from spack.llnl.util.filesystem import ( FileFilter, FileList, HeaderList, LibraryList, ancestor, can_access, change_sed_delimiter, copy, copy_tree, filter_file, find, find_all_headers, find_all_libraries, find_first, find_headers, find_libraries, find_system_libraries, force_remove, force_symlink, has_shebang, install, install_tree, is_exe, join_path, keep_modification_time, library_extensions, mkdirp, path_contains_subdirectory, readlink, remove_directory_contents, remove_linked_tree, rename, safe_remove, set_executable, set_install_permissions, symlink, touch, windows_sfn, working_dir, ) from spack.llnl.util.lang import ClassProperty, classproperty, dedupe, memoized from spack.llnl.util.link_tree import LinkTree from spack.mixins import filter_compiler_wrappers from spack.multimethod import default_args, when from spack.operating_systems.linux_distro import kernel_version from spack.operating_systems.mac_os import macos_version from spack.package_base import PackageBase, make_package_test_rpath, on_package_attributes from spack.package_completions import ( bash_completion_path, fish_completion_path, zsh_completion_path, ) from spack.package_test import compare_output, compare_output_file, compile_c_and_execute from spack.paths import spack_script from spack.phase_callbacks import run_after, run_before from spack.platforms import host as host_platform from spack.spec import Spec from spack.url import substitute_version as substitute_version_in_url from spack.user_environment import environment_modifications_for_specs from spack.util.elf import delete_needed_from_elf, delete_rpath, get_elf_compat, parse_elf from spack.util.environment import EnvironmentModifications, set_env from spack.util.environment import filter_system_paths as _filter_system_paths from spack.util.environment import is_system_path as _is_system_path from spack.util.executable import Executable, ProcessError, which, which_string from spack.util.filesystem import fix_darwin_install_name from spack.util.libc import libc_from_dynamic_linker, parse_dynamic_linker from spack.util.module_cmd import get_path_args_from_module_line from spack.util.module_cmd import module as module_command from spack.util.path import get_user from spack.util.prefix import Prefix from spack.util.url import join as join_url from spack.util.windows_registry import HKEY, WindowsRegistryView from spack.variant import any_combination_of, auto_or_any_combination_of, disjoint_sets from spack.version import Version, ver #: Alias for :data:`os.environ` env = environ #: Alias for :func:`os.chdir` cd = chdir #: Alias for :func:`os.getcwd` pwd = getcwd #: Alias for :func:`os.rename` rename = rename #: Alias for :func:`os.makedirs` makedirs = makedirs #: Alias for :func:`os.mkdir` mkdir = mkdir #: Alias for :func:`os.remove` remove = remove #: Alias for :func:`os.removedirs` removedirs = removedirs #: Alias for :func:`shutil.move` move = move #: Alias for :func:`shutil.rmtree` rmtree = rmtree #: Alias for :func:`os.readlink` (with certain Windows-specific changes) readlink = readlink #: Alias for :func:`os.rename` (with certain Windows-specific changes) rename = rename #: Alias for :func:`os.symlink` (with certain Windows-specific changes) symlink = symlink # Not an import alias because black and isort disagree about style create_builder = spack.builder.create #: MachO class from the ``macholib`` package (vendored in Spack). MachO = MachO #: Constant for MachO ``LC_ID_DYLIB`` load command, from the ``macholib`` package (vendored in #: Spack). LC_ID_DYLIB = LC_ID_DYLIB class tty: debug = _tty.debug error = _tty.error info = _tty.info msg = _tty.msg warn = _tty.warn def is_system_path(path: str) -> bool: """Returns :obj:`True` iff the argument is a system path. .. deprecated:: v2.0 """ warnings.warn( "spack.package.is_system_path is deprecated", category=SpackAPIWarning, stacklevel=2 ) return _is_system_path(path) def filter_system_paths(paths: Iterable[str]) -> List[str]: """Returns a copy of the input where system paths are filtered out. .. deprecated:: v2.0 """ warnings.warn( "spack.package.filter_system_paths is deprecated", category=SpackAPIWarning, stacklevel=2 ) return _filter_system_paths(paths) #: Assigning this to :attr:`spack.package_base.PackageBase.flag_handler` means that compiler flags #: are passed to the build system. This can be used in any package that derives from a build system #: class that implements :meth:`spack.package_base.PackageBase.flags_to_build_system_args`. #: #: See also :func:`env_flags` and :func:`inject_flags`. #: #: Example:: #: #: from spack.package import * #: #: class MyPackage(CMakePackage): #: flag_handler = build_system_flags build_system_flags = PackageBase.build_system_flags #: Assigning this to :attr:`spack.package_base.PackageBase.flag_handler` means that compiler flags #: are set as canonical environment variables. #: #: See also :func:`build_system_flags` and :func:`inject_flags`. #: #: Example:: #: #: from spack.package import * #: #: class MyPackage(MakefilePackage): #: flag_handler = env_flags env_flags = PackageBase.env_flags #: This is the default value of :attr:`spack.package_base.PackageBase.flag_handler`, which tells #: Spack to inject compiler flags through the compiler wrappers, which means that the build system #: will not see them directly. This is typically a good default, but in rare case you may need to #: use :func:`env_flags` or :func:`build_system_flags` instead. #: #: See also :func:`build_system_flags` and :func:`env_flags`. #: #: Example:: #: #: from spack.package import * #: #: class MyPackage(MakefilePackage): #: flag_handler = inject_flags inject_flags = PackageBase.inject_flags api: Dict[str, Tuple[str, ...]] = { "v2.0": ( "BaseBuilder", "Builder", "Dict", "EnvironmentModifications", "Executable", "FileFilter", "FileList", "HeaderList", "InstallError", "LibraryList", "List", "MakeExecutable", "NoHeadersError", "NoLibrariesError", "Optional", "PackageBase", "Prefix", "ProcessError", "SkipTest", "Spec", "Version", "all_deptypes", "ancestor", "any_combination_of", "auto_or_any_combination_of", "bash_completion_path", "build_system_flags", "build_system", "cache_extra_test_sources", "can_access", "can_splice", "cd", "change_sed_delimiter", "check_outputs", "conditional", "conflicts", "copy_tree", "copy", "default_args", "depends_on", "determine_number_of_jobs", "disjoint_sets", "env_flags", "env", "extends", "filter_compiler_wrappers", "filter_file", "find_all_headers", "find_first", "find_headers", "find_libraries", "find_required_file", "find_system_libraries", "find", "fish_completion_path", "fix_darwin_install_name", "force_remove", "force_symlink", "get_escaped_text_output", "inject_flags", "install_test_root", "install_tree", "install", "is_exe", "join_path", "keep_modification_time", "library_extensions", "license", "maintainers", "makedirs", "mkdir", "mkdirp", "move", "on_package_attributes", "patch", "provides", "pwd", "redistribute", "register_builder", "remove_directory_contents", "remove_linked_tree", "remove", "removedirs", "rename", "requires", "resource", "rmtree", "run_after", "run_before", "set_executable", "set_install_permissions", "symlink", "test_part", "touch", "tty", "variant", "ver", "version", "when", "which_string", "which", "working_dir", "zsh_completion_path", ), "v2.1": ("CompilerError", "SpackError"), "v2.2": ( "BuilderWithDefaults", "ClassProperty", "CompilerPropertyDetector", "GenericBuilder", "HKEY", "LC_ID_DYLIB", "LinkTree", "MachO", "ModuleChangePropagator", "Package", "WindowsRegistryView", "apply_macos_rpath_fixups", "classproperty", "compare_output_file", "compare_output", "compile_c_and_execute", "compiler_spec", "create_builder", "dedupe", "delete_needed_from_elf", "delete_rpath", "environment_modifications_for_specs", "execute_install_time_tests", "filter_shebang", "filter_system_paths", "find_all_libraries", "find_compilers", "get_cmake_prefix_path", "get_effective_jobs", "get_elf_compat", "get_path_args_from_module_line", "get_user", "has_shebang", "host_platform", "is_system_path", "join_url", "kernel_version", "libc_from_dynamic_linker", "macos_version", "make_package_test_rpath", "memoized", "microarchitecture_flags_from_target", "microarchitecture_flags", "module_command", "parse_dynamic_linker", "parse_elf", "path_contains_subdirectory", "readlink", "safe_remove", "sbang_install_path", "sbang_shebang_line", "set_env", "shared_library_suffix", "spack_script", "static_library_suffix", "substitute_version_in_url", "windows_sfn", ), } # Splatting does not work for static analysis tools. __all__ = [ # v2.0 "BaseBuilder", "Builder", "Dict", "EnvironmentModifications", "Executable", "FileFilter", "FileList", "HeaderList", "InstallError", "LibraryList", "List", "MakeExecutable", "NoHeadersError", "NoLibrariesError", "Optional", "PackageBase", "Prefix", "ProcessError", "SkipTest", "Spec", "Version", "all_deptypes", "ancestor", "any_combination_of", "auto_or_any_combination_of", "bash_completion_path", "build_system_flags", "build_system", "cache_extra_test_sources", "can_access", "can_splice", "cd", "change_sed_delimiter", "check_outputs", "conditional", "conflicts", "copy_tree", "copy", "default_args", "depends_on", "determine_number_of_jobs", "disjoint_sets", "env_flags", "env", "extends", "filter_compiler_wrappers", "filter_file", "find_all_headers", "find_first", "find_headers", "find_libraries", "find_required_file", "find_system_libraries", "find", "fish_completion_path", "fix_darwin_install_name", "force_remove", "force_symlink", "get_escaped_text_output", "inject_flags", "install_test_root", "install_tree", "install", "is_exe", "join_path", "keep_modification_time", "library_extensions", "license", "maintainers", "makedirs", "mkdir", "mkdirp", "move", "on_package_attributes", "patch", "provides", "pwd", "redistribute", "register_builder", "remove_directory_contents", "remove_linked_tree", "remove", "removedirs", "rename", "requires", "resource", "rmtree", "run_after", "run_before", "set_executable", "set_install_permissions", "symlink", "test_part", "touch", "tty", "variant", "ver", "version", "when", "which_string", "which", "working_dir", "zsh_completion_path", # v2.1 "CompilerError", "SpackError", # v2.2 "BuilderWithDefaults", "ClassProperty", "CompilerPropertyDetector", "GenericBuilder", "HKEY", "LC_ID_DYLIB", "LinkTree", "MachO", "ModuleChangePropagator", "Package", "WindowsRegistryView", "apply_macos_rpath_fixups", "classproperty", "compare_output_file", "compare_output", "compile_c_and_execute", "compiler_spec", "create_builder", "dedupe", "delete_needed_from_elf", "delete_rpath", "environment_modifications_for_specs", "execute_install_time_tests", "filter_shebang", "filter_system_paths", "find_all_libraries", "find_compilers", "get_cmake_prefix_path", "get_effective_jobs", "get_elf_compat", "get_path_args_from_module_line", "get_user", "has_shebang", "host_platform", "is_system_path", "join_url", "kernel_version", "libc_from_dynamic_linker", "macos_version", "make_package_test_rpath", "memoized", "microarchitecture_flags_from_target", "microarchitecture_flags", "module_command", "parse_dynamic_linker", "parse_elf", "path_contains_subdirectory", "readlink", "safe_remove", "sbang_install_path", "sbang_shebang_line", "set_env", "shared_library_suffix", "spack_script", "static_library_suffix", "substitute_version_in_url", "windows_sfn", ] # These are just here for editor support; they may be set when the build env is set up. configure: Executable make_jobs: int make: MakeExecutable nmake: Executable ninja: MakeExecutable python_include: str python_platlib: str python_purelib: str python: Executable spack_cc: str spack_cxx: str spack_f77: str spack_fc: str prefix: Prefix dso_suffix: str ================================================ FILE: lib/spack/spack/package_base.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Base class for all Spack packages.""" import base64 import collections import copy import errno import functools import glob import hashlib import io import itertools import os import pathlib import re import sys import textwrap import time import traceback from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, TypeVar, Union from spack.vendor.typing_extensions import Literal import spack.config import spack.dependency import spack.deptypes as dt import spack.directives_meta import spack.error import spack.fetch_strategy as fs import spack.hooks import spack.llnl.util.filesystem as fsys import spack.llnl.util.tty as tty import spack.mirrors.layout import spack.mirrors.mirror import spack.multimethod import spack.patch import spack.phase_callbacks import spack.repo import spack.spec import spack.stage as stg import spack.store import spack.url import spack.util.archive import spack.util.environment import spack.util.executable import spack.util.git import spack.util.naming import spack.util.path import spack.util.web import spack.variant from spack.compilers.adaptor import DeprecatedCompiler from spack.error import InstallError, NoURLError, PackageError from spack.filesystem_view import YamlFilesystemView from spack.llnl.util.filesystem import ( AlreadyExistsError, find_all_shared_libraries, islink, symlink, ) from spack.llnl.util.lang import ClassProperty, classproperty, dedupe, memoized from spack.resource import Resource from spack.util.package_hash import package_hash from spack.util.typing import SupportsRichComparison from spack.version import GitVersion, StandardVersion, VersionError, is_git_version FLAG_HANDLER_RETURN_TYPE = Tuple[ Optional[Iterable[str]], Optional[Iterable[str]], Optional[Iterable[str]] ] FLAG_HANDLER_TYPE = Callable[[str, Iterable[str]], FLAG_HANDLER_RETURN_TYPE] #: Filename for the Spack build/install log. _spack_build_logfile = "spack-build-out.txt" #: Filename for the Spack build/install environment file. _spack_build_envfile = "spack-build-env.txt" #: Filename for the Spack build/install environment modifications file. _spack_build_envmodsfile = "spack-build-env-mods.txt" #: Filename for the Spack configure args file. _spack_configure_argsfile = "spack-configure-args.txt" #: Filename of json with total build and phase times (seconds) spack_times_log = "install_times.json" NO_DEFAULT = object() class WindowsRPath: """Collection of functionality surrounding Windows RPATH specific features This is essentially meaningless for all other platforms due to their use of RPATH. All methods within this class are no-ops on non Windows. Packages can customize and manipulate this class as they would a genuine RPATH, i.e. adding directories that contain runtime library dependencies""" def win_add_library_dependent(self): """Return extra set of directories that require linking for package This method should be overridden by packages that produce binaries/libraries/python extension modules/etc that are installed into directories outside a package's ``bin``, ``lib``, and ``lib64`` directories, but still require linking against one of the packages dependencies, or other components of the package itself. No-op otherwise. Returns: List of additional directories that require linking """ return [] def win_add_rpath(self): """Return extra set of rpaths for package This method should be overridden by packages needing to include additional paths to be searched by rpath. No-op otherwise Returns: List of additional rpaths """ return [] def windows_establish_runtime_linkage(self): """Establish RPATH on Windows Performs symlinking to incorporate rpath dependencies to Windows runtime search paths """ # If spec is an external, we should not be modifying its bin directory, as we would # be doing in this method # Spack should in general not modify things it has not installed # we can reasonably expect externals to have their link interface properly established if sys.platform == "win32" and not self.spec.external: win_rpath = WindowsSimulatedRPath(self) win_rpath.add_library_dependent(*self.win_add_library_dependent()) win_rpath.add_rpath(*self.win_add_rpath()) win_rpath.establish_link() #: Registers which are the detectable packages, by repo and package name #: Need a pass of package repositories to be filled. detectable_packages = collections.defaultdict(list) class DetectablePackageMeta(type): """Check if a package is detectable and add default implementations for the detection function. """ TAG = "detectable" def __init__(cls, name, bases, attr_dict): if hasattr(cls, "executables") and hasattr(cls, "libraries"): msg = "a package can have either an 'executables' or 'libraries' attribute" raise spack.error.SpackError(f"{msg} [package '{name}' defines both]") # On windows, extend the list of regular expressions to look for # filenames ending with ".exe" # (in some cases these regular expressions include "$" to avoid # pulling in filenames with unexpected suffixes, but this allows # for example detecting "foo.exe" when the package writer specified # that "foo" was a possible executable. # If a package has the executables or libraries attribute then it's # assumed to be detectable. Add a tag, so finding them is faster if hasattr(cls, "executables") or hasattr(cls, "libraries"): # To add the tag, we need to copy the tags attribute, and attach it to # the current class. We don't use append, since it might modify base classes, # if "tags" is retrieved following the MRO. cls.tags = getattr(cls, "tags", []) + [DetectablePackageMeta.TAG] @classmethod def platform_executables(cls): def to_windows_exe(exe): if exe.endswith("$"): exe = exe.replace("$", "%s$" % spack.util.path.win_exe_ext()) else: exe += spack.util.path.win_exe_ext() return exe plat_exe = [] if hasattr(cls, "executables"): for exe in cls.executables: if sys.platform == "win32": exe = to_windows_exe(exe) plat_exe.append(exe) return plat_exe @classmethod def determine_spec_details(cls, prefix, objs_in_prefix): """Allow ``spack external find ...`` to locate installations. Args: prefix (str): the directory containing the executables or libraries objs_in_prefix (set): the executables or libraries that match the regex Returns: The list of detected specs for this package """ objs_by_version = collections.defaultdict(list) # The default filter function is the identity function for the # list of executables filter_fn = getattr(cls, "filter_detected_exes", lambda x, exes: exes) objs_in_prefix = filter_fn(prefix, objs_in_prefix) for obj in objs_in_prefix: try: version_str = cls.determine_version(obj) if version_str: objs_by_version[version_str].append(obj) except Exception as e: tty.debug(f"Cannot detect the version of '{obj}' [{str(e)}]") specs = [] for version_str, objs in objs_by_version.items(): variants = cls.determine_variants(objs, version_str) # Normalize output to list if not isinstance(variants, list): variants = [variants] for variant in variants: if isinstance(variant, str): variant = (variant, {}) variant_str, extra_attributes = variant spec_str = f"{cls.name}@{version_str} {variant_str}" # Pop a few reserved keys from extra attributes, since # they have a different semantics external_path = extra_attributes.pop("prefix", None) external_modules = extra_attributes.pop("modules", None) try: spec = spack.spec.Spec.from_detection( spec_str, external_path=external_path, external_modules=external_modules, extra_attributes=extra_attributes, ) except Exception as e: tty.debug(f'Parsing failed [spec_str="{spec_str}", error={str(e)}]') else: specs.append(spec) return sorted(specs) @classmethod def determine_variants(cls, objs, version_str): return "" # Register the class as a detectable package detectable_packages[cls.namespace].append(cls.name) # Attach function implementations to the detectable class default = False if not hasattr(cls, "determine_spec_details"): default = True cls.determine_spec_details = determine_spec_details if default and not hasattr(cls, "determine_version"): msg = ( 'the package "{0}" in the "{1}" repo needs to define' ' the "determine_version" method to be detectable' ) NotImplementedError(msg.format(cls.name, cls.namespace)) if default and not hasattr(cls, "determine_variants"): cls.determine_variants = determine_variants # This function should not be overridden by subclasses, # as it is not designed for bespoke pkg detection but rather # on a per-platform basis if "platform_executables" in cls.__dict__.keys(): raise PackageError("Packages should not override platform_executables") cls.platform_executables = platform_executables super(DetectablePackageMeta, cls).__init__(name, bases, attr_dict) class PackageMeta( spack.phase_callbacks.PhaseCallbacksMeta, DetectablePackageMeta, spack.directives_meta.DirectiveMeta, spack.multimethod.MultiMethodMeta, ): """ Package metaclass for supporting directives (e.g., depends_on) and phases """ def __new__(cls, name, bases, attr_dict): """ FIXME: REWRITE Instance creation is preceded by phase attribute transformations. Conveniently transforms attributes to permit extensible phases by iterating over the attribute 'phases' and creating / updating private InstallPhase attributes in the class that will be initialized in __init__. """ attr_dict["_name"] = None return super(PackageMeta, cls).__new__(cls, name, bases, attr_dict) def on_package_attributes(**attr_dict): """Decorator: executes instance function only if object has attr values. Executes the decorated method only if at the moment of calling the instance has attributes that are equal to certain values. Args: attr_dict (dict): dictionary mapping attribute names to their required values """ def _execute_under_condition(func): @functools.wraps(func) def _wrapper(instance, *args, **kwargs): # If all the attributes have the value we require, then execute has_all_attributes = all([hasattr(instance, key) for key in attr_dict]) if has_all_attributes: has_the_right_values = all( [getattr(instance, key) == value for key, value in attr_dict.items()] # NOQA: ignore=E501 ) if has_the_right_values: func(instance, *args, **kwargs) return _wrapper return _execute_under_condition class PackageViewMixin: """This collects all functionality related to adding installed Spack package to views. Packages can customize how they are added to views by overriding these functions. """ spec: spack.spec.Spec def view_source(self): """The source root directory that will be added to the view: files are added such that their path relative to the view destination matches their path relative to the view source. """ return self.spec.prefix def view_destination(self, view): """The target root directory: each file is added relative to this directory. """ return view.get_projection_for_spec(self.spec) def view_file_conflicts(self, view, merge_map): """Report any files which prevent adding this package to the view. The default implementation looks for any files which already exist. Alternative implementations may allow some of the files to exist in the view (in this case they would be omitted from the results). """ return set(dst for dst in merge_map.values() if os.path.lexists(dst)) def add_files_to_view(self, view, merge_map, skip_if_exists=True): """Given a map of package files to destination paths in the view, add the files to the view. By default this adds all files. Alternative implementations may skip some files, for example if other packages linked into the view already include the file. Args: view (spack.filesystem_view.FilesystemView): the view that's updated merge_map (dict): maps absolute source paths to absolute dest paths for all files in from this package. skip_if_exists (bool): when True, don't link files in view when they already exist. When False, always link files, without checking if they already exist. """ if skip_if_exists: for src, dst in merge_map.items(): if not os.path.lexists(dst): view.link(src, dst, spec=self.spec) else: for src, dst in merge_map.items(): view.link(src, dst, spec=self.spec) def remove_files_from_view(self, view, merge_map): """Given a map of package files to files currently linked in the view, remove the files from the view. The default implementation removes all files. Alternative implementations may not remove all files. For example if two packages include the same file, it should only be removed when both packages are removed. """ view.remove_files(merge_map.values()) Pb = TypeVar("Pb", bound="PackageBase") # Some typedefs for dealing with when-indexed dictionaries # # Many of the dictionaries on PackageBase are of the form: # { Spec: { K: V } } # # K might be a variant name, a version, etc. V is a definition of some Spack object. # The methods below transform these types of dictionaries. K = TypeVar("K", bound=SupportsRichComparison) V = TypeVar("V") def _by_subkey( when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]], when: bool = False ) -> Dict[K, Union[List[V], Dict[spack.spec.Spec, List[V]]]]: """Convert a dict of dicts keyed by when/subkey into a dict of lists keyed by subkey. Optional Arguments: when: if ``True``, don't discard the ``when`` specs; return a 2-level dictionary keyed by subkey and when spec. """ # very hard to define this type to be conditional on `when` all_by_subkey: Dict[K, Any] = {} for when_spec, by_key in when_indexed_dictionary.items(): for key, value in by_key.items(): if when: when_dict = all_by_subkey.setdefault(key, {}) when_dict.setdefault(when_spec, []).append(value) else: all_by_subkey.setdefault(key, []).append(value) # this needs to preserve the insertion order of whens return dict(sorted(all_by_subkey.items())) def _subkeys(when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]]) -> List[K]: """Get sorted names from dicts keyed by when/name.""" all_keys = set() for when, by_key in when_indexed_dictionary.items(): for key in by_key: all_keys.add(key) return sorted(all_keys) def _has_subkey(when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]], key: K) -> bool: return any(key in dictionary for dictionary in when_indexed_dictionary.values()) def _num_definitions(when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]]) -> int: return sum(len(dictionary) for dictionary in when_indexed_dictionary.values()) def _remove_overridden_defs(defs: List[Tuple[spack.spec.Spec, Any]]) -> None: """Remove definitions from the list if their when specs are satisfied by later ones. Any such definitions are *always* overridden by their successor, as they will match everything the predecessor matches, and the solver will prefer them because of their higher precedence. We can just remove these defs and avoid putting them in the solver. This is also useful for, e.g., `spack info`, where we don't want to show a variant from a superclass if it is always overridden by a variant defined in a subclass. Example:: class ROCmPackage: variant("amdgpu_target", ..., when="+rocm") class Hipblas: variant("amdgpu_target", ...) The subclass definition *always* overrides the superclass definition here, but they have different when specs and the subclass def won't just replace the one in the superclass. In this situation, the subclass should *probably* also have ``when="+rocm"``, but we can't guarantee that will always happen when a vdef is overridden. So we use this method to remove any overrides we can know statically. """ i = 0 while i < len(defs): when, _ = defs[i] if any(when.satisfies(successor) for successor, _ in defs[i + 1 :]): del defs[i] else: i += 1 def _definitions( when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]], key: K ) -> List[Tuple[spack.spec.Spec, V]]: """Iterator over (when_spec, Value) for all values with a particular Key.""" # construct a list of defs sorted by precedence defs: List[Tuple[spack.spec.Spec, V]] = [] for when, values_by_key in when_indexed_dictionary.items(): value_def = values_by_key.get(key) if value_def: defs.append((when, value_def)) # With multiple definitions, ensure precedence order and simplify overrides if len(defs) > 1: defs.sort(key=lambda v: getattr(v[1], "precedence", 0)) _remove_overridden_defs(defs) return defs #: Store whether a given Spec source/binary should not be redistributed. class DisableRedistribute: def __init__(self, source, binary): self.source = source self.binary = binary class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta): """This is the universal base class for all Spack packages. At its core, a package consists of a set of software to be installed. A package may focus on a piece of software and its associated software dependencies or it may simply be a set, or bundle, of software. The former requires defining how to fetch, verify (via, e.g., ``sha256``), build, and install that software and the packages it depends on, so that dependencies can be installed along with the package itself. The latter, sometimes referred to as a "no-source" package, requires only defining the packages to be built. There are two main parts of a Spack package: 1. **The package class**. Classes contain *directives*, which are functions such as :py:func:`spack.package.version`, :py:func:`spack.package.patch`, and :py:func:`spack.package.depends_on`, that store metadata on the package class. Directives provide the constraints that are used as input to the concretizer. 2. **Package instances**. Once instantiated with a concrete spec, a package can be passed to the :py:class:`spack.installer.PackageInstaller`. It calls methods like :meth:`do_stage` on the package instance, and it uses those to drive user-implemented methods like ``def patch`` and install phases like ``def configure`` and ``def install``. Packages are imported from package repositories (see :py:mod:`spack.repo`). For most use cases, package creators typically just add attributes like ``homepage`` and, for a code-based package, ``url``, or installation phases such as ``install()``. There are many custom ``PackageBase`` subclasses in the ``spack_repo.builtin.build_systems`` package that make things even easier for specific build systems. .. note:: Many methods and attributes that appear to be public interface are not meant to be overridden by packagers. They are "final", but we currently have not adopted the ``@final`` decorator in the Spack codebase. For example, the ``do_*`` functions are intended only to be called internally by Spack commands. These aren't for package writers to override, and doing so may break the functionality of the ``PackageBase`` class.""" compiler = DeprecatedCompiler() #: Class level dictionary populated by :func:`~spack.directives.version` directives versions: Dict[StandardVersion, Dict[str, Any]] #: Class level dictionary populated by :func:`~spack.directives.resource` directives resources: Dict[spack.spec.Spec, List[Resource]] #: Class level dictionary populated by :func:`~spack.directives.depends_on` and #: :func:`~spack.directives.extends` directives dependencies: Dict[spack.spec.Spec, Dict[str, spack.dependency.Dependency]] #: Class level dictionary populated by :func:`~spack.directives.extends` directives extendees: Dict[str, Tuple[spack.spec.Spec, spack.spec.Spec]] #: Class level dictionary populated by :func:`~spack.directives.conflicts` directives conflicts: Dict[spack.spec.Spec, List[Tuple[spack.spec.Spec, Optional[str]]]] #: Class level dictionary populated by :func:`~spack.directives.requires` directives requirements: Dict[ spack.spec.Spec, List[Tuple[Tuple[spack.spec.Spec, ...], str, Optional[str]]] ] #: Class level dictionary populated by :func:`~spack.directives.provides` directives provided: Dict[spack.spec.Spec, Set[spack.spec.Spec]] #: Class level dictionary populated by :func:`~spack.directives.provides` directives provided_together: Dict[spack.spec.Spec, List[Set[str]]] #: Class level dictionary populated by :func:`~spack.directives.patch` directives patches: Dict[spack.spec.Spec, List[spack.patch.Patch]] #: Class level dictionary populated by :func:`~spack.directives.variant` directives variants: Dict[spack.spec.Spec, Dict[str, spack.variant.Variant]] #: Class level dictionary populated by :func:`~spack.directives.license` directives licenses: Dict[spack.spec.Spec, str] #: Class level dictionary populated by :func:`~spack.directives.can_splice` directives splice_specs: Dict[spack.spec.Spec, Tuple[spack.spec.Spec, Union[None, str, List[str]]]] #: Class level dictionary populated by :func:`~spack.directives.redistribute` directives disable_redistribute: Dict[spack.spec.Spec, DisableRedistribute] #: Must be defined as a fallback for old specs that don't have the ``build_system`` variant default_buildsystem: str #: Use :attr:`~spack.package_base.PackageBase.default_buildsystem` instead of this attribute, #: which is deprecated legacy_buildsystem: str #: Used when reporting the build system to users build_system_class: str = "PackageBase" #: By default, packages are not virtual #: Virtual packages override this attribute virtual: bool = False #: Most Spack packages are used to install source or binary code while #: those that do not can be used to install a set of other Spack packages. has_code: bool = True #: By default we build in parallel. Subclasses can override this. parallel: bool = True #: By default do not run tests within package's install() run_tests: bool = False #: Most packages are NOT extendable. Set to True if you want extensions. extendable: bool = False #: When True, add RPATHs for the entire DAG. When False, add RPATHs only #: for immediate dependencies. transitive_rpaths: bool = True #: List of shared objects that should be replaced with a different library at #: runtime. Typically includes stub libraries like ``libcuda.so``. When linking #: against a library listed here, the dependent will only record its soname #: or filename, not its absolute path, so that the dynamic linker will search #: for it. Note: accepts both file names and directory names, for example #: ``["libcuda.so", "stubs"]`` will ensure ``libcuda.so`` and all libraries in the #: ``stubs`` directory are not bound by path. non_bindable_shared_objects: List[str] = [] #: List of fnmatch patterns of library file names (specifically DT_NEEDED entries) that are not #: expected to be locatable in RPATHs. Generally this is a problem, and Spack install with #: config:shared_linking:strict will cause install failures if such libraries are found. #: However, in certain cases it can be hard if not impossible to avoid accidental linking #: against system libraries; until that is resolved, this attribute can be used to suppress #: errors. unresolved_libraries: List[str] = [] #: List of prefix-relative file paths (or a single path). If these do #: not exist after install, or if they exist but are not files, #: sanity checks fail. sanity_check_is_file: List[str] = [] #: List of prefix-relative directory paths (or a single path). If #: these do not exist after install, or if they exist but are not #: directories, sanity checks will fail. sanity_check_is_dir: List[str] = [] #: Boolean. Set to ``True`` for packages that require a manual download. #: This is currently used by package sanity tests and generation of a #: more meaningful fetch failure error. manual_download: bool = False #: Set of additional options used when fetching package versions. fetch_options: Dict[str, Any] = {} # # Set default licensing information # #: If set to ``True``, this software requires a license. #: If set to ``False``, all of the ``license_*`` attributes will #: be ignored. Defaults to ``False``. license_required: bool = False #: Contains the symbol used by the license manager to denote #: a comment. Defaults to ``#``. license_comment: str = "#" #: These are files that the software searches for when #: looking for a license. All file paths must be relative to the #: installation directory. More complex packages like Intel may require #: multiple licenses for individual components. Defaults to the empty list. license_files: List[str] = [] #: Environment variables that can be set to tell the #: software where to look for a license if it is not in the usual location. #: Defaults to the empty list. license_vars: List[str] = [] #: A URL pointing to license setup instructions for the software. #: Defaults to the empty string. license_url: str = "" #: Verbosity level, preserved across installs. _verbose = None #: Package homepage where users can find more information about the package homepage: ClassProperty[Optional[str]] = None #: Default list URL (place to find available versions) list_url: ClassProperty[Optional[str]] = None #: Link depth to which list_url should be searched for new versions list_depth: int = 0 #: List of GitHub usernames of package maintainers. #: Do not include @ here in order not to unnecessarily ping the users. maintainers: List[str] = [] #: Set to ``True`` to indicate the stand-alone test requires a compiler. #: It is used to ensure a compiler and build dependencies like ``cmake`` #: are available to build a custom test code. test_requires_compiler: bool = False #: TestSuite instance used to manage stand-alone tests for 1+ specs. test_suite: Optional[Any] = None def __init__(self, spec: spack.spec.Spec) -> None: # this determines how the package should be built. self.spec = spec # Allow custom staging paths for packages self.path = None # Keep track of whether or not this package was installed from # a binary cache. self.installed_from_binary_cache = False # Ensure that only one of these two attributes are present if getattr(self, "url", None) and getattr(self, "urls", None): msg = "a package can have either a 'url' or a 'urls' attribute" msg += " [package '{0.name}' defines both]" raise ValueError(msg.format(self)) # init internal variables self._stage: Optional[stg.StageComposite] = None # need to track patch stages separately, in order to apply them self._patch_stages: List[stg.Stage] = [] self._fetcher = None self._tester: Optional[Any] = None # Set up timing variables self._fetch_time = 0.0 super().__init__() def __getitem__(self, key: str) -> "PackageBase": return self.spec[key].package @classmethod def dependency_names(cls): return _subkeys(cls.dependencies) @classmethod def dependencies_by_name(cls, when: bool = False): return _by_subkey(cls.dependencies, when=when) # Accessors for variants # External code working with Variants should go through the methods below @classmethod def variant_names(cls) -> List[str]: return _subkeys(cls.variants) @classmethod def has_variant(cls, name) -> bool: return _has_subkey(cls.variants, name) @classmethod def num_variant_definitions(cls) -> int: """Total number of variant definitions in this class so far.""" return _num_definitions(cls.variants) @classmethod def variant_definitions(cls, name: str) -> List[Tuple[spack.spec.Spec, spack.variant.Variant]]: """Iterator over (when_spec, Variant) for all variant definitions for a particular name.""" return _definitions(cls.variants, name) @classmethod def variant_items(cls) -> Iterable[Tuple[spack.spec.Spec, Dict[str, spack.variant.Variant]]]: """Iterate over ``cls.variants.items()`` with overridden definitions removed.""" # Note: This is quadratic in the average number of variant definitions per name. # That is likely close to linear in practice, as there are few variants with # multiple definitions (but it matters when they are there). exclude = { name: [id(vdef) for _, vdef in cls.variant_definitions(name)] for name in cls.variant_names() } for when, variants_by_name in cls.variants.items(): filtered_variants_by_name = { name: vdef for name, vdef in variants_by_name.items() if id(vdef) in exclude[name] } if filtered_variants_by_name: yield when, filtered_variants_by_name def get_variant(self, name: str) -> spack.variant.Variant: """Get the highest precedence variant definition matching this package's spec. Arguments: name: name of the variant definition to get """ try: highest_to_lowest = reversed(self.variant_definitions(name)) return next(vdef for when, vdef in highest_to_lowest if self.spec.satisfies(when)) except StopIteration: raise ValueError(f"No variant '{name}' on spec: {self.spec}") @classmethod def validate_variant_names(self, spec: spack.spec.Spec): """Check that all variant names on Spec exist in this package. Raises ``UnknownVariantError`` if invalid variants are on the spec. """ names = self.variant_names() for v in spec.variants: if v not in names: raise spack.variant.UnknownVariantError( f"No such variant '{v}' in package {self.name}", [v] ) @classproperty def package_dir(cls): """Directory where the package.py file lives.""" return os.path.abspath(os.path.dirname(cls.module.__file__)) @classproperty def module(cls): """Module instance that this package class is defined in. We use this to add variables to package modules. This makes install() methods easier to write (e.g., can call configure()) """ return sys.modules[cls.__module__] @classproperty def namespace(cls): """Spack namespace for the package, which identifies its repo.""" return spack.repo.namespace_from_fullname(cls.__module__) @classproperty def fullname(cls): """Name of this package, including the namespace""" return "%s.%s" % (cls.namespace, cls.name) @classproperty def fullnames(cls): """Fullnames for this package and any packages from which it inherits.""" fullnames = [] for base in cls.__mro__: if not spack.repo.is_package_module(base.__module__): break fullnames.append(base.fullname) return fullnames @classproperty def name(cls): """The name of this package.""" if cls._name is None: # We cannot know the exact package API version, but we can distinguish between v1 # v2 based on the module. We don't want to figure out the exact package API version # since it requires parsing the repo.yaml. module = cls.__module__ if module.startswith(spack.repo.PKG_MODULE_PREFIX_V1): version = (1, 0) elif module.startswith(spack.repo.PKG_MODULE_PREFIX_V2): version = (2, 0) else: raise ValueError(f"Package {cls.__qualname__} is not a known Spack package") if version < (2, 0): # spack.pkg.builtin.package_name. _, _, pkg_module = module.rpartition(".") else: # spack_repo.builtin.packages.package_name.package pkg_module = module.rsplit(".", 2)[-2] cls._name = spack.util.naming.pkg_dir_to_pkg_name(pkg_module, version) return cls._name @classproperty def global_license_dir(cls): """Returns the directory where license files for all packages are stored.""" return spack.util.path.canonicalize_path(spack.config.get("config:license_dir")) @property def global_license_file(self): """Returns the path where a global license file for this particular package should be stored.""" if not self.license_files: return return os.path.join( self.global_license_dir, self.name, os.path.basename(self.license_files[0]) ) # Source redistribution must be determined before concretization (because source mirrors work # with abstract specs). @classmethod def redistribute_source(cls, spec): """Whether it should be possible to add the source of this package to a Spack mirror.""" for when_spec, disable_redistribute in cls.disable_redistribute.items(): if disable_redistribute.source and spec.satisfies(when_spec): return False return True @property def redistribute_binary(self): """Whether it should be possible to create a binary out of an installed instance of this package.""" for when_spec, disable_redistribute in self.disable_redistribute.items(): if disable_redistribute.binary and self.spec.satisfies(when_spec): return False return True @property def keep_werror(self) -> Optional[Literal["all", "specific", "none"]]: """Keep ``-Werror`` flags, matches ``config:flags:keep_werror`` to override config. Valid return values are: * ``"all"``: keep all ``-Werror`` flags. * ``"specific"``: keep only ``-Werror=specific-warning`` flags. * ``"none"``: filter out all ``-Werror*`` flags. * :data:`None`: respect the user's configuration (``"none"`` by default). """ if self.spec.satisfies("%nvhpc@:23.3"): # Filtering works by replacing -Werror with -Wno-error, but older nvhpc and # PGI do not understand -Wno-error, so we disable filtering. return "all" elif self.spec.satisfies("%nvhpc@23.4:"): # newer nvhpc supports -Wno-error but can't disable specific warnings with # -Wno-error=warning. Skip -Werror=warning, but still filter -Werror. return "specific" else: # use -Werror disablement by default for other compilers return None @property def version(self): if not self.spec.versions.concrete: raise ValueError( "Version requested for a package that does not have a concrete version." ) return self.spec.versions[0] @classmethod @memoized def version_urls(cls) -> Dict[StandardVersion, str]: """Dict of explicitly defined URLs for versions of this package. Return: An dict mapping version to url, ordered by version. A version's URL only appears in the result if it has an an explicitly defined ``url`` argument. So, this list may be empty if a package only defines ``url`` at the top level. """ return {v: args["url"] for v, args in sorted(cls.versions.items()) if "url" in args} def nearest_url(self, version): """Finds the URL with the "closest" version to ``version``. This uses the following precedence order: 1. Find the next lowest or equal version with a URL. 2. If no lower URL, return the next *higher* URL. 3. If no higher URL, return None. """ version_urls = self.version_urls() if version in version_urls: return version_urls[version] last_url = None for v, u in self.version_urls().items(): if v > version: if last_url: return last_url last_url = u return last_url def url_for_version(self, version: Union[str, StandardVersion]) -> str: """Returns a URL from which the specified version of this package may be downloaded. Arguments: version: The version for which a URL is sought.""" return self._implement_all_urls_for_version(version)[0] def _update_external_dependencies( self, extendee_spec: Optional[spack.spec.Spec] = None ) -> None: """ Method to override in package classes to handle external dependencies """ pass def detect_dev_src_change(self) -> bool: """ Method for checking for source code changes to trigger rebuild/reinstall """ dev_path_var = self.spec.variants.get("dev_path", None) _, record = spack.store.STORE.db.query_by_spec_hash(self.spec.dag_hash()) assert dev_path_var and record, "dev_path variant and record must be present" return fsys.recursive_mtime_greater_than(dev_path_var.value, record.installation_time) @classmethod def version_or_package_attr(cls, attr, version, default=NO_DEFAULT): """ Get an attribute that could be on the version or package with preference to the version """ version_attrs = cls.versions.get(version) if version_attrs and attr in version_attrs: return version_attrs.get(attr) if default is NO_DEFAULT and not hasattr(cls, attr): raise PackageError(f"{attr} attribute not defined on {cls.name}") return getattr(cls, attr, default) @classmethod def needs_commit(cls, version) -> bool: """ Method for checking if the package instance needs a commit sha to be found """ if isinstance(version, GitVersion): return True ver_attrs = cls.versions.get(version) if ver_attrs: return bool(ver_attrs.get("commit") or ver_attrs.get("tag") or ver_attrs.get("branch")) return False @classmethod def _resolve_git_provenance(cls, spec) -> None: # early return cases, don't overwrite user intention # commit pre-assigned or develop specs don't need commits changed # since this would create un-necessary churn if "commit" in spec.variants or spec.is_develop: return if is_git_version(str(spec.version)): ref = spec.version.ref else: v_attrs = cls.versions.get(spec.version, {}) if "commit" in v_attrs: spec.variants["commit"] = spack.variant.SingleValuedVariant( "commit", v_attrs["commit"] ) return ref = v_attrs.get("tag") or v_attrs.get("branch") if not ref: raise VersionError( f"{spec.name}'s version {str(spec.version)} " "is missing a git ref (commit, tag or branch)" ) # Look for commits in the following places: # 1) mirror archive file, (cheapish, local, staticish) # 2) URL (cheap, remote, dynamic) # # If users pre-stage (_LOCAL_CACHE), or use a mirror they can expect # consistent commit resolution sha = None # construct a package instance to get fetch/staging together pkg_instance = cls(spec.copy()) try: pkg_instance.do_fetch(mirror_only=True) except spack.error.FetchError: pass if pkg_instance.stage.archive_file: sha = spack.util.archive.retrieve_commit_from_archive( pkg_instance.stage.archive_file, ref ) if not sha: url = cls.version_or_package_attr("git", spec.version) sha = spack.util.git.get_commit_sha(url, ref) if sha: spec.variants["commit"] = spack.variant.SingleValuedVariant("commit", sha) def resolve_binary_provenance(self): """ Method to ensure concrete spec has binary provenance. Base implementation will look up git commits when appropriate. Packages may override this implementation for custom implementations """ self._resolve_git_provenance(self.spec) def all_urls_for_version(self, version: StandardVersion) -> List[str]: """Return all URLs derived from version_urls(), url, urls, and list_url (if it contains a version) in a package in that order. Args: version: the version for which a URL is sought """ uf = None if type(self).url_for_version != PackageBase.url_for_version: uf = self.url_for_version return self._implement_all_urls_for_version(version, uf) def _implement_all_urls_for_version( self, version: Union[str, StandardVersion], custom_url_for_version: Optional[Callable[[StandardVersion], Optional[str]]] = None, ) -> List[str]: version = StandardVersion.from_string(version) if isinstance(version, str) else version urls: List[str] = [] # If we have a specific URL for this version, don't extrapolate. url = self.version_urls().get(version) if url: urls.append(url) # if there is a custom url_for_version, use it if custom_url_for_version is not None: u = custom_url_for_version(version) if u is not None and u not in urls: urls.append(u) def sub_and_add(u: Optional[str]) -> None: if u is None: return # skip the url if there is no version to replace try: spack.url.parse_version(u) except spack.url.UndetectableVersionError: return urls.append(spack.url.substitute_version(u, self.url_version(version))) # If no specific URL, use the default, class-level URL sub_and_add(getattr(self, "url", None)) for u in getattr(self, "urls", []): sub_and_add(u) sub_and_add(getattr(self, "list_url", None)) # if no version-bearing URLs can be found, try them raw if not urls: default_url = getattr(self, "url", getattr(self, "urls", [None])[0]) # if no exact match AND no class-level default, use the nearest URL if not default_url: default_url = self.nearest_url(version) # if there are NO URLs to go by, then we can't do anything if not default_url: raise NoURLError(self.__class__) urls.append(spack.url.substitute_version(default_url, self.url_version(version))) return urls def find_valid_url_for_version(self, version: StandardVersion) -> Optional[str]: """Returns a URL from which the specified version of this package may be downloaded after testing whether the url is valid. Will try ``url``, ``urls``, and :attr:`list_url` before failing. Arguments: version: The version for which a URL is sought. """ urls = self.all_urls_for_version(version) for u in urls: if spack.util.web.url_exists(u): return u return None def _make_resource_stage(self, root_stage, resource): pretty_resource_name = fsys.polite_filename(f"{resource.name}-{self.version}") return stg.ResourceStage( resource.fetcher, root=root_stage, resource=resource, name=self._resource_stage(resource), mirror_paths=spack.mirrors.layout.default_mirror_layout( resource.fetcher, os.path.join(self.name, pretty_resource_name) ), mirrors=spack.mirrors.mirror.MirrorCollection(source=True).values(), path=self.path, ) def _download_search(self): dynamic_fetcher = fs.from_list_url(self) return [dynamic_fetcher] if dynamic_fetcher else [] def _make_root_stage(self, fetcher): # Construct a mirror path (TODO: get this out of package.py) format_string = "{name}-{version}" pretty_name = self.spec.format_path(format_string) mirror_paths = spack.mirrors.layout.default_mirror_layout( fetcher, os.path.join(self.name, pretty_name), self.spec ) # Construct a path where the stage should build.. s = self.spec stage_name = stg.compute_stage_name(s) stage = stg.Stage( fetcher, mirror_paths=mirror_paths, mirrors=spack.mirrors.mirror.MirrorCollection(source=True).values(), name=stage_name, path=self.path, search_fn=self._download_search, ) return stage def _make_stages(self) -> Tuple[stg.StageComposite, List[stg.Stage]]: """Create stages for this package, its resources, and any patches to be applied. Returns: A StageComposite containing all stages created, as well as a list of patch stages for any patches that need to be fetched remotely. The StageComposite is used to manage (create destroy, etc.) the stages. The list of patch stages will be in the same order that patches are to be applied to the package's staged source code. This is needed in order to apply the patches later. """ # If it's a dev package (not transitively), use a DIY stage object dev_path_var = self.spec.variants.get("dev_path", None) if dev_path_var: dev_path = dev_path_var.value link_format = spack.config.get("config:develop_stage_link") if not link_format: link_format = "build-{arch}-{hash:7}" if link_format == "None": stage_link = None else: stage_link = self.spec.format_path(link_format) source_stage = stg.DevelopStage( stg.compute_stage_name(self.spec), dev_path, stage_link ) else: source_stage = self._make_root_stage(self.fetcher) # all_stages is source + resources + patches all_stages = stg.StageComposite() all_stages.append(source_stage) all_stages.extend( self._make_resource_stage(source_stage, r) for r in self._get_needed_resources() ) def make_patch_stage(patch: spack.patch.UrlPatch, uniqe_part: str): # UrlPatches can make their own fetchers fetcher = patch.fetcher() # The same package can have multiple patches with the same name but # with different contents, therefore apply a subset of the hash. fetch_digest = patch.archive_sha256 or patch.sha256 name = f"{os.path.basename(patch.url)}-{fetch_digest[:7]}" per_package_ref = os.path.join(patch.owner.split(".")[-1], name) mirror_ref = spack.mirrors.layout.default_mirror_layout(fetcher, per_package_ref) return stg.Stage( fetcher, name=f"{stg.stage_prefix}-{uniqe_part}-patch-{fetch_digest}", mirror_paths=mirror_ref, mirrors=spack.mirrors.mirror.MirrorCollection(source=True).values(), ) if self.spec.concrete: patches = self.spec.patches uniqe_part = self.spec.dag_hash(7) else: # The only code path that gets here is `spack mirror create --all`, # which needs all matching patches. patch_lists = [ plist for when, plist in self.patches.items() if self.spec.intersects(when) ] patches = sum(patch_lists, []) uniqe_part = self.name patch_stages = [ make_patch_stage(p, uniqe_part) for p in patches if isinstance(p, spack.patch.UrlPatch) ] all_stages.extend(patch_stages) return all_stages, patch_stages @property def stage(self): """Get the build staging area for this package. This automatically instantiates a ``Stage`` object if the package doesn't have one yet, but it does not create the Stage directory on the filesystem. """ if not self.spec.versions.concrete: raise ValueError("Cannot retrieve stage for package without concrete version.") if self._stage is None: self._stage, self._patch_stages = self._make_stages() return self._stage @stage.setter def stage(self, stage: stg.StageComposite): """Allow a stage object to be set to override the default.""" self._stage = stage @property def env_path(self): """Return the build environment file path associated with staging.""" return os.path.join(self.stage.path, _spack_build_envfile) @property def env_mods_path(self): """ Return the build environment modifications file path associated with staging. """ return os.path.join(self.stage.path, _spack_build_envmodsfile) @property def metadata_dir(self): """Return the install metadata directory.""" return spack.store.STORE.layout.metadata_path(self.spec) @property def install_env_path(self): """ Return the build environment file path on successful installation. """ # Backward compatibility: Return the name of an existing log path; # otherwise, return the current install env path name. old_filename = os.path.join(self.metadata_dir, "build.env") if os.path.exists(old_filename): return old_filename else: return os.path.join(self.metadata_dir, _spack_build_envfile) @property def log_path(self): """Return the build log file path associated with staging.""" return os.path.join(self.stage.path, _spack_build_logfile) @property def phase_log_files(self): """Find sorted phase log files written to the staging directory""" logs_dir = os.path.join(self.stage.path, "spack-build-*-out.txt") log_files = glob.glob(logs_dir) log_files.sort() return log_files @property def install_log_path(self): """Return the (compressed) build log file path on successful installation""" # Backward compatibility: Return the name of an existing install log. for filename in [_spack_build_logfile, "build.out", "build.txt"]: old_log = os.path.join(self.metadata_dir, filename) if os.path.exists(old_log): return old_log # Otherwise, return the current install log path name. return os.path.join(self.metadata_dir, _spack_build_logfile + ".gz") @property def configure_args_path(self): """Return the configure args file path associated with staging.""" return os.path.join(self.stage.path, _spack_configure_argsfile) @property def times_log_path(self): """Return the times log json file.""" return os.path.join(self.metadata_dir, spack_times_log) @property def install_configure_args_path(self): """Return the configure args file path on successful installation.""" return os.path.join(self.metadata_dir, _spack_configure_argsfile) def archive_install_test_log(self): """Archive the install-phase test log, if present.""" if getattr(self, "tester", None): self.tester.archive_install_test_log(self.metadata_dir) @property def tester(self): import spack.install_test if not self.spec.versions.concrete: raise ValueError("Cannot retrieve tester for package without concrete version.") if not self._tester: self._tester = spack.install_test.PackageTest(self) return self._tester @property def fetcher(self): if not self.spec.versions.concrete: raise ValueError("Cannot retrieve fetcher for package without concrete version.") if not self._fetcher: # assign private member with the public setter api for error checking self.fetcher = fs.for_package_version(self) return self._fetcher @fetcher.setter def fetcher(self, f): self._fetcher = f self._fetcher.set_package(self) @classmethod def dependencies_of_type(cls, deptypes: dt.DepFlag): """Get names of dependencies that can possibly have these deptypes. This analyzes the package and determines which dependencies *can* be a certain kind of dependency. Note that they may not *always* be this kind of dependency, since dependencies can be optional, so something may be a build dependency in one configuration and a run dependency in another. """ return { name for name, dependencies in cls.dependencies_by_name().items() if any(deptypes & dep.depflag for dep in dependencies) } # TODO: allow more than one active extendee. @property def extendee_spec(self) -> Optional[spack.spec.Spec]: """Spec of the extendee of this package, or None if it is not an extension.""" if not self.extendees: return None # If the extendee is in the spec's deps already, return that. deps = [ dep for dep in self.spec.dependencies(deptype=("link", "run")) for d, when in self.extendees.values() if dep.satisfies(d) and self.spec.satisfies(when) ] if deps: assert len(deps) == 1 return deps[0] # if the spec is concrete already, then it extends something # that is an *optional* dependency, and the dep isn't there. if self.spec._concrete: return None else: # If it's not concrete, then return the spec from the # extends() directive since that is all we know so far. spec_str = next(iter(self.extendees)) return spack.spec.Spec(spec_str) @property def is_extension(self): # if it is concrete, it's only an extension if it actually # dependes on the extendee. if self.spec._concrete: return self.extendee_spec is not None else: # If not, then it's an extension if it *could* be an extension return bool(self.extendees) def extends(self, spec: spack.spec.Spec) -> bool: """ Returns True if this package extends the given spec. If ``self.spec`` is concrete, this returns whether this package extends the given spec. If ``self.spec`` is not concrete, this returns whether this package may extend the given spec. """ if spec.name not in self.extendees: return False s = self.extendee_spec return s is not None and spec.satisfies(s) def provides(self, vpkg_name: str) -> bool: """ True if this package provides a virtual package with the specified name """ return any( any(spec.name == vpkg_name for spec in provided) for when_spec, provided in self.provided.items() if self.spec.intersects(when_spec) ) def intersects(self, spec: spack.spec.Spec) -> bool: """Context-ful intersection that takes into account package information. By design, ``Spec.intersects()`` does not know anything about package metadata. This avoids unnecessary package lookups and keeps things efficient where extra information is not needed, and it decouples ``Spec`` from ``PackageBase``. In many cases, though, we can rule more cases out in ``intersects()`` if we know, for example, that certain variants are always single-valued, or that certain variants are conditional on other variants. This adds logic for such cases when they are knowable. Note that because ``intersects()`` is conservative, it can only give false positives ("i.e., the two specs *may* overlap"), not false negatives. This method can fix false positives (i.e. it may return ``False`` when ``Spec.intersects()`` would return ``True``, but it will never return ``True`` when ``Spec.intersects()`` returns ``False``. """ # Spec.intersects() is right when False if not self.spec.intersects(spec): return False def sv_variant_conflicts(spec, variant): name = variant.name return ( variant.name in spec.variants and all(not d[name].multi for when, d in self.variants.items() if name in d) and spec.variants[name].value != variant.value ) # Specs don't know if a variant is single- or multi-valued (concretization handles this) # But, we know if the spec has a value for a single-valued variant, it *has* to equal the # value in self.spec, if there is one. for v, variant in spec.variants.items(): if sv_variant_conflicts(self.spec, variant): return False # if there is no intersecting condition for a conditional variant, it can't exist. e.g.: # - cuda_arch= can't be satisfied when ~cuda. # - generator= can't be satisfied when build_system=autotools def mutually_exclusive(spec, variant_name): return all( not spec.intersects(when) or any(sv_variant_conflicts(spec, wv) for wv in when.variants.values()) for when, d in self.variants.items() if variant_name in d ) names = self.variant_names() for v in set(itertools.chain(spec.variants, self.spec.variants)): if v not in names: # treat unknown variants as intersecting continue if mutually_exclusive(self.spec, v) or mutually_exclusive(spec, v): return False return True @property def virtuals_provided(self): """ virtual packages provided by this package with its spec """ return [ vspec for when_spec, provided in self.provided.items() for vspec in sorted(provided) if self.spec.satisfies(when_spec) ] @classmethod def provided_virtual_names(cls): """Return sorted list of names of virtuals that can be provided by this package.""" return sorted( set(vpkg.name for virtuals in cls.provided.values() for vpkg in sorted(virtuals)) ) @property def prefix(self): """Get the prefix into which this package should be installed.""" return self.spec.prefix @property def home(self): return self.prefix @property def command(self) -> spack.util.executable.Executable: """Returns the main executable for this package.""" path = os.path.join(self.home.bin, self.spec.name) if fsys.is_exe(path): return spack.util.executable.Executable(path) raise RuntimeError(f"Unable to locate {self.spec.name} command in {self.home.bin}") def url_version(self, version): """ Given a version, this returns a string that should be substituted into the package's URL to download that version. By default, this just returns the version string. Subclasses may need to override this, e.g. for boost versions where you need to ensure that there are _'s in the download URL. """ return str(version) def remove_prefix(self): """ Removes the prefix for a package along with any empty parent directories """ spack.store.STORE.layout.remove_install_directory(self.spec) @property def download_instr(self) -> str: """ Defines the default manual download instructions. Packages can override the property to provide more information. Returns: default manual download instructions """ required = ( f"Manual download is required for {self.spec.name}. " if self.manual_download else "" ) return f"{required}Refer to {self.homepage} for download instructions." def do_fetch(self, mirror_only=False): """ Creates a stage directory and downloads the tarball for this package. Working directory will be set to the stage directory. """ if not self.has_code or self.spec.external: tty.debug("No fetch required for {0}".format(self.name)) return checksum = spack.config.get("config:checksum") if ( checksum and (self.version not in self.versions) and (not isinstance(self.version, GitVersion)) and ("dev_path" not in self.spec.variants) ): tty.warn( "There is no checksum on file to fetch %s safely." % self.spec.cformat("{name}{@version}") ) # Ask the user whether to skip the checksum if we're # interactive, but just fail if non-interactive. ck_msg = "Add a checksum or use --no-checksum to skip this check." ignore_checksum = False if sys.stdout.isatty(): ignore_checksum = tty.get_yes_or_no(" Fetch anyway?", default=False) if ignore_checksum: tty.debug("Fetching with no checksum. {0}".format(ck_msg)) if not ignore_checksum: raise spack.error.FetchError( "Will not fetch %s" % self.spec.format("{name}{@version}"), ck_msg ) deprecated = spack.config.get("config:deprecated") if not deprecated and self.versions.get(self.version, {}).get("deprecated", False): tty.warn( "{0} is deprecated and may be removed in a future Spack release.".format( self.spec.format("{name}{@version}") ) ) # Ask the user whether to install deprecated version if we're # interactive, but just fail if non-interactive. dp_msg = ( "If you are willing to be a maintainer for this version " "of the package, submit a PR to remove `deprecated=False" "`, or use `--deprecated` to skip this check." ) ignore_deprecation = False if sys.stdout.isatty(): ignore_deprecation = tty.get_yes_or_no(" Fetch anyway?", default=False) if ignore_deprecation: tty.debug("Fetching deprecated version. {0}".format(dp_msg)) if not ignore_deprecation: raise spack.error.FetchError( "Will not fetch {0}".format(self.spec.format("{name}{@version}")), dp_msg ) self.stage.create() err_msg = None if not self.manual_download else self.download_instr start_time = time.time() self.stage.fetch(mirror_only, err_msg=err_msg) self._fetch_time = time.time() - start_time if checksum and self.version in self.versions: self.stage.check() self.stage.cache_local() def do_stage(self, mirror_only=False): """Unpacks and expands the fetched tarball.""" # Always create the stage directory at this point. Why? A no-code # package may want to use the installation process to install metadata. self.stage.create() # Fetch/expand any associated code. user_dev_path = spack.config.get(f"develop:{self.name}:path", None) skip = user_dev_path and os.path.exists(user_dev_path) if skip: tty.debug("Skipping staging because develop path exists") if self.has_code and not self.spec.external and not skip: self.do_fetch(mirror_only) self.stage.expand_archive() else: # Support for post-install hooks requires a stage.source_path fsys.mkdirp(self.stage.source_path) def do_patch(self): """Applies patches if they haven't been applied already.""" if not self.spec.concrete: raise ValueError("Can only patch concrete packages.") # Kick off the stage first. This creates the stage. self.do_stage() # Package can add its own patch function. has_patch_fun = hasattr(self, "patch") and callable(self.patch) # Get the patches from the spec (this is a shortcut for the MV-variant) patches = self.spec.patches # If there are no patches, note it. if not patches and not has_patch_fun: tty.msg("No patches needed for {0}".format(self.name)) return # Construct paths to special files in the archive dir used to # keep track of whether patches were successfully applied. archive_dir = self.stage.source_path good_file = os.path.join(archive_dir, ".spack_patched") no_patches_file = os.path.join(archive_dir, ".spack_no_patches") bad_file = os.path.join(archive_dir, ".spack_patch_failed") # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): if self.stage.requires_patch_success: tty.debug("Patching failed last time. Restaging.") self.stage.restage() else: # develop specs may have patch failures but should never be restaged tty.warn( f"A patch failure was detected in {self.name}." " Build errors may occur due to this." ) return # If this file exists, then we already applied all the patches. if os.path.isfile(good_file): tty.msg("Already patched {0}".format(self.name)) return elif os.path.isfile(no_patches_file): tty.msg("No patches needed for {0}".format(self.name)) return errors = [] # Apply all the patches for specs that match this one patched = False patch_stages = iter(self._patch_stages) for patch in patches: try: with fsys.working_dir(self.stage.source_path): # get the path either from the stage where it was fetched, or from the Patch if isinstance(patch, spack.patch.UrlPatch): patch_stage = next(patch_stages) patch_path = patch_stage.single_file else: patch_path = patch.path spack.patch.apply_patch( self.stage.source_path, patch_path, patch.level, patch.working_dir, patch.reverse, ) tty.msg(f"Applied patch {patch.path_or_url}") patched = True except spack.error.SpackError as e: # Touch bad file if anything goes wrong. fsys.touch(bad_file) error_msg = f"Patch {patch.path_or_url} failed." if self.stage.requires_patch_success: tty.msg(error_msg) raise else: tty.debug(error_msg) tty.debug(e) errors.append(e) if has_patch_fun: try: with fsys.working_dir(self.stage.source_path): self.patch() tty.msg("Ran patch() for {0}".format(self.name)) patched = True except spack.multimethod.NoSuchMethodError: # We are running a multimethod without a default case. # If there's no default it means we don't need to patch. if not patched: # if we didn't apply a patch from a patch() # directive, AND the patch function didn't apply, say # no patches are needed. Otherwise, we already # printed a message for each patch. tty.msg("No patches needed for {0}".format(self.name)) except spack.error.SpackError as e: # Touch bad file if anything goes wrong. fsys.touch(bad_file) error_msg = f"patch() function failed for {self.name}" if self.stage.requires_patch_success: tty.msg(error_msg) raise else: tty.debug(error_msg) tty.debug(e) errors.append(e) if not errors: # Get rid of any old failed file -- patches have either succeeded # or are not needed. This is mostly defensive -- it's needed # if we didn't restage if os.path.isfile(bad_file): os.remove(bad_file) # touch good or no patches file so that we skip next time. if patched: fsys.touch(good_file) else: fsys.touch(no_patches_file) @classmethod def all_patches(cls): """Retrieve all patches associated with the package. Retrieves patches on the package itself as well as patches on the dependencies of the package.""" patches = [] for _, patch_list in cls.patches.items(): for patch in patch_list: patches.append(patch) pkg_deps = cls.dependencies for dep_name in pkg_deps: for _, dependency in pkg_deps[dep_name].items(): for _, patch_list in dependency.patches.items(): for patch in patch_list: patches.append(patch) return patches def content_hash(self, content: Optional[bytes] = None) -> str: """Create a hash based on the artifacts and patches used to build this package. This includes: * source artifacts (tarballs, repositories) used to build; * content hashes (``sha256``'s) of all patches applied by Spack; and * canonicalized contents the ``package.py`` recipe used to build. This hash is only included in Spack's DAG hash for concrete specs, but if it happens to be called on a package with an abstract spec, only applicable (i.e., determinable) portions of the hash will be included. """ # list of components to make up the hash hash_content = [] # source artifacts/repositories # TODO: resources if self.spec.versions.concrete: try: source_id = fs.for_package_version(self).source_id() except (fs.ExtrapolationError, fs.InvalidArgsError, spack.error.NoURLError): # ExtrapolationError happens if the package has no fetchers defined. # InvalidArgsError happens when there are version directives with args, # but none of them identifies an actual fetcher. # NoURLError happens if the package is external-only with no url source_id = None if not source_id: # TODO? in cases where a digest or source_id isn't available, # should this attempt to download the source and set one? This # probably only happens for source repositories which are # referenced by branch name rather than tag or commit ID. from_local_sources = "dev_path" in self.spec.variants if self.has_code and not self.spec.external and not from_local_sources: message = "Missing a source id for {s.name}@{s.version}" tty.debug(message.format(s=self)) hash_content.append("".encode("utf-8")) else: hash_content.append(source_id.encode("utf-8")) # patch sha256's # Only include these if they've been assigned by the concretizer. # We check spec._patches_assigned instead of spec.concrete because # we have to call package_hash *before* marking specs concrete if self.spec._patches_assigned(): hash_content.extend( ":".join((p.sha256, str(p.level))).encode("utf-8") for p in self.spec.patches ) # package.py contents hash_content.append(package_hash(self.spec, source=content).encode("utf-8")) # put it all together and encode as base32 b32_hash = base64.b32encode( hashlib.sha256(bytes().join(sorted(hash_content))).digest() ).lower() b32_hash = b32_hash.decode("utf-8") return b32_hash @property def cmake_prefix_paths(self) -> List[str]: """Return a list of paths to be used in CMake's ``CMAKE_PREFIX_PATH``.""" return [self.prefix] def _has_make_target(self, target): """Checks to see if 'target' is a valid target in a Makefile. Parameters: target (str): the target to check for Returns: bool: True if 'target' is found, else False """ # Check if we have a Makefile for makefile in ["GNUmakefile", "Makefile", "makefile"]: if os.path.exists(makefile): break else: tty.debug("No Makefile found in the build directory") return False # Prevent altering LC_ALL for 'make' outside this function make = copy.deepcopy(self.module.make) # Use English locale for missing target message comparison make.add_default_env("LC_ALL", "C") # Check if 'target' is a valid target. # # `make -n target` performs a "dry run". It prints the commands that # would be run but doesn't actually run them. If the target does not # exist, you will see one of the following error messages: # # GNU Make: # make: *** No rule to make target `test'. Stop. # *** No rule to make target 'test'. Stop. # # BSD Make: # make: don't know how to make test. Stop # # Note: "Stop." is not printed when running a Make jobserver (spack env depfile) that runs # with `make -k/--keep-going` missing_target_msgs = [ "No rule to make target `{0}'.", "No rule to make target '{0}'.", "don't know how to make {0}.", ] kwargs = { "fail_on_error": False, "output": os.devnull, "error": str, # Remove MAKEFLAGS to avoid inherited flags from Make jobserver (spack env depfile) "extra_env": {"MAKEFLAGS": ""}, } stderr = make("-n", target, **kwargs) for missing_target_msg in missing_target_msgs: if missing_target_msg.format(target) in stderr: tty.debug("Target '{0}' not found in {1}".format(target, makefile)) return False return True def _if_make_target_execute(self, target, *args, **kwargs): """Runs ``make target`` if 'target' is a valid target in the Makefile. Parameters: target (str): the target to potentially execute """ if self._has_make_target(target): # Execute target self.module.make(target, *args, **kwargs) def _has_ninja_target(self, target): """Checks to see if 'target' is a valid target in a Ninja build script. Parameters: target (str): the target to check for Returns: bool: True if 'target' is found, else False """ ninja = self.module.ninja # Check if we have a Ninja build script if not os.path.exists("build.ninja"): tty.debug("No Ninja build script found in the build directory") return False # Get a list of all targets in the Ninja build script # https://ninja-build.org/manual.html#_extra_tools all_targets = ninja("-t", "targets", "all", output=str).split("\n") # Check if 'target' is a valid target matches = [line for line in all_targets if line.startswith(target + ":")] if not matches: tty.debug("Target '{0}' not found in build.ninja".format(target)) return False return True def _if_ninja_target_execute(self, target, *args, **kwargs): """Runs ``ninja target`` if 'target' is a valid target in the Ninja build script. Parameters: target (str): the target to potentially execute """ if self._has_ninja_target(target): # Execute target self.module.ninja(target, *args, **kwargs) def _get_needed_resources(self): # We use intersects here cause it would also work if self.spec is abstract resources = [ resource for when_spec, resource_list in self.resources.items() if self.spec.intersects(when_spec) for resource in resource_list ] # Sorts the resources by the length of the string representing their destination. Since any # nested resource must contain another resource's path, that should work return sorted(resources, key=lambda res: len(res.destination)) def _resource_stage(self, resource): pieces = ["resource", resource.name, self.spec.dag_hash()] resource_stage_folder = "-".join(pieces) return resource_stage_folder def do_test(self, *, dirty=False, externals=False, timeout: Optional[int] = None): if self.test_requires_compiler and not any( lang in self.spec for lang in ("c", "cxx", "fortran") ): tty.error( f"Skipping tests for package {self.spec}, since a compiler is required, " f"but not available" ) return kwargs = { "dirty": dirty, "fake": False, "context": "test", "externals": externals, "verbose": tty.is_verbose(), } self.tester.stand_alone_tests(kwargs, timeout=timeout) def _unit_test_check(self) -> bool: """Hook for Spack's own unit tests to assert things about package internals. Unit tests can override this function to perform checks after ``Package.install`` and all post-install hooks run, but before the database is updated. The overridden function may indicate that the install procedure should terminate early (before updating the database) by returning :data:`False` (or any value such that ``bool(result)`` is :data:`False`). Return: :data:`True` to continue, :data:`False` to skip ``install()`` """ return True @classmethod def inject_flags(cls: Type[Pb], name: str, flags: Iterable[str]) -> FLAG_HANDLER_RETURN_TYPE: """See :func:`spack.package.inject_flags`.""" return flags, None, None @classmethod def env_flags(cls: Type[Pb], name: str, flags: Iterable[str]) -> FLAG_HANDLER_RETURN_TYPE: """See :func:`spack.package.env_flags`.""" return None, flags, None @classmethod def build_system_flags( cls: Type[Pb], name: str, flags: Iterable[str] ) -> FLAG_HANDLER_RETURN_TYPE: """See :func:`spack.package.build_system_flags`.""" return None, None, flags def setup_run_environment(self, env: spack.util.environment.EnvironmentModifications) -> None: """Sets up the run environment for a package. Args: env: environment modifications to be applied when the package is run. Package authors can call methods on it to alter the run environment. """ pass def setup_dependent_run_environment( self, env: spack.util.environment.EnvironmentModifications, dependent_spec: spack.spec.Spec ) -> None: """Sets up the run environment of packages that depend on this one. This is similar to ``setup_run_environment``, but it is used to modify the run environment of a package that *depends* on this one. This gives packages like Python and others that follow the extension model a way to implement common environment or run-time settings for dependencies. Args: env: environment modifications to be applied when the dependent package is run. Package authors can call methods on it to alter the build environment. dependent_spec: The spec of the dependent package about to be run. This allows the extendee (self) to query the dependent's state. Note that *this* package's spec is available as ``self.spec`` """ pass def setup_dependent_package(self, module, dependent_spec: spack.spec.Spec) -> None: """Set up module-scope global variables for dependent packages. This function is called when setting up the build and run environments of a DAG. Examples: 1. Extensions often need to invoke the ``python`` interpreter from the Python installation being extended. This routine can put a ``python`` Executable as a global in the module scope for the extension package to simplify extension installs. 2. MPI compilers could set some variables in the dependent's scope that point to ``mpicc``, ``mpicxx``, etc., allowing them to be called by common name regardless of which MPI is used. Args: module: The Python ``module`` object of the dependent package. Packages can use this to set module-scope variables for the dependent to use. dependent_spec: The spec of the dependent package about to be built. This allows the extendee (self) to query the dependent's state. Note that *this* package's spec is available as ``self.spec``. """ pass _flag_handler: Optional[FLAG_HANDLER_TYPE] = None @property def flag_handler(self) -> FLAG_HANDLER_TYPE: if self._flag_handler is None: self._flag_handler = PackageBase.inject_flags return self._flag_handler @flag_handler.setter def flag_handler(self, var: FLAG_HANDLER_TYPE) -> None: self._flag_handler = var # The flag handler method is called for each of the allowed compiler flags. # It returns a triple of inject_flags, env_flags, build_system_flags. # The flags returned as inject_flags are injected through the spack # compiler wrappers. # The flags returned as env_flags are passed to the build system through # the environment variables of the same name. # The flags returned as build_system_flags are passed to the build system # package subclass to be turned into the appropriate part of the standard # arguments. This is implemented for build system classes where # appropriate and will otherwise raise a NotImplementedError. def flags_to_build_system_args(self, flags: Dict[str, List[str]]) -> None: # Takes flags as a dict name: list of values if any(v for v in flags.values()): msg = "The {0} build system".format(self.__class__.__name__) msg += " cannot take command line arguments for compiler flags" raise NotImplementedError(msg) @staticmethod def uninstall_by_spec(spec, force=False, deprecator=None): if not os.path.isdir(spec.prefix): # prefix may not exist, but DB may be inconsistent. Try to fix by # removing, but omit hooks. specs = spack.store.STORE.db.query(spec, installed=True) if specs: if deprecator: spack.store.STORE.db.deprecate(specs[0], deprecator) tty.debug("Deprecating stale DB entry for {0}".format(spec.short_spec)) else: spack.store.STORE.db.remove(specs[0]) tty.debug("Removed stale DB entry for {0}".format(spec.short_spec)) return else: raise InstallError(str(spec) + " is not installed.") if not force: dependents = spack.store.STORE.db.installed_relatives( spec, direction="parents", transitive=True, deptype=("link", "run") ) if dependents: raise PackageStillNeededError(spec, dependents) # Try to get the package for the spec try: pkg = spec.package except spack.repo.UnknownEntityError: pkg = None # Pre-uninstall hook runs first. with spack.store.STORE.prefix_locker.write_lock(spec): if pkg is not None: try: spack.hooks.pre_uninstall(spec) except Exception as error: if force: error_msg = ( "One or more pre_uninstall hooks have failed" " for {0}, but Spack is continuing with the" " uninstall".format(str(spec)) ) if isinstance(error, spack.error.SpackError): error_msg += "\n\nError message: {0}".format(str(error)) tty.warn(error_msg) # Note that if the uninstall succeeds then we won't be # seeing this error again and won't have another chance # to run the hook. else: raise # Uninstalling in Spack only requires removing the prefix. if not spec.external: msg = "Deleting package prefix [{0}]" tty.debug(msg.format(spec.short_spec)) # test if spec is already deprecated, not whether we want to # deprecate it now deprecated = bool(spack.store.STORE.db.deprecator(spec)) spack.store.STORE.layout.remove_install_directory(spec, deprecated) # Delete DB entry if deprecator: msg = "deprecating DB entry [{0}] in favor of [{1}]" tty.debug(msg.format(spec.short_spec, deprecator.short_spec)) spack.store.STORE.db.deprecate(spec, deprecator) else: msg = "Deleting DB entry [{0}]" tty.debug(msg.format(spec.short_spec)) spack.store.STORE.db.remove(spec) if pkg is not None: try: spack.hooks.post_uninstall(spec) except Exception: # If there is a failure here, this is our only chance to do # something about it: at this point the Spec has been removed # from the DB and prefix, so the post-uninstallation hooks # will not have another chance to run. error_msg = ( "One or more post-uninstallation hooks failed for" " {0}, but the prefix has been removed (if it is not" " external).".format(str(spec)) ) tb_msg = traceback.format_exc() error_msg += "\n\nThe error:\n\n{0}".format(tb_msg) tty.warn(error_msg) tty.msg("Successfully uninstalled {0}".format(spec.short_spec)) def do_uninstall(self, force=False): """Uninstall this package by spec.""" # delegate to instance-less method. PackageBase.uninstall_by_spec(self.spec, force) def view(self): """Create a view with the prefix of this package as the root. Extensions added to this view will modify the installation prefix of this package. """ return YamlFilesystemView(self.prefix, spack.store.STORE.layout) def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() def do_clean(self): """Removes the package's build stage and source tarball.""" self.stage.destroy() @classmethod def format_doc(cls, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get("indent", 0) if not cls.__doc__: return "" doc = re.sub(r"\s+", " ", cls.__doc__) lines = textwrap.wrap(doc, 72) results = io.StringIO() for line in lines: results.write((" " * indent) + line + "\n") return results.getvalue() @property def all_urls(self) -> List[str]: """A list of all URLs in a package. Check both class-level and version-specific URLs. Returns a list of URLs """ urls: List[str] = [] if hasattr(self, "url") and self.url: urls.append(self.url) # fetch from first entry in urls to save time if hasattr(self, "urls") and self.urls: urls.append(self.urls[0]) for args in self.versions.values(): if "url" in args: urls.append(args["url"]) return urls def fetch_remote_versions( self, concurrency: Optional[int] = None ) -> Dict[StandardVersion, str]: """Find remote versions of this package. Uses :attr:`list_url` and any other URLs listed in the package file. Returns: a dictionary mapping versions to URLs """ if not self.all_urls: return {} try: return spack.url.find_versions_of_archive( self.all_urls, self.list_url, self.list_depth, concurrency, reference_package=self ) except spack.util.web.NoNetworkConnectionError as e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) @property def rpath(self): """Get the rpath this package links with, as a list of paths.""" deps = self.spec.dependencies(deptype="link") # on Windows, libraries of runtime interest are typically # stored in the bin directory # Do not include Windows system libraries in the rpath interface # these libraries are handled automatically by VS/VCVARS and adding # Spack derived system libs into the link path or address space of a program # can result in conflicting versions, which makes Spack packages less usable if sys.platform == "win32": rpaths = [self.prefix.bin] rpaths.extend( d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin) and "windows-system" not in getattr(d.package, "tags", []) ) else: rpaths = [self.prefix.lib, self.prefix.lib64] rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64)) return rpaths @property def rpath_args(self): """ Get the rpath args as a string, with -Wl,-rpath, for each element """ return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) class WindowsSimulatedRPath: """Class representing Windows filesystem rpath analog One instance of this class is associated with a package (only on Windows) For each lib/binary directory in an associated package, this class introduces a symlink to any/all dependent libraries/binaries. This includes the packages own bin/lib directories, meaning the libraries are linked to the binary directory and vis versa. """ def __init__( self, package: PackageBase, base_modification_prefix: Optional[Union[str, pathlib.Path]] = None, link_install_prefix: bool = True, ): """ Args: package: Package requiring links base_modification_prefix: Path representation indicating the root directory in which to establish the simulated rpath, ie where the symlinks that comprise the "rpath" behavior will be installed. Note: This is a mutually exclusive option with `link_install_prefix` using both is an error. Default: None link_install_prefix: Link against package's own install or stage root. Packages that run their own executables during build and require rpaths to the build directory during build time require this option. Default: install root Note: This is a mutually exclusive option with `base_modification_prefix`, using both is an error. """ self.pkg = package self._addl_rpaths: set[str] = set() if link_install_prefix and base_modification_prefix: raise RuntimeError( "Invalid combination of arguments given to WindowsSimulated RPath.\n" "Select either `link_install_prefix` to create an install prefix rpath" " or specify a `base_modification_prefix` for any other link type. " "Specifying both arguments is invalid." ) if not (link_install_prefix or base_modification_prefix): raise RuntimeError( "Insufficient arguments given to WindowsSimulatedRpath.\n" "WindowsSimulatedRPath requires one of link_install_prefix" " or base_modification_prefix to be specified." " Neither was provided." ) self.link_install_prefix = link_install_prefix if base_modification_prefix: self.base_modification_prefix = pathlib.Path(base_modification_prefix) else: self.base_modification_prefix = pathlib.Path(self.pkg.prefix) self._additional_library_dependents: set[pathlib.Path] = set() if not self.link_install_prefix: tty.debug(f"Generating rpath for non install context: {base_modification_prefix}") @property def library_dependents(self): """ Set of directories where package binaries/libraries are located. """ base_pths = set() if self.link_install_prefix: base_pths.add(pathlib.Path(self.pkg.prefix.bin)) base_pths |= self._additional_library_dependents return base_pths def add_library_dependent(self, *dest: Union[str, pathlib.Path]): """ Add paths to directories or libraries/binaries to set of common paths that need to link against other libraries Specified paths should fall outside of a package's common link paths, i.e. the bin directories. """ for pth in dest: if os.path.isfile(pth): new_pth = pathlib.Path(pth).parent else: new_pth = pathlib.Path(pth) path_is_in_prefix = new_pth.is_relative_to(self.base_modification_prefix) if not path_is_in_prefix: raise RuntimeError( f"Attempting to generate rpath symlink out of rpath context:\ {str(self.base_modification_prefix)}" ) self._additional_library_dependents.add(new_pth) @property def rpaths(self): """ Set of libraries this package needs to link against during runtime These packages will each be symlinked into the packages lib and binary dir """ dependent_libs = [] for path in self.pkg.rpath: dependent_libs.extend(list(find_all_shared_libraries(path, recursive=True))) for extra_path in self._addl_rpaths: dependent_libs.extend(list(find_all_shared_libraries(extra_path, recursive=True))) return set([pathlib.Path(x) for x in dependent_libs]) def add_rpath(self, *paths: str): """ Add libraries found at the root of provided paths to runtime linking These are libraries found outside of the typical scope of rpath linking that require manual inclusion in a runtime linking scheme. These links are unidirectional, and are only intended to bring outside dependencies into this package Args: *paths : arbitrary number of paths to be added to runtime linking """ self._addl_rpaths = self._addl_rpaths | set(paths) def _link(self, path: pathlib.Path, dest_dir: pathlib.Path): """Perform link step of simulated rpathing, installing simlinks of file in path to the dest_dir location. This method deliberately prevents the case where a path points to a file inside the dest_dir. This is because it is both meaningless from an rpath perspective, and will cause an error when Developer mode is not enabled""" def report_already_linked(): # We have either already symlinked or we are encountering a naming clash # either way, we don't want to overwrite existing libraries already_linked = islink(str(dest_file)) tty.debug( "Linking library %s to %s failed, " % (str(path), str(dest_file)) + "already linked." if already_linked else "library with name %s already exists at location %s." % (str(file_name), str(dest_dir)) ) file_name = path.name dest_file = dest_dir / file_name if not dest_file.exists() and dest_dir.exists() and not dest_file == path: try: symlink(str(path), str(dest_file)) # For py2 compatibility, we have to catch the specific Windows error code # associate with trying to create a file that already exists (winerror 183) # Catch OSErrors missed by the SymlinkError checks except OSError as e: if sys.platform == "win32" and e.errno == errno.EEXIST: report_already_linked() else: raise e # catch errors we raise ourselves from Spack except AlreadyExistsError: report_already_linked() def establish_link(self): """ (sym)link packages to runtime dependencies based on RPath configuration for Windows heuristics """ # from build_environment.py:463 # The top-level package is always RPATHed. It hasn't been installed yet # so the RPATHs are added unconditionally # for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib) # install a symlink to each dependent library # do not rpath for system libraries included in the dag # we should not be modifying libraries managed by the Windows system # as this will negatively impact linker behavior and can result in permission # errors if those system libs are not modifiable by Spack if "windows-system" not in getattr(self.pkg, "tags", []): for library, lib_dir in itertools.product(self.rpaths, self.library_dependents): self._link(library, lib_dir) def make_package_test_rpath(pkg: PackageBase, test_dir: Union[str, pathlib.Path]) -> None: """Establishes a temp Windows simulated rpath for the pkg in the testing directory so an executable can test the libraries/executables with proper access to dependent dlls. Note: this is a no-op on all other platforms besides Windows Args: pkg: the package for which the rpath should be computed test_dir: the testing directory in which we should construct an rpath """ # link_install_prefix as false ensures we're not linking into the install prefix mini_rpath = WindowsSimulatedRPath(pkg, link_install_prefix=False) # add the testing directory as a location to install rpath symlinks mini_rpath.add_library_dependent(test_dir) # check for whether build_directory is available, if not # assume the stage root is the build dir build_dir_attr = getattr(pkg, "build_directory", None) build_directory = build_dir_attr if build_dir_attr else pkg.stage.path # add the build dir & build dir bin mini_rpath.add_rpath(os.path.join(build_directory, "bin")) mini_rpath.add_rpath(os.path.join(build_directory)) # construct rpath mini_rpath.establish_link() def deprecated_version(pkg: PackageBase, version: Union[str, StandardVersion]) -> bool: """Return True iff the version is deprecated. Arguments: pkg: The package whose version is to be checked. version: The version being checked """ if not isinstance(version, StandardVersion): version = StandardVersion.from_string(version) details = pkg.versions.get(version) return details is not None and details.get("deprecated", False) def preferred_version( pkg: Union[PackageBase, Type[PackageBase]], ) -> Union[StandardVersion, GitVersion]: """Returns the preferred versions of the package according to package.py. Accounts for version deprecation in the package recipe. Doesn't account for any user configuration in packages.yaml. Arguments: pkg: The package whose versions are to be assessed. """ def _version_order(version_info): version, info = version_info deprecated_key = not info.get("deprecated", False) return (deprecated_key, *concretization_version_order(version_info)) version, _ = max(pkg.versions.items(), key=_version_order) return version def non_preferred_version(node: spack.spec.Spec) -> bool: """Returns True if the spec version is not the preferred one, according to the package.py""" if not node.versions.concrete: return False try: return node.version != preferred_version(node.package) except ValueError: return False def non_default_variant(node: spack.spec.Spec, variant_name: str) -> bool: """Returns True if the variant in the spec has a non-default value.""" try: default_variant = node.package.get_variant(variant_name).make_default() return not node.satisfies(str(default_variant)) except ValueError: # This is the case for special variants like "patches" etc. return False def sort_by_pkg_preference( versions: Iterable[Union[GitVersion, StandardVersion]], *, pkg: Union[PackageBase, Type[PackageBase]], ) -> List[Union[GitVersion, StandardVersion]]: """Sorts the list of versions passed in input according to the preferences in the package. The return value does not contain duplicate versions. Most preferred versions first. """ s = [(v, pkg.versions.get(v, {})) for v in dedupe(versions)] return [v for v, _ in sorted(s, reverse=True, key=concretization_version_order)] def concretization_version_order( version_info: Tuple[Union[GitVersion, StandardVersion], dict], ) -> Tuple[bool, bool, bool, bool, Union[GitVersion, StandardVersion]]: """Version order key for concretization, where preferred > not preferred, finite > any infinite component; only if all are the same, do we use default version ordering. Version deprecation needs to be accounted for separately. """ version, info = version_info return ( info.get("preferred", False), not isinstance(version, GitVersion), not version.isdevelop(), not version.is_prerelease(), version, ) class PackageStillNeededError(InstallError): """Raised when package is still needed by another on uninstall.""" def __init__(self, spec, dependents): spec_fmt = spack.spec.DEFAULT_FORMAT + " /{hash:7}" dep_fmt = "{name}{@versions} /{hash:7}" super().__init__( f"Cannot uninstall {spec.format(spec_fmt)}, " f"needed by {[dep.format(dep_fmt) for dep in dependents]}" ) self.spec = spec self.dependents = dependents class InvalidPackageOpError(PackageError): """Raised when someone tries perform an invalid operation on a package.""" class ExtensionError(PackageError): """Superclass for all errors having to do with extension packages.""" class ActivationError(ExtensionError): """Raised when there are problems activating an extension.""" def __init__(self, msg, long_msg=None): super().__init__(msg, long_msg) class DependencyConflictError(spack.error.SpackError): """Raised when the dependencies cannot be flattened as asked for.""" def __init__(self, conflict): super().__init__("%s conflicts with another file in the flattened directory." % (conflict)) class ManualDownloadRequiredError(InvalidPackageOpError): """Raised when attempting an invalid operation on a package that requires a manual download.""" ================================================ FILE: lib/spack/spack/package_completions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from pathlib import Path from typing import Union """Functions relating to shell autocompletion scripts for packages.""" def bash_completion_path(root: Union[str, Path]) -> Path: """ Return standard path for bash completion scripts under root. Args: root: The prefix root under which to generate the path. Returns: Standard path for bash completion scripts under root. """ return Path(root) / "share" / "bash-completion" / "completions" def zsh_completion_path(root: Union[str, Path]) -> Path: """ Return standard path for zsh completion scripts under root. Args: root: The prefix root under which to generate the path. Returns: Standard path for zsh completion scripts under root. """ return Path(root) / "share" / "zsh" / "site-functions" def fish_completion_path(root: Union[str, Path]) -> Path: """ Return standard path for fish completion scripts under root. Args: root: The prefix root under which to generate the path. Returns: Standard path for fish completion scripts under root. """ return Path(root) / "share" / "fish" / "vendor_completions.d" ================================================ FILE: lib/spack/spack/package_prefs.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import stat import warnings import spack.config import spack.error import spack.repo import spack.spec from spack.error import ConfigError from spack.version import Version _lesser_spec_types = {"compiler": spack.spec.CompilerSpec, "version": Version} def _spec_type(component): """Map from component name to spec type for package prefs.""" return _lesser_spec_types.get(component, spack.spec.Spec) class PackagePrefs: """Defines the sort order for a set of specs. Spack's package preference implementation uses PackagePrefs to define sort order. The PackagePrefs class looks at Spack's packages.yaml configuration and, when called on a spec, returns a key that can be used to sort that spec in order of the user's preferences. You can use it like this:: # key function sorts CompilerSpecs for `mpich` in order of preference kf = PackagePrefs("mpich", "compiler") compiler_list.sort(key=kf) Or like this:: # key function to sort VersionLists for OpenMPI in order of preference. kf = PackagePrefs("openmpi", "version") version_list.sort(key=kf) Optionally, you can sort in order of preferred virtual dependency providers. To do that, provide ``"providers"`` and a third argument denoting the virtual package (e.g., ``mpi``):: kf = PackagePrefs("trilinos", "providers", "mpi") provider_spec_list.sort(key=kf) """ def __init__(self, pkgname, component, vpkg=None, all=True): self.pkgname = pkgname self.component = component self.vpkg = vpkg self.all = all self._spec_order = None def __call__(self, spec): """Return a key object (an index) that can be used to sort spec. Sort is done in package order. We don't cache the result of this function as Python's sort functions already ensure that the key function is called at most once per sorted element. """ if self._spec_order is None: self._spec_order = self._specs_for_pkg( self.pkgname, self.component, self.vpkg, self.all ) spec_order = self._spec_order # integer is the index of the first spec in order that satisfies # spec, or it's a number larger than any position in the order. match_index = next( (i for i, s in enumerate(spec_order) if spec.intersects(s)), len(spec_order) ) if match_index < len(spec_order) and spec_order[match_index] == spec: # If this is called with multiple specs that all satisfy the same # minimum index in spec_order, the one which matches that element # of spec_order exactly is considered slightly better. Note # that because this decreases the value by less than 1, it is not # better than a match which occurs at an earlier index. match_index -= 0.5 return match_index @classmethod def order_for_package(cls, pkgname, component, vpkg=None, all=True): """Given a package name, sort component (e.g, version, compiler, ...), and an optional vpkg, return the list from the packages config. """ pkglist = [pkgname] if all: pkglist.append("all") packages = spack.config.CONFIG.get_config("packages") for pkg in pkglist: pkg_entry = packages.get(pkg) if not pkg_entry: continue order = pkg_entry.get(component) if not order: continue # vpkg is one more level if vpkg is not None: order = order.get(vpkg) if order: ret = [str(s).strip() for s in order] if component == "target": ret = ["target=%s" % tname for tname in ret] return ret return [] @classmethod def _specs_for_pkg(cls, pkgname, component, vpkg=None, all=True): """Given a sort order specified by the pkgname/component/second_key, return a list of CompilerSpecs, VersionLists, or Specs for that sorting list. """ pkglist = cls.order_for_package(pkgname, component, vpkg, all) spec_type = _spec_type(component) return [spec_type(s) for s in pkglist] @classmethod def has_preferred_providers(cls, pkgname, vpkg): """Whether specific package has a preferred vpkg providers.""" return bool(cls.order_for_package(pkgname, "providers", vpkg, False)) @classmethod def has_preferred_targets(cls, pkg_name): """Whether specific package has a preferred vpkg providers.""" return bool(cls.order_for_package(pkg_name, "target")) @classmethod def preferred_variants(cls, pkg_name): """Return a VariantMap of preferred variants/values for a spec.""" packages = spack.config.CONFIG.get_config("packages") for pkg_cls in (pkg_name, "all"): variants = packages.get(pkg_cls, {}).get("variants", "") if variants: break # allow variants to be list or string if not isinstance(variants, str): variants = " ".join(variants) # Only return variants that are actually supported by the package pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) spec = spack.spec.Spec(f"{pkg_name} {variants}") return { name: variant for name, variant in spec.variants.items() if name in pkg_cls.variant_names() } def is_spec_buildable(spec): """Return true if the spec is configured as buildable""" allpkgs = spack.config.get("packages") all_buildable = allpkgs.get("all", {}).get("buildable", True) so_far = all_buildable # the default "so far" def _package(s): pkg_cls = spack.repo.PATH.get_pkg_class(s.name) return pkg_cls(s) # check whether any providers for this package override the default if any( _package(spec).provides(name) and entry.get("buildable", so_far) != so_far for name, entry in allpkgs.items() ): so_far = not so_far spec_buildable = allpkgs.get(spec.name, {}).get("buildable", so_far) return spec_buildable def get_package_dir_permissions(spec): """Return the permissions configured for the spec. Include the GID bit if group permissions are on. This makes the group attribute sticky for the directory. Package-specific settings take precedent over settings for ``all``""" perms = get_package_permissions(spec) if perms & stat.S_IRWXG and spack.config.get("config:allow_sgid", True): perms |= stat.S_ISGID if spec.concrete and "/afs/" in spec.prefix: warnings.warn( "Directory {0} seems to be located on AFS. If you" " encounter errors, try disabling the allow_sgid option" " using: spack config add 'config:allow_sgid:false'".format(spec.prefix) ) return perms def get_package_permissions(spec): """Return the permissions configured for the spec. Package-specific settings take precedence over settings for ``all``""" # Get read permissions level for name in (spec.name, "all"): try: readable = spack.config.get("packages:%s:permissions:read" % name, "") if readable: break except AttributeError: readable = "world" # Get write permissions level for name in (spec.name, "all"): try: writable = spack.config.get("packages:%s:permissions:write" % name, "") if writable: break except AttributeError: writable = "user" perms = stat.S_IRWXU if readable in ("world", "group"): # world includes group perms |= stat.S_IRGRP | stat.S_IXGRP if readable == "world": perms |= stat.S_IROTH | stat.S_IXOTH if writable in ("world", "group"): if readable == "user": raise ConfigError( "Writable permissions may not be more" + " permissive than readable permissions.\n" + " Violating package is %s" % spec.name ) perms |= stat.S_IWGRP if writable == "world": if readable != "world": raise ConfigError( "Writable permissions may not be more" + " permissive than readable permissions.\n" + " Violating package is %s" % spec.name ) perms |= stat.S_IWOTH return perms def get_package_group(spec): """Return the unix group associated with the spec. Package-specific settings take precedence over settings for ``all``""" for name in (spec.name, "all"): try: group = spack.config.get("packages:%s:permissions:group" % name, "") if group: break except AttributeError: group = "" return group class VirtualInPackagesYAMLError(spack.error.SpackError): """Raised when a disallowed virtual is found in packages.yaml""" ================================================ FILE: lib/spack/spack/package_test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os from typing import List from spack.util.executable import Executable, which def compile_c_and_execute( source_file: str, include_flags: List[str], link_flags: List[str] ) -> str: """Compile a C source file with the given include and link flags, execute the resulting binary, and return its output as a string. Used in package tests.""" cc = which("cc", required=True) flags = include_flags flags.extend([source_file]) cc("-c", *flags) name = os.path.splitext(os.path.basename(source_file))[0] cc("-o", "check", "%s.o" % name, *link_flags) check = Executable("./check") return check(output=str) def compare_output(current_output: str, blessed_output: str) -> None: """Compare blessed and current output of executables. Used in package tests.""" if not (current_output == blessed_output): print("Produced output does not match expected output.") print("Expected output:") print("-" * 80) print(blessed_output) print("-" * 80) print("Produced output:") print("-" * 80) print(current_output) print("-" * 80) raise RuntimeError("Output check failed.", "See spack_output.log for details") def compare_output_file(current_output: str, blessed_output_file: str) -> None: """Same as above, but when the blessed output is given as a file. Used in package tests.""" with open(blessed_output_file, "r", encoding="utf-8") as f: blessed_output = f.read() compare_output(current_output, blessed_output) ================================================ FILE: lib/spack/spack/patch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import hashlib import os import pathlib import sys from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Tuple, Type, Union import spack import spack.error import spack.fetch_strategy import spack.llnl.util.filesystem import spack.util.spack_json as sjson from spack.llnl.url import allowed_archive from spack.util.crypto import Checker, checksum from spack.util.executable import which, which_string if TYPE_CHECKING: import spack.package_base import spack.repo def apply_patch( source_path: str, patch_path: str, level: int = 1, working_dir: str = ".", reverse: bool = False, ) -> None: """Apply the patch at patch_path to code in the stage. Args: stage: stage with code that will be patched patch_path: filesystem location for the patch to apply level: patch level working_dir: relative path *within* the stage to change to reverse: reverse the patch """ if not patch_path or not os.path.isfile(patch_path): raise spack.error.NoSuchPatchError(f"No such patch: {patch_path}") git_utils_path = os.environ.get("PATH", "") if sys.platform == "win32": git = which_string("git") if git: git = pathlib.Path(git) git_root = git.parent.parent git_root = git_root / "usr" / "bin" git_utils_path = os.pathsep.join([str(git_root), git_utils_path]) args = ["-s", "-p", str(level), "-i", patch_path, "-d", working_dir] if reverse: args.append("-R") # TODO: Decouple Spack's patch support on Windows from Git # for Windows, and instead have Spack directly fetch, install, and # utilize that patch. # Note for future developers: The GNU port of patch to windows # has issues handling CRLF line endings unless the --binary # flag is passed. patch = which("patch", required=True, path=git_utils_path) with spack.llnl.util.filesystem.working_dir(source_path): patch(*args) PatchPackageType = Union["spack.package_base.PackageBase", Type["spack.package_base.PackageBase"]] class Patch: """Base class for patches. The owning package is not necessarily the package to apply the patch to -- in the case where a dependent package patches its dependency, it is the dependent's fullname. """ sha256: str def __init__( self, pkg: PatchPackageType, path_or_url: str, level: int, working_dir: str, reverse: bool = False, ordering_key: Optional[Tuple[str, int]] = None, ) -> None: """Initialize a new Patch instance. Args: pkg: the package that owns the patch path_or_url: the relative path or URL to a patch file level: patch level working_dir: relative path *within* the stage to change to reverse: reverse the patch ordering_key: key used to ensure patches are applied in a consistent order """ # validate level (must be an integer >= 0) if not isinstance(level, int) or not level >= 0: raise ValueError("Patch level needs to be a non-negative integer.") # Attributes shared by all patch subclasses self.owner = pkg.fullname self.path_or_url = path_or_url # needed for debug output self.path: Optional[str] = None # must be set before apply() self.level = level self.working_dir = working_dir self.reverse = reverse # The ordering key is passed when executing package.py directives, and is only relevant # after a solve to build concrete specs with consistently ordered patches. For concrete # specs read from a file, we add patches in the order of its patches variants and the # ordering_key is irrelevant. In that case, use a default value so we don't need to branch # on whether ordering_key is None where it's used, just to make static analysis happy. self.ordering_key: Tuple[str, int] = ordering_key or ("", 0) # TODO: Use TypedDict once Spack supports Python 3.8+ only def to_dict(self) -> Dict[str, Any]: """Dictionary representation of the patch. Returns: A dictionary representation. """ return { "owner": self.owner, "sha256": self.sha256, "level": self.level, "working_dir": self.working_dir, "reverse": self.reverse, } def __eq__(self, other: object) -> bool: """Equality check. Args: other: another patch Returns: True if both patches have the same checksum, else False """ if not isinstance(other, Patch): return NotImplemented return self.sha256 == other.sha256 def __hash__(self) -> int: """Unique hash. Returns: A unique hash based on the sha256. """ return hash(self.sha256) class FilePatch(Patch): """Describes a patch that is retrieved from a file in the repository.""" _sha256: Optional[str] = None def __init__( self, pkg: PatchPackageType, relative_path: str, level: int, working_dir: str, reverse: bool = False, ordering_key: Optional[Tuple[str, int]] = None, ) -> None: """Initialize a new FilePatch instance. Args: pkg: the class object for the package that owns the patch relative_path: path to patch, relative to the repository directory for a package. level: level to pass to patch command working_dir: path within the source directory where patch should be applied reverse: reverse the patch ordering_key: key used to ensure patches are applied in a consistent order """ self.relative_path = relative_path # patches may be defined by relative paths to parent classes # search mro to look for the file abs_path: Optional[str] = None # At different times we call FilePatch on instances and classes pkg_cls = pkg if isinstance(pkg, type) else pkg.__class__ for cls in pkg_cls.__mro__: # type: ignore if not hasattr(cls, "module"): # We've gone too far up the MRO break # Cannot use pkg.package_dir because it's a property and we have # classes, not instances. pkg_dir = os.path.abspath(os.path.dirname(cls.module.__file__)) path = os.path.join(pkg_dir, self.relative_path) if os.path.exists(path): abs_path = path break if abs_path is None: msg = "FilePatch: Patch file %s for " % relative_path msg += "package %s.%s does not exist." % (pkg.namespace, pkg.name) raise ValueError(msg) super().__init__(pkg, abs_path, level, working_dir, reverse, ordering_key) self.path = abs_path @property def sha256(self) -> str: """Get the patch checksum. Returns: The sha256 of the patch file. """ if self._sha256 is None and self.path is not None: self._sha256 = checksum(hashlib.sha256, self.path) assert isinstance(self._sha256, str) return self._sha256 @sha256.setter def sha256(self, value: str) -> None: """Set the patch checksum. Args: value: the sha256 """ self._sha256 = value def to_dict(self) -> Dict[str, Any]: """Dictionary representation of the patch. Returns: A dictionary representation. """ data = super().to_dict() data["relative_path"] = self.relative_path return data class UrlPatch(Patch): """Describes a patch that is retrieved from a URL.""" def __init__( self, pkg: PatchPackageType, url: str, level: int = 1, *, working_dir: str = ".", reverse: bool = False, sha256: str, # This is required for UrlPatch ordering_key: Optional[Tuple[str, int]] = None, archive_sha256: Optional[str] = None, ) -> None: """Initialize a new UrlPatch instance. Arguments: pkg: the package that owns the patch url: URL where the patch can be fetched level: level to pass to patch command working_dir: path within the source directory where patch should be applied reverse: reverse the patch ordering_key: key used to ensure patches are applied in a consistent order sha256: sha256 sum of the patch, used to verify the patch archive_sha256: sha256 sum of the *archive*, if the patch is compressed (only required for compressed URL patches) """ super().__init__(pkg, url, level, working_dir, reverse, ordering_key) self.url = url if allowed_archive(self.url) and not archive_sha256: raise spack.error.PatchDirectiveError( "Compressed patches require 'archive_sha256' " "and patch 'sha256' attributes: %s" % self.url ) self.archive_sha256 = archive_sha256 if not sha256: raise spack.error.PatchDirectiveError("URL patches require a sha256 checksum") self.sha256 = sha256 def fetcher(self) -> spack.fetch_strategy.FetchStrategy: """Construct a fetcher that can download (and unpack) this patch.""" # Two checksums, one for compressed file, one for its contents if self.archive_sha256 and self.sha256: return spack.fetch_strategy.FetchAndVerifyExpandedFile( self.url, archive_sha256=self.archive_sha256, expanded_sha256=self.sha256 ) else: return spack.fetch_strategy.URLFetchStrategy( url=self.url, sha256=self.sha256, expand=False ) def to_dict(self) -> Dict[str, Any]: """Dictionary representation of the patch. Returns: A dictionary representation. """ data = super().to_dict() data["url"] = self.url if self.archive_sha256: data["archive_sha256"] = self.archive_sha256 return data def from_dict(dictionary: Dict[str, Any], repository: "spack.repo.RepoPath") -> Patch: """Create a patch from json dictionary. Args: dictionary: dictionary representation of a patch repository: repository containing package Returns: A patch object. Raises: ValueError: If *owner* or *url*/*relative_path* are missing in the dictionary. """ owner = dictionary.get("owner") if owner is None: raise ValueError(f"Invalid patch dictionary: {dictionary}") assert isinstance(owner, str) pkg_cls = repository.get_pkg_class(owner) if "url" in dictionary: return UrlPatch( pkg_cls, dictionary["url"], dictionary["level"], working_dir=dictionary["working_dir"], # Added in v0.22, fallback required for backwards compatibility reverse=dictionary.get("reverse", False), sha256=dictionary["sha256"], archive_sha256=dictionary.get("archive_sha256"), ) elif "relative_path" in dictionary: patch = FilePatch( pkg_cls, dictionary["relative_path"], dictionary["level"], dictionary["working_dir"], # Added in v0.22, fallback required for backwards compatibility dictionary.get("reverse", False), ) # If the patch in the repo changes, we cannot get it back, so we # just check it and fail here. # TODO: handle this more gracefully. sha256 = dictionary["sha256"] checker = Checker(sha256) if patch.path and not checker.check(patch.path): raise spack.fetch_strategy.ChecksumError( "sha256 checksum failed for %s" % patch.path, "Expected %s but got %s " % (sha256, checker.sum) + "Patch may have changed since concretization.", ) return patch else: raise ValueError("Invalid patch dictionary: %s" % dictionary) class PatchCache: """Index of patches used in a repository, by sha256 hash. This allows us to look up patches without loading all packages. It's also needed to properly implement dependency patching, as need a way to look up patches that come from packages not in the Spec sub-DAG. The patch index is structured like this in a file (this is YAML, but we write JSON):: patches: sha256: namespace1.package1: namespace2.package2: ... etc. ... """ def __init__( self, repository: "spack.repo.RepoPath", data: Optional[Dict[str, Any]] = None ) -> None: """Initialize a new PatchCache instance. Args: repository: repository containing package data: nested dictionary of patches """ if data is None: self.index = {} else: if "patches" not in data: raise IndexError("invalid patch index; try `spack clean -m`") self.index = data["patches"] self.repository = repository @classmethod def from_json(cls, stream: Any, repository: "spack.repo.RepoPath") -> "PatchCache": """Initialize a new PatchCache instance from JSON. Args: stream: stream of data repository: repository containing package Returns: A new PatchCache instance. """ return PatchCache(repository=repository, data=sjson.load(stream)) def to_json(self, stream: Any) -> None: """Dump a JSON representation to a stream. Args: stream: stream of data """ sjson.dump({"patches": self.index}, stream) def patch_for_package( self, sha256: str, pkg: Type["spack.package_base.PackageBase"], *, validate: bool = False ) -> Patch: """Look up a patch in the index and build a patch object for it. We build patch objects lazily because building them requires that we have information about the package's location in its repo. Args: sha256: sha256 hash to look up pkg: Package class to get patch for. validate: if True, validate the cached entry against the owner's current package class and raise ``PatchLookupError`` if the entry is missing or stale. Returns: The patch object. """ sha_index = self.index.get(sha256) if not sha_index: raise spack.error.PatchLookupError( f"Couldn't find patch for package {pkg.fullname} with sha256: {sha256}" ) # Find patches for this class or any class it inherits from for fullname in pkg.fullnames: patch_dict = sha_index.get(fullname) if patch_dict: break else: raise spack.error.PatchLookupError( f"Couldn't find patch for package {pkg.fullname} with sha256: {sha256}" ) if validate: # Validate the cached entry against the owner's current package class owner = patch_dict.get("owner") if not owner: raise spack.error.PatchLookupError( f"Patch for {pkg.fullname} with sha256 {sha256} has no owner in cache" ) try: owner_pkg_cls = self.repository.get_pkg_class(owner) current_index = PatchCache._index_patches(owner_pkg_cls, self.repository) except Exception as e: raise spack.error.PatchLookupError( f"Could not validate patch cache for {pkg.fullname}: {e}" ) from e current_sha_index = current_index.get(sha256) if not current_sha_index or current_sha_index.get(fullname) != patch_dict: raise spack.error.PatchLookupError( f"Stale patch cache entry for {pkg.fullname} with sha256: {sha256}" ) # add the sha256 back (we take it out on write to save space, # because it's the index key) patch_dict = dict(patch_dict) patch_dict["sha256"] = sha256 return from_dict(patch_dict, repository=self.repository) def update_packages(self, pkgs_fullname: Set[str]) -> None: """Update the patch cache. Args: pkg_fullname: package to update. """ # remove this package from any patch entries that reference it. if self.index: empty = [] for sha256, package_to_patch in self.index.items(): remove = [] for fullname, patch_dict in package_to_patch.items(): if patch_dict["owner"] in pkgs_fullname: remove.append(fullname) for fullname in remove: package_to_patch.pop(fullname) if not package_to_patch: empty.append(sha256) # remove any entries that are now empty for sha256 in empty: del self.index[sha256] # update the index with per-package patch indexes for pkg_fullname in pkgs_fullname: pkg_cls = self.repository.get_pkg_class(pkg_fullname) partial_index = self._index_patches(pkg_cls, self.repository) for sha256, package_to_patch in partial_index.items(): p2p = self.index.setdefault(sha256, {}) p2p.update(package_to_patch) def update(self, other: "PatchCache") -> None: """Update this cache with the contents of another. Args: other: another patch cache to merge """ for sha256, package_to_patch in other.index.items(): p2p = self.index.setdefault(sha256, {}) p2p.update(package_to_patch) @staticmethod def _index_patches( pkg_class: Type["spack.package_base.PackageBase"], repository: "spack.repo.RepoPath" ) -> Dict[Any, Any]: """Patch index for a specific patch. Args: pkg_class: package object to get patches for repository: repository containing the package Returns: The patch index for that package. """ index = {} # Add patches from the class for cond, patch_list in pkg_class.patches.items(): for patch in patch_list: patch_dict = patch.to_dict() patch_dict.pop("sha256") # save some space index[patch.sha256] = {pkg_class.fullname: patch_dict} if not pkg_class._patches_dependencies: return index for deps_by_name in pkg_class.dependencies.values(): for dependency in deps_by_name.values(): for patch_list in dependency.patches.values(): for patch in patch_list: dspec_cls = repository.get_pkg_class(dependency.spec.name) patch_dict = patch.to_dict() patch_dict.pop("sha256") # save some space index[patch.sha256] = {dspec_cls.fullname: patch_dict} return index ================================================ FILE: lib/spack/spack/paths.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Defines paths that are part of Spack's directory structure. Do not import other ``spack`` modules here. This module is used throughout Spack and should bring in a minimal number of external dependencies. """ import os from pathlib import PurePath import spack.llnl.util.filesystem import spack.util.hash as hash #: This file lives in $prefix/lib/spack/spack/__file__ prefix = str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4))) #: synonym for prefix spack_root = prefix #: bin directory in the spack prefix bin_path = os.path.join(prefix, "bin") #: The spack script itself spack_script = os.path.join(bin_path, "spack") #: The sbang script in the spack installation sbang_script = os.path.join(bin_path, "sbang") # spack directory hierarchy lib_path = os.path.join(prefix, "lib", "spack") module_path = os.path.join(lib_path, "spack") vendor_path = os.path.join(module_path, "vendor") command_path = os.path.join(module_path, "cmd") analyzers_path = os.path.join(module_path, "analyzers") platform_path = os.path.join(module_path, "platforms") compilers_path = os.path.join(module_path, "compilers") operating_system_path = os.path.join(module_path, "operating_systems") test_path = os.path.join(module_path, "test") hooks_path = os.path.join(module_path, "hooks") opt_path = os.path.join(prefix, "opt") share_path = os.path.join(prefix, "share", "spack") etc_path = os.path.join(prefix, "etc", "spack") # # Things in $spack/etc/spack # default_license_dir = os.path.join(etc_path, "licenses") # # Things in $spack/var/spack # var_path = os.path.join(prefix, "var", "spack") # read-only things in $spack/var/spack repos_path = os.path.join(var_path, "repos") test_repos_path = os.path.join(var_path, "test_repos") mock_packages_path = os.path.join(test_repos_path, "spack_repo", "builtin_mock") # # Writable things in $spack/var/spack # TODO: Deprecate these, as we want a read-only spack prefix by default. # TODO: These should probably move to user cache, or some other location. # # fetch cache for downloaded files default_fetch_cache_path = os.path.join(var_path, "cache") # GPG paths. gpg_keys_path = os.path.join(var_path, "gpg") mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data") mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys") gpg_path = os.path.join(opt_path, "spack", "gpg") #: Not a location itself, but used for when Spack instances #: share the same cache base directory for caches that should #: not be shared between those instances. spack_instance_id = hash.b32_hash(spack_root)[:7] # Below paths are where Spack can write information for the user. # Some are caches, some are not exactly caches. # # The options that start with `default_` below are overridable in # `config.yaml`, but they default to use `user_cache_path/`. # # You can override the top-level directory (the user cache path) by # setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack. # def _get_user_cache_path(): return os.path.expanduser(os.getenv("SPACK_USER_CACHE_PATH") or "~%s.spack" % os.sep) user_cache_path = str(PurePath(_get_user_cache_path())) #: junit, cdash, etc. reports about builds reports_path = os.path.join(user_cache_path, "reports") #: installation test (spack test) output default_test_path = os.path.join(user_cache_path, "test") #: spack monitor analysis directories default_monitor_path = os.path.join(reports_path, "monitor") #: git repositories fetched to compare commits to versions user_repos_cache_path = os.path.join(user_cache_path, "git_repos") #: default location where remote package repositories are cloned package_repos_path = os.path.join(user_cache_path, "package_repos") #: bootstrap store for bootstrapping clingo and other tools default_user_bootstrap_path = os.path.join(user_cache_path, "bootstrap") #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.) default_misc_cache_path = os.path.join(user_cache_path, spack_instance_id, "cache") # Below paths pull configuration from the host environment. # # There are three environment variables you can use to isolate spack from # the host environment: # - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches) # - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope. # - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations. # User configuration and caches in $HOME/.spack def _get_user_config_path(): return os.path.expanduser(os.getenv("SPACK_USER_CONFIG_PATH") or "~%s.spack" % os.sep) # Configuration in /etc/spack on the system def _get_system_config_path(): return os.path.expanduser( os.getenv("SPACK_SYSTEM_CONFIG_PATH") or os.sep + os.path.join("etc", "spack") ) #: User configuration location user_config_path = _get_user_config_path() #: System configuration location system_config_path = _get_system_config_path() #: Recorded directory where spack command was originally invoked spack_working_dir = None def set_working_dir(): """Change the working directory to getcwd, or spack prefix if no cwd.""" global spack_working_dir try: spack_working_dir = os.getcwd() except OSError: os.chdir(prefix) spack_working_dir = prefix ================================================ FILE: lib/spack/spack/phase_callbacks.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections from typing import Optional import spack.llnl.util.lang as lang #: An object of this kind is a shared global state used to collect callbacks during #: class definition time, and is flushed when the class object is created at the end #: of the class definition #: #: Args: #: attribute_name (str): name of the attribute that will be attached to the builder #: callbacks (list): container used to temporarily aggregate the callbacks CallbackTemporaryStage = collections.namedtuple( "CallbackTemporaryStage", ["attribute_name", "callbacks"] ) #: Shared global state to aggregate "@run_before" callbacks _RUN_BEFORE = CallbackTemporaryStage(attribute_name="_run_before_callbacks", callbacks=[]) #: Shared global state to aggregate "@run_after" callbacks _RUN_AFTER = CallbackTemporaryStage(attribute_name="_run_after_callbacks", callbacks=[]) class PhaseCallbacksMeta(type): """Permit to register arbitrary functions during class definition and run them later, before or after a given install phase. Each method decorated with ``run_before`` or ``run_after`` gets temporarily stored in a global shared state when a class being defined is parsed by the Python interpreter. At class definition time that temporary storage gets flushed and a list of callbacks is attached to the class being defined. """ def __new__(mcs, name, bases, attr_dict): for temporary_stage in (_RUN_BEFORE, _RUN_AFTER): staged_callbacks = temporary_stage.callbacks # Here we have an adapter from an old-style package. This means there is no # hierarchy of builders, and every callback that had to be combined between # *Package and *Builder has been combined already by _PackageAdapterMeta if name == "Adapter": continue # If we are here we have callbacks. To get a complete list, we accumulate all the # callbacks from base classes, we deduplicate them, then prepend what we have # registered here. # # The order should be: # 1. Callbacks are registered in order within the same class # 2. Callbacks defined in derived classes precede those defined in base # classes callbacks_from_base = [] for base in bases: current_callbacks = getattr(base, temporary_stage.attribute_name, None) if not current_callbacks: continue callbacks_from_base.extend(current_callbacks) callbacks_from_base = list(lang.dedupe(callbacks_from_base)) # Set the callbacks in this class and flush the temporary stage attr_dict[temporary_stage.attribute_name] = staged_callbacks[:] + callbacks_from_base del temporary_stage.callbacks[:] return super(PhaseCallbacksMeta, mcs).__new__(mcs, name, bases, attr_dict) @staticmethod def run_after(phase: str, when: Optional[str] = None): """Decorator to register a function for running after a given phase. Example:: @run_after("install", when="@2:") def install_missing_files(self): # Do something after the install phase for versions 2.x and above pass Args: phase: phase after which the function must run. when: condition under which the function is run (if :obj:`None`, it is always run). """ def _decorator(fn): key = (phase, when) item = (key, fn) _RUN_AFTER.callbacks.append(item) return fn return _decorator @staticmethod def run_before(phase: str, when: Optional[str] = None): """Decorator to register a function for running before a given phase. Example:: @run_before("build", when="@2:") def patch_generated_source_file(pkg): # Do something before the build phase for versions 2.x and above pass Args: phase: phase before which the function must run. when: condition under which the function is run (if :obj:`None`, it is always run). """ def _decorator(fn): key = (phase, when) item = (key, fn) _RUN_BEFORE.callbacks.append(item) return fn return _decorator # Export these names as standalone to be used in packages run_after = PhaseCallbacksMeta.run_after run_before = PhaseCallbacksMeta.run_before ================================================ FILE: lib/spack/spack/platforms/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import contextlib from typing import Callable from ._functions import _host, by_name, platforms, reset from ._platform import Platform from .darwin import Darwin from .freebsd import FreeBSD from .linux import Linux from .test import Test from .windows import Windows __all__ = [ "Platform", "Darwin", "Linux", "FreeBSD", "Test", "Windows", "platforms", "host", "by_name", "reset", ] #: The "real" platform of the host running Spack. This should not be changed #: by any method and is here as a convenient way to refer to the host platform. real_host = _host #: The current platform used by Spack. May be swapped by the use_platform #: context manager. host: Callable[[], Platform] = _host class _PickleableCallable: """Class used to pickle a callable that may substitute either _platform or _all_platforms. Lambda or nested functions are not pickleable. """ def __init__(self, return_value): self.return_value = return_value def __call__(self): return self.return_value @contextlib.contextmanager def use_platform(new_platform): global host import spack.config assert isinstance(new_platform, Platform), f'"{new_platform}" must be an instance of Platform' original_host_fn = host try: host = _PickleableCallable(new_platform) spack.config.CONFIG.clear_caches() yield new_platform finally: host = original_host_fn spack.config.CONFIG.clear_caches() ================================================ FILE: lib/spack/spack/platforms/_functions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.llnl.util.lang from .darwin import Darwin from .freebsd import FreeBSD from .linux import Linux from .test import Test from .windows import Windows #: List of all the platform classes known to Spack platforms = [Darwin, Linux, Windows, FreeBSD, Test] @spack.llnl.util.lang.memoized def _host(): """Detect and return the platform for this machine or None if detection fails.""" for platform_cls in sorted(platforms, key=lambda plt: plt.priority): if platform_cls.detect(): return platform_cls() assert False, "No platform detected. Spack cannot run without a platform." def reset(): """The result of the host search is memoized. In case it needs to be recomputed we must clear the cache, which is what this function does. """ _host.cache_clear() @spack.llnl.util.lang.memoized def cls_by_name(name): """Return a platform class that corresponds to the given name or None if there is no match. Args: name (str): name of the platform """ for platform_cls in sorted(platforms, key=lambda plt: plt.priority): if name.replace("_", "").lower() == platform_cls.__name__.lower(): return platform_cls return None def by_name(name): """Return a platform object that corresponds to the given name or None if there is no match. Args: name (str): name of the platform """ platform_cls = cls_by_name(name) return platform_cls() if platform_cls else None ================================================ FILE: lib/spack/spack/platforms/_platform.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import warnings from typing import Optional import spack.vendor.archspec.cpu import spack.llnl.util.lang @spack.llnl.util.lang.lazy_lexicographic_ordering class Platform: """Platform is an abstract class extended by subclasses. Platform also contain a priority class attribute. A lower number signifies higher priority. These numbers are arbitrarily set and can be changed though often there isn't much need unless a new platform is added and the user wants that to be detected first. """ # Subclass sets number. Controls detection order priority: Optional[int] = None #: binary formats used on this platform; used by relocation logic binary_formats = ["elf"] default: str default_os: str reserved_targets = ["default_target", "frontend", "fe", "backend", "be"] reserved_oss = ["default_os", "frontend", "fe", "backend", "be"] deprecated_names = ["frontend", "fe", "backend", "be"] def __init__(self, name): self.targets = {} self.operating_sys = {} self.name = name self._init_targets() def add_target(self, name: str, target: spack.vendor.archspec.cpu.Microarchitecture) -> None: if name in Platform.reserved_targets: msg = f"{name} is a spack reserved alias and cannot be the name of a target" raise ValueError(msg) self.targets[name] = target def _init_targets(self): self.default = spack.vendor.archspec.cpu.host().name for name, microarchitecture in spack.vendor.archspec.cpu.TARGETS.items(): self.add_target(name, microarchitecture) def target(self, name): name = str(name) if name in Platform.deprecated_names: warnings.warn(f"target={name} is deprecated, use target={self.default} instead") if name in Platform.reserved_targets: name = self.default return self.targets.get(name, None) def add_operating_system(self, name, os_class): if name in Platform.reserved_oss + Platform.deprecated_names: msg = f"{name} is a spack reserved alias and cannot be the name of an OS" raise ValueError(msg) self.operating_sys[name] = os_class def default_target(self): return self.target(self.default) def default_operating_system(self): return self.operating_system(self.default_os) def operating_system(self, name): if name in Platform.deprecated_names: warnings.warn(f"os={name} is deprecated, use os={self.default_os} instead") if name in Platform.reserved_oss: name = self.default_os return self.operating_sys.get(name, None) def setup_platform_environment(self, pkg, env): """Platform-specific build environment modifications. This method is meant to be overridden by subclasses, when needed. """ pass @classmethod def detect(cls): """Returns True if the host platform is detected to be the current Platform class, False otherwise. Derived classes are responsible for implementing this method. """ raise NotImplementedError() def __repr__(self): return self.__str__() def __str__(self): return self.name def _cmp_iter(self): yield self.name yield self.default yield self.default_os def targets(): for t in sorted(self.targets.values()): yield t._cmp_iter yield targets def oses(): for o in sorted(self.operating_sys.values()): yield o._cmp_iter yield oses ================================================ FILE: lib/spack/spack/platforms/cray.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os def slingshot_network(): return os.path.exists("/opt/cray/pe") and ( os.path.exists("/lib64/libcxi.so") or os.path.exists("/usr/lib64/libcxi.so") ) ================================================ FILE: lib/spack/spack/platforms/darwin.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform as py_platform from spack.operating_systems.mac_os import MacOs from spack.version import Version from ._platform import Platform class Darwin(Platform): priority = 89 binary_formats = ["macho"] def __init__(self): super().__init__("darwin") mac_os = MacOs() self.default_os = str(mac_os) self.add_operating_system(str(mac_os), mac_os) @classmethod def detect(cls): return "darwin" in py_platform.system().lower() def setup_platform_environment(self, pkg, env): """Specify deployment target based on target OS version. The ``MACOSX_DEPLOYMENT_TARGET`` environment variable provides a default ``-mmacosx-version-min`` argument for GCC and Clang compilers, as well as the default value of ``CMAKE_OSX_DEPLOYMENT_TARGET`` for CMake-based build systems. The default value for the deployment target is usually the major version (11, 10.16, ...) for CMake and Clang, but some versions of GCC specify a minor component as well (11.3), leading to numerous link warnings about inconsistent or incompatible target versions. Setting the environment variable ensures consistent versions for an install toolchain target, even when the host macOS version changes. TODO: it may be necessary to add SYSTEM_VERSION_COMPAT for older versions of the macosx developer tools; see https://github.com/spack/spack/pull/26290 for discussion. """ os = self.operating_sys[pkg.spec.os] version = Version(os.version) if len(version) == 1: # Version has only one component: add a minor version to prevent # potential errors with `ld`, # which fails with `-macosx_version_min 11` # but succeeds with `-macosx_version_min 11.0`. # Most compilers seem to perform this translation automatically, # but older GCC does not. version = str(version) + ".0" env.set("MACOSX_DEPLOYMENT_TARGET", str(version)) ================================================ FILE: lib/spack/spack/platforms/freebsd.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform from spack.operating_systems.freebsd import FreeBSDOs from ._platform import Platform class FreeBSD(Platform): priority = 102 def __init__(self): super().__init__("freebsd") os = FreeBSDOs() self.default_os = str(os) self.add_operating_system(str(os), os) @classmethod def detect(cls): return platform.system().lower() == "freebsd" ================================================ FILE: lib/spack/spack/platforms/linux.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform from spack.operating_systems.linux_distro import LinuxDistro from ._platform import Platform class Linux(Platform): priority = 90 def __init__(self): super().__init__("linux") linux_dist = LinuxDistro() self.default_os = str(linux_dist) self.add_operating_system(str(linux_dist), linux_dist) @classmethod def detect(cls): return "linux" in platform.system().lower() ================================================ FILE: lib/spack/spack/platforms/test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform import spack.vendor.archspec.cpu import spack.operating_systems from ._platform import Platform class Test(Platform): priority = 1000000 if platform.system().lower() == "darwin": binary_formats = ["macho"] default_os = "debian6" default = "m1" if platform.machine() == "arm64" else "core2" def __init__(self, name=None): name = name or "test" super().__init__(name) self.add_operating_system("debian6", spack.operating_systems.OperatingSystem("debian", 6)) self.add_operating_system("redhat6", spack.operating_systems.OperatingSystem("redhat", 6)) def _init_targets(self): targets = ("aarch64", "m1") if platform.machine() == "arm64" else ("x86_64", "core2") for t in targets: self.add_target(t, spack.vendor.archspec.cpu.TARGETS[t]) @classmethod def detect(cls): return True ================================================ FILE: lib/spack/spack/platforms/windows.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform from spack.operating_systems.windows_os import WindowsOs from ._platform import Platform class Windows(Platform): priority = 101 def __init__(self): super().__init__("windows") windows_os = WindowsOs() self.default_os = str(windows_os) self.add_operating_system(str(windows_os), windows_os) @classmethod def detect(cls): plat = platform.system().lower() return "cygwin" in plat or "win32" in plat or "windows" in plat ================================================ FILE: lib/spack/spack/projections.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.util.path def get_projection(projections, spec): """ Get the projection for a spec from a projections dict. """ all_projection = None for spec_like, projection in projections.items(): if spec.satisfies(spec_like): return spack.util.path.substitute_path_variables(projection) elif spec_like == "all": all_projection = spack.util.path.substitute_path_variables(projection) return all_projection ================================================ FILE: lib/spack/spack/provider_index.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes and functions to manage providers of virtual dependencies""" from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Union import spack.error import spack.util.spack_json as sjson if TYPE_CHECKING: import spack.repo import spack.spec class ProviderIndex: #: This is a dict of dicts used for finding providers of particular #: virtual dependencies. The dict of dicts looks like: #: #: { vpkg name : #: { full vpkg spec : set(packages providing spec) } } #: #: Callers can use this to first find which packages provide a vpkg, #: then find a matching full spec. e.g., in this scenario: #: #: { 'mpi' : #: { mpi@:1.1 : set([mpich]), #: mpi@:2.3 : set([mpich2@1.9:]) } } #: #: Calling providers_for(spec) will find specs that provide a #: matching implementation of MPI. Derived class need to construct #: this attribute according to the semantics above. providers: Dict[str, Dict["spack.spec.Spec", Set["spack.spec.Spec"]]] def __init__( self, repository: "spack.repo.RepoType", specs: Optional[Iterable["spack.spec.Spec"]] = None, restrict: bool = False, ): """Provider index based on a single mapping of providers. Args: specs: if provided, will call update on each single spec to initialize this provider index. restrict: "restricts" values to the verbatim input specs; do not pre-apply package's constraints. TODO: rename this. It is intended to keep things as broad TODO: as possible without overly restricting results, so it is TODO: not the best name. """ self.repository = repository self.restrict = restrict self.providers = {} if specs: self.update_packages(specs) def providers_for(self, virtual: Union[str, "spack.spec.Spec"]) -> List["spack.spec.Spec"]: """Return a list of specs of all packages that provide virtual packages with the supplied spec. Args: virtual: either a Spec or a string name of a virtual package """ result: Set["spack.spec.Spec"] = set() if isinstance(virtual, str): # In the common case where just a package name is passed, we can avoid running the # spec parser and intersects, since intersects is always true. if virtual.isalnum(): if virtual in self.providers: for p_spec, spec_set in self.providers[virtual].items(): result.update(spec_set) return list(result) from spack.spec import Spec virtual = Spec(virtual) # Add all the providers that satisfy the vpkg spec. if virtual.name in self.providers: for p_spec, spec_set in self.providers[virtual.name].items(): if p_spec.intersects(virtual, deps=False): result.update(spec_set) return list(result) def __contains__(self, name): return name in self.providers def __eq__(self, other): return self.providers == other.providers def _transform(self, transform_fun, out_mapping_type=dict): """Transform this provider index dictionary and return it. Args: transform_fun: transform_fun takes a (vpkg, pset) mapping and runs it on each pair in nested dicts. out_mapping_type: type to be used internally on the transformed (vpkg, pset) Returns: Transformed mapping """ return _transform(self.providers, transform_fun, out_mapping_type) def __str__(self): return str(self.providers) def __repr__(self): return repr(self.providers) def update_packages(self, specs: Iterable[Union[str, "spack.spec.Spec"]]): """Update the provider index with additional virtual specs. Args: spec: spec potentially providing additional virtual specs """ from spack.spec import Spec for spec in specs: if not isinstance(spec, Spec): spec = Spec(spec) if not spec.name or self.repository.is_virtual_safe(spec.name): # Only non-virtual packages with name can provide virtual specs. continue pkg_cls = self.repository.get_pkg_class(spec.name) for provider_spec_readonly, provided_specs in pkg_cls.provided.items(): for provided_spec in provided_specs: # TODO: fix this comment. # We want satisfaction other than flags provider_spec = provider_spec_readonly.copy() provider_spec.compiler_flags = spec.compiler_flags.copy() if spec.intersects(provider_spec, deps=False): provided_name = provided_spec.name provider_map = self.providers.setdefault(provided_name, {}) if provided_spec not in provider_map: provider_map[provided_spec] = set() if self.restrict: provider_set = provider_map[provided_spec] # If this package existed in the index before, # need to take the old versions out, as they're # now more constrained. old = {s for s in provider_set if s.name == spec.name} provider_set.difference_update(old) # Now add the new version. provider_set.add(spec) else: # Before putting the spec in the map, constrain # it so that it provides what was asked for. constrained = spec.copy() constrained.constrain(provider_spec) provider_map[provided_spec].add(constrained) def to_json(self, stream=None): """Dump a JSON representation of this object. Args: stream: stream where to dump """ provider_list = self._transform( lambda vpkg, pset: [vpkg.to_node_dict(), [p.to_node_dict() for p in pset]], list ) sjson.dump({"provider_index": {"providers": provider_list}}, stream) def merge(self, other): """Merge another provider index into this one. Args: other (ProviderIndex): provider index to be merged """ other = other.copy() # defensive copy. for pkg in other.providers: if pkg not in self.providers: self.providers[pkg] = other.providers[pkg] continue spdict, opdict = self.providers[pkg], other.providers[pkg] for provided_spec in opdict: if provided_spec not in spdict: spdict[provided_spec] = opdict[provided_spec] continue spdict[provided_spec] = spdict[provided_spec].union(opdict[provided_spec]) def remove_providers(self, pkg_names: Set[str]): """Remove a provider from the ProviderIndex.""" empty_pkg_dict = [] for pkg, pkg_dict in self.providers.items(): empty_pset = [] for provided, pset in pkg_dict.items(): to_remove = {spec for spec in pset if spec.name in pkg_names} pset.difference_update(to_remove) if not pset: empty_pset.append(provided) for provided in empty_pset: del pkg_dict[provided] if not pkg_dict: empty_pkg_dict.append(pkg) for pkg in empty_pkg_dict: del self.providers[pkg] def copy(self): """Return a deep copy of this index.""" clone = ProviderIndex(repository=self.repository) clone.providers = self._transform(lambda vpkg, pset: (vpkg, set((p.copy() for p in pset)))) return clone @staticmethod def from_json(stream, repository): """Construct a provider index from its JSON representation. Args: stream: stream where to read from the JSON data """ data = sjson.load(stream) if not isinstance(data, dict): raise ProviderIndexError("JSON ProviderIndex data was not a dict.") if "provider_index" not in data: raise ProviderIndexError("YAML ProviderIndex does not start with 'provider_index'") index = ProviderIndex(repository=repository) providers = data["provider_index"]["providers"] from spack.spec import SpecfileLatest index.providers = _transform( providers, lambda vpkg, plist: ( SpecfileLatest.from_node_dict(vpkg), set(SpecfileLatest.from_node_dict(p) for p in plist), ), ) return index def _transform(providers, transform_fun, out_mapping_type=dict): """Syntactic sugar for transforming a providers dict. Args: providers: provider dictionary transform_fun: transform_fun takes a (vpkg, pset) mapping and runs it on each pair in nested dicts. out_mapping_type: type to be used internally on the transformed (vpkg, pset) Returns: Transformed mapping """ def mapiter(mappings): if isinstance(mappings, dict): return mappings.items() else: return iter(mappings) return dict( (name, out_mapping_type([transform_fun(vpkg, pset) for vpkg, pset in mapiter(mappings)])) for name, mappings in providers.items() ) class ProviderIndexError(spack.error.SpackError): """Raised when there is a problem with a ProviderIndex.""" ================================================ FILE: lib/spack/spack/relocate.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import itertools import os import re import sys from typing import Dict, Iterable, List, Optional import spack.vendor.macholib.mach_o import spack.vendor.macholib.MachO import spack.llnl.util.filesystem as fs import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.store import spack.util.elf as elf import spack.util.executable as executable from spack.llnl.util.filesystem import readlink, symlink from spack.llnl.util.lang import memoized from .relocate_text import BinaryFilePrefixReplacer, PrefixToPrefix, TextFilePrefixReplacer @memoized def _patchelf() -> Optional[executable.Executable]: """Return the full path to the patchelf binary, if available, else None.""" import spack.bootstrap if sys.platform == "darwin": return None with spack.bootstrap.ensure_bootstrap_configuration(): return spack.bootstrap.ensure_patchelf_in_path_or_raise() def _decode_macho_data(bytestring): return bytestring.rstrip(b"\x00").decode("ascii") def _macho_find_paths(orig_rpaths, deps, idpath, prefix_to_prefix): """ Inputs original rpaths from mach-o binaries dependency libraries for mach-o binaries id path of mach-o libraries old install directory layout root prefix_to_prefix dictionary which maps prefixes in the old directory layout to directories in the new directory layout Output paths_to_paths dictionary which maps all of the old paths to new paths """ paths_to_paths = dict() # Sort from longest path to shortest, to ensure we try /foo/bar/baz before /foo/bar prefix_iteration_order = sorted(prefix_to_prefix, key=len, reverse=True) for orig_rpath in orig_rpaths: for old_prefix in prefix_iteration_order: new_prefix = prefix_to_prefix[old_prefix] if orig_rpath.startswith(old_prefix): new_rpath = re.sub(re.escape(old_prefix), new_prefix, orig_rpath) paths_to_paths[orig_rpath] = new_rpath break else: paths_to_paths[orig_rpath] = orig_rpath if idpath: for old_prefix in prefix_iteration_order: new_prefix = prefix_to_prefix[old_prefix] if idpath.startswith(old_prefix): paths_to_paths[idpath] = re.sub(re.escape(old_prefix), new_prefix, idpath) break for dep in deps: for old_prefix in prefix_iteration_order: new_prefix = prefix_to_prefix[old_prefix] if dep.startswith(old_prefix): paths_to_paths[dep] = re.sub(re.escape(old_prefix), new_prefix, dep) break if dep.startswith("@"): paths_to_paths[dep] = dep return paths_to_paths def _modify_macho_object(cur_path, rpaths, deps, idpath, paths_to_paths): """ This function is used to make machO buildcaches on macOS by replacing old paths with new paths using install_name_tool Inputs: mach-o binary to be modified original rpaths original dependency paths original id path if a mach-o library dictionary mapping paths in old install layout to new install layout """ # avoid error message for libgcc_s if "libgcc_" in cur_path: return args = [] if idpath: new_idpath = paths_to_paths.get(idpath, None) if new_idpath and not idpath == new_idpath: args += [("-id", new_idpath)] for dep in deps: new_dep = paths_to_paths.get(dep) if new_dep and dep != new_dep: args += [("-change", dep, new_dep)] new_rpaths = [] for orig_rpath in rpaths: new_rpath = paths_to_paths.get(orig_rpath) if new_rpath and not orig_rpath == new_rpath: args_to_add = ("-rpath", orig_rpath, new_rpath) if args_to_add not in args and new_rpath not in new_rpaths: args += [args_to_add] new_rpaths.append(new_rpath) # Deduplicate and flatten args = list(itertools.chain.from_iterable(spack.llnl.util.lang.dedupe(args))) install_name_tool = executable.Executable("install_name_tool") if args: with fs.edit_in_place_through_temporary_file(cur_path) as temp_path: install_name_tool(*args, temp_path) def _macholib_get_paths(cur_path): """Get rpaths, dependent libraries, and library id of mach-o objects.""" headers = [] try: headers = spack.vendor.macholib.MachO.MachO(cur_path).headers except ValueError: pass if not headers: tty.warn("Failed to read Mach-O headers: {0}".format(cur_path)) commands = [] else: if len(headers) > 1: # Reproduce original behavior of only returning the last mach-O # header section tty.warn("Encountered fat binary: {0}".format(cur_path)) if headers[-1].filetype == "dylib_stub": tty.warn("File is a stub, not a full library: {0}".format(cur_path)) commands = headers[-1].commands LC_ID_DYLIB = spack.vendor.macholib.mach_o.LC_ID_DYLIB LC_LOAD_DYLIB = spack.vendor.macholib.mach_o.LC_LOAD_DYLIB LC_RPATH = spack.vendor.macholib.mach_o.LC_RPATH ident = None rpaths = [] deps = [] for load_command, dylib_command, data in commands: cmd = load_command.cmd if cmd == LC_RPATH: rpaths.append(_decode_macho_data(data)) elif cmd == LC_LOAD_DYLIB: deps.append(_decode_macho_data(data)) elif cmd == LC_ID_DYLIB: ident = _decode_macho_data(data) return (rpaths, deps, ident) def _set_elf_rpaths_and_interpreter( target: str, rpaths: List[str], interpreter: Optional[str] = None ) -> Optional[str]: """Replace the original RPATH of the target with the paths passed as arguments. Args: target: target executable. Must be an ELF object. rpaths: paths to be set in the RPATH interpreter: optionally set the interpreter Returns: A string concatenating the stdout and stderr of the call to ``patchelf`` if it was invoked """ # Join the paths using ':' as a separator rpaths_str = ":".join(rpaths) try: # TODO: error handling is not great here? # TODO: revisit the use of --force-rpath as it might be conditional # TODO: if we want to support setting RUNPATH from binary packages args = ["--force-rpath", "--set-rpath", rpaths_str] if interpreter: args.extend(["--set-interpreter", interpreter]) args.append(target) return _patchelf()(*args, output=str, error=str) except executable.ProcessError as e: tty.warn(str(e)) return None def relocate_macho_binaries(path_names, prefix_to_prefix): """ Use macholib python package to get the rpaths, dependent libraries and library identity for libraries from the MachO object. Modify them with the replacement paths queried from the dictionary mapping old layout prefixes to hashes and the dictionary mapping hashes to the new layout prefixes. """ for path_name in path_names: # Corner case where macho object file ended up in the path name list if path_name.endswith(".o"): continue # get the paths in the old prefix rpaths, deps, idpath = _macholib_get_paths(path_name) # get the mapping of paths in the old prerix to the new prefix paths_to_paths = _macho_find_paths(rpaths, deps, idpath, prefix_to_prefix) # replace the old paths with new paths _modify_macho_object(path_name, rpaths, deps, idpath, paths_to_paths) def relocate_elf_binaries(binaries: Iterable[str], prefix_to_prefix: Dict[str, str]) -> None: """Take a list of binaries, and an ordered prefix to prefix mapping, and update the rpaths accordingly.""" # Transform to binary string prefix_to_prefix_bin = { k.encode("utf-8"): v.encode("utf-8") for k, v in prefix_to_prefix.items() } for path in binaries: try: elf.substitute_rpath_and_pt_interp_in_place_or_raise(path, prefix_to_prefix_bin) except elf.ElfCStringUpdatesFailed as e: # Fall back to `patchelf --set-rpath ... --set-interpreter ...` rpaths = e.rpath.new_value.decode("utf-8").split(":") if e.rpath else [] interpreter = e.pt_interp.new_value.decode("utf-8") if e.pt_interp else None _set_elf_rpaths_and_interpreter(path, rpaths=rpaths, interpreter=interpreter) def relocate_links(links: Iterable[str], prefix_to_prefix: Dict[str, str]) -> None: """Relocate links to a new install prefix.""" regex = re.compile("|".join(re.escape(p) for p in prefix_to_prefix.keys())) for link in links: old_target = readlink(link) if not os.path.isabs(old_target): continue match = regex.match(old_target) if match is None: continue new_target = prefix_to_prefix[match.group()] + old_target[match.end() :] os.unlink(link) symlink(new_target, link) def relocate_text(files: Iterable[str], prefix_to_prefix: PrefixToPrefix) -> None: """Relocate text file from the original installation prefix to the new prefix. Relocation also affects the the path in Spack's sbang script. Args: files: Text files to be relocated prefix_to_prefix: ordered prefix to prefix mapping """ TextFilePrefixReplacer.from_strings_or_bytes(prefix_to_prefix).apply(files) def relocate_text_bin(binaries: Iterable[str], prefix_to_prefix: PrefixToPrefix) -> List[str]: """Replace null terminated path strings hard-coded into binaries. The new install prefix must be shorter than the original one. Args: binaries: paths to binaries to be relocated prefix_to_prefix: ordered prefix to prefix mapping Raises: spack.relocate_text.BinaryTextReplaceError: when the new path is longer than the old path """ return BinaryFilePrefixReplacer.from_strings_or_bytes(prefix_to_prefix).apply(binaries) def is_macho_magic(magic: bytes) -> bool: return ( # In order of popularity: 64-bit mach-o le/be, 32-bit mach-o le/be. magic.startswith(b"\xcf\xfa\xed\xfe") or magic.startswith(b"\xfe\xed\xfa\xcf") or magic.startswith(b"\xce\xfa\xed\xfe") or magic.startswith(b"\xfe\xed\xfa\xce") # universal binaries: 0xcafebabe be (most common?) or 0xbebafeca le (not sure if exists). # Here we need to disambiguate mach-o and JVM class files. In mach-o the next 4 bytes are # the number of binaries; in JVM class files it's the java version number. We assume there # are less than 10 binaries in a universal binary. or (magic.startswith(b"\xca\xfe\xba\xbe") and int.from_bytes(magic[4:8], "big") < 10) or (magic.startswith(b"\xbe\xba\xfe\xca") and int.from_bytes(magic[4:8], "little") < 10) ) def is_elf_magic(magic: bytes) -> bool: return magic.startswith(b"\x7fELF") def is_binary(filename: str) -> bool: """Returns true iff a file is likely binary""" with open(filename, "rb") as f: magic = f.read(8) return is_macho_magic(magic) or is_elf_magic(magic) # Memoize this due to repeated calls to libraries in the same directory. @spack.llnl.util.lang.memoized def _exists_dir(dirname): return os.path.isdir(dirname) def is_macho_binary(path: str) -> bool: try: with open(path, "rb") as f: return is_macho_magic(f.read(4)) except OSError: return False def fixup_macos_rpath(root, filename): """Apply rpath fixups to the given file. Args: root: absolute path to the parent directory filename: relative path to the library or binary Returns: True if fixups were applied, else False """ abspath = os.path.join(root, filename) if not is_macho_binary(abspath): return False # Get Mach-O header commands (rpath_list, deps, id_dylib) = _macholib_get_paths(abspath) # Convert rpaths list to (name -> number of occurrences) add_rpaths = set() del_rpaths = set() rpaths = collections.defaultdict(int) for rpath in rpath_list: rpaths[rpath] += 1 args = [] # Check dependencies for non-rpath entries spack_root = spack.store.STORE.layout.root for name in deps: if name.startswith(spack_root): tty.debug("Spack-installed dependency for {0}: {1}".format(abspath, name)) (dirname, basename) = os.path.split(name) if dirname != root or dirname in rpaths: # Only change the rpath if it's a dependency *or* if the root # rpath was already added to the library (this is to prevent # GCC or similar getting rpaths when they weren't at all # configured) args += ["-change", name, "@rpath/" + basename] add_rpaths.add(dirname.rstrip("/")) # Check for nonexistent rpaths (often added by spack linker overzealousness # with both lib/ and lib64/) and duplicate rpaths for rpath, count in rpaths.items(): if rpath.startswith("@loader_path") or rpath.startswith("@executable_path"): # Allowable relative paths pass elif not _exists_dir(rpath): tty.debug("Nonexistent rpath in {0}: {1}".format(abspath, rpath)) del_rpaths.add(rpath) elif count > 1: # Rpath should only be there once, but it can sometimes be # duplicated between Spack's compiler and libtool. If there are # more copies of the same one, something is very odd.... tty_debug = tty.debug if count == 2 else tty.warn tty_debug("Rpath appears {0} times in {1}: {2}".format(count, abspath, rpath)) del_rpaths.add(rpath) # Delete bad rpaths for rpath in del_rpaths: args += ["-delete_rpath", rpath] # Add missing rpaths that are not set for deletion for rpath in add_rpaths - del_rpaths - set(rpaths): args += ["-add_rpath", rpath] if not args: # No fixes needed return False with fs.edit_in_place_through_temporary_file(abspath) as temp_path: executable.Executable("install_name_tool")(*args, temp_path) return True def fixup_macos_rpaths(spec): """Remove duplicate and nonexistent rpaths. Some autotools packages write their own ``-rpath`` entries in addition to those implicitly added by the Spack compiler wrappers. On Linux these duplicate rpaths are eliminated, but on macOS they result in multiple entries which makes it harder to adjust with ``install_name_tool -delete_rpath``. """ if spec.external or not spec.concrete: tty.warn("external/abstract spec cannot be fixed up: {0!s}".format(spec)) return False if "platform=darwin" not in spec: raise NotImplementedError("fixup_macos_rpaths requires macOS") applied = 0 libs = frozenset(["lib", "lib64", "libexec", "plugins", "Library", "Frameworks"]) prefix = spec.prefix if not os.path.exists(prefix): raise RuntimeError( "Could not fix up install prefix spec {0} because it does not exist: {1!s}".format( prefix, spec.name ) ) # Explore the installation prefix of the spec for root, dirs, files in os.walk(prefix, topdown=True): dirs[:] = set(dirs) & libs for name in files: try: needed_fix = fixup_macos_rpath(root, name) except Exception as e: tty.warn("Failed to apply library fixups to: {0}/{1}: {2!s}".format(root, name, e)) needed_fix = False if needed_fix: applied += 1 specname = spec.format("{name}{/hash:7}") if applied: tty.info( "Fixed rpaths for {0:d} {1} installed to {2}".format( applied, "binary" if applied == 1 else "binaries", specname ) ) else: tty.debug("No rpath fixup needed for " + specname) ================================================ FILE: lib/spack/spack/relocate_text.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module contains pure-Python classes and functions for replacing paths inside text files and binaries.""" import re from typing import IO, Dict, Iterable, List, Union import spack.error from spack.llnl.util.lang import PatternBytes Prefix = Union[str, bytes] PrefixToPrefix = Union[Dict[str, str], Dict[bytes, bytes]] def encode_path(p: Prefix) -> bytes: return p if isinstance(p, bytes) else p.encode("utf-8") def _prefix_to_prefix_as_bytes(prefix_to_prefix: PrefixToPrefix) -> Dict[bytes, bytes]: return {encode_path(k): encode_path(v) for (k, v) in prefix_to_prefix.items()} def utf8_path_to_binary_regex(prefix: str) -> PatternBytes: """Create a binary regex that matches the input path in utf8""" prefix_bytes = re.escape(prefix).encode("utf-8") return re.compile(b"(? PatternBytes: all_prefixes = b"|".join(re.escape(p) for p in prefixes) return re.compile(b"(? PatternBytes: """Create a (binary) regex that matches any input path in utf8""" return _byte_strings_to_single_binary_regex(p.encode("utf-8") for p in prefixes) def filter_identity_mappings(prefix_to_prefix: Dict[bytes, bytes]) -> Dict[bytes, bytes]: """Drop mappings that are not changed.""" # NOTE: we don't guard against the following case: # [/abc/def -> /abc/def, /abc -> /x] *will* be simplified to # [/abc -> /x], meaning that after this simplification /abc/def will be # mapped to /x/def instead of /abc/def. This should not be a problem. return {k: v for k, v in prefix_to_prefix.items() if k != v} class PrefixReplacer: """Base class for applying a prefix to prefix map to a list of binaries or text files. Derived classes implement _apply_to_file to do the actual work, which is different when it comes to binaries and text files.""" def __init__(self, prefix_to_prefix: Dict[bytes, bytes]) -> None: """ Arguments: prefix_to_prefix: An ordered mapping from prefix to prefix. The order is relevant to support substring fallbacks, for example ``[("/first/sub", "/x"), ("/first", "/y")]`` will ensure /first/sub is matched and replaced before /first. """ self.prefix_to_prefix = filter_identity_mappings(prefix_to_prefix) @property def is_noop(self) -> bool: """Returns true when the prefix to prefix map is mapping everything to the same location (identity) or there are no prefixes to replace.""" return not self.prefix_to_prefix def apply(self, filenames: Iterable[str]) -> List[str]: """Returns a list of files that were modified""" changed_files = [] if self.is_noop: return [] for filename in filenames: if self.apply_to_filename(filename): changed_files.append(filename) return changed_files def apply_to_filename(self, filename: str) -> bool: if self.is_noop: return False with open(filename, "rb+") as f: return self.apply_to_file(f) def apply_to_file(self, f: IO[bytes]) -> bool: if self.is_noop: return False return self._apply_to_file(f) def _apply_to_file(self, f: IO) -> bool: raise NotImplementedError("Derived classes must implement this method") class TextFilePrefixReplacer(PrefixReplacer): """This class applies prefix to prefix mappings for relocation on text files. Note that UTF-8 encoding is assumed.""" def __init__(self, prefix_to_prefix: Dict[bytes, bytes]): """ prefix_to_prefix (OrderedDict): OrderedDictionary where the keys are bytes representing the old prefixes and the values are the new. """ super().__init__(prefix_to_prefix) # Single regex for all paths. self.regex = _byte_strings_to_single_binary_regex(self.prefix_to_prefix.keys()) @classmethod def from_strings_or_bytes(cls, prefix_to_prefix: PrefixToPrefix) -> "TextFilePrefixReplacer": """Create a TextFilePrefixReplacer from an ordered prefix to prefix map.""" return cls(_prefix_to_prefix_as_bytes(prefix_to_prefix)) def _apply_to_file(self, f: IO) -> bool: """Text replacement implementation simply reads the entire file in memory and applies the combined regex.""" replacement = lambda m: m.group(1) + self.prefix_to_prefix[m.group(2)] + m.group(3) data = f.read() new_data = re.sub(self.regex, replacement, data) if id(data) == id(new_data): return False f.seek(0) f.write(new_data) f.truncate() return True class BinaryFilePrefixReplacer(PrefixReplacer): def __init__(self, prefix_to_prefix: Dict[bytes, bytes], suffix_safety_size: int = 7) -> None: """ prefix_to_prefix: Ordered dictionary where the keys are bytes representing the old prefixes and the values are the new suffix_safety_size: in case of null terminated strings, what size of the suffix should remain to avoid aliasing issues? """ assert suffix_safety_size >= 0 super().__init__(prefix_to_prefix) self.suffix_safety_size = suffix_safety_size self.regex = self.binary_text_regex(self.prefix_to_prefix.keys(), suffix_safety_size) @classmethod def binary_text_regex( cls, binary_prefixes: Iterable[bytes], suffix_safety_size: int = 7 ) -> PatternBytes: """Create a regex that looks for exact matches of prefixes, and also tries to match a C-string type null terminator in a small lookahead window. Arguments: binary_prefixes: Iterable of byte strings of prefixes to match suffix_safety_size: Sizeof the lookahed for null-terminated string. """ # Note: it's important not to use capture groups for the prefix, since it destroys # performance due to common prefix optimization. return re.compile( b"(" + b"|".join(re.escape(p) for p in binary_prefixes) + b")([^\0]{0,%d}\0)?" % suffix_safety_size ) @classmethod def from_strings_or_bytes( cls, prefix_to_prefix: PrefixToPrefix, suffix_safety_size: int = 7 ) -> "BinaryFilePrefixReplacer": """Create a BinaryFilePrefixReplacer from an ordered prefix to prefix map. Arguments: prefix_to_prefix: Ordered mapping of prefix to prefix. suffix_safety_size: Number of bytes to retain at the end of a C-string to avoid binary string-aliasing issues. """ return cls(_prefix_to_prefix_as_bytes(prefix_to_prefix), suffix_safety_size) def _apply_to_file(self, f: IO[bytes]) -> bool: """ Given a file opened in rb+ mode, apply the string replacements as specified by an ordered dictionary of prefix to prefix mappings. This method takes special care of null-terminated C-strings. C-string constants are problematic because compilers and linkers optimize readonly strings for space by aliasing those that share a common suffix (only suffix since all of them are null terminated). See https://github.com/spack/spack/pull/31739 and https://github.com/spack/spack/pull/32253 for details. Our logic matches the original prefix with a ``suffix_safety_size + 1`` lookahead for null bytes. If no null terminator is found, we simply pad with leading /, assuming that it's a long C-string; the full C-string after replacement has a large suffix in common with its original value. If there *is* a null terminator we can do the same as long as the replacement has a sufficiently long common suffix with the original prefix. As a last resort when the replacement does not have a long enough common suffix, we can try to shorten the string, but this only works if the new length is sufficiently short (typically the case when going from large padding -> normal path) If the replacement string is longer, or all of the above fails, we error out. Arguments: f: file opened in rb+ mode Returns: bool: True if file was modified """ assert f.tell() == 0 # We *could* read binary data in chunks to avoid loading all in memory, but it's nasty to # deal with matches across boundaries, so let's stick to something simple. modified = False for match in self.regex.finditer(f.read()): # The matching prefix (old) and its replacement (new) old = match.group(1) new = self.prefix_to_prefix[old] # Did we find a trailing null within a N + 1 bytes window after the prefix? null_terminated = match.end(0) > match.end(1) # Suffix string length, excluding the null byte. Only makes sense if null_terminated suffix_strlen = match.end(0) - match.end(1) - 1 # How many bytes are we shrinking our string? bytes_shorter = len(old) - len(new) # We can't make strings larger. if bytes_shorter < 0: raise CannotGrowString(old, new) # If we don't know whether this is a null terminated C-string (we're looking only N + 1 # bytes ahead), or if it is and we have a common suffix, we can simply pad with leading # dir separators. elif ( not null_terminated or suffix_strlen >= self.suffix_safety_size # == is enough, but let's be defensive or old[-self.suffix_safety_size + suffix_strlen :] == new[-self.suffix_safety_size + suffix_strlen :] ): replacement = b"/" * bytes_shorter + new # If it *was* null terminated, all that matters is that we can leave N bytes of old # suffix in place. Note that > is required since we also insert an additional null # terminator. elif bytes_shorter > self.suffix_safety_size: replacement = new + match.group(2) # includes the trailing null # Otherwise... we can't :( else: raise CannotShrinkCString(old, new, match.group()[:-1]) f.seek(match.start()) f.write(replacement) modified = True return modified class BinaryTextReplaceError(spack.error.SpackError): def __init__(self, msg): msg += ( " To fix this, compile with more padding " "(config:install_tree:padded_length), or install to a shorter prefix." ) super().__init__(msg) class CannotGrowString(BinaryTextReplaceError): def __init__(self, old, new): return super().__init__( f"Cannot replace {old!r} with {new!r} because the new prefix is longer." ) class CannotShrinkCString(BinaryTextReplaceError): def __init__(self, old, new, full_old_string): # Just interpolate binary string to not risk issues with invalid unicode, which would be # really bad user experience: error in error. We have no clue if we actually deal with a # real C-string nor what encoding it has. super().__init__( f"Cannot replace {old!r} with {new!r} in the C-string {full_old_string!r}." ) ================================================ FILE: lib/spack/spack/repo.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import abc import contextlib import difflib import errno import functools import importlib import importlib.machinery import importlib.util import itertools import math import os import re import shutil import stat import sys import traceback import types import uuid import warnings from typing import ( TYPE_CHECKING, Any, Callable, Dict, Generator, Iterator, List, Mapping, Optional, Set, Tuple, Type, Union, cast, ) import spack import spack.caches import spack.config import spack.error import spack.llnl.path import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.patch import spack.paths import spack.provider_index import spack.tag import spack.util.executable import spack.util.file_cache import spack.util.git import spack.util.hash import spack.util.lock import spack.util.naming as nm import spack.util.path import spack.util.spack_yaml as syaml from spack.llnl.util.filesystem import working_dir from spack.llnl.util.lang import Singleton, memoized if TYPE_CHECKING: import spack.package_base import spack.spec PKG_MODULE_PREFIX_V1 = "spack.pkg." PKG_MODULE_PREFIX_V2 = "spack_repo." _API_REGEX = re.compile(r"^v(\d+)\.(\d+)$") SPACK_REPO_INDEX_FILE_NAME = "spack-repo-index.yaml" def package_repository_lock() -> spack.util.lock.Lock: """Lock for process safety when cloning remote package repositories""" return spack.util.lock.Lock( os.path.join(spack.paths.user_cache_path, "package-repository.lock") ) def is_package_module(fullname: str) -> bool: """Check if the given module is a package module.""" return fullname.startswith(PKG_MODULE_PREFIX_V1) or ( fullname.startswith(PKG_MODULE_PREFIX_V2) and fullname.endswith(".package") ) def namespace_from_fullname(fullname: str) -> str: """Return the repository namespace only for the full module name. For instance:: namespace_from_fullname("spack.pkg.builtin.hdf5") == "builtin" namespace_from_fullname("spack_repo.x.y.z.packages.pkg_name.package") == "x.y.z" Args: fullname: full name for the Python module """ if fullname.startswith(PKG_MODULE_PREFIX_V1): namespace, _, _ = fullname.rpartition(".") return namespace[len(PKG_MODULE_PREFIX_V1) :] elif fullname.startswith(PKG_MODULE_PREFIX_V2) and fullname.endswith(".package"): return ".".join(fullname.split(".")[1:-3]) return fullname def name_from_fullname(fullname: str) -> str: """Return the package name for the full module name. For instance:: name_from_fullname("spack.pkg.builtin.hdf5") == "hdf5" name_from_fullname("spack_repo.x.y.z.packages.pkg_name.package") == "pkg_name" Args: fullname: full name for the Python module """ if fullname.startswith(PKG_MODULE_PREFIX_V1): _, _, pkg_module = fullname.rpartition(".") return pkg_module elif fullname.startswith(PKG_MODULE_PREFIX_V2) and fullname.endswith(".package"): return fullname.rsplit(".", 2)[-2] return fullname class _PrependFileLoader(importlib.machinery.SourceFileLoader): def __init__(self, fullname: str, repo: "Repo", package_name: str) -> None: self.repo = repo self.package_name = package_name path = repo.filename_for_package_name(package_name) self.fullname = fullname self.prepend = b"from spack_repo.builtin.build_systems._package_api_v1 import *\n" super().__init__(self.fullname, path) def path_stats(self, path): stats = dict(super().path_stats(path)) stats["size"] += len(self.prepend) return stats def get_data(self, path): data = super().get_data(path) return self.prepend + data if path == self.path else data class SpackNamespaceLoader: def create_module(self, spec): return SpackNamespace(spec.name) def exec_module(self, module): module.__loader__ = self class ReposFinder: """MetaPathFinder class that loads a Python module corresponding to an API v1 Spack package. Returns a loader based on the inspection of the current repository list. """ #: The current list of repositories. repo_path: "RepoPath" def find_spec(self, fullname, python_path, target=None): # "target" is not None only when calling importlib.reload() if target is not None: raise RuntimeError('cannot reload module "{0}"'.format(fullname)) # Preferred API from https://peps.python.org/pep-0451/ if not fullname.startswith(PKG_MODULE_PREFIX_V1) and fullname != "spack.pkg": return None loader = self.compute_loader(fullname) if loader is None: return None return importlib.util.spec_from_loader(fullname, loader) def compute_loader(self, fullname: str): # namespaces are added to repo, and package modules are leaves. namespace, dot, module_name = fullname.rpartition(".") # If it's a module in some repo, or if it is the repo's namespace, let the repo handle it. if not hasattr(self, "repo_path"): return None for repo in self.repo_path.repos: # We are using the namespace of the repo and the repo contains the package if namespace == repo.full_namespace: # With 2 nested conditionals we can call "repo.real_name" only once package_name = repo.real_name(module_name) if package_name: return _PrependFileLoader(fullname, repo, package_name) # We are importing a full namespace like 'spack.pkg.builtin' if fullname == repo.full_namespace: return SpackNamespaceLoader() # No repo provides the namespace, but it is a valid prefix of # something in the RepoPath. if self.repo_path.by_namespace.is_prefix(fullname[len(PKG_MODULE_PREFIX_V1) :]): return SpackNamespaceLoader() return None # # These names describe how repos should be laid out in the filesystem. # repo_config_name = "repo.yaml" # Top-level filename for repo config. repo_index_name = "index.yaml" # Top-level filename for repository index. packages_dir_name = "packages" # Top-level repo directory containing pkgs. package_file_name = "package.py" # Filename for packages in a repository. #: Guaranteed unused default value for some functions. NOT_PROVIDED = object() def builtin_repo() -> "Repo": """Get the test repo if it is active, otherwise the builtin repo.""" try: return PATH.get_repo("builtin_mock") except UnknownNamespaceError: return PATH.get_repo("builtin") class GitExe: # Wrapper around Executable for git to set working directory for all # invocations. # # Not using -C as that is not supported for git < 1.8.5. def __init__(self, packages_path: str): self._git_cmd = spack.util.git.git(required=True) self.packages_dir = packages_path def __call__(self, *args, **kwargs) -> str: with working_dir(self.packages_dir): return self._git_cmd(*args, **kwargs, output=str) def list_packages(rev: str, repo: "Repo") -> List[str]: """List all packages associated with the given revision""" git = GitExe(repo.packages_path) # git ls-tree does not support ... merge-base syntax, so do it manually if rev.endswith("..."): ref = rev.replace("...", "") rev = git("merge-base", ref, "HEAD").strip() output = git("ls-tree", "-r", "--name-only", rev) # recursively list the packages directory package_paths = [ line.split(os.sep) for line in output.split("\n") if line.endswith("package.py") ] # take the directory names with one-level-deep package files package_names = [ nm.pkg_dir_to_pkg_name(line[0], repo.package_api) for line in package_paths if len(line) == 2 ] return sorted(set(package_names)) def diff_packages(rev1: str, rev2: str, repo: "Repo") -> Tuple[Set[str], Set[str]]: """Compute packages lists for the two revisions and return a tuple containing all the packages in rev1 but not in rev2 and all the packages in rev2 but not in rev1.""" p1 = set(list_packages(rev1, repo)) p2 = set(list_packages(rev2, repo)) return p1.difference(p2), p2.difference(p1) def get_all_package_diffs(type: str, repo: "Repo", rev1="HEAD^1", rev2="HEAD") -> Set[str]: """Get packages changed, added, or removed (or any combination of those) since a commit. Arguments: type: String containing one or more of ``A``, ``R``, ``C``. rev1: Revision to compare against, default is ``"HEAD^"`` rev2: Revision to compare to rev1, default is ``"HEAD"`` """ lower_type = type.lower() if not re.match("^[arc]*$", lower_type): tty.die( f"Invalid change type: '{type}'. " "Can contain only A (added), R (removed), or C (changed)" ) removed, added = diff_packages(rev1, rev2, repo) git = GitExe(repo.packages_path) out = git("diff", "--relative", "--name-only", rev1, rev2).strip() lines = [] if not out else re.split(r"\s+", out) changed: Set[str] = set() for path in lines: dir_name, _, _ = path.partition("/") if not nm.valid_module_name(dir_name, repo.package_api): continue pkg_name = nm.pkg_dir_to_pkg_name(dir_name, repo.package_api) if pkg_name not in added and pkg_name not in removed: changed.add(pkg_name) packages: Set[str] = set() if "a" in lower_type: packages |= added if "r" in lower_type: packages |= removed if "c" in lower_type: packages |= changed return packages def add_package_to_git_stage(packages: List[str], repo: "Repo") -> None: """add a package to the git stage with ``git add``""" git = GitExe(repo.packages_path) for pkg_name in packages: filename = PATH.filename_for_package_name(pkg_name) if not os.path.isfile(filename): tty.die(f"No such package: {pkg_name}. Path does not exist:", filename) git("add", filename) def autospec(function): """Decorator that automatically converts the first argument of a function to a Spec. """ @functools.wraps(function) def converter(self, spec_like, *args, **kwargs): from spack.spec import Spec if not isinstance(spec_like, Spec): spec_like = Spec(spec_like) return function(self, spec_like, *args, **kwargs) return converter class SpackNamespace(types.ModuleType): """Allow lazy loading of modules.""" def __init__(self, namespace): super().__init__(namespace) self.__file__ = "(spack namespace)" self.__path__ = [] self.__name__ = namespace self.__package__ = namespace self.__modules = {} def __getattr__(self, name): """Getattr lazily loads modules if they're not already loaded.""" submodule = f"{self.__package__}.{name}" try: setattr(self, name, importlib.import_module(submodule)) except ImportError: msg = "'{0}' object has no attribute {1}" raise AttributeError(msg.format(type(self), name)) return getattr(self, name) @contextlib.contextmanager def _directory_fd(path: str) -> Generator[Optional[int], None, None]: if sys.platform == "win32": yield None return fd = os.open(path, os.O_RDONLY) try: yield fd finally: os.close(fd) class FastPackageChecker(Mapping[str, float]): """Cache that maps package names to the modification times of their ``package.py`` files. For each repository a cache is maintained at class level, and shared among all instances referring to it. Update of the global cache is done lazily during instance initialization.""" #: Global cache, reused by every instance _paths_cache: Dict[str, Dict[str, float]] = {} def __init__(self, packages_path: str, package_api: Tuple[int, int]) -> None: # The path of the repository managed by this instance self.packages_path = packages_path self.package_api = package_api # If the cache we need is not there yet, then build it appropriately if packages_path not in self._paths_cache: self._paths_cache[packages_path] = self._create_new_cache() #: Reference to the appropriate entry in the global cache self._packages_to_mtime = self._paths_cache[packages_path] def invalidate(self) -> None: """Regenerate cache for this checker.""" self._paths_cache[self.packages_path] = self._create_new_cache() self._packages_to_mtime = self._paths_cache[self.packages_path] def _create_new_cache(self) -> Dict[str, float]: """Create a new cache for packages in a repo. The implementation here should try to minimize filesystem calls. At the moment, it makes one stat call per package. This is reasonably fast, and avoids actually importing packages in Spack, which is slow.""" # Create a dictionary that will store the mapping between a # package name and its mtime cache: Dict[str, float] = {} # Don't use os.path.join in the loop cause it's slow and redundant. package_py_suffix = f"{os.path.sep}{package_file_name}" # Use a file descriptor for the packages directory to avoid repeated path resolution. with _directory_fd(self.packages_path) as fd, os.scandir(self.packages_path) as entries: for entry in entries: # Construct the file name from the directory if sys.platform == "win32": pkg_file = f"{entry.path}{package_py_suffix}" else: pkg_file = f"{entry.name}{package_py_suffix}" try: sinfo = os.stat(pkg_file, dir_fd=fd) except OSError as e: if e.errno in (errno.ENOENT, errno.ENOTDIR): # No package.py file here. continue elif e.errno == errno.EACCES: pkg_file = os.path.join(self.packages_path, entry.name, package_file_name) tty.warn(f"Can't read package file {pkg_file}.") continue raise # If it's not a file, skip it. if not stat.S_ISREG(sinfo.st_mode): continue # Only consider package.py files in directories that are valid module names under # the current package API if not nm.valid_module_name(entry.name, self.package_api): x, y = self.package_api pkg_file = os.path.join(self.packages_path, entry.name, package_file_name) tty.warn( f"Package {pkg_file} cannot be used because `{entry.name}` is not a valid " f"Spack package module name for Package API v{x}.{y}." ) continue # Store the mtime by package name. cache[nm.pkg_dir_to_pkg_name(entry.name, self.package_api)] = sinfo.st_mtime return cache def last_mtime(self) -> float: return max(self._packages_to_mtime.values()) def modified_since(self, since: float) -> List[str]: return [name for name, mtime in self._packages_to_mtime.items() if mtime > since] def __getitem__(self, item: str) -> float: return self._packages_to_mtime[item] def __iter__(self) -> Iterator[str]: return iter(self._packages_to_mtime) def __len__(self) -> int: return len(self._packages_to_mtime) class Indexer(metaclass=abc.ABCMeta): """Adaptor for indexes that need to be generated when repos are updated.""" def __init__(self, repository): self.repository = repository self.index = None def create(self): self.index = self._create() @abc.abstractmethod def _create(self): """Create an empty index and return it.""" def needs_update(self, pkg) -> bool: """Whether an update is needed when the package file hasn't changed. Returns: ``True`` iff this package needs its index updated. We already automatically update indexes when package files change, but other files (like patches) may change underneath the package file. This method can be used to check additional package-specific files whenever they're loaded, to tell the RepoIndex to update the index *just* for that package. """ return False @abc.abstractmethod def read(self, stream): """Read this index from a provided file object.""" @abc.abstractmethod def update(self, pkgs_fullname: Set[str]): """Update the index in memory with information about a package.""" @abc.abstractmethod def write(self, stream): """Write the index to a file object.""" class TagIndexer(Indexer): """Lifecycle methods for a TagIndex on a Repo.""" def _create(self) -> spack.tag.TagIndex: return spack.tag.TagIndex() def read(self, stream): self.index = spack.tag.TagIndex.from_json(stream) def update(self, pkgs_fullname: Set[str]): self.index.update_packages({p.split(".")[-1] for p in pkgs_fullname}, self.repository) def write(self, stream): self.index.to_json(stream) class ProviderIndexer(Indexer): """Lifecycle methods for virtual package providers.""" def _create(self) -> "spack.provider_index.ProviderIndex": return spack.provider_index.ProviderIndex(repository=self.repository) def read(self, stream): self.index = spack.provider_index.ProviderIndex.from_json(stream, self.repository) def update(self, pkgs_fullname: Set[str]): is_virtual = lambda name: ( not self.repository.exists(name) or self.repository.get_pkg_class(name).virtual ) non_virtual_pkgs_fullname = {p for p in pkgs_fullname if not is_virtual(p.split(".")[-1])} non_virtual_pkgs_names = {p.split(".")[-1] for p in non_virtual_pkgs_fullname} self.index.remove_providers(non_virtual_pkgs_names) self.index.update_packages(non_virtual_pkgs_fullname) def write(self, stream): self.index.to_json(stream) class PatchIndexer(Indexer): """Lifecycle methods for patch cache.""" def _create(self) -> spack.patch.PatchCache: return spack.patch.PatchCache(repository=self.repository) def needs_update(self): # TODO: patches can change under a package and we should handle # TODO: it, but we currently punt. This should be refactored to # TODO: check whether patches changed each time a package loads, # TODO: tell the RepoIndex to reindex them. return False def read(self, stream): self.index = spack.patch.PatchCache.from_json(stream, repository=self.repository) def write(self, stream): self.index.to_json(stream) def update(self, pkgs_fullname: Set[str]): self.index.update_packages(pkgs_fullname) class RepoIndex: """Container class that manages a set of Indexers for a Repo. This class is responsible for checking packages in a repository for updates (using ``FastPackageChecker``) and for regenerating indexes when they're needed. ``Indexers`` should be added to the ``RepoIndex`` using ``add_indexer(name, indexer)``, and they should support the interface defined by ``Indexer``, so that the ``RepoIndex`` can read, generate, and update stored indices. Generated indexes are accessed by name via ``__getitem__()``.""" def __init__( self, packages_path: str, package_checker: "Callable[[], FastPackageChecker]", namespace: str, cache: spack.util.file_cache.FileCache, ): self._get_checker = package_checker self._checker: Optional[FastPackageChecker] = None self.packages_path = packages_path if sys.platform == "win32": self.packages_path = spack.llnl.path.convert_to_posix_path(self.packages_path) self.namespace = namespace self.indexers: Dict[str, Indexer] = {} self.indexes: Dict[str, Any] = {} self.cache = cache #: Whether the indexes are up to date with the package repository. self.is_fresh = False @property def checker(self) -> FastPackageChecker: if self._checker is None: self._checker = self._get_checker() return self._checker def add_indexer(self, name: str, indexer: Indexer): """Add an indexer to the repo index. Arguments: name: name of this indexer indexer: object implementing the ``Indexer`` interface""" self.indexers[name] = indexer def __getitem__(self, name): """Get an up-to-date index with the specified name.""" return self.get_index(name, allow_stale=False) def get_index(self, name, allow_stale: bool = False): """Get the index with the specified name. The index will be updated if it is stale, unless allow_stale is True, in which case its contents are not validated against the package repository. When no cache is available, the index will be updated regardless of the value of allow_stale.""" indexer = self.indexers.get(name) if not indexer: raise KeyError("no such index: %s" % name) if name not in self.indexes or (not allow_stale and not self.is_fresh): self._build_all_indexes(allow_stale=allow_stale) return self.indexes[name] def _build_all_indexes(self, allow_stale: bool = False) -> None: """Build all the indexes at once. We regenerate *all* indexes whenever *any* index needs an update, because the main bottleneck here is loading all the packages. It can take tens of seconds to regenerate sequentially, and we'd rather only pay that cost once rather than on several invocations.""" is_fresh = True for name, indexer in self.indexers.items(): is_fresh &= self._update_index(name, indexer, allow_stale=allow_stale) self.is_fresh = is_fresh def _update_index(self, name: str, indexer: Indexer, allow_stale: bool = False) -> bool: """Determine which packages need an update, and update indexes. Returns true if the index is fresh.""" # Filename of the provider index cache (we assume they're all json) from spack.spec import SPECFILE_FORMAT_VERSION cache_filename = f"{name}/{self.namespace}-specfile_v{SPECFILE_FORMAT_VERSION}-index.json" with self.cache.read_transaction(cache_filename) as f: # Get the mtime of the cache if it exists, of -inf. index_mtime = os.fstat(f.fileno()).st_mtime if f is not None else -math.inf if f is not None and allow_stale: # Cache exists and caller accepts stale data: skip the expensive modified_since. indexer.read(f) self.indexes[name] = indexer.index return False needs_update = self.checker.modified_since(index_mtime) if f is not None and not needs_update: # Cache exists and is up to date. indexer.read(f) self.indexes[name] = indexer.index return True # Cache is missing or stale: acquire write lock and rebuild. with self.cache.write_transaction(cache_filename) as (old, new): old_mtime = os.fstat(old.fileno()).st_mtime if old is not None else -math.inf # Re-check in case another writer updated the index while we waited for the lock. if old_mtime != index_mtime: needs_update = self.checker.modified_since(old_mtime) indexer.read(old) if old is not None else indexer.create() indexer.update({f"{self.namespace}.{pkg_name}" for pkg_name in needs_update}) indexer.write(new) self.indexes[name] = indexer.index return True class RepoPath: """A RepoPath is a list of Repo instances that function as one. It functions exactly like a Repo, but it operates on the combined results of the Repos in its list instead of on a single package repository. """ def __init__(self, *repos: "Repo") -> None: self.repos: List[Repo] = [] self.by_namespace = nm.NamespaceTrie() self._provider_index: Optional[spack.provider_index.ProviderIndex] = None self._patch_index: Optional[spack.patch.PatchCache] = None self._index_is_fresh: bool = False self._tag_index: Optional[spack.tag.TagIndex] = None for repo in repos: self.put_last(repo) @staticmethod def from_descriptors( descriptors: "RepoDescriptors", cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None, ) -> "RepoPath": repo_path, errors = descriptors.construct(cache=cache, fetch=True, overrides=overrides) # Merely warn if package repositories from config could not be constructed. if errors: for path, error in errors.items(): tty.warn(f"Error constructing repository '{path}': {error}") return repo_path @staticmethod def from_config(config: spack.config.Configuration) -> "RepoPath": """Create a RepoPath from a configuration object.""" overrides = { pkg_name: data["package_attributes"] for pkg_name, data in config.get_config("packages").items() if pkg_name != "all" and "package_attributes" in data } return RepoPath.from_descriptors( descriptors=RepoDescriptors.from_config(lock=package_repository_lock(), config=config), cache=spack.caches.MISC_CACHE, overrides=overrides, ) def enable(self) -> None: """Set the relevant search paths for package module loading""" REPOS_FINDER.repo_path = self for p in reversed(self.python_paths()): if p not in sys.path: sys.path.insert(0, p) def disable(self) -> None: """Disable the search paths for package module loading""" if hasattr(REPOS_FINDER, "repo_path"): del REPOS_FINDER.repo_path for p in self.python_paths(): if p in sys.path: sys.path.remove(p) def ensure_unwrapped(self) -> "RepoPath": """Ensure we unwrap this object from any dynamic wrapper (like Singleton)""" return self def put_first(self, repo: Union["Repo", "RepoPath"]) -> None: """Add repo first in the search path.""" if isinstance(repo, RepoPath): for r in reversed(repo.repos): self.put_first(r) return self.repos.insert(0, repo) self.by_namespace[repo.namespace] = repo def put_last(self, repo): """Add repo last in the search path.""" if isinstance(repo, RepoPath): for r in repo.repos: self.put_last(r) return self.repos.append(repo) # don't mask any higher-precedence repos with same namespace if repo.namespace not in self.by_namespace: self.by_namespace[repo.namespace] = repo def remove(self, repo): """Remove a repo from the search path.""" if repo in self.repos: self.repos.remove(repo) def get_repo(self, namespace: str) -> "Repo": """Get a repository by namespace.""" if namespace not in self.by_namespace: raise UnknownNamespaceError(namespace) return self.by_namespace[namespace] def first_repo(self) -> Optional["Repo"]: """Get the first repo in precedence order.""" return self.repos[0] if self.repos else None @memoized def _all_package_names_set(self, include_virtuals) -> Set[str]: return {name for repo in self.repos for name in repo.all_package_names(include_virtuals)} @memoized def _all_package_names(self, include_virtuals: bool) -> List[str]: """Return all unique package names in all repositories.""" return sorted(self._all_package_names_set(include_virtuals), key=lambda n: n.lower()) def all_package_names(self, include_virtuals: bool = False) -> List[str]: return self._all_package_names(include_virtuals) def package_path(self, name: str) -> str: """Get path to package.py file for this repo.""" return self.repo_for_pkg(name).package_path(name) def all_package_paths(self) -> Generator[str, None, None]: for name in self.all_package_names(): yield self.package_path(name) def packages_with_tags(self, *tags: str, full: bool = False) -> Set[str]: """Returns a set of packages matching any of the tags in input. Args: full: if True the package names in the output are fully-qualified """ return { f"{repo.namespace}.{pkg}" if full else pkg for repo in self.repos for pkg in repo.packages_with_tags(*tags) } def all_package_classes(self) -> Generator[Type["spack.package_base.PackageBase"], None, None]: for name in self.all_package_names(): yield self.get_pkg_class(name) @property def provider_index(self) -> spack.provider_index.ProviderIndex: """Merged ProviderIndex from all Repos in the RepoPath.""" if self._provider_index is None: self._provider_index = spack.provider_index.ProviderIndex(repository=self) for repo in reversed(self.repos): self._provider_index.merge(repo.provider_index) return self._provider_index @property def tag_index(self) -> spack.tag.TagIndex: """Merged TagIndex from all Repos in the RepoPath.""" if self._tag_index is None: self._tag_index = spack.tag.TagIndex() for repo in reversed(self.repos): self._tag_index.merge(repo.tag_index) return self._tag_index def get_patch_index(self, allow_stale: bool = False) -> spack.patch.PatchCache: """Return the merged patch index for all repos in this path. Args: allow_stale: if True, return a possibly out-of-date index from cache files, avoiding filesystem calls to check whether the index is up to date. """ if self._patch_index is not None and (self._index_is_fresh or allow_stale): return self._patch_index index = spack.patch.PatchCache(repository=self) for repo in reversed(self.repos): index.update(repo.get_patch_index(allow_stale=allow_stale)) self._patch_index = index self._index_is_fresh = not allow_stale return self._patch_index def get_patches_for_package( self, sha256s: List[str], pkg_cls: Type["spack.package_base.PackageBase"] ) -> List["spack.patch.Patch"]: """Look up patches by sha256, trying stale cache first to avoid stat calls. Args: sha256s: ordered list of patch sha256 hashes pkg_cls: package class the patches belong to Returns: List of Patch objects in the same order as sha256s. Raises: spack.error.PatchLookupError: if a sha256 cannot be found even after a full rebuild. """ stale_index = self.get_patch_index(allow_stale=True) try: return [ stale_index.patch_for_package(sha256, pkg_cls, validate=True) for sha256 in sha256s ] except spack.error.PatchLookupError: pass current_index = self.get_patch_index(allow_stale=False) return [current_index.patch_for_package(sha256, pkg_cls) for sha256 in sha256s] def providers_for(self, virtual: Union[str, "spack.spec.Spec"]) -> List["spack.spec.Spec"]: all_packages = self._all_package_names_set(include_virtuals=False) providers = [ spec for spec in self.provider_index.providers_for(virtual) if spec.name in all_packages ] if not providers: raise UnknownPackageError(virtual if isinstance(virtual, str) else virtual.fullname) return providers @autospec def extensions_for( self, extendee_spec: "spack.spec.Spec" ) -> List["spack.package_base.PackageBase"]: from spack.spec import Spec return [ pkg_cls(Spec(pkg_cls.name)) for pkg_cls in self.all_package_classes() if pkg_cls(Spec(pkg_cls.name)).extends(extendee_spec) ] def last_mtime(self): """Time a package file in this repo was last updated.""" return max(repo.last_mtime() for repo in self.repos) def repo_for_pkg(self, spec: Union[str, "spack.spec.Spec"]) -> "Repo": """Given a spec, get the repository for its package.""" # We don't @_autospec this function b/c it's called very frequently # and we want to avoid parsing str's into Specs unnecessarily. from spack.spec import Spec if isinstance(spec, Spec): namespace = spec.namespace name = spec.name else: # handle strings directly for speed instead of @_autospec'ing namespace, _, name = spec.rpartition(".") # If the spec already has a namespace, then return the # corresponding repo if we know about it. if namespace: if namespace not in self.by_namespace: raise UnknownNamespaceError(namespace, name=name) return self.by_namespace[namespace] # If there's no namespace, search in the RepoPath. for repo in self.repos: if name in repo: return repo # If the package isn't in any repo, return the one with # highest precedence. This is for commands like `spack edit` # that can operate on packages that don't exist yet. selected = self.first_repo() if selected is None: raise UnknownPackageError(name) return selected def get(self, spec: "spack.spec.Spec") -> "spack.package_base.PackageBase": """Returns the package associated with the supplied spec.""" from spack.spec import Spec msg = "RepoPath.get can only be called on concrete specs" assert isinstance(spec, Spec) and spec.concrete, msg return self.repo_for_pkg(spec).get(spec) def python_paths(self) -> List[str]: """Return a list of all the Python paths in the repos.""" return [repo.python_path for repo in self.repos if repo.python_path] def get_pkg_class(self, pkg_name: str) -> Type["spack.package_base.PackageBase"]: """Find a class for the spec's package and return the class object.""" return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) @autospec def dump_provenance(self, spec, path): """Dump provenance information for a spec to a particular path. This dumps the package file and any associated patch files. Raises UnknownPackageError if not found. """ return self.repo_for_pkg(spec).dump_provenance(spec, path) def dirname_for_package_name(self, pkg_name: str) -> str: return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) def filename_for_package_name(self, pkg_name: str) -> str: return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name) def exists(self, pkg_name: str) -> bool: """Whether package with the give name exists in the path's repos. Note that virtual packages do not "exist". """ return any(repo.exists(pkg_name) for repo in self.repos) def is_virtual(self, pkg_name: str) -> bool: """Return True if the package with this name is virtual, False otherwise. This function use the provider index. If calling from a code block that is used to construct the provider index use the ``is_virtual_safe`` function. Args: pkg_name: name of the package we want to check """ have_name = bool(pkg_name) return have_name and pkg_name in self.provider_index def is_virtual_safe(self, pkg_name: str) -> bool: """Return True if the package with this name is virtual, False otherwise. This function doesn't use the provider index. Args: pkg_name: name of the package we want to check """ have_name = bool(pkg_name) return have_name and (not self.exists(pkg_name) or self.get_pkg_class(pkg_name).virtual) def __contains__(self, pkg_name): return self.exists(pkg_name) def marshal(self): return (self.repos,) @staticmethod def unmarshal(repos): return RepoPath(*repos) def __reduce__(self): return RepoPath.unmarshal, self.marshal() def _parse_package_api_version( config: Dict[str, Any], min_api: Tuple[int, int] = spack.min_package_api_version, max_api: Tuple[int, int] = spack.package_api_version, ) -> Tuple[int, int]: api = config.get("api") if api is None: package_api = (1, 0) else: if not isinstance(api, str): raise BadRepoError(f"Invalid Package API version '{api}'. Must be of the form vX.Y") api_match = _API_REGEX.match(api) if api_match is None: raise BadRepoError(f"Invalid Package API version '{api}'. Must be of the form vX.Y") package_api = (int(api_match.group(1)), int(api_match.group(2))) if min_api <= package_api <= max_api: return package_api min_str = ".".join(str(i) for i in min_api) max_str = ".".join(str(i) for i in max_api) curr_str = ".".join(str(i) for i in package_api) raise BadRepoVersionError( api, f"Package API v{curr_str} is not supported by this version of Spack (" f"must be between v{min_str} and v{max_str})", ) def _validate_and_normalize_subdir(subdir: Any, root: str, package_api: Tuple[int, int]) -> str: if not isinstance(subdir, str): raise BadRepoError(f"Invalid subdirectory '{subdir}' in '{root}'. Must be a string") if package_api < (2, 0): return subdir # In v1.x we did not validate subdir names if subdir in (".", ""): raise BadRepoError( f"Invalid subdirectory '{subdir}' in '{root}'. Use a symlink packages -> . instead" ) # Otherwise we expect a directory name (not path) that can be used as a Python module. if os.sep in subdir: raise BadRepoError( f"Invalid subdirectory '{subdir}' in '{root}'. Expected a directory name, not a path" ) if not nm.valid_module_name(subdir, package_api): raise BadRepoError( f"Invalid subdirectory '{subdir}' in '{root}'. Must be a valid Python module name" ) return subdir class Repo: """Class representing a package repository in the filesystem. Each package repository must have a top-level configuration file called ``repo.yaml``. It contains the following keys: ``namespace`` A Python namespace where the repository's packages should live. ``subdirectory`` An optional subdirectory name where packages are placed ``api`` A string of the form vX.Y that indicates the Package API version. The default is ``v1.0``. For the repo to be compatible with the current version of Spack, the version must be greater than or equal to :py:data:`spack.min_package_api_version` and less than or equal to :py:data:`spack.package_api_version`. """ namespace: str def __init__( self, root: str, *, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None, ) -> None: """Instantiate a package repository from a filesystem path. Args: root: the root directory of the repository cache: file cache associated with this repository overrides: dict mapping package name to class attribute overrides for that package """ # Root directory, containing _repo.yaml and package dirs # Allow roots to by spack-relative by starting with '$spack' self.root = spack.util.path.canonicalize_path(root) # check and raise BadRepoError on fail. def check(condition, msg): if not condition: raise BadRepoError(msg) # Validate repository layout. self.config_file = os.path.join(self.root, repo_config_name) check(os.path.isfile(self.config_file), f"No {repo_config_name} found in '{root}'") # Read configuration and validate namespace config = self._read_config() self.package_api = _parse_package_api_version(config) self.subdirectory = _validate_and_normalize_subdir( config.get("subdirectory", packages_dir_name), root, self.package_api ) self.packages_path = os.path.join(self.root, self.subdirectory) self.build_systems_path = os.path.join(self.root, "build_systems") check( os.path.isdir(self.packages_path), f"No directory '{self.subdirectory}' found in '{root}'", ) # The parent dir of spack_repo/ which should be added to sys.path for api v2.x self.python_path: Optional[str] = None if self.package_api < (2, 0): check( "namespace" in config, f"{os.path.join(root, repo_config_name)} must define a namespace.", ) self.namespace = config["namespace"] # Note: for Package API v1.x the namespace validation always had bugs, which won't be # fixed for compatibility reasons. The regex is missing "$" at the end, and it claims # to test for valid identifiers, but fails to split on `.` first. check( isinstance(self.namespace, str) and re.match(r"[a-zA-Z][a-zA-Z0-9_.]+", self.namespace), f"Invalid namespace '{self.namespace}' in repo '{self.root}'. " "Namespaces must be valid python identifiers separated by '.'", ) else: # From Package API v2.0 the namespace follows from the directory structure. check( f"{os.sep}spack_repo{os.sep}" in self.root, f"Invalid repository path '{self.root}'. Path must contain 'spack_repo{os.sep}'", ) derived_namespace = self.root.rpartition(f"spack_repo{os.sep}")[2].replace(os.sep, ".") if "namespace" in config: self.namespace = config["namespace"] check( isinstance(self.namespace, str) and self.namespace == derived_namespace, f"Namespace '{self.namespace}' should be {derived_namespace} or omitted in " f"{os.path.join(root, repo_config_name)}", ) else: self.namespace = derived_namespace # strip the namespace directories from the root path to get the python path # e.g. /my/pythonpath/spack_repo/x/y/z -> /my/pythonpath python_path = self.root for _ in self.namespace.split("."): python_path = os.path.dirname(python_path) self.python_path = os.path.dirname(python_path) # check that all subdirectories are valid module names check( all(nm.valid_module_name(x, self.package_api) for x in self.namespace.split(".")), f"Invalid namespace '{self.namespace}' in repo '{self.root}'", ) # Set up 'full_namespace' to include the super-namespace if self.package_api < (2, 0): self.full_namespace = f"{PKG_MODULE_PREFIX_V1}{self.namespace}" elif self.subdirectory == ".": self.full_namespace = f"{PKG_MODULE_PREFIX_V2}{self.namespace}" else: self.full_namespace = f"{PKG_MODULE_PREFIX_V2}{self.namespace}.{self.subdirectory}" # Keep name components around for checking prefixes. self._names = self.full_namespace.split(".") # Class attribute overrides by package name self.overrides = overrides or {} # Maps that goes from package name to corresponding file stat self._fast_package_checker: Optional[FastPackageChecker] = None # Indexes for this repository, computed lazily self._repo_index: Optional[RepoIndex] = None self._cache = cache @property def package_api_str(self) -> str: return f"v{self.package_api[0]}.{self.package_api[1]}" def real_name(self, import_name: str) -> Optional[str]: """Allow users to import Spack packages using Python identifiers. In Package API v1.x, there was no canonical module name for a package, and package's dir was not necessarily a valid Python module name. For that case we have to guess the actual package directory. From Package API v2.0 there is a one-to-one mapping between Spack package names and Python module names, so there is no guessing. For Package API v1.x we support the following one-to-many mappings: * ``num3proxy`` -> ``3proxy`` * ``foo_bar`` -> ``foo_bar``, ``foo-bar`` * ``foo_bar_baz`` -> ``foo_bar_baz``, ``foo-bar-baz``, ``foo_bar-baz``, ``foo-bar_baz`` """ if self.package_api >= (2, 0): if nm.pkg_dir_to_pkg_name(import_name, package_api=self.package_api) in self: return import_name return None if import_name in self: return import_name # For v1 generate the possible package names from a module name, and return the first # package name that exists in this repo. options = nm.possible_spack_module_names(import_name) try: options.remove(import_name) except ValueError: pass for name in options: if name in self: return name return None def is_prefix(self, fullname: str) -> bool: """True if fullname is a prefix of this Repo's namespace.""" parts = fullname.split(".") return self._names[: len(parts)] == parts def _read_config(self) -> Dict[str, Any]: """Check for a YAML config file in this db's root directory.""" try: with open(self.config_file, encoding="utf-8") as reponame_file: yaml_data = syaml.load(reponame_file) if ( not yaml_data or "repo" not in yaml_data or not isinstance(yaml_data["repo"], dict) ): tty.die(f"Invalid {repo_config_name} in repository {self.root}") return yaml_data["repo"] except OSError: tty.die(f"Error reading {self.config_file} when opening {self.root}") def get(self, spec: "spack.spec.Spec") -> "spack.package_base.PackageBase": """Returns the package associated with the supplied spec.""" from spack.spec import Spec msg = "Repo.get can only be called on concrete specs" assert isinstance(spec, Spec) and spec.concrete, msg # NOTE: we only check whether the package is None here, not whether it # actually exists, because we have to load it anyway, and that ends up # checking for existence. We avoid constructing FastPackageChecker, # which will stat all packages. if not spec.name: raise UnknownPackageError(None, self) if spec.namespace and spec.namespace != self.namespace: raise UnknownPackageError(spec.name, self.namespace) package_class = self.get_pkg_class(spec.name) try: return package_class(spec) except spack.error.SpackError: # pass these through as their error messages will be fine. raise except Exception as e: # Make sure other errors in constructors hit the error # handler by wrapping them tty.debug(e) raise FailedConstructorError(spec.fullname, *sys.exc_info()) from e @autospec def dump_provenance(self, spec: "spack.spec.Spec", path: str) -> None: """Dump provenance information for a spec to a particular path. This dumps the package file and any associated patch files. Raises UnknownPackageError if not found. """ if spec.namespace and spec.namespace != self.namespace: raise UnknownPackageError( f"Repository {self.namespace} does not contain package {spec.fullname}." ) package_path = self.filename_for_package_name(spec.name) if not os.path.exists(package_path): # Spec has no files (e.g., package, patches) to copy tty.debug(f"{spec.name} does not have a package to dump") return # Install patch files needed by the (concrete) package. fs.mkdirp(path) if spec.concrete: for patch in itertools.chain.from_iterable(spec.package.patches.values()): if patch.path: if os.path.exists(patch.path): fs.install(patch.path, path) else: warnings.warn(f"Patch file did not exist: {patch.path}") # Install the package.py file itself. fs.install(self.filename_for_package_name(spec.name), path) @property def index(self) -> RepoIndex: """Construct the index for this repo lazily.""" if self._repo_index is None: self._repo_index = RepoIndex( self.packages_path, lambda: self._pkg_checker, self.namespace, cache=self._cache ) self._repo_index.add_indexer("providers", ProviderIndexer(self)) self._repo_index.add_indexer("tags", TagIndexer(self)) self._repo_index.add_indexer("patches", PatchIndexer(self)) return self._repo_index @property def provider_index(self) -> spack.provider_index.ProviderIndex: """A fresh provider index with names *specific* to this repo.""" return self.index["providers"] @property def tag_index(self) -> spack.tag.TagIndex: """Fresh index of tags and which packages they're defined on.""" return self.index["tags"] def get_patch_index(self, allow_stale: bool = False) -> spack.patch.PatchCache: """Index of patches and packages they're defined on. Set allow_stale is True to bypass cache validation and return a potentially stale index.""" return self.index.get_index("patches", allow_stale=allow_stale) def providers_for(self, virtual: Union[str, "spack.spec.Spec"]) -> List["spack.spec.Spec"]: providers = self.provider_index.providers_for(virtual) if not providers: raise UnknownPackageError(virtual if isinstance(virtual, str) else virtual.fullname) return providers @autospec def extensions_for( self, extendee_spec: "spack.spec.Spec" ) -> List["spack.package_base.PackageBase"]: from spack.spec import Spec result = [pkg_cls(Spec(pkg_cls.name)) for pkg_cls in self.all_package_classes()] return [x for x in result if x.extends(extendee_spec)] def dirname_for_package_name(self, pkg_name: str) -> str: """Given a package name, get the directory containing its package.py file.""" _, unqualified_name = self.partition_package_name(pkg_name) return os.path.join( self.packages_path, nm.pkg_name_to_pkg_dir(unqualified_name, self.package_api) ) def filename_for_package_name(self, pkg_name: str) -> str: """Get the filename for the module we should load for a particular package. Packages for a Repo live in ``$root//package.py`` This will return a proper package.py path even if the package doesn't exist yet, so callers will need to ensure the package exists before importing. """ pkg_dir = self.dirname_for_package_name(pkg_name) return os.path.join(pkg_dir, package_file_name) @property def _pkg_checker(self) -> FastPackageChecker: if self._fast_package_checker is None: self._fast_package_checker = FastPackageChecker(self.packages_path, self.package_api) return self._fast_package_checker def all_package_names(self, include_virtuals: bool = False) -> List[str]: """Returns a sorted list of all package names in the Repo.""" names = sorted(self._pkg_checker.keys()) if include_virtuals: return names return [x for x in names if not self.is_virtual(x)] def package_path(self, name: str) -> str: """Get path to package.py file for this repo.""" return os.path.join( self.packages_path, nm.pkg_name_to_pkg_dir(name, self.package_api), package_file_name ) def all_package_paths(self) -> Generator[str, None, None]: for name in self.all_package_names(): yield self.package_path(name) def packages_with_tags(self, *tags: str) -> Set[str]: v = set(self.all_package_names()) for tag in tags: v.intersection_update(self.tag_index.get_packages(tag.lower())) return v def all_package_classes(self) -> Generator[Type["spack.package_base.PackageBase"], None, None]: """Iterator over all package *classes* in the repository. Use this with care, because loading packages is slow. """ for name in self.all_package_names(): yield self.get_pkg_class(name) def exists(self, pkg_name: str) -> bool: """Whether a package with the supplied name exists.""" if pkg_name is None: return False # if the FastPackageChecker is already constructed, use it if self._fast_package_checker: return pkg_name in self._pkg_checker # if not, check for the package.py file path = self.filename_for_package_name(pkg_name) return os.path.exists(path) def last_mtime(self): """Time a package file in this repo was last updated.""" return self._pkg_checker.last_mtime() def is_virtual(self, pkg_name: str) -> bool: """Return True if the package with this name is virtual, False otherwise. This function use the provider index. If calling from a code block that is used to construct the provider index use the ``is_virtual_safe`` function. """ return pkg_name in self.provider_index def is_virtual_safe(self, pkg_name: str) -> bool: """Return True if the package with this name is virtual, False otherwise. This function doesn't use the provider index. """ return not self.exists(pkg_name) or self.get_pkg_class(pkg_name).virtual def get_pkg_class(self, pkg_name: str) -> Type["spack.package_base.PackageBase"]: """Get the class for the package out of its module. First loads (or fetches from cache) a module for the package. Then extracts the package class from the module according to Spack's naming convention. """ _, pkg_name = self.partition_package_name(pkg_name) fullname = f"{self.full_namespace}.{nm.pkg_name_to_pkg_dir(pkg_name, self.package_api)}" if self.package_api >= (2, 0): fullname += ".package" class_name = nm.pkg_name_to_class_name(pkg_name) if not self.exists(pkg_name): raise UnknownPackageError(fullname, self) try: if self.python_path: sys.path.insert(0, self.python_path) module = importlib.import_module(fullname) except Exception as e: msg = f"cannot load package '{pkg_name}' from the '{self.namespace}' repository: {e}" raise RepoError(msg) from e finally: if self.python_path: sys.path.remove(self.python_path) cls = getattr(module, class_name) if not isinstance(cls, type): tty.die(f"{pkg_name}.{class_name} is not a class") # Early exit if no overrides to apply or undo if ( not self.overrides.get(pkg_name) and not hasattr(cls, "overridden_attrs") and not hasattr(cls, "attrs_exclusively_from_config") ): return cls def defining_class(myclass, name): return next((c for c in myclass.__mro__ if name in c.__dict__), None) # Clear any prior changes to class attributes in case the class was loaded from the # same repo, but with different overrides overridden_attrs = getattr(cls, "overridden_attrs", {}) attrs_exclusively_from_config = getattr(cls, "attrs_exclusively_from_config", []) defclass_attrs = defining_class(cls, "overridden_attrs") defclass_exclusively_from_config = defining_class(cls, "attrs_exclusively_from_config") for key, val in overridden_attrs.items(): setattr(defclass_attrs, key, val) for key in attrs_exclusively_from_config: delattr(defclass_exclusively_from_config, key) # Keep track of every class attribute that is overridden: if different overrides # dictionaries are used on the same physical repo, we make sure to restore the original # config values new_overridden_attrs = {} new_attrs_exclusively_from_config = set() for key, val in self.overrides.get(pkg_name, {}).items(): if hasattr(cls, key): new_overridden_attrs[key] = getattr(cls, key) else: new_attrs_exclusively_from_config.add(key) setattr(cls, key, val) if new_overridden_attrs: setattr(cls, "overridden_attrs", dict(new_overridden_attrs)) elif hasattr(cls, "overridden_attrs"): delattr(defclass_attrs, "overridden_attrs") if new_attrs_exclusively_from_config: setattr(cls, "attrs_exclusively_from_config", new_attrs_exclusively_from_config) elif hasattr(cls, "attrs_exclusively_from_config"): delattr(defclass_exclusively_from_config, "attrs_exclusively_from_config") return cls def partition_package_name(self, pkg_name: str) -> Tuple[str, str]: namespace, pkg_name = partition_package_name(pkg_name) if namespace and (namespace != self.namespace): raise InvalidNamespaceError( f"Invalid namespace for the '{self.namespace}' repo: {namespace}" ) return namespace, pkg_name def __str__(self) -> str: return f"Repo '{self.namespace}' at {self.root}" def __repr__(self) -> str: return self.__str__() def __contains__(self, pkg_name: str) -> bool: return self.exists(pkg_name) @staticmethod def unmarshal(root, cache, overrides): """Helper method to unmarshal keyword arguments""" return Repo(root, cache=cache, overrides=overrides) def marshal(self): cache = self._cache if isinstance(cache, Singleton): cache = cache.instance return self.root, cache, self.overrides def __reduce__(self): return Repo.unmarshal, self.marshal() RepoType = Union[Repo, RepoPath] def partition_package_name(pkg_name: str) -> Tuple[str, str]: """Given a package name that might be fully-qualified, returns the namespace part, if present and the unqualified package name. If the package name is unqualified, the namespace is an empty string. Args: pkg_name: a package name, either unqualified like ``llvm``, or fully-qualified, like ``builtin.llvm`` """ namespace, _, pkg_name = pkg_name.rpartition(".") return namespace, pkg_name def get_repo_yaml_dir( root: str, namespace: Optional[str], package_api: Tuple[int, int] ) -> Tuple[str, str]: """Returns the directory where repo.yaml is located and the effective namespace.""" if package_api < (2, 0): namespace = namespace or os.path.basename(root) # This ad-hoc regex is left for historical reasons, and should not have a breaking change. if not re.match(r"\w[\.\w-]*", namespace): raise InvalidNamespaceError(f"'{namespace}' is not a valid namespace.") return root, namespace # Package API v2 has /spack_repo// structure and requires a namespace if namespace is None: raise InvalidNamespaceError("Namespace must be provided.") # if namespace has dots those translate to subdirs of further namespace packages. namespace_components = namespace.split(".") if not all(nm.valid_module_name(n, package_api=package_api) for n in namespace_components): raise InvalidNamespaceError(f"'{namespace}' is not a valid namespace." % namespace) return os.path.join(root, "spack_repo", *namespace_components), namespace def create_repo( root, namespace: Optional[str] = None, subdir: str = packages_dir_name, package_api: Tuple[int, int] = spack.package_api_version, ) -> Tuple[str, str]: """Create a new repository in root with the specified namespace. If the namespace is not provided, use basename of root. Return the canonicalized path and namespace of the created repository. """ root = spack.util.path.canonicalize_path(root) repo_yaml_dir, namespace = get_repo_yaml_dir(os.path.abspath(root), namespace, package_api) existed = True try: dir_entry = next(os.scandir(repo_yaml_dir), None) except OSError as e: if e.errno == errno.ENOENT: existed = False dir_entry = None else: raise BadRepoError(f"Cannot create new repo in {root}: {e}") if dir_entry is not None: raise BadRepoError(f"Cannot create new repo in {root}: directory is not empty.") config_path = os.path.join(repo_yaml_dir, repo_config_name) subdir = _validate_and_normalize_subdir(subdir, root, package_api) packages_path = os.path.join(repo_yaml_dir, subdir) try: fs.mkdirp(packages_path) with open(config_path, "w", encoding="utf-8") as config: config.write("repo:\n") config.write(f" namespace: '{namespace}'\n") if subdir != packages_dir_name: config.write(f" subdirectory: '{subdir}'\n") x, y = package_api config.write(f" api: v{x}.{y}\n") except OSError as e: # try to clean up. if existed: shutil.rmtree(config_path, ignore_errors=True) shutil.rmtree(packages_path, ignore_errors=True) else: shutil.rmtree(root, ignore_errors=True) raise BadRepoError( "Failed to create new repository in %s." % root, "Caused by %s: %s" % (type(e), e) ) from e return repo_yaml_dir, namespace def from_path(path: str) -> Repo: """Constructs a Repo using global misc cache.""" return Repo(path, cache=spack.caches.MISC_CACHE) MaybeExecutable = Optional[spack.util.executable.Executable] class RepoDescriptor: """Abstract base class for repository data.""" def __init__(self, name: Optional[str]) -> None: self.name = name @property def _maybe_name(self) -> str: """Return the name if it exists, otherwise an empty string.""" return f"{self.name}: " if self.name else "" def initialize(self, fetch: bool = True, git: MaybeExecutable = None) -> None: return None def update(self, git: MaybeExecutable = None, remote: str = "origin") -> None: return None def construct( self, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None ) -> Dict[str, Union[Repo, Exception]]: """Construct Repo instances from the descriptor.""" raise RuntimeError("construct() must be implemented in subclasses") class LocalRepoDescriptor(RepoDescriptor): def __init__(self, name: Optional[str], path: str) -> None: super().__init__(name) self.path = path def __repr__(self) -> str: return f"{self.__class__.__name__}(name={self.name!r}, path={self.path!r})" def construct( self, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None ) -> Dict[str, Union[Repo, Exception]]: try: return {self.path: Repo(self.path, cache=cache, overrides=overrides)} except RepoError as e: return {self.path: e} class RemoteRepoDescriptor(RepoDescriptor): def __init__( self, *, name: Optional[str], repository: str, branch: Optional[str], commit: Optional[str], tag: Optional[str], destination: str, relative_paths: Optional[List[str]], lock: spack.util.lock.Lock, ) -> None: super().__init__(name) self.repository = repository self.branch = branch self.commit = commit self.tag = tag self.destination = destination self.relative_paths = relative_paths self.error: Optional[str] = None self.write_transaction = spack.util.lock.WriteTransaction(lock) self.read_transaction = spack.util.lock.ReadTransaction(lock) def _fetched(self) -> bool: """Check if the repository has been fetched by looking for the .git directory or file (when a submodule).""" return os.path.exists(os.path.join(self.destination, ".git")) def fetched(self) -> bool: with self.read_transaction: return self._fetched() return False def get_commit(self, git: MaybeExecutable = None): git = git or spack.util.git.git(required=True) with self.read_transaction: if not self._fetched(): return None with fs.working_dir(self.destination): return git("rev-parse", "HEAD", output=str).strip() def _clone_or_pull( self, git: spack.util.executable.Executable, update: bool = False, remote: str = "origin", depth: Optional[int] = None, ) -> None: with self.write_transaction: try: with fs.working_dir(self.destination, create=True): # do not fetch if the package repository was fetched by another # process while we were waiting for the lock fetched = self._fetched() if fetched and not update: self.read_index_file() return # If depth is not provided, default to: # 1. The first time the repo is loaded, download a partial clone. # This speeds up CI/CD and other cases where the user never # updates the repository. # 2. When *updating* an already cloned copy of the repository, # perform a full fetch (unshallowing the repo if necessary) to # optimize for full history. if depth is None and not fetched: depth = 2 # setup the repository if it does not exist if not fetched: spack.util.git.init_git_repo(self.repository, remote=remote, git_exe=git) # determine the default branch from ls-remote # (if no branch, tag, or commit is specified) if not (self.commit or self.tag or self.branch): # Get HEAD and all branches. On more recent versions of git, this can # be done with a single call to `git ls-remote --symref remote HEAD`. refs = git("ls-remote", remote, "HEAD", "refs/heads/*", output=str) head_match = re.search(r"^([0-9a-f]+)\s+HEAD$", refs, re.MULTILINE) if not head_match: self.error = f"Unable to locate HEAD for {self.repository}" return head_sha = head_match.group(1) # Find the first branch that matches this SHA branch_match = re.search( rf"^{re.escape(head_sha)}\s+refs/heads/(\S+)$", refs, re.MULTILINE ) if not branch_match: self.error = ( f"Unable to locate a default branch for {self.repository}" ) return self.branch = branch_match.group(1) # determine the branch and remote if no config values exist elif not (self.commit or self.tag or self.branch): self.branch = git("rev-parse", "--abbrev-ref", "HEAD", output=str).strip() remote = git("config", f"branch.{self.branch}.remote", output=str).strip() if self.commit: spack.util.git.pull_checkout_commit( self.commit, remote=remote, depth=depth, git_exe=git ) elif self.tag: spack.util.git.pull_checkout_tag( self.tag, remote=remote, depth=depth, git_exe=git ) elif self.branch: # if the branch already exists we should use the # previously configured remote try: output = git("config", f"branch.{self.branch}.remote", output=str) remote = output.strip() except spack.util.executable.ProcessError: pass spack.util.git.pull_checkout_branch( self.branch, remote=remote, depth=depth, git_exe=git ) except spack.util.executable.ProcessError: self.error = f"Failed to {'update' if update else 'clone'} repository {self.name}" return self.read_index_file() def update(self, git: MaybeExecutable = None, remote: str = "origin") -> None: if git is None: raise RepoError("Git executable not found") self._clone_or_pull(git, update=True, remote=remote) if self.error: raise RepoError(self.error) def initialize(self, fetch: bool = True, git: MaybeExecutable = None) -> None: """Clone the remote repository if it has not been fetched yet and read the index file if necessary.""" if self.fetched(): self.read_index_file() return if not fetch: return if not git: self.error = "Git executable not found" return self._clone_or_pull(git) def read_index_file(self) -> None: if self.relative_paths is not None: return repo_index_file = os.path.join(self.destination, SPACK_REPO_INDEX_FILE_NAME) try: with open(repo_index_file, encoding="utf-8") as f: index_data = syaml.load(f) assert "repo_index" in index_data, "missing 'repo_index' key" repo_index = index_data["repo_index"] assert isinstance(repo_index, dict), "'repo_index' must be a dictionary" assert "paths" in repo_index, "missing 'paths' key in 'repo_index'" sub_paths = repo_index["paths"] assert isinstance(sub_paths, list), "'paths' under 'repo_index' must be a list" except (OSError, syaml.SpackYAMLError, AssertionError) as e: self.error = f"failed to read {repo_index_file}: {e}" return # validate that this is a list of relative paths. if not isinstance(sub_paths, list) or not all(isinstance(p, str) for p in sub_paths): self.error = "invalid repo index file format: expected a list of relative paths." return self.relative_paths = sub_paths def __repr__(self): return ( f"RemoteRepoDescriptor(name={self.name!r}, " f"repository={self.repository!r}, " f"destination={self.destination!r}, " f"relative_paths={self.relative_paths!r})" ) def construct( self, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None ) -> Dict[str, Union[Repo, Exception]]: if self.error: return {self.destination: Exception(self.error)} repos: Dict[str, Union[Repo, Exception]] = {} for subpath in self.relative_paths or []: if os.path.isabs(subpath): repos[self.destination] = Exception( f"Repository subpath '{subpath}' must be relative" ) continue path = os.path.join(self.destination, subpath) try: repos[path] = Repo(path, cache=cache, overrides=overrides) except RepoError as e: repos[path] = e return repos class BrokenRepoDescriptor(RepoDescriptor): """A descriptor for a broken repository, used to indicate errors in the configuration that aren't fatal until the repository is used.""" def __init__(self, name: Optional[str], error: str) -> None: super().__init__(name) self.error = error def initialize( self, fetch: bool = True, git: Optional[spack.util.executable.Executable] = None ) -> None: pass def construct( self, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None ) -> Dict[str, Union[Repo, Exception]]: return {self.name or "": Exception(self.error)} class RepoDescriptors(Mapping[str, RepoDescriptor]): """A collection of repository descriptors.""" def __init__(self, descriptors: Dict[str, RepoDescriptor]) -> None: self.descriptors = descriptors def __getitem__(self, name: str) -> RepoDescriptor: return self.descriptors[name] def __iter__(self): return iter(self.descriptors.keys()) def __len__(self): return len(self.descriptors) def __contains__(self, name) -> bool: return name in self.descriptors def __repr__(self): return f"RepoDescriptors({self.descriptors!r})" @staticmethod def from_config( lock: spack.util.lock.Lock, config: spack.config.Configuration, scope=None ) -> "RepoDescriptors": return RepoDescriptors( { name: parse_config_descriptor(name, cfg, lock) for name, cfg in config.get_config("repos", scope=scope).items() } ) def construct( self, cache: spack.util.file_cache.FileCache, fetch: bool = True, find_git: Callable[[], MaybeExecutable] = lambda: spack.util.git.git(required=True), overrides: Optional[Dict[str, Any]] = None, ) -> Tuple[RepoPath, Dict[str, Exception]]: """Construct a RepoPath from the descriptors. If init is True, initialize all remote repositories that have not been fetched yet. Returns: A tuple containing a RepoPath instance with all constructed Repos and a dictionary mapping paths to exceptions that occurred during construction. """ repos: List[Repo] = [] errors: Dict[str, Exception] = {} git: MaybeExecutable = None for descriptor in self.descriptors.values(): if fetch and isinstance(descriptor, RemoteRepoDescriptor): git = git or find_git() descriptor.initialize(fetch=True, git=git) else: descriptor.initialize(fetch=False) for path, result in descriptor.construct(cache=cache, overrides=overrides).items(): if isinstance(result, Repo): repos.append(result) else: errors[path] = result return RepoPath(*repos), errors def parse_config_descriptor( name: Optional[str], descriptor: Any, lock: spack.util.lock.Lock ) -> RepoDescriptor: """Parse a repository descriptor from validated configuration. This does not instantiate Repo objects, but merely turns the config into a more useful RepoDescriptor instance. Args: name: the name of the repository, used for error messages descriptor: the configuration for the repository, which can be a string (local path), or a dictionary with ``git`` key containing git URL and other options. Returns: A RepoDescriptor instance, either LocalRepoDescriptor or RemoteRepoDescriptor. Raises: BadRepoError: if the descriptor is invalid or cannot be parsed. RuntimeError: if the descriptor is of an unexpected type. """ if isinstance(descriptor, str): return LocalRepoDescriptor(name, spack.util.path.canonicalize_path(descriptor)) # Should be the case due to config validation. assert isinstance(descriptor, dict), "Repository descriptor must be a string or a dictionary" # Configuration validation works per scope, and we want to allow overriding e.g. destination # in user config without the user having to repeat the `git` key and value again. This is a # hard error, since config validation is a hard error. if "git" not in descriptor: raise RuntimeError( f"Invalid configuration for repository '{name}': {descriptor!r}. A `git` attribute is " "required for remote repositories." ) repository = descriptor["git"] assert isinstance(repository, str), "Package repository git URL must be a string" destination = descriptor.get("destination", None) if destination is None: # use a default destination dir_name = spack.util.hash.b32_hash(repository)[-7:] destination = os.path.join(spack.paths.package_repos_path, dir_name) else: destination = spack.util.path.canonicalize_path(destination) return RemoteRepoDescriptor( name=name, repository=repository, branch=descriptor.get("branch"), commit=descriptor.get("commit"), tag=descriptor.get("tag"), destination=destination, relative_paths=descriptor.get("paths"), lock=lock, ) def create_or_construct( root: str, namespace: Optional[str] = None, package_api: Tuple[int, int] = spack.package_api_version, ) -> Repo: """Create a repository, or just return a Repo if it already exists.""" repo_yaml_dir, _ = get_repo_yaml_dir(root, namespace, package_api) if not os.path.exists(repo_yaml_dir): fs.mkdirp(root) create_repo(root, namespace=namespace, package_api=package_api) return from_path(repo_yaml_dir) def create_and_enable(config: spack.config.Configuration) -> RepoPath: """Immediately call enable() on the created RepoPath instance.""" repo_path = RepoPath.from_config(config) repo_path.enable() return repo_path #: Global package repository instance. PATH = cast(RepoPath, Singleton(lambda: create_and_enable(spack.config.CONFIG))) # Add the finder to sys.meta_path REPOS_FINDER = ReposFinder() sys.meta_path.append(REPOS_FINDER) def all_package_names(include_virtuals=False): """Convenience wrapper around ``spack.repo.all_package_names()``.""" return PATH.all_package_names(include_virtuals) @contextlib.contextmanager def use_repositories( *paths_and_repos: Union[str, Repo], override: bool = True ) -> Generator[RepoPath, None, None]: """Use the repositories passed as arguments within the context manager. Args: *paths_and_repos: paths to the repositories to be used, or already constructed Repo objects override: if True use only the repositories passed as input, if False add them to the top of the list of current repositories. Returns: Corresponding RepoPath object """ paths = {getattr(x, "root", x): getattr(x, "root", x) for x in paths_and_repos} scope_name = f"use-repo-{uuid.uuid4()}" repos_key = "repos:" if override else "repos" spack.config.CONFIG.push_scope( spack.config.InternalConfigScope(name=scope_name, data={repos_key: paths}) ) old_repo, new_repo = PATH, RepoPath.from_config(spack.config.CONFIG) old_repo.disable() enable_repo(new_repo) try: yield new_repo finally: spack.config.CONFIG.remove_scope(scope_name=scope_name) new_repo.disable() enable_repo(old_repo) def enable_repo(repo_path: RepoPath) -> None: """Set the global package repository and make them available in module search paths.""" global PATH PATH = repo_path PATH.enable() class RepoError(spack.error.SpackError): """Superclass for repository-related errors.""" class NoRepoConfiguredError(RepoError): """Raised when there are no repositories configured.""" class InvalidNamespaceError(RepoError): """Raised when an invalid namespace is encountered.""" class BadRepoError(RepoError): """Raised when repo layout is invalid.""" class BadRepoVersionError(BadRepoError): """Raised when repo API version is too high or too low for Spack.""" def __init__(self, api, *args, **kwargs): self.api = api super().__init__(*args, **kwargs) class UnknownEntityError(RepoError): """Raised when we encounter a package spack doesn't have.""" class UnknownPackageError(UnknownEntityError): """Raised when we encounter a package spack doesn't have.""" def __init__( self, name, repo: Optional[Union[Repo, RepoPath, str]] = None, *, get_close_matches=difflib.get_close_matches, ): msg = "Attempting to retrieve anonymous package." long_msg = None if name: msg = f"Package '{name}' not found" if repo: if isinstance(repo, Repo): msg += f" in repository '{repo.root}'" elif isinstance(repo, str): msg += f" in repository '{repo}'" # Special handling for specs that may have been intended as # filenames: prompt the user to ask whether they intended to write # './'. if name.endswith(".yaml"): long_msg = "Did you mean to specify a filename with './{0}'?" long_msg = long_msg.format(name) else: long_msg = "Use 'spack create' to create a new package." if not repo: repo = PATH.ensure_unwrapped() # We need to compare the base package name pkg_name = name_from_fullname(name) similar = [] if isinstance(repo, (Repo, RepoPath)): try: similar = get_close_matches(pkg_name, repo.all_package_names()) except Exception: pass if 1 <= len(similar) <= 5: long_msg += "\n\nDid you mean one of the following packages?\n " long_msg += "\n ".join(similar) super().__init__(msg, long_msg) self.name = name class UnknownNamespaceError(UnknownEntityError): """Raised when we encounter an unknown namespace""" def __init__(self, namespace, name=None): msg, long_msg = f"Unknown namespace: {namespace}", None if name == "yaml": long_msg = f"Did you mean to specify a filename with './{namespace}.{name}'?" super().__init__(msg, long_msg) class FailedConstructorError(RepoError): """Raised when a package's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb): super().__init__( "Class constructor failed for package '%s'." % name, "\nCaused by:\n" + ("%s: %s\n" % (exc_type.__name__, exc_obj)) + "".join(traceback.format_tb(exc_tb)), ) self.name = name ================================================ FILE: lib/spack/spack/repo_migrate.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import ast import difflib import os import re import shutil import sys from typing import IO, Dict, List, Optional, Set, Tuple import spack.repo import spack.util.naming import spack.util.spack_yaml def _same_contents(f: str, g: str) -> bool: """Return True if the files have the same contents.""" try: with open(f, "rb") as f1, open(g, "rb") as f2: while True: b1 = f1.read(4096) b2 = f2.read(4096) if b1 != b2: return False if not b1 and not b2: break return True except OSError: return False def migrate_v1_to_v2( repo: spack.repo.Repo, *, patch_file: Optional[IO[bytes]], err: IO[str] = sys.stderr ) -> Tuple[bool, Optional[spack.repo.Repo]]: """To upgrade a repo from Package API v1 to v2 we need to: 1. ensure ``spack_repo/`` parent dirs to the ``repo.yaml`` file. 2. rename /package.py to /package.py. 3. bump the version in ``repo.yaml``. """ if not (1, 0) <= repo.package_api < (2, 0): raise RuntimeError(f"Cannot upgrade from {repo.package_api_str} to v2.0") with open(os.path.join(repo.root, "repo.yaml"), encoding="utf-8") as f: updated_config = spack.util.spack_yaml.load(f) updated_config["repo"]["api"] = "v2.0" namespace = repo.namespace.split(".") if not all( spack.util.naming.valid_module_name(part, package_api=(2, 0)) for part in namespace ): print( f"Cannot upgrade from v1 to v2, because the namespace '{repo.namespace}' is not a " "valid Python module", file=err, ) return False, None try: subdirectory = spack.repo._validate_and_normalize_subdir( repo.subdirectory, repo.root, package_api=(2, 0) ) except spack.repo.BadRepoError: print( f"Cannot upgrade from v1 to v2, because the subdirectory '{repo.subdirectory}' is not " "a valid Python module", file=err, ) return False, None new_root = os.path.join(repo.root, "spack_repo", *namespace) ino_to_relpath: Dict[int, str] = {} symlink_to_ino: Dict[str, int] = {} prefix_len = len(repo.root) + len(os.sep) rename: Dict[str, str] = {} dirs_to_create: List[str] = [] files_to_copy: List[str] = [] errors = False stack: List[Tuple[str, int]] = [(repo.packages_path, 0)] while stack: path, depth = stack.pop() try: entries = os.scandir(path) except OSError: continue for entry in entries: rel_path = entry.path[prefix_len:] if depth == 0 and entry.name in ("spack_repo", "repo.yaml"): continue ino_to_relpath[entry.inode()] = entry.path[prefix_len:] if entry.is_symlink(): try: symlink_to_ino[rel_path] = entry.stat(follow_symlinks=True).st_ino except OSError: symlink_to_ino[rel_path] = -1 # dangling or no access continue elif entry.is_dir(follow_symlinks=False): if entry.name == "__pycache__": continue # check if this is a package if depth == 0 and os.path.exists(os.path.join(entry.path, "package.py")): if "_" in entry.name: print( f"Invalid package name '{entry.name}': underscores are not allowed in " "package names, rename the package with hyphens as separators", file=err, ) errors = True continue pkg_dir = spack.util.naming.pkg_name_to_pkg_dir(entry.name, package_api=(2, 0)) if pkg_dir != entry.name: rename[f"{subdirectory}{os.sep}{entry.name}"] = ( f"{subdirectory}{os.sep}{pkg_dir}" ) dirs_to_create.append(rel_path) stack.append((entry.path, depth + 1)) continue files_to_copy.append(rel_path) if errors: return False, None rename_regex = re.compile("^(" + "|".join(re.escape(k) for k in rename.keys()) + ")") if not patch_file: os.makedirs(os.path.join(new_root, repo.subdirectory), exist_ok=True) def _relocate(rel_path: str) -> Tuple[str, str]: old = os.path.join(repo.root, rel_path) if rename: new_rel = rename_regex.sub(lambda m: rename[m.group(0)], rel_path) else: new_rel = rel_path new = os.path.join(new_root, new_rel) return old, new if patch_file: patch_file.write(b"The following directories, files and symlinks will be created:\n") for rel_path in dirs_to_create: _, new_path = _relocate(rel_path) if not patch_file: try: os.mkdir(new_path) except FileExistsError: # not an error if the directory already exists continue else: patch_file.write(b"create directory ") patch_file.write(new_path.encode("utf-8")) patch_file.write(b"\n") for rel_path in files_to_copy: old_path, new_path = _relocate(rel_path) if os.path.lexists(new_path): # if we already copied this file, don't error. if not _same_contents(old_path, new_path): print( f"Cannot upgrade from v1 to v2, because the file '{new_path}' already exists", file=err, ) return False, None continue if not patch_file: shutil.copy2(old_path, new_path) else: patch_file.write(b"copy ") patch_file.write(old_path.encode("utf-8")) patch_file.write(b" -> ") patch_file.write(new_path.encode("utf-8")) patch_file.write(b"\n") for rel_path, ino in symlink_to_ino.items(): old_path, new_path = _relocate(rel_path) if ino in ino_to_relpath: # link by path relative to the new root _, new_target = _relocate(ino_to_relpath[ino]) tgt = os.path.relpath(new_target, new_path) else: tgt = os.path.realpath(old_path) # no-op if the same, error if different if os.path.lexists(new_path): if not os.path.islink(new_path) or os.readlink(new_path) != tgt: print( f"Cannot upgrade from v1 to v2, because the file '{new_path}' already exists", file=err, ) return False, None continue if not patch_file: os.symlink(tgt, new_path) else: patch_file.write(b"create symlink ") patch_file.write(new_path.encode("utf-8")) patch_file.write(b" -> ") patch_file.write(tgt.encode("utf-8")) patch_file.write(b"\n") if not patch_file: with open(os.path.join(new_root, "repo.yaml"), "w", encoding="utf-8") as f: spack.util.spack_yaml.dump(updated_config, f) updated_repo = spack.repo.from_path(new_root) else: patch_file.write(b"\n") updated_repo = repo # compute the import diff on the v1 repo since v2 doesn't exist yet result = migrate_v2_imports( updated_repo.packages_path, updated_repo.root, patch_file=patch_file, err=err ) return result, (updated_repo if patch_file else None) def _pkg_module_update(pkg_module: str) -> str: return re.sub(r"^num(\d)", r"_\1", pkg_module) # num7zip -> _7zip. def _spack_pkg_to_spack_repo(modulename: str) -> str: # rewrite spack.pkg.builtin.foo -> spack_repo.builtin.packages.foo.package parts = modulename.split(".") assert parts[:2] == ["spack", "pkg"] parts[0:2] = ["spack_repo"] parts.insert(2, "packages") parts[3] = _pkg_module_update(parts[3]) parts.append("package") return ".".join(parts) def migrate_v2_imports( packages_dir: str, root: str, patch_file: Optional[IO[bytes]], err: IO[str] = sys.stderr ) -> bool: """In Package API v2.0, packages need to explicitly import package classes and a few other symbols from the build_systems module. This function automatically adds the missing imports to each package.py file in the repository.""" symbol_to_module = { "AspellDictPackage": "spack_repo.builtin.build_systems.aspell_dict", "AutotoolsPackage": "spack_repo.builtin.build_systems.autotools", "BundlePackage": "spack_repo.builtin.build_systems.bundle", "CachedCMakePackage": "spack_repo.builtin.build_systems.cached_cmake", "cmake_cache_filepath": "spack_repo.builtin.build_systems.cached_cmake", "cmake_cache_option": "spack_repo.builtin.build_systems.cached_cmake", "cmake_cache_path": "spack_repo.builtin.build_systems.cached_cmake", "cmake_cache_string": "spack_repo.builtin.build_systems.cached_cmake", "CargoPackage": "spack_repo.builtin.build_systems.cargo", "CMakePackage": "spack_repo.builtin.build_systems.cmake", "generator": "spack_repo.builtin.build_systems.cmake", "CompilerPackage": "spack_repo.builtin.build_systems.compiler", "CudaPackage": "spack_repo.builtin.build_systems.cuda", "Package": "spack_repo.builtin.build_systems.generic", "GNUMirrorPackage": "spack_repo.builtin.build_systems.gnu", "GoPackage": "spack_repo.builtin.build_systems.go", "LuaPackage": "spack_repo.builtin.build_systems.lua", "MakefilePackage": "spack_repo.builtin.build_systems.makefile", "MavenPackage": "spack_repo.builtin.build_systems.maven", "MesonPackage": "spack_repo.builtin.build_systems.meson", "MSBuildPackage": "spack_repo.builtin.build_systems.msbuild", "NMakePackage": "spack_repo.builtin.build_systems.nmake", "OctavePackage": "spack_repo.builtin.build_systems.octave", "INTEL_MATH_LIBRARIES": "spack_repo.builtin.build_systems.oneapi", "IntelOneApiLibraryPackage": "spack_repo.builtin.build_systems.oneapi", "IntelOneApiLibraryPackageWithSdk": "spack_repo.builtin.build_systems.oneapi", "IntelOneApiPackage": "spack_repo.builtin.build_systems.oneapi", "IntelOneApiStaticLibraryList": "spack_repo.builtin.build_systems.oneapi", "PerlPackage": "spack_repo.builtin.build_systems.perl", "PythonExtension": "spack_repo.builtin.build_systems.python", "PythonPackage": "spack_repo.builtin.build_systems.python", "QMakePackage": "spack_repo.builtin.build_systems.qmake", "RPackage": "spack_repo.builtin.build_systems.r", "RacketPackage": "spack_repo.builtin.build_systems.racket", "ROCmPackage": "spack_repo.builtin.build_systems.rocm", "RubyPackage": "spack_repo.builtin.build_systems.ruby", "SConsPackage": "spack_repo.builtin.build_systems.scons", "SIPPackage": "spack_repo.builtin.build_systems.sip", "SourceforgePackage": "spack_repo.builtin.build_systems.sourceforge", "SourcewarePackage": "spack_repo.builtin.build_systems.sourceware", "WafPackage": "spack_repo.builtin.build_systems.waf", "XorgPackage": "spack_repo.builtin.build_systems.xorg", } success = True for f in os.scandir(packages_dir): pkg_path = os.path.join(f.path, "package.py") if ( f.name in ("__init__.py", "__pycache__") or not f.is_dir(follow_symlinks=False) or os.path.islink(pkg_path) ): print(f"Skipping {f.path}", file=err) continue try: with open(pkg_path, "rb") as file: tree = ast.parse(file.read()) except FileNotFoundError: continue except (OSError, SyntaxError) as e: print(f"Skipping {pkg_path}: {e}", file=err) continue #: Symbols that are referenced in the package and may need to be imported. referenced_symbols: Set[str] = set() #: Set of symbols of interest that are already defined through imports, assignments, or #: function definitions. defined_symbols: Set[str] = set() best_line: Optional[int] = None seen_import = False module_replacements: Dict[str, str] = {} parent: Dict[int, ast.AST] = {} #: List of (line, col start, old, new) tuples of strings to be replaced inline. inline_updates: List[Tuple[int, int, str, str]] = [] #: List of (line from, line to, new lines) tuples of line replacements multiline_updates: List[Tuple[int, int, List[str]]] = [] try: with open(pkg_path, "r", encoding="utf-8", newline="") as file: original_lines = file.readlines() except (OSError, UnicodeDecodeError) as e: success = False print(f"Skipping {pkg_path}: {e}", file=err) continue if len(original_lines) < 2: # assume package.py files have at least 2 lines... continue if original_lines[0].endswith("\r\n"): newline = "\r\n" elif original_lines[0].endswith("\n"): newline = "\n" elif original_lines[0].endswith("\r"): newline = "\r" else: success = False print(f"{pkg_path}: unknown line ending, cannot fix", file=err) continue updated_lines = original_lines.copy() for node in ast.walk(tree): for child in ast.iter_child_nodes(node): if isinstance(child, ast.Attribute): parent[id(child)] = node # Get the last import statement from the first block of top-level imports if isinstance(node, ast.Module): for child in ast.iter_child_nodes(node): # if we never encounter an import statement, the best line to add is right # before the first node under the module if best_line is None and isinstance(child, ast.stmt): best_line = child.lineno # prefer adding right before `from spack.package import ...` if isinstance(child, ast.ImportFrom) and child.module == "spack.package": seen_import = True best_line = child.lineno # add it right before spack.package break is_import = isinstance(child, (ast.Import, ast.ImportFrom)) if is_import: if isinstance(child, (ast.stmt, ast.expr)): end_lineno = getattr(child, "end_lineno", None) if end_lineno is not None: # put it right after the last import statement best_line = end_lineno + 1 else: # old versions of python don't have end_lineno; put it before. best_line = child.lineno if not seen_import and is_import: seen_import = True elif seen_import and not is_import: break # Function definitions or assignments to variables whose name is a symbol of interest # are considered as redefinitions, so we skip them. elif isinstance(node, ast.FunctionDef): if node.name in symbol_to_module: print( f"{pkg_path}:{node.lineno}: redefinition of `{node.name}` skipped", file=err, ) defined_symbols.add(node.name) elif isinstance(node, ast.Assign): for target in node.targets: if isinstance(target, ast.Name) and target.id in symbol_to_module: print( f"{pkg_path}:{target.lineno}: redefinition of `{target.id}` skipped", file=err, ) defined_symbols.add(target.id) # Register symbols that are not imported. elif isinstance(node, ast.Name) and node.id in symbol_to_module: referenced_symbols.add(node.id) # Find lines where spack.pkg is used. elif ( isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == "spack" and node.attr == "pkg" ): # go as many attrs up until we reach a known module name to be replaced known_module = "spack.pkg" ancestor = node while True: next_parent = parent.get(id(ancestor)) if next_parent is None or not isinstance(next_parent, ast.Attribute): break ancestor = next_parent known_module = f"{known_module}.{ancestor.attr}" if known_module in module_replacements: break inline_updates.append( ( ancestor.lineno, ancestor.col_offset, known_module, module_replacements[known_module], ) ) elif isinstance(node, ast.ImportFrom): # Keep track of old style spack.pkg imports, to be replaced. if node.module and node.module.startswith("spack.pkg.") and node.level == 0: depth = node.module.count(".") # not all python versions have end_lineno for ImportFrom end_lineno = getattr(node, "end_lineno", None) # simple case of find and replace # from spack.pkg.builtin.my_pkg import MyPkg # -> from spack_repo.builtin.packages.my_pkg.package import MyPkg if depth == 3: module_replacements[node.module] = _spack_pkg_to_spack_repo(node.module) inline_updates.append( ( node.lineno, node.col_offset, node.module, module_replacements[node.module], ) ) # non-trivial possible multiline case # from spack.pkg.builtin import (boost, cmake as foo) # -> import spack_repo.builtin.packages.boost.package as boost # -> import spack_repo.builtin.packages.cmake.package as foo elif depth == 2: if end_lineno is None: success = False print( f"{pkg_path}:{node.lineno}: cannot rewrite {node.module} " "import statement, since this Python version does not " "provide end_lineno. Best to update to Python 3.8+", file=err, ) continue _, _, namespace = node.module.rpartition(".") indent = original_lines[node.lineno - 1][: node.col_offset] new_lines = [] for alias in node.names: pkg_module = _pkg_module_update(alias.name) new_lines.append( f"{indent}import spack_repo.{namespace}.packages." f"{pkg_module}.package as {alias.asname or pkg_module}" f"{newline}" ) multiline_updates.append((node.lineno, end_lineno + 1, new_lines)) else: success = False print( f"{pkg_path}:{node.lineno}: don't know how to rewrite `{node.module}`", file=err, ) continue elif node.module is not None and node.level == 1 and "." not in node.module: # rewrite `from .blt import ...` -> `from ..blt.package import ...` pkg_module = _pkg_module_update(node.module) inline_updates.append( ( node.lineno, node.col_offset, f".{node.module}", f"..{pkg_module}.package", ) ) # Subtract the symbols that are imported so we don't repeatedly add imports. for alias in node.names: if alias.name in symbol_to_module: if alias.asname is None: defined_symbols.add(alias.name) # error when symbols are explicitly imported that are no longer available if node.module == "spack.package" and node.level == 0: success = False print( f"{pkg_path}:{node.lineno}: `{alias.name}` is imported from " "`spack.package`, which no longer provides this symbol", file=err, ) if alias.asname and alias.asname in symbol_to_module: defined_symbols.add(alias.asname) elif isinstance(node, ast.Import): # normal imports are easy find and replace since they are single lines. for alias in node.names: if alias.asname and alias.asname in symbol_to_module: defined_symbols.add(alias.name) elif alias.asname is None and alias.name.startswith("spack.pkg."): module_replacements[alias.name] = _spack_pkg_to_spack_repo(alias.name) inline_updates.append( ( node.lineno, node.col_offset, alias.name, module_replacements[alias.name], ) ) # Remove imported symbols from the referenced symbols referenced_symbols.difference_update(defined_symbols) # Sort from last to first so we can modify without messing up the line / col offsets inline_updates.sort(reverse=True) # Nothing to change here. if not inline_updates and not referenced_symbols: continue # First do module replacements of spack.pkg imports for line, col, old, new in inline_updates: updated_lines[line - 1] = updated_lines[line - 1][:col] + updated_lines[line - 1][ col: ].replace(old, new, 1) # Then insert new imports for symbols referenced in the package if referenced_symbols: if best_line is None: print(f"{pkg_path}: failed to update imports", file=err) success = False continue # Group missing symbols by their module missing_imports_by_module: Dict[str, list] = {} for symbol in referenced_symbols: module = symbol_to_module[symbol] if module not in missing_imports_by_module: missing_imports_by_module[module] = [] missing_imports_by_module[module].append(symbol) new_lines = [ f"from {module} import {', '.join(sorted(symbols))}{newline}" for module, symbols in sorted(missing_imports_by_module.items()) ] if not seen_import: new_lines.extend((newline, newline)) multiline_updates.append((best_line, best_line, new_lines)) multiline_updates.sort(reverse=True) for start, end, new_lines in multiline_updates: updated_lines[start - 1 : end - 1] = new_lines if patch_file: rel_pkg_path = os.path.relpath(pkg_path, start=root).replace(os.sep, "/") diff = difflib.unified_diff( original_lines, updated_lines, n=3, fromfile=f"a/{rel_pkg_path}", tofile=f"b/{rel_pkg_path}", lineterm="\n", ) for _line in diff: patch_file.write(_line.encode("utf-8")) continue tmp_file = pkg_path + ".tmp" # binary mode to avoid newline conversion issues; utf-8 was already required upon read. with open(tmp_file, "wb") as file: for _line in updated_lines: file.write(_line.encode("utf-8")) os.replace(tmp_file, pkg_path) return success ================================================ FILE: lib/spack/spack/report.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tools to produce reports of spec installations or tests""" import collections import gzip import os import time import traceback import spack.error reporter = None report_file = None Property = collections.namedtuple("Property", ["name", "value"]) class Record(dict): """Data class that provides attr-style access to a dictionary Attributes beginning with ``_`` are reserved for the Record class itself.""" def __getattr__(self, name): # only called if no attribute exists if name in self: return self[name] raise AttributeError(f"Record for {self.name} has no attribute {name}") def __setattr__(self, name, value): if name.startswith("_"): super().__setattr__(name, value) else: self[name] = value class RequestRecord(Record): """Data class for recording outcomes for an entire DAG Each BuildRequest in the installer and each root spec in a TestSuite generates a RequestRecord. The ``packages`` list of the RequestRecord is a list of SpecRecord objects recording individual data for each node in the Spec represented by the RequestRecord. These data classes are collated by the reporters in lib/spack/spack/reporters """ def __init__(self, spec): super().__init__() self._spec = spec self.name = spec.name self.nerrors = None self.nfailures = None self.npackages = None self.time = None self.timestamp = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()) self.properties = [ Property("architecture", spec.architecture) # Property("compiler", spec.compiler), ] self.packages = [] def skip_installed(self): """Insert records for all nodes in the DAG that are no-ops for this request""" for dep in filter(lambda x: x.installed or x.external, self._spec.traverse()): record = InstallRecord(dep) record.skip(msg="Spec external or already installed") self.packages.append(record) def append_record(self, record): self.packages.append(record) def summarize(self): """Construct request-level summaries of the individual records""" self.npackages = len(self.packages) self.nfailures = len([r for r in self.packages if r.result == "failure"]) self.nerrors = len([r for r in self.packages if r.result == "error"]) self.time = sum(float(r.elapsed_time or 0.0) for r in self.packages) class SpecRecord(Record): """Individual record for a single spec within a request""" def __init__(self, spec): super().__init__() self._spec = spec self._package = spec.package self._start_time = None self.name = spec.name self.id = spec.dag_hash() self.elapsed_time = None def start(self): self._start_time = time.time() def skip(self, msg): self.result = "skipped" self.elapsed_time = 0.0 self.message = msg def fail(self, exc): """Record failure based on exception type Errors wrapped by spack.error.InstallError are "failures" Other exceptions are "errors". """ if isinstance(exc, spack.error.InstallError): self.result = "failure" self.message = exc.message or "Installation failure" self.exception = exc.traceback else: self.result = "error" self.message = str(exc) or "Unknown error" self.exception = traceback.format_exc() self.stdout = self.fetch_log() + self.message assert self._start_time, "Start time is None" self.elapsed_time = time.time() - self._start_time def succeed(self): """Record success for this spec""" self.result = "success" self.stdout = self.fetch_log() assert self._start_time, "Start time is None" self.elapsed_time = time.time() - self._start_time class InstallRecord(SpecRecord): """Record class with specialization for install logs.""" def __init__(self, spec): super().__init__(spec) self.installed_from_binary_cache = None def fetch_log(self): """Install log comes from install prefix on success, or stage dir on failure.""" try: if os.path.exists(self._package.install_log_path): stream = gzip.open( self._package.install_log_path, "rt", encoding="utf-8", errors="replace" ) else: stream = open(self._package.log_path, encoding="utf-8", errors="replace") with stream as f: return f.read() except OSError: return f"Cannot open log for {self._spec.cshort_spec}" def succeed(self): super().succeed() self.installed_from_binary_cache = self._package.installed_from_binary_cache class NullInstallRecord(InstallRecord): """No-op drop-in for InstallRecord when no reporter is configured. Avoids reading log files from disk on every completed build.""" def start(self) -> None: pass def succeed(self) -> None: pass def fail(self, exc) -> None: pass def skip(self, msg: str = "") -> None: pass class NullRequestRecord(RequestRecord): """No-op drop-in for RequestRecord when no reporter is configured. Avoids traversing the DAG and accumulating data that will not be reported.""" def __init__(self) -> None: dict.__init__(self) def skip_installed(self) -> None: pass def append_record(self, record) -> None: pass def summarize(self) -> None: pass class TestRecord(SpecRecord): """Record class with specialization for test logs.""" def __init__(self, spec, directory): super().__init__(spec) self.directory = directory def fetch_log(self): """Get output from test log""" log_file = os.path.join(self.directory, self._package.test_suite.test_log_name(self._spec)) try: with open(log_file, "r", encoding="utf-8", errors="replace") as stream: return "".join(stream.readlines()) except Exception: return f"Cannot open log for {self._spec.cshort_spec}" def succeed(self, externals): """Test reports skip externals by default.""" if self._spec.external and not externals: return self.skip(msg="Skipping test of external package") super().succeed() ================================================ FILE: lib/spack/spack/reporters/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from .base import Reporter from .cdash import CDash, CDashConfiguration from .junit import JUnit __all__ = ["JUnit", "CDash", "CDashConfiguration", "Reporter"] ================================================ FILE: lib/spack/spack/reporters/base.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from typing import Any, Dict, List class Reporter: """Base class for report writers.""" def build_report(self, filename: str, specs: List[Dict[str, Any]]): raise NotImplementedError("must be implemented by derived classes") def test_report(self, filename: str, specs: List[Dict[str, Any]]): raise NotImplementedError("must be implemented by derived classes") def concretization_report(self, filename: str, msg: str): raise NotImplementedError("must be implemented by derived classes") ================================================ FILE: lib/spack/spack/reporters/cdash.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import hashlib import io import os import platform import posixpath import re import socket import time import warnings import xml.sax.saxutils from typing import Dict, Optional from urllib.parse import urlencode from urllib.request import Request import spack import spack.llnl.util.tty as tty import spack.paths import spack.platforms import spack.spec import spack.tengine import spack.util.git import spack.util.web as web_util from spack.error import SpackError from spack.llnl.util.filesystem import working_dir from spack.util.crypto import checksum from spack.util.log_parse import parse_log_events from .base import Reporter from .extract import extract_test_parts # Mapping Spack phases to the corresponding CTest/CDash phase. # TODO: Some of the phases being lumped into configure in the CDash tables # TODO: really belong in a separate column, such as "Setup". # TODO: Would also be nice to have `stage` as a separate phase that could # TODO: be lumped into that new column instead of configure, for example. MAP_PHASES_TO_CDASH = { "autoreconf": "configure", # AutotoolsBuilder "bootstrap": "configure", # CMakeBuilder "build": "build", "build_processes": "build", # Openloops "cmake": "configure", # CMakeBuilder "configure": "configure", "edit": "configure", # MakefileBuilder "generate_luarocks_config": "configure", # LuaBuilder "hostconfig": "configure", # Lvarray "initconfig": "configure", # CachedCMakeBuilder "install": "build", "meson": "configure", # MesonBuilder "preprocess": "configure", # LuaBuilder "qmake": "configure", # QMakeBuilder "unpack": "configure", # LuaBuilder } # Initialize data structures common to each phase's report. CDASH_PHASES = set(MAP_PHASES_TO_CDASH.values()) CDASH_PHASES.add("update") # CDash request timeout in seconds SPACK_CDASH_TIMEOUT = 45 CDashConfiguration = collections.namedtuple( "CDashConfiguration", ["upload_url", "packages", "build", "site", "buildstamp", "track"] ) def build_stamp(track, timestamp): buildstamp_format = "%Y%m%d-%H%M-{0}".format(track) return time.strftime(buildstamp_format, time.localtime(timestamp)) class CDash(Reporter): """Generate reports of spec installations for CDash. To use this reporter, pass the ``--cdash-upload-url`` argument to ``spack install``:: spack install --cdash-upload-url=\\ https://example.com/cdash/submit.php?project=Spack In this example, results will be uploaded to the *Spack* project on the CDash instance hosted at ``https://example.com/cdash``. """ def __init__(self, configuration: CDashConfiguration): #: Set to False if any error occurs when building the CDash report self.success = True # Jinja2 expects `/` path separators self.template_dir = "reports/cdash" self.cdash_upload_url = configuration.upload_url if self.cdash_upload_url: self.buildid_regexp = re.compile("([0-9]+)") self.phase_regexp = re.compile(r"Executing phase: '(.*)'") self.authtoken = None if "SPACK_CDASH_AUTH_TOKEN" in os.environ: tty.verbose("Using CDash auth token from environment") self.authtoken = os.environ.get("SPACK_CDASH_AUTH_TOKEN") self.install_command = " ".join(configuration.packages) self.base_buildname = configuration.build or self.install_command self.site = configuration.site or socket.gethostname() self.osname = platform.system() self.osrelease = platform.release() self.target = spack.platforms.host().default_target() self.starttime = int(time.time()) self.endtime = self.starttime self.buildstamp = ( configuration.buildstamp if configuration.buildstamp else build_stamp(configuration.track, self.starttime) ) self.buildIds: Dict[str, str] = {} self.revision = "" git = spack.util.git.git(required=True) with working_dir(spack.paths.spack_root): self.revision = git("rev-parse", "HEAD", output=str).strip() self.generator = "spack-{0}".format(spack.get_version()) self.multiple_packages = False def report_build_name(self, pkg_name): buildname = ( "{0} - {1}".format(self.base_buildname, pkg_name) if self.multiple_packages else self.base_buildname ) if len(buildname) > 190: warnings.warn("Build name exceeds CDash 190 character maximum and will be truncated.") buildname = buildname[:190] return buildname def build_report_for_package(self, report_dir, package, duration): if "stdout" not in package: # Skip reporting on packages that do not generate output. return self.current_package_name = package["name"] self.buildname = self.report_build_name(self.current_package_name) report_data = self.initialize_report(report_dir) for phase in CDASH_PHASES: report_data[phase] = {} report_data[phase]["loglines"] = [] report_data[phase]["status"] = 0 report_data[phase]["starttime"] = self.starttime # Track the phases we perform so we know what reports to create. # We always report the update step because this is how we tell CDash # what revision of Spack we are using. phases_encountered = ["update"] # Generate a report for this package. current_phase = "" cdash_phase = "" for line in package["stdout"].splitlines(): match = None if line.find("Executing phase: '") != -1: match = self.phase_regexp.search(line) if match: current_phase = match.group(1) if current_phase not in MAP_PHASES_TO_CDASH: current_phase = "" continue cdash_phase = MAP_PHASES_TO_CDASH[current_phase] if cdash_phase not in phases_encountered: phases_encountered.append(cdash_phase) report_data[cdash_phase]["loglines"].append( str("{0} output for {1}:".format(cdash_phase, package["name"])) ) elif cdash_phase: report_data[cdash_phase]["loglines"].append(xml.sax.saxutils.escape(line)) # something went wrong pre-cdash "configure" phase b/c we have an exception and only # "update" was encountered. # dump the report in the configure line so teams can see what the issue is if len(phases_encountered) == 1 and package.get("exception"): # TODO this mapping is not ideal since these are pre-configure errors # we need to determine if a more appropriate cdash phase can be utilized # for now we will add a message to the log explaining this cdash_phase = "configure" phases_encountered.append(cdash_phase) log_message = ( "Pre-configure errors occurred in Spack's process that terminated the " "build process prematurely.\nSpack output::\n{0}".format( xml.sax.saxutils.escape(package["exception"]) ) ) report_data[cdash_phase]["loglines"].append(log_message) # Move the build phase to the front of the list if it occurred. # This supports older versions of CDash that expect this phase # to be reported before all others. if "build" in phases_encountered: build_pos = phases_encountered.index("build") phases_encountered.insert(0, phases_encountered.pop(build_pos)) self.endtime = self.starttime + duration for phase in phases_encountered: report_data[phase]["endtime"] = self.endtime report_data[phase]["log"] = "\n".join(report_data[phase]["loglines"]) errors, warnings = parse_log_events(report_data[phase]["loglines"]) # Convert errors to warnings if the package reported success. if package["result"] == "success": warnings = errors + warnings errors = [] # Cap the number of errors and warnings at 50 each. errors = errors[:50] warnings = warnings[:50] nerrors = len(errors) if nerrors > 0: self.success = False if phase == "configure": report_data[phase]["status"] = 1 if phase == "build": # Convert log output from ASCII to Unicode and escape for XML. def clean_log_event(event): event = vars(event) event["text"] = xml.sax.saxutils.escape(event["text"]) event["pre_context"] = xml.sax.saxutils.escape("\n".join(event["pre_context"])) event["post_context"] = xml.sax.saxutils.escape( "\n".join(event["post_context"]) ) # source_file and source_line_no are either strings or # the tuple (None,). Distinguish between these two cases. if event["source_file"][0] is None: event["source_file"] = "" event["source_line_no"] = "" else: event["source_file"] = xml.sax.saxutils.escape(event["source_file"]) return event report_data[phase]["errors"] = [] report_data[phase]["warnings"] = [] for error in errors: report_data[phase]["errors"].append(clean_log_event(error)) for warning in warnings: report_data[phase]["warnings"].append(clean_log_event(warning)) if phase == "update": report_data[phase]["revision"] = self.revision # Write the report. report_name = phase.capitalize() + ".xml" if self.multiple_packages: report_file_name = package["name"] + "_" + report_name else: report_file_name = report_name phase_report = os.path.join(report_dir, report_file_name) with open(phase_report, "w", encoding="utf-8") as f: env = spack.tengine.make_environment() if phase != "update": # Update.xml stores site information differently # than the rest of the CTest XML files. site_template = posixpath.join(self.template_dir, "Site.xml") t = env.get_template(site_template) f.write(t.render(report_data)) phase_template = posixpath.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report) def build_report(self, report_dir, specs): # Do an initial scan to determine if we are generating reports for more # than one package. When we're only reporting on a single package we # do not explicitly include the package's name in the CDash build name. self.multiple_packages = False num_packages = 0 for spec in specs: spec.summarize() # Do not generate reports for packages that were installed # from the binary cache. spec["packages"] = [ x for x in spec["packages"] if "installed_from_binary_cache" not in x or not x["installed_from_binary_cache"] ] for package in spec["packages"]: if "stdout" in package: num_packages += 1 if num_packages > 1: self.multiple_packages = True break if self.multiple_packages: break # Generate reports for each package in each spec. for spec in specs: duration = 0 if "time" in spec: duration = int(spec["time"]) for package in spec["packages"]: self.build_report_for_package(report_dir, package, duration) self.finalize_report() def extract_standalone_test_data(self, package, phases, report_data): """Extract stand-alone test outputs for the package.""" testing = {} report_data["testing"] = testing testing["starttime"] = self.starttime testing["endtime"] = self.starttime testing["generator"] = self.generator testing["parts"] = extract_test_parts(package["name"], package["stdout"].splitlines()) def report_test_data(self, report_dir, package, phases, report_data): """Generate and upload the test report(s) for the package.""" for phase in phases: # Write the report. report_name = phase.capitalize() + ".xml" report_file_name = "_".join([package["name"], package["id"], report_name]) phase_report = os.path.join(report_dir, report_file_name) with open(phase_report, "w", encoding="utf-8") as f: env = spack.tengine.make_environment() if phase not in ["update", "testing"]: # Update.xml stores site information differently # than the rest of the CTest XML files. site_template = posixpath.join(self.template_dir, "Site.xml") t = env.get_template(site_template) f.write(t.render(report_data)) phase_template = posixpath.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) tty.debug("Preparing to upload {0}".format(phase_report)) self.upload(phase_report) def test_report_for_package(self, report_dir, package, duration): if "stdout" not in package: # Skip reporting on packages that did not generate any output. tty.debug("Skipping report for {0}: No generated output".format(package["name"])) return self.current_package_name = package["name"] if self.base_buildname == self.install_command: # The package list is NOT all that helpful in this case self.buildname = "{0}-{1}".format(self.current_package_name, package["id"]) else: self.buildname = self.report_build_name(self.current_package_name) self.endtime = self.starttime + duration report_data = self.initialize_report(report_dir) report_data["hostname"] = socket.gethostname() phases = ["testing"] self.extract_standalone_test_data(package, phases, report_data) self.report_test_data(report_dir, package, phases, report_data) def test_report(self, report_dir, specs): """Generate reports for each package in each spec.""" tty.debug("Processing test report") for spec in specs: spec.summarize() duration = 0 if "time" in spec: duration = int(spec["time"]) for package in spec["packages"]: self.test_report_for_package(report_dir, package, duration) self.finalize_report() def test_skipped_report( self, report_dir: str, spec: spack.spec.Spec, reason: Optional[str] = None ): """Explicitly report spec as being skipped (e.g., CI). Examples are the installation failed or the package is known to have broken tests. Args: report_dir: directory where the report is to be written spec: spec being tested reason: optional reason the test is being skipped """ output = "Skipped {0} package".format(spec.name) if reason: output += "\n{0}".format(reason) package = {"name": spec.name, "id": spec.dag_hash(), "result": "skipped", "stdout": output} self.test_report_for_package(report_dir, package, duration=0.0) def concretization_report(self, report_dir, msg): self.buildname = self.base_buildname report_data = self.initialize_report(report_dir) report_data["update"] = {} report_data["update"]["starttime"] = self.starttime report_data["update"]["endtime"] = self.endtime report_data["update"]["revision"] = self.revision report_data["update"]["log"] = msg env = spack.tengine.make_environment() update_template = posixpath.join(self.template_dir, "Update.xml") t = env.get_template(update_template) output_filename = os.path.join(report_dir, "Update.xml") with open(output_filename, "w", encoding="utf-8") as f: f.write(t.render(report_data)) # We don't have a current package when reporting on concretization # errors so refer to this report with the base buildname instead. self.current_package_name = self.base_buildname self.upload(output_filename) self.success = False self.finalize_report() def initialize_report(self, report_dir): if not os.path.exists(report_dir): os.mkdir(report_dir) report_data = {} report_data["buildname"] = self.buildname report_data["buildstamp"] = self.buildstamp report_data["install_command"] = self.install_command report_data["generator"] = self.generator report_data["osname"] = self.osname report_data["osrelease"] = self.osrelease report_data["site"] = self.site report_data["target"] = self.target return report_data def upload(self, filename): if not self.cdash_upload_url: print("Cannot upload {0} due to missing upload url".format(filename)) return # Compute md5 checksum for the contents of this file. md5sum = checksum(hashlib.md5, filename, block_size=8192) with open(filename, "rb") as f: params_dict = { "build": self.buildname, "site": self.site, "stamp": self.buildstamp, "MD5": md5sum, } encoded_params = urlencode(params_dict) url = "{0}&{1}".format(self.cdash_upload_url, encoded_params) request = Request(url, data=f, method="PUT") request.add_header("Content-Type", "text/xml") request.add_header("Content-Length", os.path.getsize(filename)) if self.authtoken: request.add_header("Authorization", "Bearer {0}".format(self.authtoken)) try: with web_util.urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response: if self.current_package_name not in self.buildIds: resp_value = io.TextIOWrapper(response, encoding="utf-8").read() match = self.buildid_regexp.search(resp_value) if match: buildid = match.group(1) self.buildIds[self.current_package_name] = buildid except Exception as e: print(f"Upload to CDash failed: {e}") def finalize_report(self): if self.buildIds: tty.msg("View your build results here:") for package_name, buildid in self.buildIds.items(): # Construct and display a helpful link if CDash responded with # a buildId. build_url = self.cdash_upload_url build_url = build_url[0 : build_url.find("submit.php")] build_url += "buildSummary.php?buildid={0}".format(buildid) tty.msg("{0}: {1}".format(package_name, build_url)) if not self.success: raise SpackError("Errors encountered, see above for more details") ================================================ FILE: lib/spack/spack/reporters/extract.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import xml.sax.saxutils from datetime import datetime import spack.llnl.util.tty as tty from spack.install_test import TestStatus # The keys here represent the only recognized (ctest/cdash) status values completed = { "failed": "Completed", "passed": "Completed", "skipped": "Completed", "notrun": "No tests to run", } log_regexp = re.compile(r"^==> \[([0-9:.\-]*)(?:, [0-9]*)?\] (.*)") returns_regexp = re.compile(r"\[([0-9 ,]*)\]") skip_msgs = ["Testing package", "Results for", "Detected the following", "Warning:"] skip_regexps = [re.compile(r"{0}".format(msg)) for msg in skip_msgs] status_regexps = [re.compile(r"^({0})".format(str(stat))) for stat in TestStatus] def add_part_output(part, line): if part: part["loglines"].append(xml.sax.saxutils.escape(line)) def elapsed(current, previous): if not (current and previous): return 0 diff = current - previous tty.debug("elapsed = %s - %s = %s" % (current, previous, diff)) return diff.total_seconds() def new_part(): return { "command": None, "completed": "Unknown", "desc": None, "elapsed": None, "name": None, "loglines": [], "output": None, "status": None, } def process_part_end(part, curr_time, last_time): if part: if not part["elapsed"]: part["elapsed"] = elapsed(curr_time, last_time) stat = part["status"] if stat in completed: if part["completed"] == "Unknown": part["completed"] = completed[stat] elif stat is None or stat == "unknown": part["status"] = "passed" part["output"] = "\n".join(part["loglines"]) def timestamp(time_string): return datetime.strptime(time_string, "%Y-%m-%d-%H:%M:%S.%f") def skip(line): for regex in skip_regexps: match = regex.search(line) if match: return match def status(line): for regex in status_regexps: match = regex.search(line) if match: stat = match.group(0) stat = "notrun" if stat == "NO_TESTS" else stat return stat.lower() def extract_test_parts(default_name, outputs): parts = [] part = {} last_time = None curr_time = None for line in outputs: line = line.strip() if not line: add_part_output(part, line) continue if skip(line): continue # The spec was explicitly reported as skipped (e.g., installation # failed, package known to have failing tests, won't test external # package). if line.startswith("Skipped") and line.endswith("package"): stat = "skipped" part = new_part() part["command"] = "Not Applicable" part["completed"] = completed[stat] part["elapsed"] = 0.0 part["loglines"].append(line) part["name"] = default_name part["status"] = "notrun" parts.append(part) continue # Process Spack log messages if line.find("==>") != -1: match = log_regexp.search(line) if match: curr_time = timestamp(match.group(1)) msg = match.group(2) # Skip logged message for caching build-time data if msg.startswith("Installing"): continue # Terminate without further parsing if no more test messages if "Completed testing" in msg: # Process last lingering part IF it didn't generate status process_part_end(part, curr_time, last_time) return parts # New test parts start "test: : ". if msg.startswith("test: "): # Update the last part processed process_part_end(part, curr_time, last_time) part = new_part() desc = msg.split(":") part["name"] = desc[1].strip() part["desc"] = ":".join(desc[2:]).strip() parts.append(part) # There is no guarantee of a 1-to-1 mapping of a test part and # a (single) command (or executable) since the introduction of # PR 34236. # # Note that tests where the package does not save the output # (e.g., output=str.split, error=str.split) will not have # a command printed to the test log. elif msg.startswith("'") and msg.endswith("'"): if part: if part["command"]: part["command"] += "; " + msg.replace("'", "") else: part["command"] = msg.replace("'", "") else: part = new_part() part["command"] = msg.replace("'", "") else: # Update the last part processed since a new log message # means a non-test action process_part_end(part, curr_time, last_time) else: tty.debug("Did not recognize test output '{0}'".format(line)) # Each log message potentially represents a new test part so # save off the last timestamp last_time = curr_time continue # Check for status values stat = status(line) if stat: if part: part["status"] = stat add_part_output(part, line) else: tty.warn("No part to add status from '{0}'".format(line)) continue add_part_output(part, line) # Process the last lingering part IF it didn't generate status process_part_end(part, curr_time, last_time) # If no parts, create a skeleton to flag that the tests are not run if not parts: part = new_part() stat = "failed" if outputs[0].startswith("Cannot open log") else "notrun" part["command"] = "unknown" part["completed"] = completed[stat] part["elapsed"] = 0.0 part["name"] = default_name part["status"] = stat part["output"] = "\n".join(outputs) parts.append(part) return parts ================================================ FILE: lib/spack/spack/reporters/junit.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import spack.tengine from .base import Reporter class JUnit(Reporter): """Generate reports of spec installations for JUnit.""" _jinja_template = "reports/junit.xml" def concretization_report(self, filename, msg): pass def build_report(self, filename, specs): for spec in specs: spec.summarize() if not (os.path.splitext(filename))[1]: # Ensure the report name will end with the proper extension; # otherwise, it currently defaults to the "directory" name. filename = filename + ".xml" report_data = {"specs": specs} with open(filename, "w", encoding="utf-8") as f: env = spack.tengine.make_environment() t = env.get_template(self._jinja_template) f.write(t.render(report_data)) def test_report(self, filename, specs): self.build_report(filename, specs) ================================================ FILE: lib/spack/spack/resource.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another package to enable optional features. """ class Resource: """Represents any resource to be fetched by a package. This includes the main tarball or source archive, as well as extra archives defined by the resource() directive. Aggregates a name, a fetcher, a destination and a placement. """ def __init__(self, name, fetcher, destination, placement): self.name = name self.fetcher = fetcher self.destination = destination self.placement = placement ================================================ FILE: lib/spack/spack/rewiring.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import tempfile import spack.binary_distribution import spack.error import spack.hooks import spack.store def rewire(spliced_spec): """Given a spliced spec, this function conducts all the rewiring on all nodes in the DAG of that spec.""" assert spliced_spec.spliced for spec in spliced_spec.traverse(order="post", root=True): if not spec.build_spec.installed: # TODO: May want to change this at least for the root spec... # TODO: Also remember to import PackageInstaller # PackageInstaller([spec.build_spec.package]).install() raise PackageNotInstalledError(spliced_spec, spec.build_spec, spec) if spec.build_spec is not spec and not spec.installed: explicit = spec is spliced_spec rewire_node(spec, explicit) def rewire_node(spec, explicit): """This function rewires a single node, worrying only about references to its subgraph. Binaries, text, and links are all changed in accordance with the splice. The resulting package is then 'installed.'""" tempdir = tempfile.mkdtemp() # Copy spec.build_spec.prefix to spec.prefix through a temporary tarball tarball = os.path.join(tempdir, f"{spec.dag_hash()}.tar.gz") spack.binary_distribution.create_tarball(spec.build_spec, tarball) spack.hooks.pre_install(spec) spack.binary_distribution.extract_buildcache_tarball(tarball, destination=spec.prefix) spack.binary_distribution.relocate_package(spec) # run post install hooks and add to db spack.hooks.post_install(spec, explicit) spack.store.STORE.db.add(spec, explicit=explicit) class RewireError(spack.error.SpackError): """Raised when something goes wrong with rewiring.""" def __init__(self, message, long_msg=None): super().__init__(message, long_msg) class PackageNotInstalledError(RewireError): """Raised when the build_spec for a splice was not installed.""" def __init__(self, spliced_spec, build_spec, dep): super().__init__( """Rewire of {0} failed due to missing install of build spec {1} for spec {2}""".format(spliced_spec, build_spec, dep) ) ================================================ FILE: lib/spack/spack/schema/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module contains jsonschema files for all of Spack's YAML formats.""" import copy import typing import warnings from spack.vendor import jsonschema from spack.vendor.jsonschema import validators from spack.error import SpecSyntaxError class DeprecationMessage(typing.NamedTuple): message: str error: bool def _validate_spec(validator, is_spec, instance, schema): """Check if all additional keys are valid specs.""" import spack.spec_parser if not validator.is_type(instance, "object"): return properties = schema.get("properties") or {} for spec_str in instance: if spec_str in properties: continue try: spack.spec_parser.parse(spec_str) except SpecSyntaxError: yield jsonschema.ValidationError(f"the key '{spec_str}' is not a valid spec") def _deprecated_properties(validator, deprecated, instance, schema): if not (validator.is_type(instance, "object") or validator.is_type(instance, "array")): return if not deprecated: return deprecations = { name: DeprecationMessage(message=x["message"], error=x["error"]) for x in deprecated for name in x["names"] } # Get a list of the deprecated properties, return if there is none issues = [entry for entry in instance if entry in deprecations] if not issues: return # Process issues errors = [] for name in issues: msg = deprecations[name].message.format(name=name) if deprecations[name].error: errors.append(msg) else: warnings.warn(msg) if errors: yield jsonschema.ValidationError("\n".join(errors)) Validator = validators.extend( jsonschema.Draft7Validator, {"additionalKeysAreSpecs": _validate_spec, "deprecatedProperties": _deprecated_properties}, ) def _append(string: str) -> bool: """Test if a spack YAML string is an append. See ``spack_yaml`` for details. Keys in Spack YAML can end in ``+:``, and if they do, their values append lower-precedence configs. str, str : concatenate strings. [obj], [obj] : append lists. """ return getattr(string, "append", False) def _prepend(string: str) -> bool: """Test if a spack YAML string is an prepend. See ``spack_yaml`` for details. Keys in Spack YAML can end in ``+:``, and if they do, their values prepend lower-precedence configs. str, str : concatenate strings. [obj], [obj] : prepend lists. (default behavior) """ return getattr(string, "prepend", False) def override(string: str) -> bool: """Test if a spack YAML string is an override. See ``spack_yaml`` for details. Keys in Spack YAML can end in ``::``, and if they do, their values completely replace lower-precedence configs instead of merging into them. """ return hasattr(string, "override") and string.override def merge_yaml(dest, source, prepend=False, append=False): """Merges source into dest; entries in source take precedence over dest. This routine may modify dest and should be assigned to dest, in case dest was None to begin with, e.g.:: dest = merge_yaml(dest, source) In the result, elements from lists from ``source`` will appear before elements of lists from ``dest``. Likewise, when iterating over keys or items in merged ``OrderedDict`` objects, keys from ``source`` will appear before keys from ``dest``. Config file authors can optionally end any attribute in a dict with ``::`` instead of ``:``, and the key will override that of the parent instead of merging. ``+:`` will extend the default prepend merge strategy to include string concatenation ``-:`` will change the merge strategy to append, it also includes string concatenation """ def they_are(t): return isinstance(dest, t) and isinstance(source, t) # If source is None, overwrite with source. if source is None: return None # Source list is prepended (for precedence) if they_are(list): if append: # Make sure to copy ruamel comments dest[:] = [x for x in dest if x not in source] + source else: # Make sure to copy ruamel comments dest[:] = source + [x for x in dest if x not in source] return dest # Source dict is merged into dest. elif they_are(dict): # save dest keys to reinsert later -- this ensures that source items # come *before* dest in OrderdDicts dest_keys = [dk for dk in dest.keys() if dk not in source] for sk, sv in source.items(): # always remove the dest items. Python dicts do not overwrite # keys on insert, so this ensures that source keys are copied # into dest along with mark provenance (i.e., file/line info). merge = sk in dest old_dest_value = dest.pop(sk, None) if merge and not override(sk): dest[sk] = merge_yaml(old_dest_value, sv, _prepend(sk), _append(sk)) else: # if sk ended with ::, or if it's new, completely override dest[sk] = copy.deepcopy(sv) # reinsert dest keys so they are last in the result for dk in dest_keys: dest[dk] = dest.pop(dk) return dest elif they_are(str): # Concatenate strings in prepend mode if prepend: return source + dest elif append: return dest + source # If we reach here source and dest are either different types or are # not both lists or dicts: replace with source. return copy.copy(source) ================================================ FILE: lib/spack/spack/schema/bootstrap.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for bootstrap.yaml configuration file.""" from typing import Any, Dict #: Schema of a single source _source_schema: Dict[str, Any] = { "type": "object", "description": "Bootstrap source configuration", "properties": { "name": { "type": "string", "description": "Name of the bootstrap source (e.g., 'github-actions-v0.6', " "'spack-install')", }, "metadata": { "type": "string", "description": "Path to metadata directory containing bootstrap source configuration", }, }, "additionalProperties": False, "required": ["name", "metadata"], } #: schema for dev bootstrap configuration _dev_schema: Dict[str, Any] = { "type": "object", "description": "Dev Bootstrap configuration", "properties": { "enable_source": { "type": "boolean", "description": "Enable bootstrapping dev dependencies from source", } }, } properties: Dict[str, Any] = { "bootstrap": { "type": "object", "description": "Configure how Spack bootstraps its own dependencies when needed", "properties": { "enable": { "type": "boolean", "description": "Enable or disable bootstrapping entirely", }, "root": { "type": "string", "description": "Where to install bootstrapped dependencies", }, "sources": { "type": "array", "items": _source_schema, "description": "List of bootstrap sources tried in order. Each method may " "bootstrap different software depending on its type (e.g., pre-built binaries, " "source builds)", }, "trusted": { "type": "object", "additionalProperties": {"type": "boolean"}, "description": "Controls which sources are enabled for automatic bootstrapping", }, "dev": _dev_schema, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack bootstrap configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/buildcache_spec.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for a buildcache spec.yaml file .. literalinclude:: _spack_root/lib/spack/spack/schema/buildcache_spec.py :lines: 15- """ from typing import Any, Dict import spack.schema.spec properties: Dict[str, Any] = { # `buildinfo` is no longer needed as of Spack 0.21 "buildinfo": {"type": "object"}, "spec": {**spack.schema.spec.spec_node, "additionalProperties": True}, "buildcache_layout_version": {"type": "number"}, } schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack buildcache specfile schema", "type": "object", "additionalProperties": True, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/cdash.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for cdash.yaml configuration file. .. literalinclude:: ../spack/schema/cdash.py :lines: 13- """ from typing import Any, Dict #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "cdash": { "type": "object", "additionalProperties": False, "required": ["build-group"], "description": "Configuration for uploading build results to CDash", "properties": { "build-group": { "type": "string", "description": "Unique build group name for this stack", }, "url": {"type": "string", "description": "CDash server URL"}, "project": {"type": "string", "description": "CDash project name"}, "site": {"type": "string", "description": "Site identifier for CDash reporting"}, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack cdash configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/ci.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for gitlab-ci.yaml configuration file. .. literalinclude:: ../spack/schema/ci.py :lines: 16- """ from typing import Any, Dict # Schema for script fields # List of lists and/or strings # This is similar to what is allowed in # the gitlab schema script_schema = { "type": "array", "items": {"anyOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}]}, } # Schema for CI image image_schema = { "oneOf": [ {"type": "string"}, { "type": "object", "properties": { "name": {"type": "string"}, "entrypoint": {"type": "array", "items": {"type": "string"}}, }, }, ] } # Additional attributes are allowed and will be forwarded directly to the CI target YAML for each # job. ci_job_attributes = { "type": "object", "additionalProperties": True, "properties": { "image": image_schema, "tags": {"type": "array", "items": {"type": "string"}}, "variables": { "type": "object", "patternProperties": {r"^[\w\-\.]+$": {"type": ["string", "number"]}}, }, "before_script": script_schema, "script": script_schema, "after_script": script_schema, }, } ref_ci_job_attributes = {"$ref": "#/definitions/ci_job_attributes"} submapping_schema = { "type": "object", "additionalProperties": False, "required": ["submapping"], "properties": { "match_behavior": {"type": "string", "enum": ["first", "merge"], "default": "first"}, "submapping": { "type": "array", "items": { "type": "object", "additionalProperties": False, "required": ["match"], "properties": { "match": {"type": "array", "items": {"type": "string"}}, "build-job": ref_ci_job_attributes, "build-job-remove": ref_ci_job_attributes, }, }, }, }, } dynamic_mapping_schema = { "type": "object", "additionalProperties": False, "required": ["dynamic-mapping"], "properties": { "dynamic-mapping": { "type": "object", "required": ["endpoint"], "properties": { "name": {"type": "string"}, # "endpoint" cannot have http patternProperties constraint since it is required # Constrain is applied in code "endpoint": {"type": "string"}, "timeout": {"type": "integer", "minimum": 0}, "verify_ssl": {"type": "boolean", "default": False}, "header": {"type": "object", "additionalProperties": {"type": "string"}}, "allow": {"type": "array", "items": {"type": "string"}}, "require": {"type": "array", "items": {"type": "string"}}, "ignore": {"type": "array", "items": {"type": "string"}}, }, } }, } def job_schema(name: str): return { "type": "object", "additionalProperties": False, "properties": { f"{name}-job": ref_ci_job_attributes, f"{name}-job-remove": ref_ci_job_attributes, }, } pipeline_gen_schema = { "type": "array", "items": { "oneOf": [ submapping_schema, dynamic_mapping_schema, job_schema("any"), job_schema("build"), job_schema("cleanup"), job_schema("copy"), job_schema("noop"), job_schema("reindex"), job_schema("signing"), ] }, } #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "ci": { "type": "object", "properties": { "pipeline-gen": pipeline_gen_schema, "rebuild-index": {"type": "boolean"}, "broken-specs-url": {"type": "string"}, "broken-tests-packages": {"type": "array", "items": {"type": "string"}}, "target": {"type": "string", "default": "gitlab"}, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack CI configuration file schema", "type": "object", "additionalProperties": False, "definitions": {"ci_job_attributes": ci_job_attributes}, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/compilers.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for compilers.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/compilers.py :lines: 15- """ from typing import Any, Dict import spack.schema.environment flags: Dict[str, Any] = { "type": "object", "additionalProperties": False, "description": "Flags to pass to the compiler during compilation and linking", "properties": { "cflags": { "anyOf": [{"type": "string"}, {"type": "null"}], "description": "Flags for C compiler, e.g. -std=c11", }, "cxxflags": { "anyOf": [{"type": "string"}, {"type": "null"}], "description": "Flags for C++ compiler, e.g. -std=c++14", }, "fflags": { "anyOf": [{"type": "string"}, {"type": "null"}], "description": "Flags for Fortran 77 compiler, e.g. -ffixed-line-length-none", }, "cppflags": { "anyOf": [{"type": "string"}, {"type": "null"}], "description": "Flags for C preprocessor, e.g. -DFOO=1", }, "ldflags": { "anyOf": [{"type": "string"}, {"type": "null"}], "description": "Flags passed to the compiler driver during linking, e.g. " "-Wl,--gc-sections", }, "ldlibs": { "anyOf": [{"type": "string"}, {"type": "null"}], "description": "Flags for linker libraries, e.g. -lpthread", }, }, } extra_rpaths: Dict[str, Any] = { "type": "array", "default": [], "items": {"type": "string"}, "description": "List of extra rpaths to inject by Spack's compiler wrappers", } implicit_rpaths: Dict[str, Any] = { "anyOf": [{"type": "array", "items": {"type": "string"}}, {"type": "boolean"}], "description": "List of non-default link directories to register at runtime as rpaths", } #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "compilers": { "type": "array", "items": { "type": "object", "additionalProperties": False, "properties": { "compiler": { "type": "object", "additionalProperties": False, "required": ["paths", "spec", "modules", "operating_system"], "properties": { "paths": { "type": "object", "required": ["cc", "cxx", "f77", "fc"], "additionalProperties": False, "properties": { "cc": {"anyOf": [{"type": "string"}, {"type": "null"}]}, "cxx": {"anyOf": [{"type": "string"}, {"type": "null"}]}, "f77": {"anyOf": [{"type": "string"}, {"type": "null"}]}, "fc": {"anyOf": [{"type": "string"}, {"type": "null"}]}, }, }, "flags": flags, "spec": {"type": "string"}, "operating_system": {"type": "string"}, "target": {"type": "string"}, "alias": {"anyOf": [{"type": "string"}, {"type": "null"}]}, "modules": { "anyOf": [ {"type": "null"}, {"type": "array", "items": {"type": "string"}}, ] }, "implicit_rpaths": implicit_rpaths, "environment": spack.schema.environment.ref_env_modifications, "extra_rpaths": extra_rpaths, }, } }, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack compiler configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, "definitions": {"env_modifications": spack.schema.environment.env_modifications}, } ================================================ FILE: lib/spack/spack/schema/concretizer.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for concretizer.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/concretizer.py :lines: 12- """ from typing import Any, Dict LIST_OF_SPECS = {"type": "array", "items": {"type": "string"}} properties: Dict[str, Any] = { "concretizer": { "type": "object", "additionalProperties": False, "description": "Concretizer configuration that controls dependency selection, package " "reuse, and solver behavior", "properties": { "force": { "type": "boolean", "default": False, "description": "Force re-concretization when concretizing environments", }, "reuse": { "description": "Controls how aggressively Spack reuses installed packages and " "build caches during concretization", "oneOf": [ { "type": "boolean", "description": "If true, reuse installed packages and build caches for " "all specs; if false, always perform fresh concretization", }, { "type": "string", "enum": ["dependencies"], "description": "Reuse installed packages and build caches only for " "dependencies, not root specs", }, { "type": "object", "description": "Advanced reuse configuration with fine-grained control " "over which specs are reused", "properties": { "roots": { "type": "boolean", "description": "If true, root specs are reused; if false, only " "dependencies of root specs are reused", }, "include": { **LIST_OF_SPECS, "description": "List of spec constraints. Reusable specs must " "match at least one constraint", }, "exclude": { **LIST_OF_SPECS, "description": "List of spec constraints. Reusable specs must " "not match any constraint", }, "from": { "type": "array", "description": "List of sources from which reused specs are taken", "items": { "type": "object", "description": "Source configuration for reusable specs", "properties": { "type": { "type": "string", "enum": [ "local", "buildcache", "external", "environment", ], "description": "Type of source: 'local' (installed " "packages), 'buildcache' (remote binaries), " "'external' (system packages), or 'environment' " "(from specific environment)", }, "path": { "type": "string", "description": "Path to the source (for environment " "type sources)", }, "include": { **LIST_OF_SPECS, "description": "Spec constraints that must be " "matched for this source (overrides global include)", }, "exclude": { **LIST_OF_SPECS, "description": "Spec constraints that must not be " "matched for this source (overrides global exclude)", }, }, }, }, }, }, ], }, "targets": { "type": "object", "description": "Controls which target microarchitectures are considered " "during concretization", "properties": { "host_compatible": { "type": "boolean", "description": "If true, only allow targets compatible with the " "current host; if false, allow any target (e.g., concretize for icelake " "while running on haswell)", }, "granularity": { "type": "string", "enum": ["generic", "microarchitectures"], "description": "Target selection granularity: 'microarchitectures' " "(e.g., haswell, skylake) or 'generic' (e.g., x86_64_v3, aarch64)", }, }, }, "unify": { "description": "Controls whether environment specs are concretized together " "or separately", "oneOf": [ { "type": "boolean", "description": "If true, concretize environment root specs together " "for unified dependencies; if false, concretize each spec independently", }, { "type": "string", "enum": ["when_possible"], "description": "Maximizes reuse, while allowing multiple instances of the " "same package", }, ], }, "compiler_mixing": { "oneOf": [{"type": "boolean"}, {"type": "array"}], "description": "Whether to allow compiler mixing between link/run dependencies", }, "splice": { "type": "object", "additionalProperties": False, "description": "Configuration for spec splicing: replacing dependencies " "with ABI-compatible alternatives to improve package reuse", "properties": { "explicit": { "type": "array", "default": [], "description": "List of explicit splice configurations to replace " "specific dependencies", "items": { "type": "object", "required": ["target", "replacement"], "additionalProperties": False, "description": "Explicit splice configuration", "properties": { "target": { "type": "string", "description": "Abstract spec to be replaced (e.g., 'mpi' " "or specific package)", }, "replacement": { "type": "string", "description": "Concrete spec with hash to use as " "replacement (e.g., 'mpich/abcdef')", }, "transitive": { "type": "boolean", "default": False, "description": "If true, use transitive splice (conflicts " "resolved using replacement dependencies); if false, use " "intransitive splice (conflicts resolved using original " "dependencies)", }, }, }, }, "automatic": { "type": "boolean", "description": "Enable automatic splicing for ABI-compatible packages " "(experimental feature)", }, }, }, "duplicates": { "type": "object", "description": "Controls whether the dependency graph can contain multiple " "configurations of the same package", "properties": { "strategy": { "type": "string", "enum": ["none", "minimal", "full"], "description": "Duplication strategy: 'none' (single config per " "package), 'minimal' (allow build-tools duplicates), 'full' " "(experimental: allow full build-tool stack separation)", }, "max_dupes": { "type": "object", "description": "Maximum number of duplicates allowed per package when " "using strategies that permit duplicates", "additionalProperties": { "type": "integer", "minimum": 1, "description": "Maximum number of duplicate instances for this " "package", }, }, }, }, "static_analysis": { "type": "boolean", "description": "Enable static analysis to reduce concretization time by " "generating smaller ASP problems", }, "timeout": { "type": "integer", "minimum": 0, "description": "Maximum time in seconds for the solve phase (0 means no " "time limit)", }, "error_on_timeout": { "type": "boolean", "description": "If true, timeout always results in error; if false, use best " "suboptimal solution found before timeout (yields unreproducible results)", }, "os_compatible": { "type": "object", "additionalProperties": {"type": "array"}, "description": "Compatibility mapping between operating systems for reuse of " "compilers and packages (key: target OS, value: list of compatible source OSes)", }, "concretization_cache": { "type": "object", "description": "Configuration for caching solver outputs from successful " "concretization runs", "properties": { "enable": { "type": "boolean", "description": "Whether to utilize a cache of solver outputs from " "successful concretization runs", }, "url": { "type": "string", "description": "Path to the location where Spack will root the " "concretization cache", }, "entry_limit": { "type": "integer", "minimum": 0, "description": "Limit on the number of concretization results that " "Spack will cache (0 disables pruning)", }, }, }, "externals": { "type": "object", "description": "Configuration for how Spack handles external packages during " "concretization", "properties": { "completion": { "type": "string", "enum": ["architecture_only", "default_variants"], "description": "Controls how missing information (variants, etc.) is " "completed for external packages: 'architecture_only' completes only " "mandatory architectural information; 'default_variants' also completes " "missing variants using their default values", } }, }, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack concretizer configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/config.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for config.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/config.py :lines: 17- """ from typing import Any, Dict import spack.schema import spack.schema.projections #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "config": { "type": "object", "default": {}, "description": "Spack's basic configuration options", "properties": { "flags": { "type": "object", "description": "Build flag configuration options", "properties": { "keep_werror": { "type": "string", "enum": ["all", "specific", "none"], "description": "Whether to keep -Werror flags active in package builds", } }, }, "shared_linking": { "description": "Control how shared libraries are located at runtime on Linux", "anyOf": [ {"type": "string", "enum": ["rpath", "runpath"]}, { "type": "object", "properties": { "type": { "type": "string", "enum": ["rpath", "runpath"], "description": "Whether to use RPATH or RUNPATH for runtime " "library search paths", }, "bind": { "type": "boolean", "description": "Embed absolute paths of dependent libraries " "directly in ELF binaries (experimental)", }, "missing_library_policy": { "enum": ["error", "warn", "ignore"], "description": "How to handle missing dynamic libraries after " "installation", }, }, }, ], }, "install_tree": { "type": "object", "description": "Installation tree configuration", "properties": { "root": { "type": "string", "description": "The location where Spack will install packages and " "their dependencies", }, "padded_length": { "oneOf": [{"type": "integer", "minimum": 0}, {"type": "boolean"}], "description": "Length to pad installation paths to allow better " "relocation of binaries (true for max length, integer for specific " "length)", }, **spack.schema.projections.ref_properties, }, }, "install_hash_length": { "type": "integer", "minimum": 1, "description": "Length of hash used in installation directory names", }, "build_stage": { "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], "description": "Temporary locations Spack can try to use for builds", }, "stage_name": { "type": "string", "description": "Name format for build stage directories", }, "develop_stage_link": { "type": "string", "description": "Name for development spec build stage directories. Setting to " "None will disable develop stage links.", }, "test_stage": { "type": "string", "description": "Directory in which to run tests and store test results", }, "extensions": { "type": "array", "items": {"type": "string"}, "description": "List of Spack extensions to load", }, "template_dirs": { "type": "array", "items": {"type": "string"}, "description": "Locations where templates should be found", }, "license_dir": { "type": "string", "description": "Directory where licenses should be located", }, "source_cache": { "type": "string", "description": "Location to cache downloaded tarballs and repositories", }, "misc_cache": { "type": "string", "description": "Temporary directory to store long-lived cache files, such as " "indices of packages", }, "environments_root": { "type": "string", "description": "Directory where Spack managed environments are created and stored", }, "connect_timeout": { "type": "integer", "minimum": 0, "description": "Abort downloads after this many seconds if no data is received " "(0 disables timeout)", }, "verify_ssl": { "type": "boolean", "description": "When true, Spack will verify certificates of remote hosts when " "making SSL connections", }, "ssl_certs": { "type": "string", "description": "Path to custom certificates for SSL verification", }, "suppress_gpg_warnings": { "type": "boolean", "description": "Suppress GPG warnings from binary package verification", }, "debug": { "type": "boolean", "description": "Enable debug mode for additional logging", }, "checksum": { "type": "boolean", "description": "When true, Spack verifies downloaded source code using checksums", }, "deprecated": { "type": "boolean", "description": "If true, Spack will fetch deprecated versions without warning", }, "locks": { "type": "boolean", "description": "When true, concurrent instances of Spack will use locks to avoid " "conflicts (strongly recommended)", }, "dirty": { "type": "boolean", "description": "When true, builds will NOT clean potentially harmful variables " "from the environment", }, "build_language": { "type": "string", "description": "The language the build environment will use (C for English, " "empty string for user's environment)", }, "build_jobs": { "type": "integer", "minimum": 1, "description": "The maximum number of jobs to use for the build system (e.g. " "make -j), defaults to 16", }, "concurrent_packages": { "type": "integer", "minimum": 0, "description": "The maximum number of concurrent package builds a single Spack " "instance will run", }, "ccache": { "type": "boolean", "description": "When true, Spack's compiler wrapper will use ccache when " "compiling C and C++", }, "db_lock_timeout": { "type": "integer", "minimum": 1, "description": "How long to wait to lock the Spack installation database", }, "package_lock_timeout": { "anyOf": [{"type": "integer", "minimum": 1}, {"type": "null"}], "description": "How long to wait when attempting to modify a package (null for " "never timeout)", }, "allow_sgid": { "type": "boolean", "description": "Allow installation on filesystems that don't allow setgid bit " "manipulation", }, "install_status": { "type": "boolean", "description": "Whether to show status information in the terminal title during " "the build", }, "url_fetch_method": { "anyOf": [{"enum": ["urllib", "curl"]}, {"type": "string", "pattern": r"^curl "}], "description": "The default URL fetch method to use (urllib or curl)", }, "additional_external_search_paths": { "type": "array", "items": {"type": "string"}, "description": "Additional paths to search for external packages", }, "binary_index_ttl": { "type": "integer", "minimum": 0, "description": "Number of seconds a buildcache's index.json is cached locally " "before probing for updates", }, "aliases": { "type": "object", "additionalProperties": {"type": "string"}, "description": "A mapping of aliases that can be used to define new " "Spack commands", }, "installer": { "type": "string", "enum": ["old", "new"], "description": "Which installer to use. The new installer is experimental.", }, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack core configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, "definitions": {"projections": spack.schema.projections.projections}, } def update(data: dict) -> bool: """Update the data in place to remove deprecated properties. Args: data: dictionary to be updated Returns: True if data was changed, False otherwise """ changed = False data = data["config"] shared_linking = data.get("shared_linking", None) if isinstance(shared_linking, str): # deprecated short-form shared_linking: rpath/runpath # add value as `type` in updated shared_linking data["shared_linking"] = {"type": shared_linking, "bind": False} changed = True return changed ================================================ FILE: lib/spack/spack/schema/container.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for the ``container`` subsection of Spack environments.""" from typing import Any, Dict _stages_from_dockerhub = { "type": "object", "additionalProperties": False, "properties": { "os": {"type": "string"}, "spack": { "anyOf": [ {"type": "string"}, { "type": "object", "additionalProperties": False, "properties": { "url": {"type": "string"}, "ref": {"type": "string"}, "resolve_sha": {"type": "boolean", "default": False}, "verify": {"type": "boolean", "default": False}, }, }, ] }, }, "required": ["os", "spack"], } _custom_stages = { "type": "object", "additionalProperties": False, "properties": {"build": {"type": "string"}, "final": {"type": "string"}}, "required": ["build", "final"], } #: List of packages for the schema below _list_of_packages = {"type": "array", "items": {"type": "string"}} #: Schema for the container attribute included in Spack environments container_schema = { "type": "object", "additionalProperties": False, "properties": { # The recipe formats that are currently supported by the command "format": {"type": "string", "enum": ["docker", "singularity"]}, # Describes the base image to start from and the version # of Spack to be used "images": {"anyOf": [_stages_from_dockerhub, _custom_stages]}, # Whether or not to strip installed binaries "strip": {"type": "boolean", "default": True}, # Additional system packages that are needed at runtime "os_packages": { "type": "object", "properties": { "command": { "type": "string", "enum": ["apt", "yum", "zypper", "apk", "yum_amazon"], }, "update": {"type": "boolean"}, "build": _list_of_packages, "final": _list_of_packages, }, "additionalProperties": False, }, # Add labels to the image "labels": {"type": "object"}, # Use a custom template to render the recipe "template": {"type": "string", "default": None}, # Reserved for properties that are specific to each format "singularity": { "type": "object", "additionalProperties": False, "default": {}, "properties": { "runscript": {"type": "string"}, "startscript": {"type": "string"}, "test": {"type": "string"}, "help": {"type": "string"}, }, }, "docker": {"type": "object", "additionalProperties": False, "default": {}}, "depfile": {"type": "boolean", "default": False}, }, } properties: Dict[str, Any] = {"container": container_schema} ================================================ FILE: lib/spack/spack/schema/cray_manifest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for Cray descriptive manifest: this describes a set of installed packages on the system and also specifies dependency relationships between them (so this provides more information than external entries in packages configuration). This does not specify a configuration - it is an input format that is consumed and transformed into Spack DB records. """ from typing import Any, Dict properties: Dict[str, Any] = { "_meta": { "type": "object", "additionalProperties": False, "properties": { "file-type": {"type": "string", "minLength": 1}, "cpe-version": {"type": "string", "minLength": 1}, "system-type": {"type": "string", "minLength": 1}, "schema-version": {"type": "string", "minLength": 1}, # Older schemas use did not have "cpe-version", just the # schema version; in that case it was just called "version" "version": {"type": "string", "minLength": 1}, }, }, "compilers": { "type": "array", "items": { "type": "object", "additionalProperties": False, "properties": { "name": {"type": "string", "minLength": 1}, "version": {"type": "string", "minLength": 1}, "prefix": {"type": "string", "minLength": 1}, "executables": { "type": "object", "additionalProperties": False, "properties": { "cc": {"type": "string", "minLength": 1}, "cxx": {"type": "string", "minLength": 1}, "fc": {"type": "string", "minLength": 1}, }, }, "arch": { "type": "object", "required": ["os", "target"], "additionalProperties": False, "properties": { "os": {"type": "string", "minLength": 1}, "target": {"type": "string", "minLength": 1}, }, }, }, }, }, "specs": { "type": "array", "items": { "type": "object", "required": ["name", "version", "arch", "compiler", "prefix", "hash"], "additionalProperties": False, "properties": { "name": {"type": "string", "minLength": 1}, "version": {"type": "string", "minLength": 1}, "arch": { "type": "object", "required": ["platform", "platform_os", "target"], "additionalProperties": False, "properties": { "platform": {"type": "string", "minLength": 1}, "platform_os": {"type": "string", "minLength": 1}, "target": { "type": "object", "additionalProperties": False, "required": ["name"], "properties": {"name": {"type": "string", "minLength": 1}}, }, }, }, "compiler": { "type": "object", "required": ["name", "version"], "additionalProperties": False, "properties": { "name": {"type": "string", "minLength": 1}, "version": {"type": "string", "minLength": 1}, }, }, "dependencies": { "type": "object", "additionalProperties": { "type": "object", "required": ["hash"], "additionalProperties": False, "properties": { "hash": {"type": "string", "minLength": 1}, "type": {"type": "array", "items": {"type": "string", "minLength": 1}}, }, }, }, "prefix": {"type": "string", "minLength": 1}, "rpm": {"type": "string", "minLength": 1}, "hash": {"type": "string", "minLength": 1}, "parameters": {"type": "object"}, }, }, }, } schema = { "$schema": "http://json-schema.org/schema#", "title": "CPE manifest schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/database_index.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for database index.json file .. literalinclude:: _spack_root/lib/spack/spack/schema/database_index.py :lines: 17- """ from typing import Any, Dict import spack.schema.spec properties: Dict[str, Any] = { "database": { "type": "object", "required": ["installs", "version"], "additionalProperties": False, "properties": { "installs": { "type": "object", "patternProperties": { r"^[a-z0-9]{32}$": { "type": "object", "properties": { "spec": spack.schema.spec.spec_node, "path": {"oneOf": [{"type": "string"}, {"type": "null"}]}, "installed": {"type": "boolean"}, "ref_count": {"type": "integer", "minimum": 0}, "explicit": {"type": "boolean"}, "installation_time": {"type": "number"}, }, } }, }, "version": {"type": "string"}, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack spec schema", "type": "object", "required": ["database"], "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/definitions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for definitions .. literalinclude:: _spack_root/lib/spack/spack/schema/definitions.py :lines: 16- """ from typing import Any, Dict from .spec_list import spec_list_schema #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "definitions": { "type": "array", "default": [], "description": "Named spec lists to be referred to with $name in the specs section of " "environments", "items": { "type": "object", "description": "Named definition entry containing a named spec list and optional " "conditional 'when' clause", "properties": { "when": { "type": "string", "description": "Python code condition evaluated as boolean. Specs are " "appended to the named list only if the condition is True. Available " "variables: platform, os, target, arch, arch_str, re, env, hostname", } }, "additionalProperties": spec_list_schema, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack definitions configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/develop.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from typing import Any, Dict properties: Dict[str, Any] = { "develop": { "type": "object", "default": {}, "description": "Configuration for local development of Spack packages", "additionalProperties": { "type": "object", "additionalProperties": False, "description": "Name of a package to develop, with its spec and optional source path", "required": ["spec"], "properties": { "spec": { "type": "string", "description": "Spec of the package to develop, e.g. hdf5@1.12.0", }, "path": { "type": "string", "description": "Path to the source code for this package, can be " "absolute or relative to the environment directory", }, }, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack repository configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/env.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for env.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/env.py :lines: 19- """ import os from typing import Any, Dict import spack.schema.merged from .spec_list import spec_list_properties, spec_list_schema #: Top level key in a manifest file TOP_LEVEL_KEY = "spack" # (DEPRECATED) include concrete entries to be merged under the include key include_concrete = { "type": "array", "default": [], "description": "List of paths to other environments. Includes concrete specs " "from their spack.lock files without modifying the source environments. Useful " "for phased deployments where you want to build on existing concrete specs.", "items": {"type": "string"}, } group_name_and_deps = { "group": {"type": "string", "description": "Name for this group of specs"}, "explicit": { "type": "boolean", "default": True, "description": "When false, specs in this group are installed as implicit " "dependencies and are eligible for garbage collection.", }, "needs": { "type": "array", "description": "Groups of specs that are needed by this group", "items": {"type": "string"}, }, "override": { "type": "object", "description": "Top-most configuration scope for this group of specs", "additionalProperties": False, "properties": {**spack.schema.merged.ref_sections}, }, } properties: Dict[str, Any] = { "spack": { "type": "object", "default": {}, "description": "Spack environment configuration, including specs, view, and any other " "config section (config, packages, concretizer, mirrors, etc.)", "additionalProperties": False, "properties": { # merged configuration scope schemas **spack.schema.merged.ref_sections, # extra environment schema properties "specs": { "type": "array", "description": "List of specs to include in the environment, " "supporting both simple specs and matrix configurations", "default": [], "items": { "anyOf": [ { "type": "object", "description": "Matrix configuration for generating multiple specs" " from combinations of constraints", "additionalProperties": False, "properties": {**spec_list_properties}, }, {"type": "string", "description": "Simple spec string"}, {"type": "null"}, { "type": "object", "description": "User spec group with a single matrix", "additionalProperties": False, "properties": {**spec_list_properties, **group_name_and_deps}, }, { "type": "object", "description": "User spec group with multiple matrices", "additionalProperties": False, "properties": {**group_name_and_deps, "specs": spec_list_schema}, }, ] }, }, # (DEPRECATED) include concrete to be merged under the include key "include_concrete": include_concrete, }, } } schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack environment file schema", "type": "object", "additionalProperties": False, "properties": properties, "definitions": spack.schema.merged.defs, } def update(data: Dict[str, Any]) -> bool: """Update the spack.yaml data to the new format. Args: data: dictionary to be updated Returns: ``True`` if data was changed, ``False`` otherwise """ if not isinstance(data, dict): return False if "include_concrete" not in data: return False # Move the old 'include_concrete' paths to reside under the 'include', # ensuring that the lock file name is appended. includes = [] for path in data["include_concrete"]: if os.path.basename(path) != "spack.lock": path = os.path.join(path, "spack.lock") includes.append(path) # Now add back the includes the environment file already has. if "include" in data: for path in data["include"]: includes.append(path) data["include"] = includes del data["include_concrete"] return True ================================================ FILE: lib/spack/spack/schema/env_vars.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for env_vars.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/env_vars.py :lines: 15- """ from typing import Any, Dict import spack.schema.environment properties: Dict[str, Any] = {"env_vars": spack.schema.environment.ref_env_modifications} #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack env_vars configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, "definitions": {"env_modifications": spack.schema.environment.env_modifications}, } ================================================ FILE: lib/spack/spack/schema/environment.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for environment modifications. Meant for inclusion in other schemas. """ import collections.abc from typing import Any, Dict dictionary_of_strings_or_num = { "type": "object", "additionalProperties": {"anyOf": [{"type": "string"}, {"type": "number"}]}, } env_modifications: Dict[str, Any] = { "type": "object", "description": "Environment variable modifications to apply at runtime", "default": {}, "additionalProperties": False, "properties": { "set": { "description": "Environment variables to set to specific values", **dictionary_of_strings_or_num, }, "unset": { "description": "Environment variables to remove/unset", "default": [], "type": "array", "items": {"type": "string"}, }, "prepend_path": { "description": "Environment variables to prepend values to (typically PATH-like " "variables)", **dictionary_of_strings_or_num, }, "append_path": { "description": "Environment variables to append values to (typically PATH-like " "variables)", **dictionary_of_strings_or_num, }, "remove_path": { "description": "Values to remove from PATH-like environment variables", **dictionary_of_strings_or_num, }, }, } #: $ref pointer for use in merged schema ref_env_modifications = {"$ref": "#/definitions/env_modifications"} def parse(config_obj): """Returns an EnvironmentModifications object containing the modifications parsed from input. Args: config_obj: a configuration dictionary conforming to the schema definition for environment modifications """ import spack.util.environment as ev env = ev.EnvironmentModifications() for command, variable in config_obj.items(): # Distinguish between commands that take only a name as argument # (e.g. unset) and commands that take a name and a value. if isinstance(variable, collections.abc.Sequence): for name in variable: getattr(env, command)(name) else: for name, value in variable.items(): getattr(env, command)(name, value) return env ================================================ FILE: lib/spack/spack/schema/include.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for include.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/include.py :lines: 12- """ from typing import Any, Dict #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "include": { "type": "array", "default": [], "additionalProperties": False, "description": "Include external configuration files to pull in configuration from " "other files/URLs for modular and reusable configurations", "items": { "anyOf": [ # local, required path { "type": "string", "description": "Simple include entry specifying path to required " "configuration file/directory", }, # local or remote paths that may be optional or conditional { "type": "object", "description": "Advanced include entry with optional conditions and " "remote file support", "properties": { "when": { "type": "string", "description": "Include this config only when the condition (as " "Python code) evaluates to true", }, "name": {"type": "string"}, "path_override_env_var": {"type": "string"}, "path": { "type": "string", "description": "Path to configuration file/directory (absolute, " "relative, or URL). URLs must be raw file content (GitHub/GitLab " "raw form). Supports file, ftp, http, https schemes and " "Spack/environment variables", }, "sha256": { "type": "string", "description": "Required SHA256 hash for remote URLs to verify " "file integrity", }, "optional": { "type": "boolean", "description": "If true, include only if path exists; if false " "(default), path is required and missing files cause errors", }, "prefer_modify": {"type": "boolean"}, }, "required": ["path"], "additionalProperties": False, }, # remote git paths that may be optional or conditional { "type": "object", "description": "Include configuration files from a git repository with " "conditional and optional support", "properties": { "git": { "type": "string", "description": "URL of the git repository to clone (e.g., " "https://github.com/spack/spack-configs)", }, "branch": { "type": "string", "description": "Branch to check out from the repository", }, "commit": { "type": "string", "description": "Specific commit SHA to check out from the repository", }, "tag": { "type": "string", "description": "Tag to check out from the repository", }, "paths": { "type": "array", "items": { "type": "string", "description": "Relative path within the repository to a " "configuration file to include", }, "description": "List of relative paths within the repository where " "configuration files are located", }, "name": {"type": "string"}, "when": { "type": "string", "description": "Include this config only when the condition (as " "Python code) evaluates to true", }, "optional": { "type": "boolean", "description": "If true, include only if repository is accessible; " "if false (default), inaccessible repository causes errors", }, }, "required": ["git", "paths"], "additionalProperties": False, }, ] }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack include configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/merged.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for configuration merged into one file. .. literalinclude:: _spack_root/lib/spack/spack/schema/merged.py :lines: 32- """ from typing import Any, Dict import spack.schema.bootstrap import spack.schema.cdash import spack.schema.ci import spack.schema.compilers import spack.schema.concretizer import spack.schema.config import spack.schema.container import spack.schema.definitions import spack.schema.develop import spack.schema.env_vars import spack.schema.environment import spack.schema.include import spack.schema.mirrors import spack.schema.modules import spack.schema.packages import spack.schema.projections import spack.schema.repos import spack.schema.toolchains import spack.schema.upstreams import spack.schema.view #: Properties for inclusion in other schemas sections: Dict[str, Any] = { **spack.schema.bootstrap.properties, **spack.schema.cdash.properties, **spack.schema.compilers.properties, **spack.schema.concretizer.properties, **spack.schema.config.properties, **spack.schema.container.properties, **spack.schema.ci.properties, **spack.schema.definitions.properties, **spack.schema.develop.properties, **spack.schema.env_vars.properties, **spack.schema.include.properties, **spack.schema.mirrors.properties, **spack.schema.modules.properties, **spack.schema.packages.properties, **spack.schema.repos.properties, **spack.schema.toolchains.properties, **spack.schema.upstreams.properties, **spack.schema.view.properties, } #: Canonical definitions for JSON Schema $ref defs: Dict[str, Any] = { # Section schemas, prefixed to avoid collisions with sub-schema definitions **{f"section_{name}": schema for name, schema in sections.items()}, # Sub-schema definitions hoisted for $ref resolution in env.py "ci_job_attributes": spack.schema.ci.ci_job_attributes, "env_modifications": spack.schema.environment.env_modifications, "module_file_configuration": spack.schema.modules.module_file_configuration, "projections": spack.schema.projections.projections, } #: Properties using $ref pointers into $defs ref_sections: Dict[str, Any] = { name: {"$ref": f"#/definitions/section_{name}"} for name in sections } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack merged configuration file schema", "type": "object", "additionalProperties": False, "properties": ref_sections, "definitions": defs, } ================================================ FILE: lib/spack/spack/schema/mirrors.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for mirrors.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/mirrors.py :lines: 13- """ from typing import Any, Dict #: Common properties for connection specification connection = { "url": { "type": "string", "description": "URL pointing to the mirror directory, can be local filesystem " "(file://) or remote server (http://, https://, s3://, oci://)", }, "view": {"type": "string"}, "access_pair": { "type": "object", "description": "Authentication credentials for accessing private mirrors with ID and " "secret pairs", "required": ["secret_variable"], # Only allow id or id_variable to be set, not both "oneOf": [{"required": ["id"]}, {"required": ["id_variable"]}], "properties": { "id": { "type": "string", "description": "Static access ID or username for authentication", }, "id_variable": { "type": "string", "description": "Environment variable name containing the access ID or username", }, "secret_variable": { "type": "string", "description": "Environment variable name containing the secret key, password, " "or access token", }, }, }, "profile": { "type": ["string", "null"], "description": "AWS profile name to use for S3 mirror authentication", }, "endpoint_url": { "type": ["string", "null"], "description": "Custom endpoint URL for S3-compatible storage services", }, "access_token_variable": { "type": ["string", "null"], "description": "Environment variable containing an access token for OCI registry " "authentication", }, } #: Mirror connection inside pull/push keys fetch_and_push = { "description": "Mirror connection configuration for fetching or pushing packages, can be a" "simple URL string or detailed connection object", "anyOf": [ { "type": "string", "description": "Simple URL string for basic mirror connections without authentication", }, { "type": "object", "description": "Detailed connection configuration with authentication and custom " "settings", "additionalProperties": False, "properties": {**connection}, }, ], } #: Mirror connection when no pull/push keys are set mirror_entry = { "type": "object", "description": "Mirror configuration entry supporting both source package archives and " "binary build caches with optional authentication", "additionalProperties": False, "anyOf": [{"required": ["url"]}, {"required": ["fetch"]}, {"required": ["pull"]}], "properties": { "source": { "type": "boolean", "description": "Whether this mirror provides source package archives (tarballs) for " "building from source", }, "binary": { "type": "boolean", "description": "Whether this mirror provides binary build caches for installing " "precompiled packages", }, "signed": { "type": "boolean", "description": "Whether to require GPG signature verification for packages from " "this mirror", }, "fetch": { **fetch_and_push, "description": "Configuration for fetching/downloading packages from this mirror", }, "push": { **fetch_and_push, "description": "Configuration for pushing/uploading packages to this mirror for " "build cache creation", }, "autopush": { "type": "boolean", "description": "Automatically push packages to this build cache immediately after " "they are installed locally", }, **connection, }, } #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "mirrors": { "type": "object", "default": {}, "description": "Configure local and remote mirrors that provide repositories of source " "tarballs and binary build caches for faster package installation", "additionalProperties": { "description": "Named mirror configuration that can be a simple URL string or " "detailed mirror entry with authentication and build cache settings", "anyOf": [ { "type": "string", "description": "Simple mirror URL for basic source package or build " "cache access", }, mirror_entry, ], }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack mirror configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/modules.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for modules.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/modules.py :lines: 16- """ from typing import Any, Dict import spack.schema.environment import spack.schema.projections #: Definitions for parts of module schema array_of_strings = {"type": "array", "default": [], "items": {"type": "string"}} dependency_selection = {"type": "string", "enum": ["none", "run", "direct", "all"]} module_file_configuration = { "type": "object", "default": {}, "description": "Configuration for individual module file behavior and content customization", "additionalProperties": False, "properties": { "filter": { "type": "object", "default": {}, "description": "Filter out specific environment variable modifications from " "module files", "additionalProperties": False, "properties": { "exclude_env_vars": { "type": "array", "default": [], "items": {"type": "string"}, "description": "List of environment variable names to exclude from module " "file modifications", } }, }, "template": { "type": "string", "description": "Path to custom template file for generating module files", }, "autoload": { **dependency_selection, "description": "Automatically load dependency modules when this module is loaded", }, "prerequisites": { **dependency_selection, "description": "Mark dependency modules as prerequisites instead of autoloading them", }, "conflict": { **array_of_strings, "description": "List of modules that conflict with this one and should not be loaded " "simultaneously", }, "load": { **array_of_strings, "description": "List of additional modules to load when this module is loaded", }, "suffixes": { "type": "object", "description": "Add custom suffixes to module names based on spec matching for better " "readability", "additionalKeysAreSpecs": True, "additionalProperties": {"type": "string"}, # key }, "environment": spack.schema.environment.ref_env_modifications, }, } ref_module_file_configuration = {"$ref": "#/definitions/module_file_configuration"} projections_scheme = {"$ref": "#/definitions/projections"} common_props = { "verbose": { "type": "boolean", "default": False, "description": "Enable verbose output during module file generation", }, "hash_length": { "type": "integer", "minimum": 0, "default": 7, "description": "Length of package hash to include in module file names (0-32, shorter " "hashes may cause naming conflicts)", }, "include": { **array_of_strings, "description": "List of specs to explicitly include for module file generation, even if " "they would normally be excluded", }, "exclude": { **array_of_strings, "description": "List of specs to exclude from module file generation", }, "exclude_implicits": { "type": "boolean", "default": False, "description": "Exclude implicit dependencies from module file generation while still " "allowing autoloading", }, "defaults": { **array_of_strings, "description": "List of specs for which to create default module symlinks when multiple " "versions exist", }, "hide_implicits": { "type": "boolean", "default": False, "description": "Hide implicit dependency modules from 'module avail' but still allow " "autoloading (requires module system support)", }, "naming_scheme": { "type": "string", "description": "Custom naming scheme for module files using format strings", }, "projections": { **projections_scheme, "description": "Custom directory structure and naming convention for module files using " "projection format", }, "all": ref_module_file_configuration, } tcl_configuration = { "type": "object", "default": {}, "description": "Configuration for TCL module files compatible with Environment Modules and " "Lmod", "additionalKeysAreSpecs": True, "properties": {**common_props}, "additionalProperties": ref_module_file_configuration, } lmod_configuration = { "type": "object", "default": {}, "description": "Configuration for Lua module files compatible with Lmod hierarchical module " "system", "additionalKeysAreSpecs": True, "properties": { **common_props, "core_compilers": { **array_of_strings, "description": "List of core compilers that are always available at the top level of " "the Lmod hierarchy", }, "hierarchy": { **array_of_strings, "description": "List of packages to use for building the Lmod module hierarchy " "(typically compilers and MPI implementations)", }, "core_specs": { **array_of_strings, "description": "List of specs that should be placed in the core level of the Lmod " "hierarchy regardless of dependencies", }, "filter_hierarchy_specs": { "type": "object", "description": "Filter which specs are included at different levels of the Lmod " "hierarchy based on spec matching", "additionalKeysAreSpecs": True, "additionalProperties": array_of_strings, }, }, "additionalProperties": ref_module_file_configuration, } module_config_properties = { "use_view": { "anyOf": [{"type": "string"}, {"type": "boolean"}], "description": "Generate modules relative to an environment view instead of install " "tree (True for default view, string for named view, False to disable)", }, "arch_folder": { "type": "boolean", "description": "Whether to include architecture-specific subdirectories in module file " "paths", }, "roots": { "type": "object", "description": "Custom root directories for different module file types", "properties": { "tcl": {"type": "string", "description": "Root directory for TCL module files"}, "lmod": {"type": "string", "description": "Root directory for Lmod module files"}, }, }, "enable": { "type": "array", "default": [], "description": "List of module types to automatically generate during package " "installation", "items": {"type": "string", "enum": ["tcl", "lmod"]}, }, "lmod": { **lmod_configuration, "description": "Configuration for Lmod hierarchical module system", }, "tcl": { **tcl_configuration, "description": "Configuration for TCL module files compatible with Environment Modules", }, "prefix_inspections": { "type": "object", "description": "Control which package subdirectories are added to environment variables " "(e.g., bin to PATH, lib to LIBRARY_PATH)", "additionalProperties": { # prefix-relative path to be inspected for existence **array_of_strings, "description": "List of environment variables to update with this prefix-relative " "path if it exists", }, }, } # Properties for inclusion into other schemas (requires definitions) properties: Dict[str, Any] = { "modules": { "type": "object", "description": "Configure automatic generation of module files for Environment Modules " "and Lmod to manage user environments at HPC centers", "properties": { "prefix_inspections": { "type": "object", "description": "Global prefix inspection settings that apply to all module sets, " "controlling which subdirectories are added to environment variables", "additionalProperties": { # prefix-relative path to be inspected for existence **array_of_strings, "description": "List of environment variables to update with this " "prefix-relative path if it exists", }, } }, "additionalProperties": { "type": "object", "default": {}, "description": "Named module set configuration (e.g., 'default') defining how module " "files are generated for a specific set of packages", "additionalProperties": False, "properties": module_config_properties, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack module file configuration file schema", "type": "object", "additionalProperties": False, "definitions": { "module_file_configuration": module_file_configuration, "projections": spack.schema.projections.projections, "env_modifications": spack.schema.environment.env_modifications, }, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/packages.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for packages.yaml configuration files. .. literalinclude:: _spack_root/lib/spack/spack/schema/packages.py :lines: 14- """ from typing import Any, Dict import spack.schema.environment from .compilers import extra_rpaths, flags, implicit_rpaths permissions = { "type": "object", "description": "File permissions settings for package installations", "additionalProperties": False, "properties": { "read": { "type": "string", "enum": ["user", "group", "world"], "description": "Who can read the files installed by a package", }, "write": { "type": "string", "enum": ["user", "group", "world"], "description": "Who can write to the files installed by a package", }, "group": { "type": "string", "description": "The group that owns the files installed by a package", }, }, } variants = { "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], "description": "Soft variant preferences as a single spec string or list of variant " "specifications (ignored if the concretizer can reuse existing installations)", } requirements = { "description": "Package requirements that must be satisfied during concretization", "oneOf": [ # 'require' can be a list of requirement_groups. each requirement group is a list of one or # more specs. Either at least one or exactly one spec in the group must be satisfied # (depending on whether you use "any_of" or "one_of", respectively) { "type": "array", "items": { "oneOf": [ { "type": "object", "additionalProperties": False, "properties": { "one_of": { "type": "array", "items": {"type": "string"}, "description": "List of specs where exactly one must be satisfied", }, "any_of": { "type": "array", "items": {"type": "string"}, "description": "List of specs where at least one must be " "satisfied", }, "spec": { "type": "string", "description": "Single spec requirement that must be satisfied", }, "message": { "type": "string", "description": "Custom error message when requirement is not " "satisfiable", }, "when": { "type": "string", "description": "Conditional spec that triggers this requirement", }, }, }, {"type": "string"}, ] }, }, # Shorthand for a single requirement group with one member {"type": "string"}, ], } prefer_and_conflict = { "type": "array", "items": { "oneOf": [ { "type": "object", "additionalProperties": False, "properties": { "spec": {"type": "string", "description": "Spec constraint to apply"}, "message": { "type": "string", "description": "Custom message explaining the constraint", }, "when": { "type": "string", "description": "Conditional spec that triggers this constraint", }, }, }, {"type": "string"}, ] }, } package_attributes = { "type": "object", "description": "Class-level attributes to assign to package instances " "(accessible in package.py methods)", "additionalProperties": False, "patternProperties": {r"^[a-zA-Z_]\w*$": {}}, } REQUIREMENT_URL = "https://spack.readthedocs.io/en/latest/packages_yaml.html#package-requirements" #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "packages": { "type": "object", "description": "Package-specific build settings and external package configurations", "default": {}, "properties": { "all": { "type": "object", "description": "Default settings that apply to all packages (can be overridden " "by package-specific settings)", "default": {}, "additionalProperties": False, "properties": { "require": requirements, "prefer": { "description": "Strong package preferences that influence concretization " "without imposing hard constraints", **prefer_and_conflict, }, "conflict": { "description": "Package conflicts that prevent certain spec combinations", **prefer_and_conflict, }, # target names "target": { "type": "array", "description": "Ordered list of soft preferences for target " "architectures for all packages (ignored if the concretizer can reuse " "existing installations)", "default": [], "items": {"type": "string"}, }, # compiler specs "compiler": { "type": "array", "description": "Soft preferences for compiler specs for all packages " "(deprecated)", "default": [], "items": {"type": "string"}, }, "buildable": { "type": "boolean", "description": "Whether packages should be built from source (false " "prevents building)", "default": True, }, "permissions": permissions, # If 'get_full_repo' is promoted to a Package-level # attribute, it could be useful to set it here "package_attributes": package_attributes, "providers": { "type": "object", "description": "Soft preferences for providers of virtual packages " "(ignored if the concretizer can reuse existing installations)", "default": {}, "additionalProperties": { "type": "array", "description": "Ordered list of preferred providers for this virtual " "package", "default": [], "items": {"type": "string"}, }, }, "variants": variants, }, "deprecatedProperties": [ { "names": ["compiler"], "message": "The packages:all:compiler preference has been deprecated in " "Spack v1.0, and is currently ignored. It will be removed from config in " "Spack v1.2.", "error": False, } ], } }, # package names "additionalProperties": { "type": "object", "description": "Package-specific settings that override defaults from 'all'", "default": {}, "additionalProperties": False, "properties": { "require": requirements, "prefer": { "description": "Strong package preferences that influence concretization " "without imposing hard constraints", **prefer_and_conflict, }, "conflict": { "description": "Package conflicts that prevent certain spec combinations", **prefer_and_conflict, }, "version": { "type": "array", "description": "Ordered list of soft preferences for versions for this " "package (ignored if the concretizer can reuse existing installations)", "default": [], # version strings "items": {"anyOf": [{"type": "string"}, {"type": "number"}]}, }, "buildable": { "type": "boolean", "description": "Whether this package should be built from source (false " "prevents building)", "default": True, }, "permissions": permissions, # If 'get_full_repo' is promoted to a Package-level # attribute, it could be useful to set it here "package_attributes": package_attributes, "variants": variants, "externals": { "type": "array", "description": "List of external, system-installed instances of this package", "items": { "type": "object", "properties": { "spec": { "type": "string", "description": "Spec string describing this external package " "instance. Typically name@version and relevant variants", }, "prefix": { "type": "string", "description": "Installation prefix path for this external " "package (typically /usr, *excluding* bin/, lib/, etc.)", }, "modules": { "type": "array", "description": "Environment modules to load for this external " "package", "items": {"type": "string"}, }, "id": {"type": "string"}, "extra_attributes": { "type": "object", "description": "Additional information needed by the package " "to use this external", "additionalProperties": {"type": "string"}, "properties": { "compilers": { "type": "object", "description": "Compiler executable paths for external " "compiler packages", "properties": { "c": { "type": "string", "description": "Path to the C compiler " "executable (e.g. /usr/bin/gcc)", }, "cxx": { "type": "string", "description": "Path to the C++ compiler " "executable (e.g. /usr/bin/g++)", }, "fortran": { "type": "string", "description": "Path to the Fortran compiler " "executable (e.g. /usr/bin/gfortran)", }, }, "patternProperties": {r"^\w": {"type": "string"}}, "additionalProperties": False, }, "environment": spack.schema.environment.ref_env_modifications, "extra_rpaths": extra_rpaths, "implicit_rpaths": implicit_rpaths, "flags": flags, }, }, "dependencies": { "type": "array", "description": "List of dependencies for this external package, " "specifying dependency relationships explicitly", "items": { "type": "object", "description": "Dependency specification for an external " "package", "properties": { "id": { "type": "string", "description": "Explicit reference ID to another " "external package (provides unambiguous reference)", }, "spec": { "type": "string", "description": "Spec string that matches an " "available external package", }, "deptypes": { "oneOf": [ { "type": "string", "description": "Single dependency type " "(e.g., 'build', 'link', 'run', 'test')", }, { "type": "array", "items": { "type": "string", "description": "Dependency type (e.g., " "'build', 'link', 'run', 'test')", }, "description": "List of dependency types " "(e.g., ['build', 'link'])", }, ], "description": "Dependency types; if not specified, " "inferred from package recipe", }, "virtuals": { "type": "string", "description": "Virtual package name this dependency " "provides (e.g., 'mpi')", }, }, }, }, }, "additionalProperties": False, "required": ["spec"], }, }, }, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack package configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, "definitions": {"env_modifications": spack.schema.environment.env_modifications}, } def update(data): data = data["packages"] changed = False for key in data: version = data[key].get("version") if not version or all(isinstance(v, str) for v in version): continue data[key]["version"] = [str(v) for v in version] changed = True return changed ================================================ FILE: lib/spack/spack/schema/projections.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for projections.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/projections.py :lines: 14- """ from typing import Any, Dict #: Properties for inclusion in other schemas projections: Dict[str, Any] = { "type": "object", "description": "Customize directory structure and naming schemes by mapping specs to " "format strings.", "properties": { "all": { "type": "string", "description": "Default projection format string used as fallback for all specs " "that do not match other entries. Uses spec format syntax like " '"{name}/{version}/{hash:16}".', } }, "additionalKeysAreSpecs": True, "additionalProperties": { "type": "string", "description": "Projection format string for specs matching this key. Uses spec " "format syntax supporting tokens like {name}, {version}, {compiler.name}, " "{^dependency.name}, etc.", }, } #: $ref pointer for use in merged schema ref_properties: Dict[str, Any] = {"projections": {"$ref": "#/definitions/projections"}} #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack view projection configuration file schema", "type": "object", "additionalProperties": False, "properties": ref_properties, "definitions": {"projections": projections}, } ================================================ FILE: lib/spack/spack/schema/repos.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for repos.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/repos.py :lines: 18- """ from typing import Any, Dict #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "repos": { "description": "Configuration for package repositories that Spack searches for packages", "oneOf": [ { # old format: array of strings "type": "array", "items": { "type": "string", "description": "Path to a Spack package repository directory", }, "description": "Legacy format: list of local paths to package repository " "directories", }, { # new format: object with named repositories "type": "object", "description": "Named repositories mapping configuration names to repository " "definitions", "additionalProperties": { "oneOf": [ { # local path "type": "string", "description": "Path to a local Spack package repository directory " "containing repo.yaml and packages/", }, { # remote git repository "type": "object", "properties": { "git": { "type": "string", "description": "Git repository URL for remote package " "repository", }, "branch": { "type": "string", "description": "Git branch name to checkout (default branch " "if not specified)", }, "commit": { "type": "string", "description": "Specific git commit hash to pin the " "repository to", }, "tag": { "type": "string", "description": "Git tag name to pin the repository to", }, "destination": { "type": "string", "description": "Custom local directory path where the Git " "repository should be cloned", }, "paths": { "type": "array", "items": {"type": "string"}, "description": "List of relative paths (from the Git " "repository root) that contain Spack package repositories " "(overrides spack-repo-index.yaml)", }, }, "additionalProperties": False, }, ] }, }, ], "default": {}, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack repository configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } def update(data: Dict[str, Any]) -> bool: """Update the repos.yaml configuration data to the new format.""" if not isinstance(data["repos"], list): return False from spack.llnl.util import tty from spack.repo import from_path # Convert old format [paths...] to new format {namespace: path, ...} repos = {} for path in data["repos"]: try: repo = from_path(path) except Exception as e: tty.warn(f"package repository {path} is disabled due to: {e}") continue if repo.namespace is not None: repos[repo.namespace] = path data["repos"] = repos return True ================================================ FILE: lib/spack/spack/schema/spec.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for a spec found in spec descriptor or database index.json files .. literalinclude:: _spack_root/lib/spack/spack/schema/spec.py :lines: 15- """ from typing import Any, Dict target = { "description": "Target architecture (string for abstract specs, object for concrete specs)", "oneOf": [ { "type": "string", "description": 'Target as a string (e.g. "zen2" or "haswell:broadwell") used in ' "abstract specs", }, { "type": "object", "additionalProperties": False, "required": ["name", "vendor", "features", "generation", "parents"], "properties": { "name": {"type": "string"}, "vendor": {"type": "string"}, "features": {"type": "array", "items": {"type": "string"}}, "generation": {"type": "integer"}, "parents": {"type": "array", "items": {"type": "string"}}, "cpupart": {"type": "string"}, }, "description": "Target as an object with detailed fields, used in concrete specs", }, ], } arch = { "type": "object", "additionalProperties": False, "properties": { "platform": { "type": ["string", "null"], "description": 'Target platform (e.g. "linux" or "darwin"). May be null for abstract ' "specs", }, "platform_os": { "type": ["string", "null"], "description": 'Target operating system (e.g. "ubuntu24.04"). May be ' "null for abstract specs", }, "target": target, }, } #: Corresponds to specfile format v1 dependencies_v1 = { "type": "object", "description": "Specfile v1 style dependencies specification (package name to dependency " "info)", "additionalProperties": { "type": "object", "properties": { "hash": {"type": "string", "description": "Unique identifier of the dependency"}, "type": { "type": "array", "items": {"enum": ["build", "link", "run", "test"]}, "description": "Dependency types", }, }, }, } #: Corresponds to specfile format v2-v3 dependencies_v2_v3 = { "type": "array", "description": "Specfile v2-v3 style dependencies specification (array of dependencies)", "items": { "type": "object", "additionalProperties": False, "required": ["name", "hash", "type"], "properties": { "name": {"type": "string", "description": "Name of the dependency package"}, "hash": {"type": "string", "description": "Unique identifier of the dependency"}, "type": { "type": "array", "items": {"enum": ["build", "link", "run", "test"]}, "description": "Dependency types", }, }, }, } #: Corresponds to specfile format v4+ dependencies_v4_plus = { "type": "array", "description": "Specfile v4+ style dependencies specification (array of dependencies)", "items": { "type": "object", "additionalProperties": False, "required": ["name", "hash", "parameters"], "properties": { "name": {"type": "string", "description": "Name of the dependency package"}, "hash": {"type": "string", "description": "Unique identifier of the dependency"}, "parameters": { "type": "object", "additionalProperties": False, "required": ["deptypes", "virtuals"], "properties": { "deptypes": { "type": "array", "items": {"enum": ["build", "link", "run", "test"]}, "description": "Dependency types", }, "virtuals": { "type": "array", "items": {"type": "string"}, "description": "Virtual dependencies used by the parent", }, "direct": { "type": "boolean", "description": "Whether the dependency is direct (only on abstract specs)", }, }, }, }, }, } dependencies = {"oneOf": [dependencies_v1, dependencies_v2_v3, dependencies_v4_plus]} build_spec = { "type": "object", "additionalProperties": False, "required": ["name", "hash"], "properties": {"name": {"type": "string"}, "hash": {"type": "string"}}, "description": "Records the origin spec as it was built (used in splicing)", } #: Schema for a single spec node (used in both spec files and database entries) spec_node = { "type": "object", "additionalProperties": False, "required": ["name"], "properties": { "name": { "type": ["string", "null"], "description": "Name is a string for concrete specs, but may be null for abstract " "specs", }, "hash": {"type": "string", "description": "The DAG hash, which identifies the spec"}, "package_hash": {"type": "string", "description": "The package hash (concrete specs)"}, "full_hash": { "type": "string", "description": "This hash was used on some specs prior to 0.18", }, "build_hash": { "type": "string", "description": "This hash was used on some specs prior to 0.18", }, "version": {"type": "string", "description": "A single, concrete version (e.g. @=1.2)"}, "versions": { "type": "array", "items": {"type": "string"}, "description": "Abstract version (e.g. @1.2)", }, "propagate": { "type": "array", "items": {"type": "string"}, "description": "List of variants to propagate (for abstract specs)", }, "abstract": { "type": "array", "items": {"type": "string"}, "description": "List of multi-valued variants that are abstract, i.e. foo=bar,baz " "instead of foo:=bar,baz (for abstract specs)", }, "concrete": { "type": "boolean", "description": "Whether the spec is concrete or not, when omitted defaults to true", }, "arch": arch, "compiler": { "type": "object", "additionalProperties": False, "properties": {"name": {"type": "string"}, "version": {"type": "string"}}, "description": "Compiler name and version (in spec file v5 listed as normal " "dependencies)", }, "namespace": {"type": "string", "description": "Package repository namespace"}, "parameters": { "type": "object", "additionalProperties": True, "description": "Variants and other parameters", "properties": { "patches": {"type": "array", "items": {"type": "string"}}, "cflags": {"type": "array", "items": {"type": "string"}}, "cppflags": {"type": "array", "items": {"type": "string"}}, "cxxflags": {"type": "array", "items": {"type": "string"}}, "fflags": {"type": "array", "items": {"type": "string"}}, "ldflags": {"type": "array", "items": {"type": "string"}}, "ldlibs": {"type": "array", "items": {"type": "string"}}, }, }, "patches": { "type": "array", "items": {"type": "string"}, "description": "List of patches, similar to the patches variant under parameters", }, "dependencies": dependencies, "build_spec": build_spec, "external": { "type": "object", "additionalProperties": False, "description": "If path or module (or both) are set, the spec is an external " "system-installed package", "properties": { "path": { "type": ["string", "null"], "description": "Install prefix on the system, e.g. /usr", }, "module": { "anyOf": [{"type": "array", "items": {"type": "string"}}, {"type": "null"}], "description": 'List of module names, e.g. ["pkg/1.2"]', }, "extra_attributes": { "type": "object", "description": "Package.py specific attributes to use the external package, " "such as paths to compiler executables", }, }, }, "annotations": { "type": "object", "properties": { "original_specfile_version": {"type": "number"}, "compiler": {"type": "string"}, }, "required": ["original_specfile_version"], "additionalProperties": False, "description": "Currently used to preserve compiler information of old specs when " "upgrading to a newer spec format", }, }, } #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "spec": { "type": "object", "additionalProperties": False, "required": ["_meta", "nodes"], "properties": { "_meta": { "type": "object", "properties": {"version": {"type": "number"}}, "description": "Spec schema version metadata, used for parsing spec files", }, "nodes": { "type": "array", "items": spec_node, "description": "List of spec nodes which, combined with dependencies, induce a " "DAG", }, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack spec schema", "type": "object", "additionalProperties": False, "required": ["spec"], "properties": properties, } ================================================ FILE: lib/spack/spack/schema/spec_list.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) matrix_schema = { "type": "array", "description": "List of spec constraint lists whose cross product generates multiple specs", "items": { "type": "array", "description": "List of spec constraints for this matrix dimension", "items": {"type": "string"}, }, } spec_list_properties = { "matrix": matrix_schema, "exclude": { "type": "array", "description": "List of specific spec combinations to exclude from the matrix", "items": {"type": "string"}, }, } spec_list_schema = { "type": "array", "description": "List of specs to include in the environment, supporting both simple specs and " "matrix configurations", "default": [], "items": { "anyOf": [ { "type": "object", "description": "Matrix configuration for generating multiple specs from " "combinations of constraints", "additionalProperties": False, "properties": {**spec_list_properties}, }, {"type": "string", "description": "Simple spec string"}, {"type": "null"}, ] }, } ================================================ FILE: lib/spack/spack/schema/toolchains.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for toolchains.yaml configuration file. .. literalinclude:: _spack_root/lib/spack/spack/schema/toolchains.py :lines: 13- """ from typing import Any, Dict #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "toolchains": { "type": "object", "default": {}, "description": "Define named compiler sets (toolchains) that group compiler constraints " "under a single user-defined name for easy reference with specs like %my_toolchain", "additionalProperties": { "description": "Named toolchain definition that can be referenced in specs to apply " "a complex set of compiler choices for C, C++, and Fortran", "oneOf": [ { "type": "string", "description": "Simple toolchain alias containing a spec string directly", }, { "type": "array", "description": "List of conditional compiler constraints and specifications " "that define the toolchain behavior", "items": { "type": "object", "description": "Individual toolchain entry with a spec constraint and " "optional condition for when it applies", "properties": { "spec": { "type": "string", "description": "Spec constraint to apply such as compiler " "selection (%c=llvm), flags (cflags=-O3), or other virtual " "dependencies (%mpi=openmpi)", }, "when": { "type": "string", "description": "Condition that determines when this spec " "constraint is applied, typically checking for language " "dependencies like %c, %cxx, %fortran, or other virtual packages " "like %mpi", }, }, }, }, ], "default": [], }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack toolchain configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/upstreams.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from typing import Any, Dict #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "upstreams": { "type": "object", "default": {}, "description": "Configuration for chaining Spack installations. Point this Spack " "instance to other Spack installations to use their installed packages", "additionalProperties": { "type": "object", "default": {}, "additionalProperties": False, "description": "Named upstream Spack instance configuration", "properties": { "install_tree": { "type": "string", "description": "Path to the opt/spack directory of the upstream Spack " "installation (or the install_tree root from its config.yaml)", }, "modules": { "type": "object", "description": "Configuration to use modules generated by the upstream " "Spack instance", "properties": { "tcl": { "type": "string", "description": "Path to TCL modules directory of the upstream " "instance", }, "lmod": { "type": "string", "description": "Path to Lmod modules directory of the upstream " "instance", }, }, }, }, }, } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack core configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/url_buildcache_manifest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for buildcache entry manifest file .. literalinclude:: _spack_root/lib/spack/spack/schema/url_buildcache_manifest.py :lines: 11- """ from typing import Any, Dict properties: Dict[str, Any] = { "version": {"type": "integer"}, "data": { "type": "array", "items": { "type": "object", "required": [ "contentLength", "mediaType", "compression", "checksumAlgorithm", "checksum", ], "properties": { "contentLength": {"type": "integer"}, "mediaType": {"type": "string"}, "compression": {"type": "string"}, "checksumAlgorithm": {"type": "string"}, "checksum": {"type": "string"}, }, "additionalProperties": True, }, }, } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Buildcache manifest schema", "type": "object", "required": ["version", "data"], "additionalProperties": True, "properties": properties, } ================================================ FILE: lib/spack/spack/schema/view.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Schema for view .. literalinclude:: _spack_root/lib/spack/spack/schema/view.py :lines: 15- """ from typing import Any, Dict import spack.schema.projections #: Properties for inclusion in other schemas properties: Dict[str, Any] = { "view": { "description": "Environment filesystem view configuration for creating a directory with " "traditional structure where all files of installed packages are linked", "anyOf": [ { "type": "boolean", "description": "Enable or disable default views. If 'true', the view is " "generated under .spack-env/view", }, {"type": "string", "description": "Path where the default view should be created"}, { "type": "object", "description": "Advanced view configuration with one or more named view " "descriptors", "additionalProperties": { "description": "Named view descriptor (use 'default' for the view activated " "with environment)", "required": ["root"], "additionalProperties": False, "properties": { "root": { "type": "string", "description": "Root directory path where the view will be created", }, "group": { "oneOf": [ { "type": "array", "items": {"type": "string"}, "description": "Groups of specs to include in the view", }, { "type": "string", "description": "Groups of specs to include in the view", }, ] }, "link": { "enum": ["roots", "all", "run"], "description": "Which specs to include: 'all' (environment roots " "with transitive run+link deps), 'run' (environment roots with " "transitive run deps), 'roots' (environment roots only)", }, "link_type": { "type": "string", "enum": ["symlink", "hardlink", "copy"], "description": "How files are linked in the view: 'symlink' " "(default), 'hardlink', or 'copy'", }, "select": { "type": "array", "items": {"type": "string"}, "description": "List of specs to include in the view " "(default: select everything)", }, "exclude": { "type": "array", "items": {"type": "string"}, "description": "List of specs to exclude from the view " "(default: exclude nothing)", }, **spack.schema.projections.ref_properties, }, }, }, ], } } #: Full schema with metadata schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Spack view configuration file schema", "type": "object", "additionalProperties": False, "properties": properties, "definitions": {"projections": spack.schema.projections.projections}, } ================================================ FILE: lib/spack/spack/solver/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/solver/asp.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import collections.abc import enum import functools import gzip import io import itertools import json import os import pathlib import pprint import random import re import sys import time import warnings from typing import ( Any, Callable, Dict, Generator, Iterable, Iterator, List, NamedTuple, Optional, Sequence, Set, Tuple, Type, Union, ) import spack.vendor.archspec.cpu import spack import spack.caches import spack.compilers.config import spack.compilers.flags import spack.concretize import spack.config import spack.deptypes as dt import spack.error import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.package_base import spack.package_prefs import spack.platforms import spack.repo import spack.solver.splicing import spack.spec import spack.store import spack.util.crypto import spack.util.hash import spack.util.lock as lk import spack.util.module_cmd as md import spack.util.path import spack.util.timer import spack.variant as vt import spack.version as vn import spack.version.git_ref_lookup from spack import traverse from spack.compilers.libraries import CompilerPropertyDetector from spack.llnl.util.lang import elide_list from spack.spec import EMPTY_SPEC from spack.util.compression import GZipFileType from .core import ( AspFunction, AspVar, NodeId, SourceContext, clingo, extract_args, fn, using_libc_compatibility, ) from .input_analysis import create_counter, create_graph_analyzer from .requirements import RequirementKind, RequirementOrigin, RequirementParser, RequirementRule from .reuse import ReusableSpecsSelector, SpecFiltersFactory, create_external_parser from .runtimes import RuntimePropertyRecorder, all_libcs, external_config_with_implicit_externals from .versions import Provenance GitOrStandardVersion = Union[vn.GitVersion, vn.StandardVersion] TransformFunction = Callable[[str, spack.spec.Spec, List[AspFunction]], List[AspFunction]] class OutputConfiguration(NamedTuple): """Data class that contains configuration on what a clingo solve should output.""" #: Print out coarse timers for different solve phases timers: bool #: Whether to output Clingo's internal solver statistics stats: bool #: Optional output stream for the generated ASP program out: Optional[io.IOBase] #: If True, stop after setup and don't solve setup_only: bool #: Default output configuration for a solve DEFAULT_OUTPUT_CONFIGURATION = OutputConfiguration( timers=False, stats=False, out=None, setup_only=False ) def default_clingo_control(): """Return a control object with the default settings used in Spack""" control = clingo().Control() control.configuration.configuration = "tweety" control.configuration.solver.heuristic = "Domain" control.configuration.solver.opt_strategy = "usc" return control # Below numbers are used to map names of criteria to the order # they appear in the solution. See concretize.lp # The space of possible priorities for optimization targets # is partitioned in the following ranges: # # [0-100) Optimization criteria for software being reused # [100-200) Fixed criteria that are higher priority than reuse, but lower than build # [200-300) Optimization criteria for software being built # [300-1000) High-priority fixed criteria # [1000-inf) Error conditions # # Each optimization target is a minimization with optimal value 0. #: High fixed priority offset for criteria that supersede all build criteria high_fixed_priority_offset = 300 #: Priority offset for "build" criteria (regular criterio shifted to #: higher priority for specs we have to build) build_priority_offset = 200 #: Priority offset of "fixed" criteria (those w/o build criteria) fixed_priority_offset = 100 class OptimizationKind: """Enum for the optimization KIND of a criteria. It's not using enum.Enum since it must be serializable. """ BUILD = 0 CONCRETE = 1 OTHER = 2 class OptimizationCriteria(NamedTuple): """A named tuple describing an optimization criteria.""" priority: int value: int name: str kind: OptimizationKind def build_criteria_names(costs, arg_tuples): """Construct an ordered mapping from criteria names to costs.""" # pull optimization criteria names out of the solution priorities_names = [] for args in arg_tuples: priority, name = args[:2] priority = int(priority) # Add the priority of this opt criterion and its name if priority < fixed_priority_offset: # if the priority is less than fixed_priority_offset, then it # has an associated build priority -- the same criterion but for # nodes that we have to build. priorities_names.append((priority, name, OptimizationKind.CONCRETE)) build_priority = priority + build_priority_offset priorities_names.append((build_priority, name, OptimizationKind.BUILD)) else: priorities_names.append((priority, name, OptimizationKind.OTHER)) # sort the criteria by priority priorities_names = sorted(priorities_names, reverse=True) # We only have opt-criterion values for non-error types # error type criteria are excluded (they come first) error_criteria = len(costs) - len(priorities_names) costs = costs[error_criteria:] return [ OptimizationCriteria(priority, value, name, status) for (priority, name, status), value in zip(priorities_names, costs) ] def specify(spec): if isinstance(spec, spack.spec.Spec): return spec return spack.spec.Spec(spec) # Caching because the returned function id is used as a cache key @functools.lru_cache(maxsize=None) def remove_facts(*to_be_removed: str) -> TransformFunction: """Returns a transformation function that removes facts from the input list of facts.""" def _remove(name: str, spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]: return [x for x in facts if x.args[0] not in to_be_removed] return _remove def identity_for_facts( name: str, spec: spack.spec.Spec, facts: List[AspFunction] ) -> List[AspFunction]: return facts # Caching because the returned function id is used as a cache key @functools.lru_cache(maxsize=None) def dependency_holds( *, dependency_flags: dt.DepFlag, pkg_cls: Type[spack.package_base.PackageBase] ) -> TransformFunction: def _transform_fn( name: str, input_spec: spack.spec.Spec, requirements: List[AspFunction] ) -> List[AspFunction]: result = remove_facts("node", "virtual_node")(name, input_spec, requirements) + [ fn.attr("dependency_holds", pkg_cls.name, name, dt.flag_to_string(t)) for t in dt.ALL_FLAGS if t & dependency_flags ] if name not in pkg_cls.extendees: return result return result + [fn.attr("extends", pkg_cls.name, name)] return _transform_fn def dag_closure_by_deptype( name: str, spec: spack.spec.Spec, facts: List[AspFunction] ) -> List[AspFunction]: edges = spec.edges_to_dependencies() # Compute the "link" transitive closure with `when: root ^[deptypes=link] ` if len(edges) == 1: edge = edges[0] if not edge.direct and edge.depflag == dt.LINK | dt.RUN: root, leaf = edge.parent.name, edge.spec.name return [fn.attr("closure", root, leaf, "linkrun")] return facts def libc_is_compatible(lhs: spack.spec.Spec, rhs: spack.spec.Spec) -> bool: return ( lhs.name == rhs.name and lhs.external_path == rhs.external_path and lhs.version >= rhs.version ) def c_compiler_runs(compiler) -> bool: return CompilerPropertyDetector(compiler).compiler_verbose_output() is not None def extend_flag_list(flag_list, new_flags): """Extend a list of flags, preserving order and precedence. Add new_flags at the end of flag_list. If any flags in new_flags are already in flag_list, they are moved to the end so that they take higher precedence on the compile line. """ for flag in new_flags: if flag in flag_list: flag_list.remove(flag) flag_list.append(flag) def _reorder_flags(flag_list: List[spack.spec.CompilerFlag]) -> List[spack.spec.CompilerFlag]: """Reorder a list of flags to ensure that the order matches that of the flag group.""" if not flag_list: return [] if len({x.flag_group for x in flag_list}) != 1 or len({x.source for x in flag_list}) != 1: raise InternalConcretizerError( "internal solver error: cannot reorder compiler flags for concretized specs. " "Please report a bug at https://github.com/spack/spack/issues" ) flag_group = flag_list[0].flag_group flag_source = flag_list[0].source flag_propagate = flag_list[0].propagate # Once we have the flag_group, no need to iterate over the flag_list because the # group represents all of them return [ spack.spec.CompilerFlag( flag, propagate=flag_propagate, flag_group=flag_group, source=flag_source ) for flag, propagate in spack.compilers.flags.tokenize_flags( flag_group, propagate=flag_propagate ) ] def check_packages_exist(specs): """Ensure all packages mentioned in specs exist.""" repo = spack.repo.PATH for spec in specs: for s in spec.traverse(): try: check_passed = repo.repo_for_pkg(s).exists(s.name) or repo.is_virtual(s.name) except Exception as e: msg = "Cannot find package: {0}".format(str(e)) check_passed = False tty.debug(msg) if not check_passed: raise spack.repo.UnknownPackageError(str(s.fullname)) class Result: """Result of an ASP solve.""" def __init__(self, specs): self.satisfiable = None self.optimal = None self.warnings = None self.nmodels = 0 # specs ordered by optimization level self.answers = [] # names of optimization criteria self.criteria = [] # Abstract user requests self.abstract_specs = specs # possible dependencies self.possible_dependencies = None # Concrete specs self._concrete_specs_by_input = None self._concrete_specs = None self._unsolved_specs = None def raise_if_unsat(self): """Raise a generic internal error if the result is unsatisfiable.""" if self.satisfiable: return constraints = self.abstract_specs if len(constraints) == 1: constraints = constraints[0] raise SolverError(constraints) @property def specs(self): """List of concretized specs satisfying the initial abstract request. """ if self._concrete_specs is None: self._compute_specs_from_answer_set() return self._concrete_specs @property def unsolved_specs(self): """List of tuples pairing abstract input specs that were not solved with their associated candidate spec from the solver (if the solve completed). """ if self._unsolved_specs is None: self._compute_specs_from_answer_set() return self._unsolved_specs @property def specs_by_input(self) -> Dict[spack.spec.Spec, spack.spec.Spec]: if self._concrete_specs_by_input is None: self._compute_specs_from_answer_set() return self._concrete_specs_by_input # type: ignore def _compute_specs_from_answer_set(self): if not self.satisfiable: self._concrete_specs = [] self._unsolved_specs = list((x, None) for x in self.abstract_specs) self._concrete_specs_by_input = {} return self._concrete_specs, self._unsolved_specs = [], [] self._concrete_specs_by_input = {} best = min(self.answers) opt, _, answer = best for input_spec in self.abstract_specs: # The specs must be unified to get here, so it is safe to associate any satisfying spec # with the input. Multiple inputs may be matched to the same concrete spec node = SpecBuilder.make_node(pkg=input_spec.name) if spack.repo.PATH.is_virtual(input_spec.name): providers = [ spec.name for spec in answer.values() if spec.package.provides(input_spec.name) ] node = SpecBuilder.make_node(pkg=providers[0]) candidate = answer.get(node) if candidate and candidate.satisfies(input_spec): self._concrete_specs.append(answer[node]) self._concrete_specs_by_input[input_spec] = answer[node] elif candidate and candidate.build_spec.satisfies(input_spec): tty.warn( "explicit splice configuration has caused the concretized spec" f" {candidate} not to satisfy the input spec {input_spec}" ) self._concrete_specs.append(answer[node]) self._concrete_specs_by_input[input_spec] = answer[node] else: self._unsolved_specs.append((input_spec, candidate)) @staticmethod def format_unsolved(unsolved_specs): """Create a message providing info on unsolved user specs and for each one show the associated candidate spec from the solver (if there is one). """ msg = "Unsatisfied input specs:" for input_spec, candidate in unsolved_specs: msg += f"\n\tInput spec: {str(input_spec)}" if candidate: msg += f"\n\tCandidate spec: {candidate.long_spec}" else: msg += "\n\t(No candidate specs from solver)" return msg def to_dict(self) -> dict: """Produces dict representation of Result object Does not include anything related to unsatisfiability as we are only interested in storing satisfiable results """ serial_node_arg = lambda node_dict: ( f"""{{"id": "{node_dict.id}", "pkg": "{node_dict.pkg}"}}""" ) ret = dict() ret["criteria"] = self.criteria ret["optimal"] = self.optimal ret["warnings"] = self.warnings ret["nmodels"] = self.nmodels ret["abstract_specs"] = [str(x) for x in self.abstract_specs] ret["satisfiable"] = self.satisfiable serial_answers = [] for answer in self.answers: serial_answer = answer[:2] serial_answer_dict = {} for node, spec in answer[2].items(): serial_answer_dict[serial_node_arg(node)] = spec.to_dict() serial_answer = serial_answer + (serial_answer_dict,) serial_answers.append(serial_answer) ret["answers"] = serial_answers ret["specs_by_input"] = {} input_specs = {} if not self.specs_by_input else self.specs_by_input for input, spec in input_specs.items(): ret["specs_by_input"][str(input)] = spec.to_dict() return ret @staticmethod def from_dict(obj: dict): """Returns Result object from compatible dictionary""" def _dict_to_node_argument(dict): id = dict["id"] pkg = dict["pkg"] return NodeId(id=id, pkg=pkg) def _str_to_spec(spec_str): return spack.spec.Spec(spec_str) def _dict_to_spec(spec_dict): loaded_spec = spack.spec.Spec.from_dict(spec_dict) _ensure_external_path_if_external(loaded_spec) spack.spec.Spec.ensure_no_deprecated(loaded_spec) return loaded_spec spec_list = obj.get("abstract_specs") if not spec_list: raise RuntimeError("Invalid json for concretization Result object") if spec_list: spec_list = [_str_to_spec(x) for x in spec_list] result = Result(spec_list) criteria = obj.get("criteria") result.criteria = ( None if criteria is None else [OptimizationCriteria(*t) for t in criteria] ) result.optimal = obj.get("optimal") result.warnings = obj.get("warnings") result.nmodels = obj.get("nmodels") result.satisfiable = obj.get("satisfiable") result._unsolved_specs = [] answers = [] for answer in obj.get("answers", []): loaded_answer = answer[:2] answer_node_dict = {} for node, spec in answer[2].items(): answer_node_dict[_dict_to_node_argument(json.loads(node))] = _dict_to_spec(spec) loaded_answer.append(answer_node_dict) answers.append(tuple(loaded_answer)) result.answers = answers result._concrete_specs_by_input = {} result._concrete_specs = [] for input, spec in obj.get("specs_by_input", {}).items(): result._concrete_specs_by_input[_str_to_spec(input)] = _dict_to_spec(spec) result._concrete_specs.append(_dict_to_spec(spec)) return result def __eq__(self, other): eq = ( self.satisfiable == other.satisfiable, self.optimal == other.optimal, self.warnings == other.warnings, self.nmodels == other.nmodels, self.criteria == other.criteria, self.answers == other.answers, self.abstract_specs == other.abstract_specs, self._concrete_specs_by_input == other._concrete_specs_by_input, self._concrete_specs == other._concrete_specs, self._unsolved_specs == other._unsolved_specs, # Not considered for equality # self.control # self.possible_dependencies # self.possible_dependencies ) return all(eq) class ConcretizationCache: """Store for Spack concretization results and statistics Serializes solver result objects and statistics to json and stores at a given endpoint in a cache associated by the sha256 of the asp problem and the involved control files. """ def __init__(self, root: Union[str, None] = None): root = root or spack.config.get("concretizer:concretization_cache:url", None) if root is None: root = os.path.join(spack.caches.misc_cache_location(), "concretization") self.root = pathlib.Path(spack.util.path.canonicalize_path(root)) self.root.mkdir(parents=True, exist_ok=True) self._lockfile = self.root / ".cc_lock" def cleanup(self): """Prunes the concretization cache according to configured entry count limits. Cleanup is done in LRU ordering.""" entry_limit = spack.config.get("concretizer:concretization_cache:entry_limit", 1000) # determine if we even need to clean up entries = list(self.cache_entries()) if len(entries) <= entry_limit: return # collect stat info for mod time about all entries removal_queue = [] for entry in entries: try: entry_stat_info = entry.stat() # mtime will always be time of last use as we update it after # each read and obviously after each write mod_time = entry_stat_info.st_mtime removal_queue.append((mod_time, entry)) except FileNotFoundError: # don't need to cleanup the file, it's not there! pass removal_queue.sort() # sort items for removal, ascending, so oldest first # Try to remove the oldest half of the cache. for _, entry_to_rm in removal_queue[: entry_limit // 2]: # cache bucket was removed by another process -- that's fine; move on if not entry_to_rm.exists(): continue try: with self.write_transaction(entry_to_rm, timeout=1e-6): self._safe_remove(entry_to_rm) except lk.LockTimeoutError: # if we can't get a lock, it's either # 1) being read, so it's been used recently, i.e. not a good candidate for LRU, # 2) it's already being removed by another process, so we don't care, or # 3) system is busy, but we don't really need to wait just for cache cleanup. pass # so skip it def cache_entries(self): """Generator producing cache entries within a bucket""" for cache_entry in self.root.iterdir(): # Lockfile starts with "." # old style concretization cache entries are in directories if not cache_entry.name.startswith(".") and cache_entry.is_file(): yield cache_entry def _results_from_cache(self, cache_entry_file: str) -> Union[Result, None]: """Returns a Results object from the concretizer cache Reads the cache hit and uses `Result`'s own deserializer to produce a new Result object """ cache_entry = json.loads(cache_entry_file) result_json = cache_entry["results"] return Result.from_dict(result_json) def _stats_from_cache(self, cache_entry_file: str) -> Union[Dict, None]: """Returns concretization statistic from the concretization associated with the cache. Deserializes the the json representation of the statistics covering the cached concretization run and returns the Python data structures """ return json.loads(cache_entry_file)["statistics"] def _prefix_digest(self, problem: str) -> str: """Return the first two characters of, and the full, sha256 of the given asp problem""" return spack.util.hash.b32_hash(problem) def _cache_path_from_problem(self, problem: str) -> pathlib.Path: """Returns a Path object representing the path to the cache entry for the given problem where the problem is the sha256 of the given asp problem""" prefix = self._prefix_digest(problem) return self.root / prefix def _safe_remove(self, cache_dir: pathlib.Path) -> bool: """Removes cache entries with handling for the case where the entry has been removed already or there are multiple cache entries in a directory""" try: cache_dir.unlink() return True except FileNotFoundError: # That's fine, removal is idempotent pass except OSError as e: # Catch other timing/access related issues tty.debug( f"Exception occurred while attempting to remove Concretization Cache entry, {e}" ) pass return False def _lock(self, path: pathlib.Path) -> lk.Lock: """Returns a lock over the byte range corresponding to the hash of the asp problem. ``path`` is a path to a file in the cache, and its basename is the hash of the problem. Args: path: absolute or relative path to concretization cache entry to be locked """ return lk.Lock( str(self._lockfile), start=spack.util.hash.base32_prefix_bits( path.name, spack.util.crypto.bit_length(sys.maxsize) ), length=1, desc=f"Concretization cache lock for {path}", ) def read_transaction( self, path: pathlib.Path, timeout: Optional[float] = None ) -> lk.ReadTransaction: """Read transactions for concretization cache entries. Args: path: absolute or relative path to the concretization cache entry to be locked timeout: give up after this many seconds """ return lk.ReadTransaction(self._lock(path), timeout=timeout) def write_transaction( self, path: pathlib.Path, timeout: Optional[float] = None ) -> lk.WriteTransaction: """Write transactions for concretization cache entries Args: path: absolute or relative path to the concretization cache entry to be locked timeout: give up after this many seconds """ return lk.WriteTransaction(self._lock(path), timeout=timeout) def store(self, problem: str, result: Result, statistics: List) -> None: """Creates entry in concretization cache for problem if none exists, storing the concretization Result object and statistics in the cache as serialized json joined as a single file. Hash membership is computed based on the sha256 of the provided asp problem. """ cache_path = self._cache_path_from_problem(problem) with self.write_transaction(cache_path, timeout=30): if cache_path.exists(): # if cache path file exists, we already have a cache entry, likely created # by another process. Exit early. return with gzip.open(cache_path, "xb", compresslevel=6) as cache_entry: cache_dict = {"results": result.to_dict(), "statistics": statistics} cache_entry.write(json.dumps(cache_dict).encode()) def fetch(self, problem: str) -> Union[Tuple[Result, Dict], Tuple[None, None]]: """Returns the concretization cache result for a lookup based on the given problem. Checks the concretization cache for the given problem, and either returns the Python objects cached on disk representing the concretization results and statistics or returns none if no cache entry was found. """ cache_path = self._cache_path_from_problem(problem) if not cache_path.exists(): return None, None # if exists is false, then there's no chance of a hit cache_content = None try: with self.read_transaction(cache_path, timeout=2): try: with gzip.open(cache_path, "rb", compresslevel=6) as f: f.peek(1) # Try to read at least one byte f.seek(0) cache_content = f.read().decode("utf-8") except OSError: # Cache may have been created pre compression check if gzip, and if not, # read from plaintext otherwise re raise with open(cache_path, "rb") as f: # raise if this is a gzip file we failed to open if GZipFileType().matches_magic(f): raise cache_content = f.read().decode() except FileNotFoundError: pass # cache miss, already cleaned up except lk.LockTimeoutError: pass # if the lock times, out skip the cache if not cache_content: return None, None # update mod/access time for use w/ LRU cleanup os.utime(cache_path) return (self._results_from_cache(cache_content), self._stats_from_cache(cache_content)) # type: ignore def _is_checksummed_git_version(v): return isinstance(v, vn.GitVersion) and v.is_commit def _is_checksummed_version(version_info: Tuple[GitOrStandardVersion, dict]): """Returns true iff the version is not a moving target""" version, info = version_info if isinstance(version, vn.StandardVersion): if any(h in info for h in spack.util.crypto.hashes.keys()) or "checksum" in info: return True return "commit" in info and len(info["commit"]) == 40 return _is_checksummed_git_version(version) def _spec_with_default_name(spec_str, name): """Return a spec with a default name if none is provided, used for requirement specs""" spec = spack.spec.Spec(spec_str) if not spec.name: spec.name = name return spec class ErrorHandler: def __init__(self, model, input_specs: List[spack.spec.Spec]): self.model = model self.input_specs = input_specs self.full_model = None def multiple_values_error(self, attribute, pkg): return f'Cannot select a single "{attribute}" for package "{pkg}"' def no_value_error(self, attribute, pkg): return f'Cannot select a single "{attribute}" for package "{pkg}"' def _get_cause_tree( self, cause: Tuple[str, str], conditions: Dict[str, str], condition_causes: List[Tuple[Tuple[str, str], Tuple[str, str]]], seen: Set, indent: str = " ", ) -> List[str]: """ Implementation of recursion for self.get_cause_tree. Much of this operates on tuples (condition_id, set_id) in which the latter idea means that the condition represented by the former held in the condition set represented by the latter. """ seen.add(cause) parents = [c for e, c in condition_causes if e == cause and c not in seen] local = f"required because {conditions[cause[0]]} " return [indent + local] + [ c for parent in parents for c in self._get_cause_tree( parent, conditions, condition_causes, seen, indent=indent + " " ) ] def get_cause_tree(self, cause: Tuple[str, str]) -> List[str]: """ Get the cause tree associated with the given cause. Arguments: cause: The root cause of the tree (final condition) Returns: A list of strings describing the causes, formatted to display tree structure. """ conditions: Dict[str, str] = dict(extract_args(self.full_model, "condition_reason")) condition_causes: List[Tuple[Tuple[str, str], Tuple[str, str]]] = list( ((Effect, EID), (Cause, CID)) for Effect, EID, Cause, CID in extract_args(self.full_model, "condition_cause") ) return self._get_cause_tree(cause, conditions, condition_causes, set()) def handle_error(self, msg, *args): """Handle an error state derived by the solver.""" if msg == "multiple_values_error": return self.multiple_values_error(*args) if msg == "no_value_error": return self.no_value_error(*args) try: idx = args.index("startcauses") except ValueError: msg_args = args causes = [] else: msg_args = args[:idx] cause_args = args[idx + 1 :] cause_args_conditions = cause_args[::2] cause_args_ids = cause_args[1::2] causes = list(zip(cause_args_conditions, cause_args_ids)) msg = msg.format(*msg_args) # For variant formatting, we sometimes have to construct specs # to format values properly. Find/replace all occurrences of # Spec(...) with the string representation of the spec mentioned specs_to_construct = re.findall(r"Spec\(([^)]*)\)", msg) for spec_str in specs_to_construct: msg = msg.replace(f"Spec({spec_str})", str(spack.spec.Spec(spec_str))) for cause in set(causes): for c in self.get_cause_tree(cause): msg += f"\n{c}" return msg def message(self, errors) -> str: input_specs = ", ".join(elide_list([f"`{s}`" for s in self.input_specs], 5)) header = f"failed to concretize {input_specs} for the following reasons:" messages = ( f" {idx + 1:2}. {self.handle_error(msg, *args)}" for idx, (_, msg, args) in enumerate(errors) ) return "\n".join((header, *messages)) def raise_if_errors(self): initial_error_args = extract_args(self.model, "error") if not initial_error_args: return error_causation = clingo().Control() parent_dir = pathlib.Path(__file__).parent errors_lp = parent_dir / "error_messages.lp" def on_model(model): self.full_model = model.symbols(shown=True, terms=True) with error_causation.backend() as backend: for atom in self.model: atom_id = backend.add_atom(atom) backend.add_rule([atom_id], [], choice=False) error_causation.load(str(errors_lp)) error_causation.ground([("base", []), ("error_messages", [])]) _ = error_causation.solve(on_model=on_model) # No choices so there will be only one model error_args = extract_args(self.full_model, "error") errors = sorted( [(int(priority), msg, args) for priority, msg, *args in error_args], reverse=True ) try: msg = self.message(errors) except Exception as e: msg = ( f"unexpected error during concretization [{str(e)}]. " f"Please report a bug at https://github.com/spack/spack/issues" ) raise spack.error.SpackError(msg) from e raise UnsatisfiableSpecError(msg) class PyclingoDriver: def __init__(self, conc_cache: Optional[ConcretizationCache] = None) -> None: """Driver for the Python clingo interface. Args: conc_cache: concretization cache """ # This attribute will be reset at each call to solve self.control: Any = None # TODO: fix typing of dynamic clingo import self._conc_cache = conc_cache def _control_file_paths(self, control_files: List[str]) -> List[str]: """Get absolute paths based on relative paths of control files. Right now the control files just live next to this file in the Spack tree. """ parent_dir = os.path.dirname(__file__) return [os.path.join(parent_dir, rel_path) for rel_path in control_files] def _make_cache_key(self, asp_problem: List[str], control_file_paths: List[str]) -> str: """Make a key for fetching a solve from the concretization cache. A key comprises the entire input to clingo, i.e., the problem instance plus the control files. The problem instance is assumed to already be sorted and stripped of comments and empty lines. The control files are stripped but not sorted, so changes to the control files will cause cache misses if they modify any code. Arguments: asp_problem: list of statements in the ASP program control_file_paths: list of paths to control files we'll send to clingo """ lines = list(asp_problem) for path in control_file_paths: with open(path, "r", encoding="utf-8") as f: lines.extend(strip_asp_problem(f.readlines())) return "\n".join(lines) def _run_clingo( self, specs: List[spack.spec.Spec], setup: "SpackSolverSetup", problem_str: str, control_file_paths: List[str], timer: spack.util.timer.Timer, ) -> Result: """Actually run clingo and generate a result. This is the core solve logic once the setup is done and once we know we can't fetch a result from cache. See ``solve()`` for caching and setup logic. """ # We could just take the cache_key and add it to clingo (since it is the # full problem representation), but we load control files separately as it # makes clingo give us better, file-aware error messages. with timer.measure("load"): # Add the problem instance self.control.add("base", [], problem_str) # Load additinoal files for path in control_file_paths: self.control.load(path) # Grounding is the first step in the solve -- it turns our facts # and first-order logic rules into propositional logic. with timer.measure("ground"): self.control.ground([("base", [])]) # With a grounded program, we can run the solve. models = [] # stable models if things go well def on_model(model): models.append((model.cost, model.symbols(shown=True, terms=True))) timer.start("solve") # A timeout of 0 means no timeout time_limit = spack.config.CONFIG.get("concretizer:timeout", 0) timeout_end = time.monotonic() + time_limit if time_limit > 0 else float("inf") error_on_timeout = spack.config.CONFIG.get("concretizer:error_on_timeout", True) with self.control.solve(on_model=on_model, async_=True) as handle: # Allow handling of interrupts every second. # # pyclingo's `SolveHandle` blocks the calling thread for the duration of each # `.wait()` call. Python also requires that signal handlers must be handled in # the main thread, so any `KeyboardInterrupt` is postponed until after the # `.wait()` call exits the control of pyclingo. finished = False while not finished and time.monotonic() < timeout_end: finished = handle.wait(1.0) if not finished: specs_str = ", ".join(spack.llnl.util.lang.elide_list([str(s) for s in specs], 4)) header = f"Spack is taking more than {time_limit} seconds to solve for {specs_str}" if error_on_timeout: raise UnsatisfiableSpecError(f"{header}, stopping concretization") warnings.warn(f"{header}, using the best configuration found so far") handle.cancel() solve_result = handle.get() timer.stop("solve") # once done, construct the solve result result = Result(specs) result.satisfiable = solve_result.satisfiable if result.satisfiable: timer.start("construct_specs") # get the best model builder = SpecBuilder(specs, hash_lookup=setup.reusable_and_possible) min_cost, best_model = min(models) # first check for errors error_handler = ErrorHandler(best_model, specs) error_handler.raise_if_errors() # build specs from spec attributes in the model spec_attrs = [(name, tuple(rest)) for name, *rest in extract_args(best_model, "attr")] answers = builder.build_specs(spec_attrs) # add best spec to the results result.answers.append((list(min_cost), 0, answers)) # get optimization criteria criteria_args = extract_args(best_model, "opt_criterion") result.criteria = build_criteria_names(min_cost, criteria_args) # record the number of models the solver considered result.nmodels = len(models) # record the possible dependencies in the solve result.possible_dependencies = setup.pkgs timer.stop("construct_specs") timer.stop() result.raise_if_unsat() if result.satisfiable and result.unsolved_specs and setup.concretize_everything: raise OutputDoesNotSatisfyInputError(result.unsolved_specs) return result def solve( self, setup: "SpackSolverSetup", specs: List[spack.spec.Spec], reuse: Optional[List[spack.spec.Spec]] = None, packages_with_externals=None, output: Optional[OutputConfiguration] = None, control: Optional[Any] = None, # TODO: figure out how to annotate clingo.Control allow_deprecated: bool = False, ) -> Tuple[Result, Optional[spack.util.timer.Timer], Optional[Dict]]: """Set up the input and solve for dependencies of ``specs``. Arguments: setup: An object to set up the ASP problem. specs: List of ``Spec`` objects to solve for. reuse: list of concrete specs that can be reused output: configuration object to set the output of this solve. control: configuration for the solver. If None, default values will be used allow_deprecated: if True, allow deprecated versions in the solve Return: A tuple of the solve result, the timer for the different phases of the solve, and the internal statistics from clingo. """ from spack.bootstrap import ensure_winsdk_external_or_raise output = output or DEFAULT_OUTPUT_CONFIGURATION timer = spack.util.timer.Timer() # Initialize the control object for the solver self.control = control or default_clingo_control() # ensure core deps are present on Windows # needs to modify active config scope, so cannot be run within # bootstrap config scope if sys.platform == "win32": ensure_winsdk_external_or_raise() # assemble a list of the control files needed for this problem. Some are conditionally # included depending on what features we're using in the solve. control_files = ["concretize.lp", "heuristic.lp", "display.lp", "direct_dependency.lp"] if not setup.concretize_everything: control_files.append("when_possible.lp") if using_libc_compatibility(): control_files.append("libc_compatibility.lp") else: control_files.append("os_compatibility.lp") if setup.enable_splicing: control_files.append("splices.lp") timer.start("setup") problem_builder = setup.setup( specs, reuse=reuse, packages_with_externals=packages_with_externals, allow_deprecated=allow_deprecated, ) timer.stop("setup") timer.start("ordering") # print the output with comments, etc. if the user asked problem = problem_builder.asp_problem if output.out is not None: output.out.write("\n".join(problem)) if output.setup_only: return Result(specs), None, None # strip the problem of comments and empty lines problem = strip_asp_problem(problem) randomize = "SPACK_SOLVER_RANDOMIZATION" in os.environ if randomize: # create a shuffled copy -- useful for understanding performance variation problem = random.sample(problem, len(problem)) else: problem.sort() # sort for deterministic output timer.stop("ordering") timer.start("cache-check") # load control files to add to the input representation control_file_paths = self._control_file_paths(control_files) cache_key = self._make_cache_key(problem, control_file_paths) result, concretization_stats = None, None conc_cache_enabled = spack.config.get("concretizer:concretization_cache:enable", False) if conc_cache_enabled and self._conc_cache: result, concretization_stats = self._conc_cache.fetch(cache_key) timer.stop("cache-check") tty.debug("Starting concretizer") # run the solver and store the result, if it wasn't cached already if not result: problem_repr = "\n".join(problem) result = self._run_clingo(specs, setup, problem_repr, control_file_paths, timer) if conc_cache_enabled and self._conc_cache: self._conc_cache.store(cache_key, result, self.control.statistics) if output.timers: timer.write_tty() print() concretization_stats = concretization_stats or self.control.statistics if output.stats: print("Statistics:") pprint.pprint(concretization_stats) return result, timer, concretization_stats class ConcreteSpecsByHash(collections.abc.Mapping): """Mapping containing concrete specs keyed by DAG hash. The mapping is ensured to be consistent, i.e. if a spec in the mapping has a dependency with hash X, it is ensured to be the same object in memory as the spec keyed by X. """ def __init__(self) -> None: self.data: Dict[str, spack.spec.Spec] = {} self.explicit: Set[str] = set() def __getitem__(self, dag_hash: str) -> spack.spec.Spec: return self.data[dag_hash] def explicit_items(self) -> Iterator[Tuple[str, spack.spec.Spec]]: """Iterate on items that have been added explicitly, and not just as a dependency of other nodes. """ for h, s in self.items(): # We need to make an exception for gcc-runtime, until we can splice it. if h in self.explicit or s.name == "gcc-runtime": yield h, s def add(self, spec: spack.spec.Spec) -> bool: """Adds a new concrete spec to the mapping. Returns True if the spec was just added, False if the spec was already in the mapping. Calling this function marks the spec as added explicitly. Args: spec: spec to be added Raises: ValueError: if the spec is not concrete """ if not spec.concrete: msg = ( f"trying to store the non-concrete spec '{spec}' in a container " f"that only accepts concrete" ) raise ValueError(msg) dag_hash = spec.dag_hash() self.explicit.add(dag_hash) if dag_hash in self.data: return False # Here we need to iterate on the input and rewire the copy. self.data[spec.dag_hash()] = spec.copy(deps=False) nodes_to_reconstruct = [spec] while nodes_to_reconstruct: input_parent = nodes_to_reconstruct.pop() container_parent = self.data[input_parent.dag_hash()] for edge in input_parent.edges_to_dependencies(): input_child = edge.spec container_child = self.data.get(input_child.dag_hash()) # Copy children that don't exist yet if container_child is None: container_child = input_child.copy(deps=False) self.data[input_child.dag_hash()] = container_child nodes_to_reconstruct.append(input_child) # Rewire edges container_parent.add_dependency_edge( dependency_spec=container_child, depflag=edge.depflag, virtuals=edge.virtuals ) return True def __len__(self) -> int: return len(self.data) def __iter__(self): return iter(self.data) # types for condition caching in solver setup ConditionSpecKey = Tuple[str, Optional[TransformFunction]] ConditionIdFunctionPair = Tuple[int, List[AspFunction]] ConditionSpecCache = Dict[str, Dict[ConditionSpecKey, ConditionIdFunctionPair]] class ConstraintOrigin(enum.Enum): """Generates identifiers that can be passed into the solver attached to constraints, and then later retrieved to determine the origin of those constraints when ``SpecBuilder`` creates Specs from the solve result. """ CONDITIONAL_SPEC = 0 DEPENDS_ON = 1 REQUIRE = 2 @staticmethod def _SUFFIXES() -> Dict["ConstraintOrigin", str]: return { ConstraintOrigin.CONDITIONAL_SPEC: "_cond", ConstraintOrigin.DEPENDS_ON: "_dep", ConstraintOrigin.REQUIRE: "_req", } @staticmethod def append_type_suffix(pkg_id: str, kind: "ConstraintOrigin") -> str: """Given a package identifier and a constraint kind, generate a string ID.""" suffix = ConstraintOrigin._SUFFIXES()[kind] return f"{pkg_id}{suffix}" @staticmethod def strip_type_suffix(source: str) -> Tuple[int, Optional[str]]: """Take a combined package/type ID generated by ``append_type_suffix``, and extract the package ID and an associated weight. """ if not source: return -1, None for kind, suffix in ConstraintOrigin._SUFFIXES().items(): if source.endswith(suffix): return kind.value, source[: -len(suffix)] return -1, source class ConditionIdContext(SourceContext): """Derived from a ``ConditionContext``: for clause-sets generated by imposed/required specs, stores an associated transform. This is primarily used for tracking whether we are generating clauses in the context of a required spec, or for an imposed spec. Is not a subclass of ``ConditionContext`` because it exists in a lower-level context with less information. """ def __init__(self): super().__init__() self.transform: Optional[TransformFunction] = None class ConditionContext(SourceContext): """Tracks context in which a condition (i.e. ``SpackSolverSetup.condition``) is generated (e.g. for a ``depends_on``). This may modify the required/imposed specs generated as relevant for the context. """ def __init__(self): super().__init__() # transformation applied to facts from the required spec. Defaults # to leave facts as they are. self.transform_required: Optional[TransformFunction] = None # transformation applied to facts from the imposed spec. Defaults # to removing "node" and "virtual_node" facts. self.transform_imposed: Optional[TransformFunction] = None # Whether to wrap direct dependency facts as node requirements, # imposed by the parent. If None, the default is used, which is: # - wrap head of rules # - do not wrap body of rules self.wrap_node_requirement: Optional[bool] = None def requirement_context(self) -> ConditionIdContext: ctxt = ConditionIdContext() ctxt.source = self.source ctxt.transform = self.transform_required ctxt.wrap_node_requirement = self.wrap_node_requirement return ctxt def impose_context(self) -> ConditionIdContext: ctxt = ConditionIdContext() ctxt.source = self.source ctxt.transform = self.transform_imposed ctxt.wrap_node_requirement = self.wrap_node_requirement return ctxt class SpackSolverSetup: """Class to set up and run a Spack concretization solve.""" gen: "ProblemInstanceBuilder" possible_versions: Dict[str, Dict[GitOrStandardVersion, List[Provenance]]] def __init__(self, tests: spack.concretize.TestsType = False): self.possible_graph = create_graph_analyzer() # these are all initialized in setup() self.requirement_parser = RequirementParser(spack.config.CONFIG) self.possible_virtuals: Set[str] = set() # pkg_name -> version -> list of possible origins (package.py, installed, etc.) self.possible_versions = collections.defaultdict(lambda: collections.defaultdict(list)) self.versions_from_yaml: Dict[str, List[GitOrStandardVersion]] = {} self.git_commit_versions: Dict[str, Dict[GitOrStandardVersion, str]] = ( collections.defaultdict(dict) ) self.deprecated_versions: Dict[str, Set[GitOrStandardVersion]] = collections.defaultdict( set ) self.possible_compilers: List[spack.spec.Spec] = [] self.rejected_compilers: Set[spack.spec.Spec] = set() self.possible_oses: Set = set() self.variant_values_from_specs: Set = set() self.version_constraints: Dict[str, Set] = collections.defaultdict(set) self.target_constraints: Set = set() self.default_targets: List = [] self.variant_ids_by_def_id: Dict[int, int] = {} self.reusable_and_possible: ConcreteSpecsByHash = ConcreteSpecsByHash() self._id_counter: Iterator[int] = itertools.count() self._trigger_cache: ConditionSpecCache = collections.defaultdict(dict) self._effect_cache: ConditionSpecCache = collections.defaultdict(dict) # Caches to optimize the setup phase of the solver self.target_specs_cache = None # whether to add installed/binary hashes to the solve self.tests = tests # If False allows for input specs that are not solved self.concretize_everything = True # Set during the call to setup self.pkgs: Set[str] = set() self.explicitly_required_namespaces: Dict[str, str] = {} # list of unique libc specs targeted by compilers (or an educated guess if no compiler) self.libcs: List[spack.spec.Spec] = [] # If true, we have to load the code for synthesizing splices self.enable_splicing: bool = spack.config.CONFIG.get("concretizer:splice:automatic") def pkg_version_rules(self, pkg: Type[spack.package_base.PackageBase]) -> None: """Declares known versions, their origins, and their weights.""" version_provenance = self.possible_versions[pkg.name] ordered_versions = spack.package_base.sort_by_pkg_preference( self.possible_versions[pkg.name], pkg=pkg ) # Account for preferences in packages.yaml, if any if pkg.name in self.versions_from_yaml: ordered_versions = list( spack.llnl.util.lang.dedupe(self.versions_from_yaml[pkg.name] + ordered_versions) ) # Set the deprecation penalty, according to the package. This should be enough to move the # first version last if deprecated. if ordered_versions: self.gen.fact( fn.pkg_fact(pkg.name, fn.version_deprecation_penalty(len(ordered_versions))) ) for weight, declared_version in enumerate(ordered_versions): self.gen.fact(fn.pkg_fact(pkg.name, fn.version_declared(declared_version, weight))) for origin in version_provenance[declared_version]: self.gen.fact( fn.pkg_fact(pkg.name, fn.version_origin(declared_version, str(origin))) ) for v in self.possible_versions[pkg.name]: if pkg.needs_commit(v): commit = pkg.version_or_package_attr("commit", v, "") self.git_commit_versions[pkg.name][v] = commit # Declare deprecated versions for this package, if any deprecated = self.deprecated_versions[pkg.name] for v in sorted(deprecated): self.gen.fact(fn.pkg_fact(pkg.name, fn.deprecated_version(v))) def spec_versions( self, spec: spack.spec.Spec, *, name: Optional[str] = None ) -> List[AspFunction]: """Return list of clauses expressing spec's version constraints.""" name = spec.name or name assert name, "Internal Error: spec with no name occurred. Please file an issue." if spec.concrete: return [fn.attr("version", name, spec.version)] if spec.versions == vn.any_version: return [] # record all version constraints for later self.version_constraints[name].add(spec.versions) return [fn.attr("node_version_satisfies", name, spec.versions)] def target_ranges( self, spec: spack.spec.Spec, single_target_fn, *, name: Optional[str] = None ) -> List[AspFunction]: name = spec.name or name assert name, "Internal Error: spec with no name occurred. Please file an issue." target = spec.architecture.target # Check if the target is a concrete target if str(target) in spack.vendor.archspec.cpu.TARGETS: return [single_target_fn(name, target)] self.target_constraints.add(target) return [fn.attr("node_target_satisfies", name, target)] def conflict_rules(self, pkg): for when_spec, conflict_specs in pkg.conflicts.items(): when_spec_msg = f"conflict constraint {when_spec}" when_spec_id = self.condition(when_spec, required_name=pkg.name, msg=when_spec_msg) when_spec_str = str(when_spec) for conflict_spec, conflict_msg in conflict_specs: conflict_spec_str = str(conflict_spec) if conflict_msg is None: conflict_msg = f"{pkg.name}: " if not when_spec_str: conflict_msg += f"conflicts with '{conflict_spec_str}'" else: conflict_msg += f"'{conflict_spec_str}' conflicts with '{when_spec_str}'" if not conflict_spec_str: conflict_spec_msg = f"conflict is triggered when {pkg.name}" else: conflict_spec_msg = f"conflict is triggered when {conflict_spec_str}" conflict_spec_id = self.condition( conflict_spec, required_name=conflict_spec.name or pkg.name, msg=conflict_spec_msg, ) self.gen.fact( fn.pkg_fact( pkg.name, fn.conflict(conflict_spec_id, when_spec_id, conflict_msg) ) ) self.gen.newline() def config_compatible_os(self): """Facts about compatible os's specified in configs""" self.gen.h2("Compatible OS from concretizer config file") os_data = spack.config.get("concretizer:os_compatible", {}) for recent, reusable in os_data.items(): for old in reusable: self.gen.fact(fn.os_compatible(recent, old)) self.gen.newline() def package_requirement_rules(self, pkg): self.emit_facts_from_requirement_rules(self.requirement_parser.rules(pkg)) def pkg_rules(self, pkg, tests): pkg = self.pkg_class(pkg) # Namespace of the package self.gen.fact(fn.pkg_fact(pkg.name, fn.namespace(pkg.namespace))) # versions self.pkg_version_rules(pkg) self.gen.newline() # variants self.variant_rules(pkg) # conflicts self.conflict_rules(pkg) # virtuals self.package_provider_rules(pkg) # dependencies self.package_dependencies_rules(pkg) # splices if self.enable_splicing: self.package_splice_rules(pkg) self.package_requirement_rules(pkg) def trigger_rules(self): """Flushes all the trigger rules collected so far, and clears the cache.""" if not self._trigger_cache: return self.gen.h2("Trigger conditions") for name in self._trigger_cache: cache = self._trigger_cache[name] for (spec_str, _), (trigger_id, requirements) in cache.items(): self.gen.fact(fn.pkg_fact(name, fn.trigger_id(trigger_id))) self.gen.fact(fn.pkg_fact(name, fn.trigger_msg(spec_str))) for predicate in requirements: self.gen.fact(fn.condition_requirement(trigger_id, *predicate.args)) self.gen.newline() self._trigger_cache.clear() def effect_rules(self): """Flushes all the effect rules collected so far, and clears the cache.""" if not self._effect_cache: return self.gen.h2("Imposed requirements") for name in sorted(self._effect_cache): cache = self._effect_cache[name] for (spec_str, _), (effect_id, requirements) in cache.items(): self.gen.fact(fn.pkg_fact(name, fn.effect_id(effect_id))) self.gen.fact(fn.pkg_fact(name, fn.effect_msg(spec_str))) for predicate in requirements: self.gen.fact(fn.imposed_constraint(effect_id, *predicate.args)) self.gen.newline() self._effect_cache.clear() def define_variant( self, pkg: Type[spack.package_base.PackageBase], name: str, when: spack.spec.Spec, variant_def: vt.Variant, ): pkg_fact = lambda f: self.gen.fact(fn.pkg_fact(pkg.name, f)) # Every variant id has a unique definition (conditional or unconditional), and # higher variant id definitions take precedence when variants intersect. vid = next(self._id_counter) # used to find a variant id from its variant definition (for variant values on specs) self.variant_ids_by_def_id[id(variant_def)] = vid if when == EMPTY_SPEC: # unconditional variant pkg_fact(fn.variant_definition(name, vid)) else: # conditional variant msg = f"Package {pkg.name} has variant '{name}' when {when}" cond_id = self.condition(when, required_name=pkg.name, msg=msg) pkg_fact(fn.variant_condition(name, vid, cond_id)) # record type so we can construct the variant when we read it back in self.gen.fact(fn.variant_type(vid, variant_def.variant_type.string)) if variant_def.sticky: pkg_fact(fn.variant_sticky(vid)) # Get the default values for this variant definition as a tuple default_values: Tuple[Union[bool, str], ...] = (variant_def.default,) if variant_def.multi: default_values = variant_def.make_default().values for val in default_values: pkg_fact(fn.variant_default_value_from_package_py(vid, val)) # Deal with variants that use validator functions if variant_def.values_defined_by_validator(): for penalty, value in enumerate(default_values, 1): pkg_fact(fn.variant_possible_value(vid, value)) pkg_fact(fn.variant_penalty(vid, value, penalty)) self.gen.newline() return values = variant_def.values or default_values # If we deal with disjoint sets of values, define the sets if isinstance(values, vt.DisjointSetsOfValues): for sid, s in enumerate(values.sets): for value in s: pkg_fact(fn.variant_value_from_disjoint_sets(vid, value, sid)) # Define penalties. Put default values first, otherwise keep the order penalty = 1 for v in default_values: pkg_fact(fn.variant_penalty(vid, v, penalty)) penalty += 1 for v in values: if v not in default_values: pkg_fact(fn.variant_penalty(vid, v, penalty)) penalty += 1 # Deal with conditional values for value in values: if not isinstance(value, vt.ConditionalValue): continue # make a spec indicating whether the variant has this conditional value variant_has_value = spack.spec.Spec() variant_has_value.variants[name] = vt.VariantValue( vt.VariantType.MULTI, name, (value.value,) ) if value.when: # the conditional value is always "possible", but it imposes its when condition as # a constraint if the conditional value is taken. This may seem backwards, but it # ensures that the conditional can only occur when its condition holds. self.condition( required_spec=variant_has_value, imposed_spec=value.when, required_name=pkg.name, imposed_name=pkg.name, msg=f"{pkg.name} variant {name} has value '{value.value}' when {value.when}", ) else: vstring = f"{name}='{value.value}'" # We know the value is never allowed statically (when was None), but we can't just # ignore it b/c it could come in as a possible value and we need a good error msg. # So, it's a conflict -- if the value is somehow used, it'll trigger an error. trigger_id = self.condition( variant_has_value, required_name=pkg.name, msg=f"invalid variant value: {vstring}", ) constraint_id = self.condition( EMPTY_SPEC, required_name=pkg.name, msg="empty (total) conflict constraint" ) msg = f"variant value {vstring} is conditionally disabled" pkg_fact(fn.conflict(trigger_id, constraint_id, msg)) self.gen.newline() def define_auto_variant(self, name: str, multi: bool): self.gen.h3(f"Special variant: {name}") vid = next(self._id_counter) self.gen.fact(fn.auto_variant(name, vid)) self.gen.fact( fn.variant_type( vid, vt.VariantType.MULTI.value if multi else vt.VariantType.SINGLE.value ) ) def variant_rules(self, pkg: Type[spack.package_base.PackageBase]): for name in pkg.variant_names(): self.gen.h3(f"Variant {name} in package {pkg.name}") for when, variant_def in pkg.variant_definitions(name): self.define_variant(pkg, name, when, variant_def) def _get_condition_id( self, name: str, cond: spack.spec.Spec, cache: ConditionSpecCache, body: bool, context: ConditionIdContext, ) -> int: """Get the id for one half of a condition (either a trigger or an imposed constraint). Construct a key from the condition spec and any associated transformation, and cache the ASP functions that they imply. The saved functions will be output later in ``trigger_rules()`` and ``effect_rules()``. Returns: The id of the cached trigger or effect. """ pkg_cache = cache[name] cond_str = str(cond) if cond.name else f"{name} {cond}" named_cond_key = (cond_str, context.transform) result = pkg_cache.get(named_cond_key) if result: return result[0] cond_id = next(self._id_counter) requirements = self.spec_clauses(cond, name=name, body=body, context=context) if context.transform: requirements = context.transform(name, cond, requirements) pkg_cache[named_cond_key] = (cond_id, requirements) return cond_id def _condition_clauses( self, required_spec: spack.spec.Spec, imposed_spec: Optional[spack.spec.Spec] = None, *, required_name: Optional[str] = None, imposed_name: Optional[str] = None, msg: Optional[str] = None, context: Optional[ConditionContext] = None, ) -> Tuple[List[AspFunction], int]: clauses = [] required_name = required_spec.name or required_name if not required_name: raise ValueError(f"Must provide a name for anonymous condition: '{required_spec}'") if not context: context = ConditionContext() context.transform_imposed = remove_facts("node", "virtual_node") # Check if we can emit the requirements before updating the condition ID counter. # In this way, if a condition can't be emitted but the exception is handled in the # caller, we won't emit partial facts. condition_id = next(self._id_counter) requirement_context = context.requirement_context() trigger_id = self._get_condition_id( required_name, required_spec, cache=self._trigger_cache, body=True, context=requirement_context, ) clauses.append(fn.pkg_fact(required_name, fn.condition(condition_id))) clauses.append(fn.condition_reason(condition_id, msg)) clauses.append(fn.pkg_fact(required_name, fn.condition_trigger(condition_id, trigger_id))) if not imposed_spec: return clauses, condition_id imposed_name = imposed_spec.name or imposed_name if not imposed_name: raise ValueError(f"Must provide a name for imposed constraint: '{imposed_spec}'") impose_context = context.impose_context() effect_id = self._get_condition_id( imposed_name, imposed_spec, cache=self._effect_cache, body=False, context=impose_context, ) clauses.append(fn.pkg_fact(required_name, fn.condition_effect(condition_id, effect_id))) return clauses, condition_id def condition( self, required_spec: spack.spec.Spec, imposed_spec: Optional[spack.spec.Spec] = None, *, required_name: Optional[str] = None, imposed_name: Optional[str] = None, msg: Optional[str] = None, context: Optional[ConditionContext] = None, ) -> int: """Generate facts for a dependency or virtual provider condition. Arguments: required_spec: the constraints that triggers this condition imposed_spec: the constraints that are imposed when this condition is triggered required_name: name for ``required_spec`` (required if required_spec is anonymous, ignored if not) imposed_name: name for ``imposed_spec`` (required if imposed_spec is anonymous, ignored if not) msg: description of the condition context: if provided, indicates how to modify the clause-sets for the required/imposed specs based on the type of constraint they are generated for (e.g. ``depends_on``) Returns: int: id of the condition created by this function """ clauses, condition_id = self._condition_clauses( required_spec=required_spec, imposed_spec=imposed_spec, required_name=required_name, imposed_name=imposed_name, msg=msg, context=context, ) for clause in clauses: self.gen.fact(clause) return condition_id def package_provider_rules(self, pkg: Type[spack.package_base.PackageBase]) -> None: for vpkg_name in pkg.provided_virtual_names(): if vpkg_name not in self.possible_virtuals: continue self.gen.fact(fn.pkg_fact(pkg.name, fn.possible_provider(vpkg_name))) for when, provided in pkg.provided.items(): for vpkg in sorted(provided): # type: ignore[type-var] if vpkg.name not in self.possible_virtuals: continue msg = f"{pkg.name} provides {vpkg}{'' if when == EMPTY_SPEC else f' when {when}'}" condition_id = self.condition(when, vpkg, required_name=pkg.name, msg=msg) self.gen.fact( fn.pkg_fact(pkg.name, fn.provider_condition(condition_id, vpkg.name)) ) self.gen.newline() for when, sets_of_virtuals in pkg.provided_together.items(): condition_id = self.condition( when, required_name=pkg.name, msg="Virtuals are provided together" ) for set_id, virtuals_together in enumerate(sorted(sets_of_virtuals)): for name in sorted(virtuals_together): self.gen.fact( fn.pkg_fact(pkg.name, fn.provided_together(condition_id, set_id, name)) ) self.gen.newline() def package_dependencies_rules(self, pkg): """Translate ``depends_on`` directives into ASP logic.""" for cond, deps_by_name in pkg.dependencies.items(): cond_str = str(cond) cond_str_suffix = f" when {cond_str}" if cond_str else "" for _, dep in deps_by_name.items(): depflag = dep.depflag # Skip test dependencies if they're not requested if not self.tests: depflag &= ~dt.TEST # ... or if they are requested only for certain packages elif not isinstance(self.tests, bool) and pkg.name not in self.tests: depflag &= ~dt.TEST # if there are no dependency types to be considered # anymore, don't generate the dependency if not depflag: continue msg = f"{pkg.name} depends on {dep.spec}{cond_str_suffix}" context = ConditionContext() context.source = ConstraintOrigin.append_type_suffix( pkg.name, ConstraintOrigin.DEPENDS_ON ) context.transform_imposed = dependency_holds(dependency_flags=depflag, pkg_cls=pkg) self.condition(cond, dep.spec, required_name=pkg.name, msg=msg, context=context) self.gen.newline() def _gen_match_variant_splice_constraints( self, pkg, cond_spec: spack.spec.Spec, splice_spec: spack.spec.Spec, hash_asp_var: "AspVar", splice_node, match_variants: List[str], ): # If there are no variants to match, no constraints are needed variant_constraints = [] for i, variant_name in enumerate(match_variants): vari_defs = pkg.variant_definitions(variant_name) # the spliceable config of the package always includes the variant if vari_defs != [] and any(cond_spec.satisfies(s) for (s, _) in vari_defs): variant = vari_defs[0][1] if variant.multi: continue # cannot automatically match multi-valued variants value_var = AspVar(f"VariValue{i}") attr_constraint = fn.attr("variant_value", splice_node, variant_name, value_var) hash_attr_constraint = fn.hash_attr( hash_asp_var, "variant_value", splice_spec.name, variant_name, value_var ) variant_constraints.append(attr_constraint) variant_constraints.append(hash_attr_constraint) return variant_constraints def package_splice_rules(self, pkg): self.gen.h2("Splice rules") for i, (cond, (spec_to_splice, match_variants)) in enumerate( sorted(pkg.splice_specs.items()) ): self.version_constraints[pkg.name].add(cond.versions) self.version_constraints[spec_to_splice.name].add(spec_to_splice.versions) hash_var = AspVar("Hash") splice_node = fn.node(AspVar("NID"), pkg.name) when_spec_attrs = [ fn.attr(c.args[0], splice_node, *(c.args[2:])) for c in self.spec_clauses(cond, name=pkg.name, body=True, required_from=None) if c.args[0] != "node" ] splice_spec_hash_attrs = [ fn.hash_attr(hash_var, *(c.args)) for c in self.spec_clauses(spec_to_splice, body=True, required_from=None) if c.args[0] != "node" ] if match_variants is None: variant_constraints = [] elif match_variants == "*": filt_match_variants = set() for map in pkg.variants.values(): for k in map: filt_match_variants.add(k) filt_match_variants = sorted(filt_match_variants) variant_constraints = self._gen_match_variant_splice_constraints( pkg, cond, spec_to_splice, hash_var, splice_node, filt_match_variants ) else: if any(v in cond.variants or v in spec_to_splice.variants for v in match_variants): raise spack.error.PackageError( "Overlap between match_variants and explicitly set variants" ) variant_constraints = self._gen_match_variant_splice_constraints( pkg, cond, spec_to_splice, hash_var, splice_node, match_variants ) rule_head = fn.abi_splice_conditions_hold( i, splice_node, spec_to_splice.name, hash_var ) rule_body_components = ( [ # splice_set_fact, fn.attr("node", splice_node), fn.installed_hash(spec_to_splice.name, hash_var), ] + when_spec_attrs + splice_spec_hash_attrs + variant_constraints ) rule_body = ",\n ".join(str(r) for r in rule_body_components) rule = f"{rule_head} :-\n {rule_body}." self.gen.append(rule) self.gen.newline() def virtual_requirements_and_weights(self): virtual_preferences = spack.config.CONFIG.get("packages:all:providers", {}) self.gen.h1("Virtual requirements and weights") for virtual_str in sorted(self.possible_virtuals): self.gen.newline() self.gen.h2(f"Virtual: {virtual_str}") self.gen.fact(fn.virtual(virtual_str)) rules = self.requirement_parser.rules_from_virtual(virtual_str) if not rules and virtual_str not in virtual_preferences: continue required, preferred, removed = [], [], set() for rule in rules: # We don't deal with conditional requirements if rule.condition != EMPTY_SPEC: continue if rule.origin == RequirementOrigin.PREFER_YAML: preferred.extend(x.name for x in rule.requirements if x.name) elif rule.origin == RequirementOrigin.REQUIRE_YAML: required.extend(x.name for x in rule.requirements if x.name) elif rule.origin == RequirementOrigin.CONFLICT_YAML: conflict_spec = rule.requirements[0] # For conflicts, we take action only if just a name is used if spack.spec.Spec(conflict_spec.name).satisfies(conflict_spec): removed.add(conflict_spec.name) current_preferences = required + preferred + virtual_preferences.get(virtual_str, []) current_preferences = [x for x in current_preferences if x not in removed] for i, provider in enumerate(spack.llnl.util.lang.dedupe(current_preferences)): provider_name = spack.spec.Spec(provider).name self.gen.fact(fn.provider_weight_from_config(virtual_str, provider_name, i)) self.gen.newline() if rules: self.emit_facts_from_requirement_rules(rules) self.trigger_rules() self.effect_rules() def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]): """Generate facts to enforce requirements. Args: rules: rules for which we want facts to be emitted """ for requirement_grp_id, rule in enumerate(rules): virtual = rule.kind == RequirementKind.VIRTUAL pkg_name, policy, requirement_grp = rule.pkg_name, rule.policy, rule.requirements requirement_weight = 0 # Propagated preferences have a higher penalty that normal preferences weight_multiplier = 2 if rule.origin == RequirementOrigin.INPUT_SPECS else 1 # Write explicitly if a requirement is conditional or not if rule.condition != EMPTY_SPEC: msg = f"activate requirement {requirement_grp_id} if {rule.condition} holds" context = ConditionContext() context.transform_required = dag_closure_by_deptype try: main_condition_id = self.condition( rule.condition, required_name=pkg_name, msg=msg, context=context ) except Exception as e: if rule.kind != RequirementKind.DEFAULT: raise RuntimeError( "cannot emit requirements for the solver: " + str(e) ) from e continue self.gen.fact( fn.requirement_conditional(pkg_name, requirement_grp_id, main_condition_id) ) self.gen.fact(fn.requirement_group(pkg_name, requirement_grp_id)) self.gen.fact(fn.requirement_policy(pkg_name, requirement_grp_id, policy)) if rule.message: self.gen.fact(fn.requirement_message(pkg_name, requirement_grp_id, rule.message)) self.gen.newline() for input_spec in requirement_grp: spec = spack.spec.Spec(input_spec) spec.replace_hash() if not spec.name: spec.name = pkg_name spec.attach_git_version_lookup() when_spec = spec if virtual and spec.name != pkg_name: when_spec = spack.spec.Spec(f"^[virtuals={pkg_name}] {spec}") try: context = ConditionContext() context.source = ConstraintOrigin.append_type_suffix( pkg_name, ConstraintOrigin.REQUIRE ) context.wrap_node_requirement = True if not virtual: context.transform_required = remove_facts("depends_on") context.transform_imposed = remove_facts( "node", "virtual_node", "depends_on" ) # else: for virtuals we want to emit "node" and # "virtual_node" in imposed specs info_msg = f"{input_spec} is a requirement for package {pkg_name}" if rule.condition != EMPTY_SPEC: info_msg += f" when {rule.condition}" if rule.message: info_msg += f" ({rule.message})" member_id = self.condition( required_spec=when_spec, imposed_spec=spec, required_name=pkg_name, msg=info_msg, context=context, ) # Conditions don't handle conditional dependencies directly # Those are handled separately here self.generate_conditional_dep_conditions(spec, member_id) except Exception as e: # Do not raise if the rule comes from the 'all' subsection, since usability # would be impaired. If a rule does not apply for a specific package, just # discard it. if rule.kind != RequirementKind.DEFAULT: raise RuntimeError( "cannot emit requirements for the solver: " + str(e) ) from e continue self.gen.fact(fn.requirement_group_member(member_id, pkg_name, requirement_grp_id)) self.gen.fact( fn.requirement_has_weight(member_id, requirement_weight * weight_multiplier) ) self.gen.newline() requirement_weight += 1 def external_packages(self, packages_with_externals): """Facts on external packages, from packages.yaml and implicit externals.""" self.gen.h1("External packages") for pkg_name, data in packages_with_externals.items(): if pkg_name == "all": continue # This package is not among possible dependencies if pkg_name not in self.pkgs: continue if not data.get("buildable", True): self.gen.h2(f"External package: {pkg_name}") self.gen.fact(fn.buildable_false(pkg_name)) def preferred_variants(self, pkg_name): """Facts on concretization preferences, as read from packages.yaml""" preferences = spack.package_prefs.PackagePrefs preferred_variants = preferences.preferred_variants(pkg_name) if not preferred_variants: return self.gen.h2(f"Package preferences: {pkg_name}") for variant_name in sorted(preferred_variants): variant = preferred_variants[variant_name] # perform validation of the variant and values try: variant_defs = vt.prevalidate_variant_value(self.pkg_class(pkg_name), variant) except (vt.InvalidVariantValueError, KeyError, ValueError) as e: tty.debug( f"[SETUP]: rejected {str(variant)} as a preference for {pkg_name}: {str(e)}" ) continue for value in variant.values: for variant_def in variant_defs: self.variant_values_from_specs.add((pkg_name, id(variant_def), value)) self.gen.fact( fn.variant_default_value_from_packages_yaml(pkg_name, variant.name, value) ) def target_preferences(self): key_fn = spack.package_prefs.PackagePrefs("all", "target") if not self.target_specs_cache: self.target_specs_cache = [ spack.spec.Spec("target={0}".format(target_name)) for _, target_name in self.default_targets ] package_targets = self.target_specs_cache[:] package_targets.sort(key=key_fn) for i, preferred in enumerate(package_targets): self.gen.fact(fn.target_weight(str(preferred.architecture.target), i)) def spec_clauses( self, spec: spack.spec.Spec, *, name: Optional[str] = None, body: bool = False, transitive: bool = True, expand_hashes: bool = False, concrete_build_deps=False, include_runtimes=False, required_from: Optional[str] = None, context: Optional[SourceContext] = None, ) -> List[AspFunction]: """Wrap a call to ``_spec_clauses()`` into a try/except block with better error handling. Arguments are as for ``_spec_clauses()`` except ``required_from``. Arguments: required_from: name of package that caused this call. """ try: clauses = self._spec_clauses( spec, name=spec.name or name, body=body, transitive=transitive, expand_hashes=expand_hashes, concrete_build_deps=concrete_build_deps, include_runtimes=include_runtimes, context=context, ) except RuntimeError as exc: msg = str(exc) if required_from: msg += f" [required from package '{required_from}']" raise RuntimeError(msg) return clauses def _spec_clauses( self, spec: spack.spec.Spec, *, name: Optional[str] = None, body: bool = False, transitive: bool = True, expand_hashes: bool = False, concrete_build_deps: bool = False, include_runtimes: bool = False, context: Optional[SourceContext] = None, seen: Optional[Set[int]] = None, ) -> List[AspFunction]: """Return a list of clauses for a spec mandates are true. Arguments: spec: the spec to analyze name: optional fallback of spec.name (used for anonymous roots) body: if True, generate clauses to be used in rule bodies (final values) instead of rule heads (setters). transitive: if False, don't generate clauses from dependencies (default True) expand_hashes: if True, descend into hashes of concrete specs (default False) concrete_build_deps: if False, do not include pure build deps of concrete specs (as they have no effect on runtime constraints) include_runtimes: generate full dependency clauses from runtime libraries that are omitted from the solve. context: tracks what constraint this clause set is generated for (e.g. a ``depends_on`` constraint in a package.py file) seen: set of ids of specs that have already been processed (for internal use only) Normally, if called with ``transitive=True``, ``spec_clauses()`` just generates hashes for the dependency requirements of concrete specs. If ``expand_hashes`` is ``True``, we'll *also* output all the facts implied by transitive hashes, which are redundant during a solve but useful outside of one (e.g., for spec ``diff``). """ clauses = [] seen = seen if seen is not None else set() name = spec.name or name or "" seen.add(id(spec)) f: Union[Type[_Head], Type[_Body]] = _Body if body else _Head if name: clauses.append( f.node(name) if not spack.repo.PATH.is_virtual(name) else f.virtual_node(name) ) if spec.namespace: clauses.append(f.namespace(name, spec.namespace)) clauses.extend(self.spec_versions(spec, name=name)) # seed architecture at the root (we'll propagate later) # TODO: use better semantics. arch = spec.architecture if arch: if arch.platform: clauses.append(f.node_platform(name, arch.platform)) if arch.os: clauses.append(f.node_os(name, arch.os)) if arch.target: clauses.extend(self.target_ranges(spec, f.node_target, name=name)) # variants for vname, variant in sorted(spec.variants.items()): # TODO: variant="*" means 'variant is defined to something', which used to # be meaningless in concretization, as all variants had to be defined. But # now that variants can be conditional, it should force a variant to exist. if not variant.values: continue for value in variant.values: # ensure that the value *can* be valid for the spec if name and not spec.concrete and not spack.repo.PATH.is_virtual(name): variant_defs = vt.prevalidate_variant_value( self.pkg_class(name), variant, spec ) # Record that that this is a valid possible value. Accounts for # int/str/etc., where valid values can't be listed in the package for variant_def in variant_defs: self.variant_values_from_specs.add((name, id(variant_def), value)) if variant.propagate: clauses.append(f.propagate(name, fn.variant_value(vname, value))) if self.pkg_class(name).has_variant(vname): clauses.append(f.variant_value(name, vname, value)) else: variant_clause = f.variant_value(name, vname, value) if ( variant.concrete and variant.type == vt.VariantType.MULTI and not spec.concrete ): if body is False: variant_clause.args = ( f"concrete_{variant_clause.args[0]}", *variant_clause.args[1:], ) else: clauses.append(fn.attr("concrete_variant_request", name, vname, value)) clauses.append(variant_clause) # compiler flags source = context.source if context else "none" for flag_type, flags in spec.compiler_flags.items(): flag_group = " ".join(flags) for flag in flags: clauses.append( f.node_flag(name, fn.node_flag(flag_type, flag, flag_group, source)) ) if not spec.concrete and flag.propagate is True: clauses.append( f.propagate( name, fn.node_flag(flag_type, flag, flag_group, source), fn.edge_types("link", "run"), ) ) # Hash for concrete specs if spec.concrete: # older specs do not have package hashes, so we have to do this carefully package_hash = getattr(spec, "_package_hash", None) if package_hash: clauses.append(fn.attr("package_hash", name, package_hash)) clauses.append(fn.attr("hash", name, spec.dag_hash())) if spec.external: clauses.append(fn.attr("external", name)) # TODO: a loop over `edges_to_dependencies` is preferred over `edges_from_dependents` # since dependents can point to specs out of scope for the solver. edges = spec.edges_from_dependents() if not body and not spec.concrete: virtuals = sorted(set(itertools.chain.from_iterable(edge.virtuals for edge in edges))) for virtual in virtuals: clauses.append(fn.attr("provider_set", name, virtual)) clauses.append(fn.attr("virtual_node", virtual)) else: # direct dependencies are handled under `edges_to_dependencies()` virtual_iter = (edge.virtuals for edge in edges if not edge.direct) virtuals = sorted(set(itertools.chain.from_iterable(virtual_iter))) for virtual in virtuals: clauses.append(fn.attr("virtual_on_incoming_edges", name, virtual)) # If the spec is external and concrete, we allow all the libcs on the system if spec.external and spec.concrete and using_libc_compatibility(): clauses.append(fn.attr("needs_libc", name)) for libc in self.libcs: clauses.append(fn.attr("compatible_libc", name, libc.name, libc.version)) if not transitive: return clauses # Dependencies edge_clauses = [] for dspec in spec.edges_to_dependencies(): # Ignore conditional dependencies, they are handled by caller if dspec.when != EMPTY_SPEC: continue dep = dspec.spec if spec.concrete: # GCC runtime is solved again by clingo, even on concrete specs, to give # the possibility to reuse specs built against a different runtime. if dep.name == "gcc-runtime": edge_clauses.append( fn.attr("compatible_runtime", name, dep.name, f"{dep.version}:") ) constraint_spec = spack.spec.Spec(f"{dep.name}@{dep.version}") self.spec_versions(constraint_spec) if not include_runtimes: continue # libc is also solved again by clingo, but in this case the compatibility # is not encoded in the parent node - so we need to emit explicit facts if "libc" in dspec.virtuals: edge_clauses.append(fn.attr("needs_libc", name)) for libc in self.libcs: if libc_is_compatible(libc, dep): edge_clauses.append( fn.attr("compatible_libc", name, libc.name, libc.version) ) if not include_runtimes: continue # We know dependencies are real for concrete specs. For abstract # specs they just mean the dep is somehow in the DAG. for dtype in dt.ALL_FLAGS: if not dspec.depflag & dtype: continue # skip build dependencies of already-installed specs if concrete_build_deps or dtype != dt.BUILD: edge_clauses.append( fn.attr("depends_on", name, dep.name, dt.flag_to_string(dtype)) ) for virtual_name in dspec.virtuals: edge_clauses.append( fn.attr("virtual_on_edge", name, dep.name, virtual_name) ) edge_clauses.append(fn.attr("virtual_node", virtual_name)) # imposing hash constraints for all but pure build deps of # already-installed concrete specs. if concrete_build_deps or dspec.depflag != dt.BUILD: edge_clauses.append(fn.attr("hash", dep.name, dep.dag_hash())) elif not concrete_build_deps and dspec.depflag: edge_clauses.append( fn.attr("concrete_build_dependency", name, dep.name, dep.dag_hash()) ) for virtual_name in dspec.virtuals: edge_clauses.append( fn.attr("virtual_on_build_edge", name, dep.name, virtual_name) ) # if the spec is abstract, descend into dependencies. # if it's concrete, then the hashes above take care of dependency # constraints, but expand the hashes if asked for. if (not spec.concrete or expand_hashes) and id(dep) not in seen: dependency_clauses = self._spec_clauses( dep, body=body, expand_hashes=expand_hashes, concrete_build_deps=concrete_build_deps, context=context, seen=seen, ) ### # Dependency expressed with "^" ### if not dspec.direct: edge_clauses.extend(dependency_clauses) continue ### # Direct dependencies expressed with "%" ### for dependency_type in dt.flag_to_tuple(dspec.depflag): edge_clauses.append(fn.attr("depends_on", name, dep.name, dependency_type)) for virtual in dspec.virtuals: dependency_clauses.append(fn.attr("virtual_on_edge", name, dep.name, virtual)) # By default, wrap head of rules, unless the context says otherwise wrap_node_requirement = body is False if context and context.wrap_node_requirement is not None: wrap_node_requirement = context.wrap_node_requirement if not wrap_node_requirement: edge_clauses.extend(dependency_clauses) continue for clause in dependency_clauses: clause.name = "node_requirement" edge_clauses.append(fn.attr("direct_dependency", name, clause)) clauses.extend(edge_clauses) return clauses def define_package_versions_and_validate_preferences( self, possible_pkgs: Set[str], *, require_checksum: bool, allow_deprecated: bool ): """Declare any versions in specs not declared in packages.""" packages_yaml = spack.config.CONFIG.get_config("packages") for pkg_name in sorted(possible_pkgs): pkg_cls = self.pkg_class(pkg_name) # All the versions from the corresponding package.py file. Since concepts # like being a "develop" version or being preferred exist only at a # package.py level, sort them in this partial list here from_package_py = list(pkg_cls.versions.items()) if require_checksum and pkg_cls.has_code: from_package_py = [x for x in from_package_py if _is_checksummed_version(x)] for v, version_info in from_package_py: if version_info.get("deprecated", False): self.deprecated_versions[pkg_name].add(v) if not allow_deprecated: continue self.possible_versions[pkg_name][v].append(Provenance.PACKAGE_PY) if pkg_name not in packages_yaml or "version" not in packages_yaml[pkg_name]: continue # TODO(psakiev) Need facts about versions # - requires_commit (associated with tag or branch) from_packages_yaml: List[GitOrStandardVersion] = [] for vstr in packages_yaml[pkg_name]["version"]: cfg_ver = vn.ver(vstr) if isinstance(cfg_ver, vn.GitVersion): if not require_checksum or cfg_ver.is_commit: from_packages_yaml.append(cfg_ver) else: matches = [x for x in self.possible_versions[pkg_name] if x.satisfies(cfg_ver)] matches.sort(reverse=True) if not matches: raise spack.error.ConfigError( f"Preference for version {cfg_ver} does not match any known " f"version of {pkg_name}" ) from_packages_yaml.extend(matches) from_packages_yaml = list(spack.llnl.util.lang.dedupe(from_packages_yaml)) for v in from_packages_yaml: provenance = Provenance.PACKAGES_YAML if isinstance(v, vn.GitVersion): provenance = Provenance.PACKAGES_YAML_GIT_VERSION self.possible_versions[pkg_name][v].append(provenance) self.versions_from_yaml[pkg_name] = from_packages_yaml def define_ad_hoc_versions_from_specs( self, specs, origin, *, allow_deprecated: bool, require_checksum: bool ): """Add concrete versions to possible versions from lists of CLI/dev specs.""" for s in traverse.traverse_nodes(specs): # If there is a concrete version on the CLI *that we know nothing # about*, add it to the known versions. version = s.versions.concrete if version is None or (any((v == version) for v in self.possible_versions[s.name])): continue if require_checksum and not _is_checksummed_git_version(version): raise UnsatisfiableSpecError( s.format("No matching version for constraint {name}{@versions}") ) if not allow_deprecated and version in self.deprecated_versions[s.name]: continue self.possible_versions[s.name][version].append(origin) def _supported_targets(self, compiler_name, compiler_version, targets): """Get a list of which targets are supported by the compiler. Results are ordered most to least recent. """ supported, unsupported = [], [] for target in targets: try: with warnings.catch_warnings(): warnings.simplefilter("ignore") target.optimization_flags( compiler_name, compiler_version.dotted_numeric_string ) supported.append(target) except spack.vendor.archspec.cpu.UnsupportedMicroarchitecture: unsupported.append(target) except ValueError: unsupported.append(target) return supported, unsupported def platform_defaults(self): self.gen.h2("Default platform") platform = spack.platforms.host() self.gen.fact(fn.node_platform_default(platform)) self.gen.fact(fn.allowed_platform(platform)) def os_defaults(self, specs): self.gen.h2("Possible operating systems") platform = spack.platforms.host() # create set of OS's to consider buildable = set(platform.operating_sys.keys()) # Consider any OS's mentioned on the command line. We need this to # cross-concretize in CI, and for some tests. # TODO: OS should really be more than just a label -- rework this. for spec in specs: if spec.architecture and spec.architecture.os: buildable.add(spec.architecture.os) # make directives for buildable OS's for build_os in sorted(buildable): self.gen.fact(fn.buildable_os(build_os)) def keyfun(os): return ( os == platform.default_os, # prefer default os not in buildable, # then prefer buildables os, # then sort by name ) all_oses = buildable.union(self.possible_oses) ordered_oses = sorted(all_oses, key=keyfun, reverse=True) # output the preference order of OS's for the concretizer to choose for i, os_name in enumerate(ordered_oses): self.gen.fact(fn.os(os_name, i)) def target_defaults(self, specs): """Add facts about targets and target compatibility.""" self.gen.h2("Target compatibility") # Add targets explicitly requested from specs candidate_targets = [] for x in self.possible_graph.candidate_targets(): if all( self.possible_graph.unreachable(pkg_name=pkg_name, when_spec=f"target={x}") for pkg_name in self.pkgs ): tty.debug(f"[{__name__}] excluding target={x}, cause no package can use it") continue candidate_targets.append(x) host_compatible = spack.config.CONFIG.get("concretizer:targets:host_compatible") for spec in specs: if not spec.architecture or not spec.architecture.target: continue target_name = spec.target.name target = spack.vendor.archspec.cpu.TARGETS.get(target_name) if not target: if spec.architecture.target_concrete: raise spack.error.SpecError( f"the target '{target_name}' in '{spec} is not a known target. " f"Run 'spack arch --known-targets' to see valid targets." ) # range/list constraint (contains ':' or ','): keep existing path self.target_ranges(spec, None) continue if target not in candidate_targets and not host_compatible: candidate_targets.append(target) for ancestor in target.ancestors: if ancestor not in candidate_targets: candidate_targets.append(ancestor) platform = spack.platforms.host() uarch = spack.vendor.archspec.cpu.TARGETS.get(platform.default) best_targets = {uarch.family.name} for compiler in self.possible_compilers: supported, unsupported = self._supported_targets( compiler.name, compiler.version, candidate_targets ) for target in supported: best_targets.add(target.name) self.gen.fact(fn.target_supported(compiler.name, compiler.version, target.name)) if supported: self.gen.fact( fn.target_supported(compiler.name, compiler.version, uarch.family.name) ) for target in unsupported: self.gen.fact( fn.target_not_supported(compiler.name, compiler.version, target.name) ) self.gen.newline() i = 0 for target in candidate_targets: self.gen.fact(fn.target(target.name)) self.gen.fact(fn.target_family(target.name, target.family.name)) self.gen.fact(fn.target_compatible(target.name, target.name)) # Code for ancestor can run on target for ancestor in target.ancestors: self.gen.fact(fn.target_compatible(target.name, ancestor.name)) # prefer best possible targets; weight others poorly so # they're not used unless set explicitly # these are stored to be generated as facts later offset by the # number of preferred targets if target.name in best_targets: self.default_targets.append((i, target.name)) i += 1 else: self.default_targets.append((100, target.name)) self.gen.newline() self.default_targets = list(sorted(set(self.default_targets))) self.target_preferences() def define_version_constraints(self): """Define what version_satisfies(...) means in ASP logic.""" sorted_versions = {} for pkg_name in self.possible_versions: possible_versions = list(self.possible_versions[pkg_name]) possible_versions.sort() sorted_versions[pkg_name] = possible_versions for idx, v in enumerate(possible_versions): self.gen.fact(fn.pkg_fact(pkg_name, fn.version_order(v, idx))) if v in self.git_commit_versions[pkg_name]: sha = self.git_commit_versions[pkg_name].get(v) if sha: self.gen.fact(fn.pkg_fact(pkg_name, fn.version_has_commit(v, sha))) else: self.gen.fact(fn.pkg_fact(pkg_name, fn.version_needs_commit(v))) self.gen.newline() self.gen.newline() for pkg_name, set_of_versions in sorted(self.version_constraints.items()): possible_versions = sorted_versions.get(pkg_name) if possible_versions is None: continue for versions in sorted(set_of_versions): # Look for contiguous ranges of versions that satisfy the constraint start_idx = None for current_idx, v in enumerate(possible_versions): if v.satisfies(versions): if start_idx is None: start_idx = current_idx elif start_idx is not None: # End of a contiguous satisfying range found version_range = fn.version_range(versions, start_idx, current_idx - 1) self.gen.fact(fn.pkg_fact(pkg_name, version_range)) start_idx = None if start_idx is not None: version_range = fn.version_range( versions, start_idx, len(possible_versions) - 1 ) self.gen.fact(fn.pkg_fact(pkg_name, version_range)) self.gen.newline() def collect_virtual_constraints(self): """Define versions for constraints on virtuals. Must be called before define_version_constraints(). """ # extract all the real versions mentioned in version ranges def versions_for(v): if isinstance(v, vn.StandardVersion): yield v elif isinstance(v, vn.ClosedOpenRange): yield v.lo yield vn._prev_version(v.hi) elif isinstance(v, vn.VersionList): for e in v: yield from versions_for(e) else: raise TypeError(f"expected version type, found: {type(v)}") # Define a set of synthetic possible versions for virtuals that don't define versions in a # package.py file. This ensures that `version_satisfies(Package, Constraint, Version)` has # the same semantics for virtuals as for regular packages. for pkg_name, versions in self.version_constraints.items(): # Not a virtual package if pkg_name not in self.possible_virtuals: continue possible_versions = {pv for v in versions for pv in versions_for(v)} for version in possible_versions: self.possible_versions[pkg_name][version].append(Provenance.VIRTUAL_CONSTRAINT) def define_target_constraints(self): def _all_targets_satisfiying(single_constraint): allowed_targets = [] if ":" not in single_constraint: return [single_constraint] t_min, _, t_max = single_constraint.partition(":") for test_target in spack.vendor.archspec.cpu.TARGETS.values(): # Check lower bound if t_min and not t_min <= test_target: continue # Check upper bound if t_max and not t_max >= test_target: continue allowed_targets.append(test_target) return allowed_targets cache = {} for target_constraint in sorted(self.target_constraints, key=lambda x: x.name): # Construct the list of allowed targets for this constraint allowed_targets = [] for single_constraint in str(target_constraint).split(","): if single_constraint not in cache: cache[single_constraint] = _all_targets_satisfiying(single_constraint) allowed_targets.extend(cache[single_constraint]) for target in allowed_targets: self.gen.fact(fn.target_satisfies(target_constraint, target)) self.gen.newline() def define_variant_values(self): """Validate variant values from the command line. Add valid variant values from the command line to the possible values for variant definitions. """ # for determinism, sort by variant ids, not variant def ids (which are object ids) def_info = [] for pkg_name, variant_def_id, value in sorted(self.variant_values_from_specs): try: vid = self.variant_ids_by_def_id[variant_def_id] except KeyError: tty.debug( f"[{__name__}] cannot retrieve id of the {value} variant from {pkg_name}" ) continue def_info.append((pkg_name, vid, value)) # Tell the concretizer about possible values from specs seen in spec_clauses(). for pkg_name, vid, value in sorted(def_info): self.gen.fact(fn.pkg_fact(pkg_name, fn.variant_possible_value(vid, value))) def register_concrete_spec(self, spec, possible: set): # tell the solver about any installed packages that could # be dependencies (don't tell it about the others) if spec.name not in possible: return try: # Only consider installed packages for repo we know spack.repo.PATH.get(spec) except (spack.repo.UnknownNamespaceError, spack.repo.UnknownPackageError) as e: tty.debug(f"[REUSE] Issues when trying to reuse {spec.short_spec}: {str(e)}") return self.reusable_and_possible.add(spec) def concrete_specs(self): """Emit facts for reusable specs""" for h, spec in self.reusable_and_possible.explicit_items(): # this indicates that there is a spec like this installed self.gen.fact(fn.installed_hash(spec.name, h)) # indirection layer between hash constraints and imposition to allow for splicing for pred in self.spec_clauses(spec, body=True, required_from=None): self.gen.fact(fn.hash_attr(h, *pred.args)) self.gen.newline() # Declare as possible parts of specs that are not in package.py # - Add versions to possible versions # - Add OS to possible OS's for dep in spec.traverse(): provenance = Provenance.INSTALLED if isinstance(dep.version, vn.GitVersion): provenance = Provenance.INSTALLED_GIT_VERSION self.possible_versions[dep.name][dep.version].append(provenance) self.possible_oses.add(dep.os) def define_concrete_input_specs(self, specs: tuple, possible: set): # any concrete specs in the input spec list for input_spec in specs: for spec in input_spec.traverse(): if spec.concrete: self.register_concrete_spec(spec, possible) def impossible_dependencies_check(self, specs) -> None: for edge in traverse.traverse_edges(specs): possible_deps = self.pkgs if spack.repo.PATH.is_virtual(edge.spec.name): possible_deps = self.possible_virtuals if edge.spec.name not in possible_deps and not str(edge.when): raise InvalidDependencyError( f"'{edge.spec.name}' is not a possible dependency of any root spec" ) def input_spec_version_check(self, specs, allow_deprecated: bool) -> None: """Raise an error early if no versions available in the solve can satisfy the inputs.""" only_deprecated = [] impossible = [] for spec in traverse.traverse_nodes(specs): if spack.repo.PATH.is_virtual(spec.name): continue if spec.name not in self.pkgs: continue # conditional dependency that won't be satisfied deprecated = self.deprecated_versions.get(spec.name, set()) sat_deprecated = [v for v in deprecated if deprecated and v.satisfies(spec.versions)] possible: Iterable = self.possible_versions.get(spec.name, set()) sat_possible = [v for v in possible if possible and v.satisfies(spec.versions)] if sat_deprecated and not sat_possible: only_deprecated.append(spec) if not sat_deprecated and not sat_possible: impossible.append(spec) if not allow_deprecated and only_deprecated: raise DeprecatedVersionError( "The following input specs can only be satisfied by deprecated versions:", " " + ", ".join(str(spec) for spec in only_deprecated) + "\n" + "Run with --deprecated to allow Spack to use these versions.", ) if impossible: raise InvalidVersionError( "No version exists that satisfies these input specs:", " " + ", ".join(str(spec) for spec in impossible), ) def setup( self, specs: Sequence[spack.spec.Spec], *, reuse: Optional[List[spack.spec.Spec]] = None, packages_with_externals=None, allow_deprecated: bool = False, ) -> "ProblemInstanceBuilder": """Generate an ASP program with relevant constraints for specs. This calls methods on the solve driver to set up the problem with facts and rules from all possible dependencies of the input specs, as well as constraints from the specs themselves. Arguments: specs: list of Specs to solve reuse: list of concrete specs that can be reused packages_with_externals: precomputed packages config with implicit externals allow_deprecated: if True adds deprecated versions into the solve Return: A ProblemInstanceBuilder populated with facts and rules for an ASP solve. """ # TODO: remove this local import and get rid of dependency on globals import spack.environment as ev reuse = reuse or [] if packages_with_externals is None: packages_with_externals = external_config_with_implicit_externals(spack.config.CONFIG) check_packages_exist(specs) self.gen = ProblemInstanceBuilder() # Get compilers from buildcaches only if injected through "reuse" specs supported_compilers = spack.compilers.config.supported_compilers() compilers_from_reuse = { x for x in reuse if x.name in supported_compilers and not x.external } candidate_compilers, self.rejected_compilers = possible_compilers( configuration=spack.config.CONFIG ) reuse_from_compilers = traverse.traverse_nodes( [x for x in candidate_compilers if not x.external], deptype=("link", "run") ) reused_set = set(reuse) reuse += [x for x in reuse_from_compilers if x not in reused_set] candidate_compilers.update(compilers_from_reuse) self.possible_compilers = list(candidate_compilers) # TODO: warning is because mypy doesn't know Spec supports rich comparison via decorator self.possible_compilers.sort() # type: ignore[call-arg,call-overload] self.compiler_mixing() self.gen.h1("Runtimes") injected_dependencies = self.define_runtime_constraints() node_counter = create_counter( list(specs) + injected_dependencies, tests=self.tests, possible_graph=self.possible_graph, ) self.possible_virtuals = node_counter.possible_virtuals() self.pkgs = node_counter.possible_dependencies() self.libcs = sorted(all_libcs()) # type: ignore[type-var] for node in traverse.traverse_nodes(specs): if node.namespace is not None: self.explicitly_required_namespaces[node.name] = node.namespace self.requirement_parser.parse_rules_from_input_specs(specs) self.gen.h1("Generic information") if using_libc_compatibility(): for libc in self.libcs: self.gen.fact(fn.host_libc(libc.name, libc.version)) if not allow_deprecated: self.gen.fact(fn.deprecated_versions_not_allowed()) self.gen.newline() for pkg_name in spack.compilers.config.supported_compilers(): self.gen.fact(fn.compiler_package(pkg_name)) # Calculate develop specs # they will be used in addition to command line specs # in determining known versions/targets/os dev_specs: Tuple[spack.spec.Spec, ...] = () env = ev.active_environment() if env: dev_specs = tuple( spack.spec.Spec(info["spec"]).constrained( 'dev_path="%s"' % spack.util.path.canonicalize_path(info["path"], default_wd=env.path) ) for name, info in env.dev_specs.items() ) specs = tuple(specs) # ensure compatible types to add self.gen.h1("Reusable concrete specs") self.define_concrete_input_specs(specs, self.pkgs) if reuse: self.gen.fact(fn.optimize_for_reuse()) for reusable_spec in reuse: self.register_concrete_spec(reusable_spec, self.pkgs) self.concrete_specs() self.gen.h1("Generic statements on possible packages") node_counter.possible_packages_facts(self.gen, fn) self.gen.h1("Possible flags on nodes") for flag in spack.spec.FlagMap.valid_compiler_flags(): self.gen.fact(fn.flag_type(flag)) self.gen.newline() self.gen.h1("General Constraints") self.config_compatible_os() # architecture defaults self.platform_defaults() self.os_defaults(specs + dev_specs) self.target_defaults(specs + dev_specs) self.virtual_requirements_and_weights() self.external_packages(packages_with_externals) # TODO: make a config option for this undocumented feature checksummed = "SPACK_CONCRETIZER_REQUIRE_CHECKSUM" in os.environ self.define_package_versions_and_validate_preferences( self.pkgs, allow_deprecated=allow_deprecated, require_checksum=checksummed ) self.define_ad_hoc_versions_from_specs( specs, Provenance.SPEC, allow_deprecated=allow_deprecated, require_checksum=checksummed ) self.define_ad_hoc_versions_from_specs( dev_specs, Provenance.DEV_SPEC, allow_deprecated=allow_deprecated, require_checksum=checksummed, ) self.validate_and_define_versions_from_requirements( allow_deprecated=allow_deprecated, require_checksum=checksummed ) self.gen.h1("Package Constraints") for pkg in sorted(self.pkgs): self.gen.h2(f"Package rules: {pkg}") self.pkg_rules(pkg, tests=self.tests) self.preferred_variants(pkg) self.gen.h1("Condition Triggers and Imposed Effects") self.trigger_rules() self.effect_rules() self.gen.h1("Special variants") self.define_auto_variant("dev_path", multi=False) self.define_auto_variant("commit", multi=False) self.define_auto_variant("patches", multi=True) self.gen.h1("Develop specs") # Inject dev_path from environment for ds in dev_specs: self.condition(spack.spec.Spec(ds.name), ds, msg=f"{ds.name} is a develop spec") self.trigger_rules() self.effect_rules() self.gen.h1("Spec Constraints") self.literal_specs(specs) self.gen.h1("Variant Values defined in specs") self.define_variant_values() self.gen.h1("Version Constraints") self.collect_virtual_constraints() self.define_version_constraints() self.gen.h1("Target Constraints") self.define_target_constraints() # once we've done a full traversal and know possible versions, check that the # requested solve is at least consistent. # do not check dependency and version availability for already concrete specs # as they come from reusable specs abstract_specs = [s for s in specs if not s.concrete] self.impossible_dependencies_check(abstract_specs) self.input_spec_version_check(abstract_specs, allow_deprecated) return self.gen def compiler_mixing(self): should_mix = spack.config.get("concretizer:compiler_mixing", True) if should_mix is True: return # anything besides should_mix: true for lang in ["c", "cxx", "fortran"]: self.gen.fact(fn.no_compiler_mixing(lang)) # user specified an allow-list if isinstance(should_mix, list): for pkg_name in should_mix: self.gen.fact(fn.allow_mixing(pkg_name)) def define_runtime_constraints(self) -> List[spack.spec.Spec]: """Define the constraints to be imposed on the runtimes, and returns a list of injected packages. """ recorder = RuntimePropertyRecorder(self) for compiler in self.possible_compilers: try: compiler_cls = spack.repo.PATH.get_pkg_class(compiler.name) except spack.repo.UnknownPackageError: pass else: if hasattr(compiler_cls, "runtime_constraints"): compiler_cls.runtime_constraints(spec=compiler, pkg=recorder) # Inject default flags for compilers recorder("*").default_flags(compiler) # Add a dependency on the compiler wrapper compiler_str = f"{compiler.name} /{compiler.dag_hash()}" for language in ("c", "cxx", "fortran"): # Using compiler.name causes a bit of duplication, but that is taken care of by # clingo during grounding. recorder("*").depends_on( "compiler-wrapper", when=f"%[deptypes=build virtuals={language}] {compiler.name}", type="build", description=f"Add compiler wrapper when using {compiler.name} for {language}", ) if not using_libc_compatibility(): continue current_libc = None if compiler.external or compiler.installed: current_libc = CompilerPropertyDetector(compiler).default_libc() else: try: current_libc = compiler["libc"] except (KeyError, RuntimeError) as e: tty.debug(f"{compiler} cannot determine libc because: {e}") if current_libc: recorder("*").depends_on( "libc", when=f"%[deptypes=build] {compiler.name}", type="link", description=f"Add libc when using {compiler.name}", ) recorder("*").depends_on( f"{current_libc.name}@={current_libc.version}", when=f"%[deptypes=build] {compiler_str}", type="link", description=f"Libc is {current_libc} when using {compiler}", ) recorder.consume_facts() return sorted(recorder.injected_dependencies) def literal_specs(self, specs): for spec in sorted(specs): self.gen.h2(f"Spec: {str(spec)}") condition_id = next(self._id_counter) trigger_id = next(self._id_counter) # Special condition triggered by "literal_solved" self.gen.fact(fn.literal(trigger_id)) self.gen.fact(fn.pkg_fact(spec.name, fn.condition_trigger(condition_id, trigger_id))) self.gen.fact(fn.condition_reason(condition_id, f"{spec} requested explicitly")) imposed_spec_key = str(spec), None cache = self._effect_cache[spec.name] if imposed_spec_key in cache: effect_id, requirements = cache[imposed_spec_key] else: effect_id = next(self._id_counter) context = SourceContext() context.source = "literal" requirements = self.spec_clauses(spec, context=context) root_name = spec.name for clause in requirements: clause_name = clause.args[0] if clause_name == "variant_set": requirements.append( fn.attr("variant_default_value_from_cli", *clause.args[1:]) ) elif clause_name in ("node", "virtual_node", "hash"): # These facts are needed to compute the "condition_set" of the root pkg_name = clause.args[1] self.gen.fact(fn.mentioned_in_literal(trigger_id, root_name, pkg_name)) requirements.append( fn.attr( "virtual_root" if spack.repo.PATH.is_virtual(spec.name) else "root", spec.name ) ) requirements = [x for x in requirements if x.args[0] != "depends_on"] cache[imposed_spec_key] = (effect_id, requirements) self.gen.fact(fn.pkg_fact(spec.name, fn.condition_effect(condition_id, effect_id))) # Create subcondition with any conditional dependencies # self.spec_clauses does not do anything with conditional # dependencies self.generate_conditional_dep_conditions(spec, condition_id) if self.concretize_everything: self.gen.fact(fn.solve_literal(trigger_id)) # Trigger rules are needed to allow conditional specs self.trigger_rules() self.effect_rules() def generate_conditional_dep_conditions(self, spec: spack.spec.Spec, condition_id: int): """Generate a subcondition in the trigger for any conditional dependencies. Dependencies are always modeled by a condition. For conditional dependencies, the when-spec is added as a subcondition of the trigger to ensure the dependency is only activated when the subcondition holds. """ for dspec in spec.traverse_edges(): # Ignore unconditional deps if dspec.when == EMPTY_SPEC: continue # Cannot use "virtual_node" attr as key for condition # because reused specs do not track virtual nodes. # Instead, track whether the parent uses the virtual def virtual_handler( name: str, input_spec: spack.spec.Spec, requirements: List[AspFunction] ) -> List[AspFunction]: ret = remove_facts("virtual_node")(name, input_spec, requirements) for edge in input_spec.traverse_edges(root=False, cover="edges"): if spack.repo.PATH.is_virtual(edge.spec.name): parent_name = name if edge.parent is input_spec else edge.parent.name ret.append(fn.attr("uses_virtual", parent_name, edge.spec.name)) return ret context = ConditionContext() context.source = ConstraintOrigin.append_type_suffix( dspec.parent.name, ConstraintOrigin.CONDITIONAL_SPEC ) # Default is to remove node-like attrs, override here context.transform_required = virtual_handler context.transform_imposed = identity_for_facts try: subcondition_id = self.condition( dspec.when, spack.spec.Spec(dspec.format(unconditional=True)), required_name=dspec.parent.name, context=context, msg=f"Conditional dependency in ^[when={dspec.when}]{dspec.spec}", ) self.gen.fact(fn.subcondition(subcondition_id, condition_id)) except vt.UnknownVariantError as e: # A variant in the 'when=' condition can't apply to the parent of the edge tty.debug(f"[{__name__}] cannot emit subcondition for {dspec.format()}: {e}") def validate_and_define_versions_from_requirements( self, *, allow_deprecated: bool, require_checksum: bool ): """If package requirements mention concrete versions that are not mentioned elsewhere, then we need to collect those to mark them as possible versions. If they are abstract and statically have no match, then we need to throw an error. This function assumes all possible versions are already registered in self.possible_versions.""" for pkg_name, d in spack.config.CONFIG.get_config("packages").items(): if pkg_name == "all" or "require" not in d: continue for s in traverse.traverse_nodes(self._specs_from_requires(pkg_name, d["require"])): name, versions = s.name, s.versions if name not in self.pkgs or versions == vn.any_version: continue s.attach_git_version_lookup() v = versions.concrete if not v: # If the version is not concrete, check it's statically concretizable. If # not throw an error, which is just so that users know they need to change # their config, instead of getting a hard to decipher concretization error. if not any(x for x in self.possible_versions[name] if x.satisfies(versions)): raise spack.error.ConfigError( f"Version requirement {versions} on {pkg_name} for {name} " f"cannot match any known version from package.py or externals" ) continue if v in self.possible_versions[name]: continue if not allow_deprecated and v in self.deprecated_versions[name]: continue # If concrete an not yet defined, conditionally define it, like we do for specs # from the command line. if not require_checksum or _is_checksummed_git_version(v): self.possible_versions[name][v].append(Provenance.PACKAGE_REQUIREMENT) def _specs_from_requires(self, pkg_name, section): """Collect specs from a requirement rule""" if isinstance(section, str): yield _spec_with_default_name(section, pkg_name) return for spec_group in section: if isinstance(spec_group, str): yield _spec_with_default_name(spec_group, pkg_name) continue # Otherwise it is an object. The object can contain a single # "spec" constraint, or a list of them with "any_of" or # "one_of" policy. if "spec" in spec_group: yield _spec_with_default_name(spec_group["spec"], pkg_name) continue key = "one_of" if "one_of" in spec_group else "any_of" for s in spec_group[key]: yield _spec_with_default_name(s, pkg_name) def pkg_class(self, pkg_name: str) -> Type[spack.package_base.PackageBase]: request = pkg_name if pkg_name in self.explicitly_required_namespaces: namespace = self.explicitly_required_namespaces[pkg_name] request = f"{namespace}.{pkg_name}" return spack.repo.PATH.get_pkg_class(request) class _Head: """ASP functions used to express spec clauses in the HEAD of a rule""" node = fn.attr("node") namespace = fn.attr("namespace_set") virtual_node = fn.attr("virtual_node") node_platform = fn.attr("node_platform_set") node_os = fn.attr("node_os_set") node_target = fn.attr("node_target_set") variant_value = fn.attr("variant_set") node_flag = fn.attr("node_flag_set") propagate = fn.attr("propagate") class _Body: """ASP functions used to express spec clauses in the BODY of a rule""" node = fn.attr("node") namespace = fn.attr("namespace") virtual_node = fn.attr("virtual_node") node_platform = fn.attr("node_platform") node_os = fn.attr("node_os") node_target = fn.attr("node_target") variant_value = fn.attr("variant_value") node_flag = fn.attr("node_flag") propagate = fn.attr("propagate") def strip_asp_problem(asp_problem: Iterable[str]) -> List[str]: """Remove comments and empty lines from an ASP program.""" def strip_statement(stmt: str) -> str: lines = [line for line in stmt.split("\n") if not line.startswith("%")] return "".join(line.strip() for line in lines if line) value = [strip_statement(stmt) for stmt in asp_problem] value = [s for s in value if s] return value class ProblemInstanceBuilder: """Provides an interface to construct a problem instance. Once all the facts and rules have been added, the problem instance can be retrieved with: >>> builder = ProblemInstanceBuilder() >>> ... >>> problem_instance = builder.value() The problem instance can be added directly to the "control" structure of clingo. """ def __init__(self) -> None: self.asp_problem: List[str] = [] def fact(self, atom: AspFunction) -> None: self.asp_problem.append(f"{atom}.") def append(self, rule: str) -> None: self.asp_problem.append(rule) def title(self, header: str, char: str) -> None: sep = char * 76 self.newline() self.asp_problem.append(f"%{sep}") self.asp_problem.append(f"% {header}") self.asp_problem.append(f"%{sep}") def h1(self, header: str) -> None: self.title(header, "=") def h2(self, header: str) -> None: self.title(header, "-") def h3(self, header: str): self.asp_problem.append(f"% {header}") def newline(self): self.asp_problem.append("") def possible_compilers(*, configuration) -> Tuple[Set["spack.spec.Spec"], Set["spack.spec.Spec"]]: result, rejected = set(), set() # Compilers defined in configuration for c in spack.compilers.config.all_compilers_from(configuration): if using_libc_compatibility() and not c_compiler_runs(c): rejected.add(c) try: compiler = c.extra_attributes["compilers"]["c"] tty.debug( f"the C compiler {compiler} does not exist, or does not run correctly." f" The compiler {c} will not be used during concretization." ) except KeyError: tty.debug(f"the spec {c} does not provide a C compiler.") continue if using_libc_compatibility() and not CompilerPropertyDetector(c).default_libc(): rejected.add(c) warnings.warn( f"cannot detect libc from {c}. The compiler will not be used " f"during concretization." ) continue if c in result: tty.debug(f"[{__name__}] duplicate {c.long_spec} compiler found") continue result.add(c) # Compilers from the local store supported_compilers = spack.compilers.config.supported_compilers() for pkg_name in supported_compilers: result.update(spack.store.STORE.db.query(pkg_name)) return result, rejected FunctionTupleT = Tuple[str, Tuple[Union[str, NodeId], ...]] class SpecBuilder: """Class with actions to rebuild a spec from ASP results.""" #: Regex for attributes that don't need actions b/c they aren't used to construct specs. ignored_attributes = re.compile( "|".join( [ r"^.*_propagate$", r"^.*_satisfies$", r"^.*_set$", r"^compatible_libc$", r"^dependency_holds$", r"^package_hash$", r"^root$", r"^uses_virtual$", r"^variant_default_value_from_cli$", r"^virtual_node$", r"^virtual_on_incoming_edges$", r"^virtual_root$", ] ) ) @staticmethod def make_node(*, pkg: str) -> NodeId: """Given a package name, returns the string representation of the "min_dupe_id" node in the ASP encoding. Args: pkg: name of a package """ return NodeId(id="0", pkg=pkg) def __init__(self, specs, hash_lookup=None): self._specs: Dict[NodeId, spack.spec.Spec] = {} # Matches parent nodes to splice node self._splices: Dict[spack.spec.Spec, List[spack.solver.splicing.Splice]] = {} self._result = None self._command_line_specs = specs self._flag_sources: Dict[Tuple[NodeId, str], Set[str]] = collections.defaultdict( lambda: set() ) # Pass in as arguments reusable specs and plug them in # from this dictionary during reconstruction self._hash_lookup = hash_lookup or ConcreteSpecsByHash() def hash(self, node, h): if node not in self._specs: self._specs[node] = self._hash_lookup[h] def node(self, node): if node not in self._specs: self._specs[node] = spack.spec.Spec(node.pkg) for flag_type in spack.spec.FlagMap.valid_compiler_flags(): self._specs[node].compiler_flags[flag_type] = [] def _arch(self, node): arch = self._specs[node].architecture if not arch: arch = spack.spec.ArchSpec() self._specs[node].architecture = arch return arch def namespace(self, node, namespace): self._specs[node].namespace = namespace def node_platform(self, node, platform): self._arch(node).platform = platform def node_os(self, node, os): self._arch(node).os = os def node_target(self, node, target): self._arch(node).target = target def variant_selected(self, node, name: str, value: str, variant_type: str, variant_id): spec = self._specs[node] variant = spec.variants.get(name) if not variant: spec.variants[name] = vt.VariantValue.from_concretizer(name, value, variant_type) else: assert variant_type == "multi", ( f"Can't have multiple values for single-valued variant: " f"{node}, {name}, {value}, {variant_type}, {variant_id}" ) variant.append(value) def version(self, node, version): self._specs[node].versions = vn.VersionList([vn.Version(version)]) def node_flag(self, node, node_flag): self._specs[node].compiler_flags.add_flag( node_flag.flag_type, node_flag.flag, False, node_flag.flag_group, node_flag.source ) def depends_on(self, parent_node, dependency_node, type): dependency_spec = self._specs[dependency_node] depflag = dt.flag_from_string(type) self._specs[parent_node].add_dependency_edge(dependency_spec, depflag=depflag, virtuals=()) def virtual_on_edge(self, parent_node, provider_node, virtual): dependencies = self._specs[parent_node].edges_to_dependencies(name=(provider_node.pkg)) provider_spec = self._specs[provider_node] dependencies = [x for x in dependencies if id(x.spec) == id(provider_spec)] assert len(dependencies) == 1, f"{virtual}: {provider_node.pkg}" dependencies[0].update_virtuals(virtual) def reorder_flags(self): """For each spec, determine the order of compiler flags applied to it. The solver determines which flags are on nodes; this routine imposes order afterwards. The order is: 1. Flags applied in compiler definitions should come first 2. Flags applied by dependents are ordered topologically (with a dependency on ``traverse`` to resolve the partial order into a stable total order) 3. Flags from requirements are then applied (requirements always come from the package and never a parent) 4. Command-line flags should come last Additionally, for each source (requirements, compiler, command line, and dependents), flags from that source should retain their order and grouping: e.g. for ``y cflags="-z -a"`` ``-z`` and ``-a`` should never have any intervening flags inserted, and should always appear in that order. """ for node, spec in self._specs.items(): # if bootstrapping, compiler is not in config and has no flags flagmap_from_compiler = { flag_type: [x for x in values if x.source == "compiler"] for flag_type, values in spec.compiler_flags.items() } flagmap_from_cli = {} for flag_type, values in spec.compiler_flags.items(): if not values: continue flags = [x for x in values if x.source == "literal"] if not flags: continue # For compiler flags from literal specs, reorder any flags to # the input order from flag.flag_group flagmap_from_cli[flag_type] = _reorder_flags(flags) for flag_type in spec.compiler_flags.valid_compiler_flags(): ordered_flags = [] # 1. Put compiler flags first from_compiler = tuple(flagmap_from_compiler.get(flag_type, [])) extend_flag_list(ordered_flags, from_compiler) # 2. Add all sources (the compiler is one of them, so skip any # flag group that matches it exactly) flag_groups = set() for flag in self._specs[node].compiler_flags.get(flag_type, []): flag_groups.add( spack.spec.CompilerFlag( flag.flag_group, propagate=flag.propagate, flag_group=flag.flag_group, source=flag.source, ) ) # For flags that are applied by dependents, put flags from parents # before children; we depend on the stability of traverse() to # achieve a stable flag order for flags introduced in this manner. topo_order = list(s.name for s in spec.traverse(order="post", direction="parents")) lex_order = list(sorted(flag_groups)) def _order_index(flag_group): source = flag_group.source # Note: if 'require: ^dependency cflags=...' is ever possible, # this will topologically sort for require as well type_index, pkg_source = ConstraintOrigin.strip_type_suffix(source) if pkg_source in topo_order: major_index = topo_order.index(pkg_source) # If for x->y, x has multiple depends_on declarations that # are activated, and each adds cflags to y, we fall back on # alphabetical ordering to maintain a total order minor_index = lex_order.index(flag_group) else: major_index = len(topo_order) + lex_order.index(flag_group) minor_index = 0 return (type_index, major_index, minor_index) prioritized_groups = sorted(flag_groups, key=lambda x: _order_index(x)) for grp in prioritized_groups: grp_flags = tuple( x for (x, y) in spack.compilers.flags.tokenize_flags(grp.flag_group) ) if grp_flags == from_compiler: continue as_compiler_flags = list( spack.spec.CompilerFlag( x, propagate=grp.propagate, flag_group=grp.flag_group, source=grp.source, ) for x in grp_flags ) extend_flag_list(ordered_flags, as_compiler_flags) # 3. Now put cmd-line flags last if flag_type in flagmap_from_cli: extend_flag_list(ordered_flags, flagmap_from_cli[flag_type]) compiler_flags = spec.compiler_flags.get(flag_type, []) msg = f"{set(compiler_flags)} does not equal {set(ordered_flags)}" assert set(compiler_flags) == set(ordered_flags), msg spec.compiler_flags.update({flag_type: ordered_flags}) def deprecated(self, node: NodeId, version: str) -> None: tty.warn(f'using "{node.pkg}@{version}" which is a deprecated version') def splice_at_hash( self, parent_node: NodeId, splice_node: NodeId, child_name: str, child_hash: str ): parent_spec = self._specs[parent_node] splice_spec = self._specs[splice_node] splice = spack.solver.splicing.Splice( splice_spec, child_name=child_name, child_hash=child_hash ) self._splices.setdefault(parent_spec, []).append(splice) def build_specs(self, function_tuples: List[FunctionTupleT]) -> List[spack.spec.Spec]: # TODO: remove this local import and get rid of dependency on globals import spack.environment as ev attr_key = { # hash attributes are handled first, since they imply entire concrete specs "hash": -5, # node attributes are handled next, since they instantiate nodes "node": -4, # evaluated last, so all nodes are fully constructed "virtual_on_edge": 2, } # Sort functions so that directives building objects are called in the right order function_tuples.sort(key=lambda x: attr_key.get(x[0], 0)) self._specs = {} for name, args in function_tuples: if SpecBuilder.ignored_attributes.match(name): continue action = getattr(self, name, None) # print out unknown actions so we can display them for debugging if not action: msg = f'UNKNOWN SYMBOL: attr("{name}", {", ".join(str(a) for a in args)})' tty.debug(msg) continue msg = ( "Internal Error: Uncallable action found in asp.py. Please report to the spack" " maintainers." ) assert action and callable(action), msg # ignore predicates on virtual packages, as they're used for # solving but don't construct anything. Do not ignore error # predicates on virtual packages. if name != "error": node = args[0] assert isinstance(node, NodeId), ( f"internal solver error: expected a node, but got a {type(args[0])}. " "Please report a bug at https://github.com/spack/spack/issues" ) pkg = node.pkg if spack.repo.PATH.is_virtual(pkg): continue # if we've already gotten a concrete spec for this pkg, we're done, unless # we're splicing. `splice_at_hash()` is the only action we call on concrete specs. spec = self._specs.get(node) if spec and spec.concrete and name != "splice_at_hash": continue action(*args) # fix flags after all specs are constructed self.reorder_flags() # inject patches -- note that we' can't use set() to unique the # roots here, because the specs aren't complete, and the hash # function will loop forever. roots = [spec.root for spec in self._specs.values()] roots = dict((id(r), r) for r in roots) for root in roots.values(): spack.spec._inject_patches_variant(root) # Add external paths to specs with just external modules for s in self._specs.values(): _ensure_external_path_if_external(s) for s in self._specs.values(): _develop_specs_from_env(s, ev.active_environment()) # check for commits must happen after all version adaptations are complete for s in self._specs.values(): _specs_with_commits(s) # mark concrete and assign hashes to all specs in the solve for root in roots.values(): root._finalize_concretization() # Unify hashes (this is to avoid duplicates of runtimes and compilers) unifier = ConcreteSpecsByHash() keys = list(self._specs) for key in keys: current_spec = self._specs[key] unifier.add(current_spec) self._specs[key] = unifier[current_spec.dag_hash()] # Only attempt to resolve automatic splices if the solver produced any if self._splices: resolved_splices = spack.solver.splicing._resolve_collected_splices( list(self._specs.values()), self._splices ) new_specs = {} for node, spec in self._specs.items(): new_specs[node] = resolved_splices.get(spec, spec) self._specs = new_specs for s in self._specs.values(): spack.spec.Spec.ensure_no_deprecated(s) # Add git version lookup info to concrete Specs (this is generated for # abstract specs as well but the Versions may be replaced during the # concretization process) for root in self._specs.values(): for spec in root.traverse(): if isinstance(spec.version, vn.GitVersion): spec.version.attach_lookup( spack.version.git_ref_lookup.GitRefLookup(spec.fullname) ) specs = self.execute_explicit_splices() return specs def execute_explicit_splices(self): splice_config = spack.config.CONFIG.get("concretizer:splice:explicit", []) splice_triples = [] for splice_set in splice_config: target = splice_set["target"] replacement = spack.spec.Spec(splice_set["replacement"]) if not replacement.abstract_hash: location = getattr( splice_set["replacement"], "_start_mark", " at unknown line number" ) msg = f"Explicit splice replacement '{replacement}' does not include a hash.\n" msg += f"{location}\n\n" msg += " Splice replacements must be specified by hash" raise InvalidSpliceError(msg) transitive = splice_set.get("transitive", False) splice_triples.append((target, replacement, transitive)) specs = {} for key, spec in self._specs.items(): current_spec = spec for target, replacement, transitive in splice_triples: if target in current_spec: # matches root or non-root # e.g. mvapich2%gcc # The first iteration, we need to replace the abstract hash if not replacement.concrete: replacement.replace_hash() current_spec = current_spec.splice(replacement, transitive) new_key = NodeId(id=key.id, pkg=current_spec.name) specs[new_key] = current_spec return specs def _specs_with_commits(spec): pkg_class = spack.repo.PATH.get_pkg_class(spec.fullname) if not pkg_class.needs_commit(spec.version): return if isinstance(spec.version, vn.GitVersion): if "commit" not in spec.variants and spec.version.commit_sha: spec.variants["commit"] = vt.SingleValuedVariant("commit", spec.version.commit_sha) pkg_class._resolve_git_provenance(spec) if "commit" not in spec.variants: if not spec.is_develop: tty.warn( f"Unable to resolve the git commit for {spec.name}. " "An installation of this binary won't have complete binary provenance." ) return # check integrity of user specified commit shas invalid_commit_msg = ( f"Internal Error: {spec.name}'s assigned commit {spec.variants['commit'].value}" " does not meet commit syntax requirements." ) assert vn.is_git_commit_sha(spec.variants["commit"].value), invalid_commit_msg def _ensure_external_path_if_external(spec: spack.spec.Spec) -> None: if not spec.external_modules or spec.external_path: return # Get the path from the module the package can override the default # (this is mostly needed for Cray) pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) package = pkg_cls(spec) spec.external_path = getattr(package, "external_prefix", None) or md.path_from_modules( spec.external_modules ) def _develop_specs_from_env(spec, env): dev_info = env.dev_specs.get(spec.name, {}) if env else {} if not dev_info: return path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path) if "dev_path" in spec.variants: error_msg = ( "Internal Error: The dev_path for spec {name} is not connected to a valid environment" "path. Please note that develop specs can only be used inside an environment" "These paths should be the same:\n\tdev_path:{dev_path}\n\tenv_based_path:{env_path}" ).format(name=spec.name, dev_path=spec.variants["dev_path"], env_path=path) assert spec.variants["dev_path"].value == path, error_msg else: spec.variants.setdefault("dev_path", vt.SingleValuedVariant("dev_path", path)) assert spec.satisfies(dev_info["spec"]) class Solver: """This is the main external interface class for solving. It manages solver configuration and preferences in one place. It sets up the solve and passes the setup method to the driver, as well. """ def __init__(self, *, specs_factory: Optional[SpecFiltersFactory] = None): # Compute possible compilers first, so we see them as externals _ = spack.compilers.config.all_compilers(init_config=True) self._conc_cache = ConcretizationCache() self.driver = PyclingoDriver(conc_cache=self._conc_cache) # Compute packages configuration with implicit externals once and reuse it self.packages_with_externals = external_config_with_implicit_externals(spack.config.CONFIG) completion_mode = spack.config.CONFIG.get("concretizer:externals:completion") self.selector = ReusableSpecsSelector( configuration=spack.config.CONFIG, external_parser=create_external_parser(self.packages_with_externals, completion_mode), factory=specs_factory, packages_with_externals=self.packages_with_externals, ) @staticmethod def _check_input_and_extract_concrete_specs( specs: Sequence[spack.spec.Spec], ) -> List[spack.spec.Spec]: reusable: List[spack.spec.Spec] = [] analyzer = create_graph_analyzer() for root in specs: for s in root.traverse(): if s.concrete: reusable.append(s) else: if spack.repo.PATH.is_virtual(s.name): continue # Error if direct dependencies cannot be satisfied deps = { edge.spec.name for edge in s.edges_to_dependencies() if edge.direct and edge.when == EMPTY_SPEC } if deps: graph = analyzer.possible_dependencies( s, allowed_deps=dt.ALL, transitive=False ) deps.difference_update(graph.real_pkgs, graph.virtuals) if deps: start_str = f"'{root}'" if s == root else f"'{s}' in '{root}'" raise UnsatisfiableSpecError( f"{start_str} cannot depend on {', '.join(deps)}" ) try: spack.repo.PATH.get_pkg_class(s.fullname) except spack.repo.UnknownPackageError: raise UnsatisfiableSpecError( f"cannot concretize '{root}', since '{s.name}' does not exist" ) spack.spec.Spec.ensure_valid_variants(s) return reusable def solve_with_stats( self, specs: Sequence[spack.spec.Spec], out: Optional[io.IOBase] = None, timers: bool = False, stats: bool = False, tests: spack.concretize.TestsType = False, setup_only: bool = False, allow_deprecated: bool = False, ) -> Tuple[Result, Optional[spack.util.timer.Timer], Optional[Dict]]: """ Concretize a set of specs and track the timing and statistics for the solve Arguments: specs: List of ``Spec`` objects to solve for. out: Optionally write the generate ASP program to a file-like object. timers: Print out coarse timers for different solve phases. stats: Print out detailed stats from clingo. tests: If True, concretize test dependencies for all packages. If a tuple of package names, concretize test dependencies for named packages (defaults to False: do not concretize test dependencies). setup_only: if True, stop after setup and don't solve (default False). allow_deprecated: allow deprecated version in the solve """ specs = [s.lookup_hash() for s in specs] reusable_specs = self._check_input_and_extract_concrete_specs(specs) reusable_specs.extend(self.selector.reusable_specs(specs)) setup = SpackSolverSetup(tests=tests) output = OutputConfiguration(timers=timers, stats=stats, out=out, setup_only=setup_only) result = self.driver.solve( setup, specs, reuse=reusable_specs, packages_with_externals=self.packages_with_externals, output=output, allow_deprecated=allow_deprecated, ) self._conc_cache.cleanup() return result def solve(self, specs: Sequence[spack.spec.Spec], **kwargs) -> Result: """ Convenience function for concretizing a set of specs and ignoring timing and statistics. Uses the same kwargs as solve_with_stats. """ # Check upfront that the variants are admissible result, _, _ = self.solve_with_stats(specs, **kwargs) return result def solve_in_rounds( self, specs: Sequence[spack.spec.Spec], out: Optional[io.IOBase] = None, timers: bool = False, stats: bool = False, tests: spack.concretize.TestsType = False, allow_deprecated: bool = False, ) -> Generator[Result, None, None]: """Solve for a stable model of specs in multiple rounds. This relaxes the assumption of solve that everything must be consistent and solvable in a single round. Each round tries to maximize the reuse of specs from previous rounds. The function is a generator that yields the result of each round. Arguments: specs (list): list of Specs to solve. out: Optionally write the generate ASP program to a file-like object. timers (bool): print timing if set to True stats (bool): print internal statistics if set to True tests (bool): add test dependencies to the solve allow_deprecated (bool): allow deprecated version in the solve """ specs = [s.lookup_hash() for s in specs] reusable_specs = self._check_input_and_extract_concrete_specs(specs) reusable_specs.extend(self.selector.reusable_specs(specs)) setup = SpackSolverSetup(tests=tests) # Tell clingo that we don't have to solve all the inputs at once setup.concretize_everything = False input_specs = specs output = OutputConfiguration(timers=timers, stats=stats, out=out, setup_only=False) while True: result, _, _ = self.driver.solve( setup, input_specs, reuse=reusable_specs, packages_with_externals=self.packages_with_externals, output=output, allow_deprecated=allow_deprecated, ) yield result # If we don't have unsolved specs we are done if not result.unsolved_specs: break if not result.specs: # This is also a problem: no specs were solved for, which means we would be in a # loop if we tried again raise OutputDoesNotSatisfyInputError(result.unsolved_specs) input_specs = list(x for (x, y) in result.unsolved_specs) for spec in result.specs: reusable_specs.extend(spec.traverse()) self._conc_cache.cleanup() class UnsatisfiableSpecError(spack.error.UnsatisfiableSpecError): """There was an issue with the spec that was requested (i.e. a user error).""" def __init__(self, msg): super(spack.error.UnsatisfiableSpecError, self).__init__(msg) self.provided = None self.required = None self.constraint_type = None class InternalConcretizerError(spack.error.UnsatisfiableSpecError): """Errors that indicate a bug in Spack.""" def __init__(self, msg): super(spack.error.UnsatisfiableSpecError, self).__init__(msg) self.provided = None self.required = None self.constraint_type = None class OutputDoesNotSatisfyInputError(InternalConcretizerError): def __init__( self, input_to_output: List[Tuple[spack.spec.Spec, Optional[spack.spec.Spec]]] ) -> None: self.input_to_output = input_to_output super().__init__( "internal solver error: the solver completed but produced specs" " that do not satisfy the request. Please report a bug at " f"https://github.com/spack/spack/issues\n\t{Result.format_unsolved(input_to_output)}" ) class SolverError(InternalConcretizerError): """For cases where the solver is unable to produce a solution. Such cases are unexpected because we allow for solutions with errors, so for example user specs that are over-constrained should still get a solution. """ def __init__(self, provided): msg = ( "Spack concretizer internal error. Please submit a bug report at " "https://github.com/spack/spack and include the command and environment " "if applicable." f"\n {provided} is unsatisfiable" ) super().__init__(msg) # Add attribute expected of the superclass interface self.required = None self.constraint_type = None self.provided = provided class InvalidSpliceError(spack.error.SpackError): """For cases in which the splice configuration is invalid.""" class NoCompilerFoundError(spack.error.SpackError): """Raised when there is no possible compiler""" class InvalidExternalError(spack.error.SpackError): """Raised when there is no possible compiler""" class DeprecatedVersionError(spack.error.SpackError): """Raised when user directly requests a deprecated version.""" class InvalidVersionError(spack.error.SpackError): """Raised when a version can't be satisfied by any possible versions.""" class InvalidDependencyError(spack.error.SpackError): """Raised when an explicit dependency is not a possible dependency.""" ================================================ FILE: lib/spack/spack/solver/concretize.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) %============================================================================= % This logic program implements Spack's concretizer %============================================================================= % ID of the nodes in the "root" link-run sub-DAG #const min_dupe_id = 0. #const direct_link_run = 0. #const direct_build = 1. % Allow clingo to create nodes { attr("node", node(0..X-1, Package)) } :- max_dupes(Package, X), not virtual(Package). { attr("virtual_node", node(0..X-1, Package)) } :- max_dupes(Package, X), virtual(Package). %%%% % Rules to prevent symmetry on duplicates %%%% duplicate_penalty(node(ID1, Package), 1, "version") :- attr("node", node(ID1, Package)), attr("node", node(ID2, Package)), version_weight(node(ID1, Package), Weight1), version_weight(node(ID2, Package), Weight2), multiple_unification_sets(Package), ID2 = ID1 + 1, Weight2 < Weight1, max_dupes(Package, X), X > 1, ID2 < X. % We can enforce this, and hope the grounder generates fewer rules :- provider_weight(ProviderNode1, node(ID1, Virtual), Weight1), provider_weight(ProviderNode2, node(ID2, Virtual), Weight2), ID2 = ID1 + 1, Weight2 < Weight1, max_dupes(Virtual, X), X > 1, ID2 < X. % Integrity constraints on DAG nodes :- attr("root", PackageNode), not attr("node", PackageNode). :- attr("version", PackageNode, _), not attr("node", PackageNode), not attr("virtual_node", PackageNode). :- attr("node_version_satisfies", PackageNode, _), not attr("node", PackageNode), not attr("virtual_node", PackageNode). :- attr("hash", PackageNode, _), not attr("node", PackageNode). :- attr("node_platform", PackageNode, _), not attr("node", PackageNode). :- attr("node_os", PackageNode, _), not attr("node", PackageNode). :- attr("node_target", PackageNode, _), not attr("node", PackageNode). :- attr("variant_value", PackageNode, _, _), not attr("node", PackageNode). :- attr("node_flag", PackageNode, _), not attr("node", PackageNode). :- attr("depends_on", ParentNode, _, _), not attr("node", ParentNode). :- attr("depends_on", _, ChildNode, _), not attr("node", ChildNode). :- attr("virtual_node", VirtualNode), not provider(_, VirtualNode). :- provider(_, VirtualNode), not attr("virtual_node", VirtualNode). :- provider(PackageNode, _), not attr("node", PackageNode). :- node_has_variant(PackageNode, _, _), not attr("node", PackageNode). :- attr("variant_set", PackageNode, _, _), not attr("node", PackageNode). :- variant_is_propagated(PackageNode, _), not attr("node", PackageNode). :- attr("root", node(ID, PackageNode)), ID > min_dupe_id. % Nodes in the "root" unification set cannot depend on non-root nodes if the dependency is "link" or "run" :- attr("depends_on", node(min_dupe_id, Package), node(ID, _), "link"), ID != min_dupe_id, unification_set("root", node(min_dupe_id, Package)). :- attr("depends_on", node(min_dupe_id, Package), node(ID, _), "run"), ID != min_dupe_id, unification_set("root", node(min_dupe_id, Package)). % Namespaces are statically assigned by a package fact if not otherwise set error(100, "{0} does not have a namespace", Package) :- attr("node", node(ID, Package)), not attr("namespace", node(ID, Package), _). error(100, "{0} cannot come from both {1} and {2} namespaces", Package, NS1, NS2) :- attr("node", node(ID, Package)), attr("namespace", node(ID, Package), NS1), attr("namespace", node(ID, Package), NS2), NS1 < NS2. attr("namespace", node(ID, Package), Namespace) :- attr("namespace_set", node(ID, Package), Namespace). attr("namespace", node(ID, Package), Namespace) :- attr("node", node(ID, Package)), not attr("namespace_set", node(ID, Package), _), pkg_fact(Package, namespace(Namespace)). % Rules on "unification sets", i.e. on sets of nodes allowing a single configuration of any given package unify(SetID, PackageName) :- unification_set(SetID, node(_, PackageName)). error(100000, "Cannot have multiple nodes for {0} in the same unification set {1}", PackageName, SetID) :- 2 { unification_set(SetID, node(_, PackageName)) }, unify(SetID, PackageName). unification_set("root", PackageNode) :- attr("root", PackageNode). unification_set("root", PackageNode) :- attr("virtual_root", PackageNode). % A package with a dependency that *is not* build belongs to the same unification set as the parent unification_set(SetID, ChildNode) :- attr("depends_on", ParentNode, ChildNode, Type), Type != "build", unification_set(SetID, ParentNode). % A separate unification set can be created if any of the build dependencies can be duplicated needs_build_unification_set(ParentNode) :- attr("depends_on", ParentNode, _, "build"). % If any of the build dependencies can be duplicated, they can go into any ("build", ID) set unification_set(("build", ID), node(X, Child)) :- attr("depends_on", ParentNode, node(X, Child), "build"), build_set_id(ParentNode, ID), needs_build_unification_set(ParentNode). % A virtual that is on an edge of non-build type belongs to the same unification set as the parent of the provider unification_set(SetID, VirtualNode) :- virtual_on_edge(ParentNode, ProviderNode, VirtualNode, Type), Type != "build", unification_set(SetID, ParentNode). % A virtual that is on a build edge, goes in the build set id of the parent of the provider unification_set(("build", ID), VirtualNode) :- virtual_on_edge(ParentNode, ProviderNode, VirtualNode, "build"), build_set_id(ParentNode, ID), needs_build_unification_set(ParentNode). % Needed for reused dependencies. A reused dependency has its build edges trimmed, so we % only care about the non-build edges. unification_set(SetID, node(VirtualID, Virtual)) :- concrete(ParentNode), attr("virtual_on_edge", ParentNode, ProviderNode, Virtual), provider(ProviderNode, node(VirtualID, Virtual)), unification_set(SetID, ParentNode). % Limit the number of unification sets to a reasonable number to avoid combinatorial explosion #const max_build_unification_sets = 4. 1 { build_set_id(ParentNode, 0..max_build_unification_sets-1) } 1 :- needs_build_unification_set(ParentNode). % Compute sub-sets of the nodes, if requested. These can be either the nodes connected % to another node by "link" edges, or the nodes connected to another node by "link and % "run" edges. compute_closure(node(ID, Package), "linkrun") :- condition_requirement(_, "closure", Package, _, "linkrun"), attr("node", node(ID, Package)). attr("closure", PackageNode, DependencyNode, "linkrun") :- attr("depends_on", PackageNode, DependencyNode, "link"), not provider(DependencyNode, node(_, Language)) : language(Language); compute_closure(PackageNode, "linkrun"). attr("closure", PackageNode, DependencyNode, "linkrun") :- attr("depends_on", PackageNode, DependencyNode, "run"), not provider(DependencyNode, node(_, Language)) : language(Language); compute_closure(PackageNode, "linkrun"). attr("closure", PackageNode, DependencyNode, "linkrun") :- attr("depends_on", ParentNode, DependencyNode, "link"), not provider(DependencyNode, node(_, Language)) : language(Language); attr("closure", PackageNode, ParentNode, "linkrun"), compute_closure(PackageNode, "linkrun"). attr("closure", PackageNode, DependencyNode, "linkrun") :- attr("depends_on", ParentNode, DependencyNode, "run"), not provider(DependencyNode, node(_, Language)) : language(Language); attr("closure", PackageNode, ParentNode, "linkrun"), compute_closure(PackageNode, "linkrun"). related(NodeA, NodeB) :- attr("closure", NodeA, NodeB, "linkrun"). % Do not allow split dependencies, for now. This ensures that we don't construct graphs where e.g. % a python extension depends on setuptools@63.4 as a run dependency, but uses e.g. setuptools@68 % as a build dependency. % % We'll need to relax the rule before we get to actual cross-compilation :- depends_on(ParentNode, node(X, Dependency)), depends_on(ParentNode, node(Y, Dependency)), X < Y. #defined multiple_unification_sets/1. #defined runtime/1. %---- % Rules to break symmetry and speed-up searches %---- % In the "root" unification set only ID = 0 are allowed :- unification_set("root", node(ID, _)), ID != 0. % In the "root" unification set we allow only packages from the link-run possible subDAG :- unification_set("root", node(_, Package)), not possible_in_link_run(Package), not virtual(Package). % Each node must belong to at least one unification set :- attr("node", PackageNode), not unification_set(_, PackageNode). % Cannot have a node with an ID, if lower ID of the same package are not used :- attr("node", node(ID1, Package)), not attr("node", node(ID2, Package)), max_dupes(Package, X), ID1=0..X-1, ID2=0..X-1, ID2 < ID1. :- attr("virtual_node", node(ID1, Package)), not attr("virtual_node", node(ID2, Package)), max_dupes(Package, X), ID1=0..X-1, ID2=0..X-1, ID2 < ID1. % Prefer to assign lower ID to virtuals associated with a lower penalty provider :- not unification_set("root", node(X, Virtual)), not unification_set("root", node(Y, Virtual)), X < Y, provider_weight(_, node(X, Virtual), WeightX), provider_weight(_, node(Y, Virtual), WeightY), WeightY < WeightX. %----------------------------------------------------------------------------- % Map literal input specs to facts that drive the solve %----------------------------------------------------------------------------- % TODO: literals, at the moment, can only influence the "root" unification set. This needs to be extended later. % Node attributes that need custom rules in ASP, e.g. because they involve multiple nodes node_attributes_with_custom_rules("depends_on"). node_attributes_with_custom_rules("virtual_on_edge"). node_attributes_with_custom_rules("provider_set"). node_attributes_with_custom_rules("concrete_variant_set"). node_attributes_with_custom_rules("concrete_variant_request"). node_attributes_with_custom_rules("closure"). trigger_condition_holds(TriggerID, node(min_dupe_id, Package)) :- solve_literal(TriggerID), pkg_fact(Package, condition_trigger(_, TriggerID)), literal(TriggerID). trigger_node(TriggerID, Node, Node) :- trigger_condition_holds(TriggerID, Node), literal(TriggerID). % Since we trigger the existence of literal nodes from a condition, we need to construct the condition_set/2 mentioned_in_literal(Root, Mentioned) :- mentioned_in_literal(TriggerID, Root, Mentioned), solve_literal(TriggerID). literal_node(Root, node(min_dupe_id, Root)) :- mentioned_in_literal(Root, Root). 1 { literal_node(Root, node(0..Y-1, Mentioned)) : max_dupes(Mentioned, Y) } 1 :- mentioned_in_literal(Root, Mentioned), Mentioned != Root. build_dependency_of_literal_node(LiteralNode, node(X, BuildDependency)) :- literal_node(Root, LiteralNode), build(LiteralNode), build_requirement(LiteralNode, node(X, BuildDependency)), attr("direct_dependency", LiteralNode, node_requirement("node", BuildDependency)). condition_set(node(min_dupe_id, Root), LiteralNode) :- literal_node(Root, LiteralNode). condition_set(LiteralNode, BuildNode) :- build_dependency_of_literal_node(LiteralNode, BuildNode). % Generate a comprehensible error for cases like 'foo ^bar' where 'bar' is a build dependency % of a transitive dependency of 'foo' error(1, "{0} is not a direct 'build' or 'test' dependency, or transitive 'link' or 'run' dependency of any root", Literal) :- literal_node(RootPackage, node(X, Literal)), not depends_on(node(min_dupe_id, RootPackage), node(X, Literal)), not unification_set("root", node(X, Literal)). :- build_dependency_of_literal_node(LiteralNode, BuildNode), not attr("depends_on", LiteralNode, BuildNode, "build"). % Discriminate between "roots" that have been explicitly requested, and roots that are deduced from "virtual roots" explicitly_requested_root(node(min_dupe_id, Package)) :- solve_literal(TriggerID), trigger_and_effect(Package, TriggerID, EffectID), imposed_constraint(EffectID, "root", Package). % Keep track of which nodes are associated with which root DAG associated_with_root(RootNode, RootNode) :- attr("root", RootNode). associated_with_root(RootNode, ChildNode) :- depends_on(ParentNode, ChildNode), associated_with_root(RootNode, ParentNode). % We cannot have a node in the root condition set, that is not associated with that root :- attr("root", RootNode), condition_set(RootNode, node(X, Package)), not virtual(Package), not associated_with_root(RootNode, node(X, Package)). #defined concretize_everything/0. #defined literal/1. % Attributes for node packages which must have a single value attr_single_value("version"). attr_single_value("node_platform"). attr_single_value("node_os"). attr_single_value("node_target"). % Error when no attribute is selected error(10000, no_value_error, Attribute, Package) :- attr("node", node(ID, Package)), attr_single_value(Attribute), not attr(Attribute, node(ID, Package), _). % Error when multiple attr need to be selected error(100, multiple_values_error, Attribute, Package) :- attr("node", node(ID, Package)), attr_single_value(Attribute), 2 { attr(Attribute, node(ID, Package), Value) }. %----------------------------------------------------------------------------- % Version semantics %----------------------------------------------------------------------------- version_declared(Package, Version) :- pkg_fact(Package, version_order(Version, _)). % If something is a package, it has only one version and that must be a % declared version. % We allow clingo to choose any version(s), and infer an error if there % is not precisely one version chosen. Error facts are heavily optimized % against to ensure they cannot be inferred when a non-error solution is % possible version_constraint_satisfied(node(ID,Package), Constraint) :- attr("version", node(ID,Package), Version), pkg_fact(Package, version_order(Version, VersionIdx)), pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)), VersionIdx >= MinIdx, VersionIdx <= MaxIdx. % Pick a single version among the possible ones 1 { choose_version(node(ID, Package), Version) : version_declared(Package, Version) } 1 :- attr("node", node(ID, Package)). % To choose the "fake" version of virtual packages, we need a separate rule. % Note that a virtual node may or may not have a version, but cannot have more than one. { choose_version(node(ID, Package), Version) : version_declared(Package, Version) } 1 :- attr("virtual_node", node(ID, Package)), virtual(Package). #defined compiler_package/1. attr("version", node(ID, Package), Version) :- choose_version(node(ID, Package), Version). % If we select a deprecated version, mark the package as deprecated attr("deprecated", node(ID, Package), Version) :- attr("version", node(ID, Package), Version), not external(node(ID, Package)), pkg_fact(Package, deprecated_version(Version)). error(100, "Package '{0}' needs the deprecated version '{1}', and this is not allowed", Package, Version) :- deprecated_versions_not_allowed(), attr("version", node(ID, Package), Version), build(node(ID, Package)), pkg_fact(Package, deprecated_version(Version)). % we can't use a weight from an installed spec if we are building it % and vice-versa 1 { allowed_origins(Origin): pkg_fact(Package, version_origin(Version, Origin)), Origin != "installed", Origin != "packages_yaml"} :- attr("version", node(ID, Package), Version), build(node(ID, Package)). % We cannot use a version declared for an installed package if we end up building it :- not pkg_fact(Package, version_origin(Version, "installed")), not pkg_fact(Package, version_origin(Version, "installed_git_version")), attr("version", node(ID, Package), Version), concrete(node(ID, Package)). version_weight(node(ID, Package), Weight) :- attr("version", node(ID, Package), Version), attr("node", node(ID, Package)), pkg_fact(Package, version_declared(Version, Weight)). version_deprecation_penalty(node(ID, Package), Penalty) :- pkg_fact(Package, deprecated_version(Version)), pkg_fact(Package, version_deprecation_penalty(Penalty)), attr("node", node(ID, Package)), attr("version", node(ID, Package), Version), not external(node(ID, Package)). % More specific error message if the version cannot satisfy some constraint % Otherwise covered by `no_version_error` and `versions_conflict_error`. error(10000, "Cannot satisfy '{0}@{1}'", Package, Constraint) :- attr("node_version_satisfies", node(ID, Package), Constraint), not version_constraint_satisfied(node(ID,Package), Constraint). attr("node_version_satisfies", node(ID, Package), Constraint) :- attr("version", node(ID, Package), Version), pkg_fact(Package, version_order(Version, VersionIdx)), pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)), VersionIdx >= MinIdx, VersionIdx <= MaxIdx. % if a version needs a commit or has one it can use the commit variant can_accept_commit(Package, Version) :- pkg_fact(Package, version_needs_commit(Version)). can_accept_commit(Package, Version) :- pkg_fact(Package, version_has_commit(Version, _)). % Specs with a commit variant can't use versions that don't need commits error(10, "Cannot use commit variant with '{0}@={1}'", Package, Version) :- attr("version", node(ID, Package), Version), not can_accept_commit(Package, Version), attr("variant_value", node(ID, Package), "commit", _). error(10, "Commit '{0}' must match package.py value '{1}' for '{2}@={3}'", Vsha, Psha, Package, Version) :- attr("version", node(ID, Package), Version), attr("variant_value", node(ID, Package), "commit", Vsha), pkg_fact(Package, version_has_commit(Version, Psha)), Vsha != Psha. #defined deprecated_versions_not_allowed/0. %----------------------------------------------------------------------------- % Spec conditions and imposed constraints % % Given Spack directives like these: % depends_on("foo@1.0+bar", when="@2.0+variant") % provides("mpi@2:", when="@1.9:") % % The conditions are `@2.0+variant` and `@1.9:`, and the imposed constraints % are `@1.0+bar` on `foo` and `@2:` on `mpi`. %----------------------------------------------------------------------------- % conditions are specified with `condition_requirement` and hold when % corresponding spec attributes hold. % A "condition_set(PackageNode, _)" is the set of nodes on which PackageNode can require / impose conditions condition_set(PackageNode, PackageNode) :- attr("node", PackageNode). condition_set(PackageNode, VirtualNode) :- provider(PackageNode, VirtualNode). condition_set(PackageNode, DependencyNode) :- condition_set(PackageNode, PackageNode), depends_on(PackageNode, DependencyNode). condition_set(ID, VirtualNode) :- condition_set(ID, PackageNode), provider(PackageNode, VirtualNode). condition_set(VirtualNode, X) :- provider(PackageNode, VirtualNode), condition_set(PackageNode, X). condition_packages(ID, A1) :- condition_requirement(ID, _, A1). condition_packages(ID, A1) :- condition_requirement(ID, _, A1, _). condition_packages(ID, A1) :- condition_requirement(ID, _, A1, _, _). condition_packages(ID, A1) :- condition_requirement(ID, _, A1, _, _, _). trigger_node(ID, node(PackageID, Package), node(PackageID, Package)) :- pkg_fact(Package, trigger_id(ID)), attr("node", node(PackageID, Package)). trigger_node(ID, node(PackageID, Package), node(VirtualID, Virtual)) :- pkg_fact(Virtual, trigger_id(ID)), provider(node(PackageID, Package), node(VirtualID, Virtual)). % This is the "real node" that triggers the request, e.g. if the request started from "mpi" this is the mpi provider trigger_real_node(ID, PackageNode) :- trigger_node(ID, PackageNode, _). % This is the requestor node, which may be a "real" or a "virtual" node trigger_requestor_node(ID, RequestorNode) :- trigger_node(ID, _, RequestorNode). trigger_package_requirement(TriggerNode, A1) :- trigger_real_node(ID, TriggerNode), condition_packages(ID, A1). condition_nodes(PackageNode, node(X, A1)) :- trigger_package_requirement(PackageNode, A1), condition_set(PackageNode, node(X, A1)), not self_build_requirement(PackageNode, node(X, A1)). cannot_hold(TriggerID, PackageNode) :- condition_packages(TriggerID, A1), not condition_set(PackageNode, node(_, A1)), trigger_real_node(TriggerID, PackageNode), attr("node", PackageNode). % Aggregates generic condition requirements with TriggerID, to see if a condition holds trigger_condition_holds(ID, RequestorNode) :- trigger_node(ID, PackageNode, RequestorNode); satisfied(trigger(PackageNode), condition_requirement(Name, A1)) : condition_requirement(ID, Name, A1); satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2)) : condition_requirement(ID, Name, A1, A2); satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2, A3)) : condition_requirement(ID, Name, A1, A2, A3); satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2, A3, A4)) : condition_requirement(ID, Name, A1, A2, A3, A4); not cannot_hold(ID, PackageNode). %%%% % Conditions verified on actual nodes in the DAG %%%% % Here we project out the trigger ID from condition requirements, so that we can reduce the space % of satisfied facts below. All we care about below is if a condition is met (e.g. a node exists % in the DAG), we don't care instead about *which* rule is requesting that. generic_condition_requirement(Name, A1) :- condition_requirement(ID, Name, A1). generic_condition_requirement(Name, A1, A2) :- condition_requirement(ID, Name, A1, A2). generic_condition_requirement(Name, A1, A2, A3) :- condition_requirement(ID, Name, A1, A2, A3). generic_condition_requirement(Name, A1, A2, A3, A4) :- condition_requirement(ID, Name, A1, A2, A3, A4). satisfied(trigger(PackageNode), condition_requirement(Name, A1)) :- attr(Name, node(X, A1)), generic_condition_requirement(Name, A1), condition_nodes(PackageNode, node(X, A1)). satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2)) :- attr(Name, node(X, A1), A2), generic_condition_requirement(Name, A1, A2), condition_nodes(PackageNode, node(X, A1)). satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2, A3)) :- attr(Name, node(X, A1), A2, A3), generic_condition_requirement(Name, A1, A2, A3), condition_nodes(PackageNode, node(X, A1)), not node_attributes_with_custom_rules(Name). satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2, A3, A4)) :- attr(Name, node(X, A1), A2, A3, A4), generic_condition_requirement(Name, A1, A2, A3, A4), condition_nodes(PackageNode, node(X, A1)). satisfied(trigger(PackageNode), condition_requirement("depends_on", A1, A2, A3)) :- attr("depends_on", node(X, A1), node(Y, A2), A3), generic_condition_requirement("depends_on", A1, A2, A3), condition_nodes(PackageNode, node(X, A1)), condition_nodes(PackageNode, node(Y, A2)). satisfied(trigger(PackageNode), condition_requirement("concrete_variant_request", A1, A2, A3)) :- generic_condition_requirement("concrete_variant_request", A1, A2, A3), condition_nodes(PackageNode, node(X, A1)). satisfied(trigger(PackageNode), condition_requirement("closure", A1, A2, A3)) :- attr("closure", node(X, A1), node(_, A2), A3), generic_condition_requirement("closure", A1, A2, A3), condition_nodes(PackageNode, node(X, A1)). satisfied(trigger(PackageNode), condition_requirement("virtual_on_edge", A1, A2, A3)) :- attr("virtual_on_edge", node(X, A1), node(Y, A2), A3), generic_condition_requirement("virtual_on_edge", A1, A2, A3), condition_nodes(PackageNode, node(X, A1)), condition_nodes(PackageNode, node(Y, A2)). %%%% % Conditions verified on pure build deps of reused nodes %%%% satisfied(trigger(PackageNode), condition_requirement("node", A1)) :- trigger_real_node(ID, PackageNode), reused_provider(PackageNode, _), condition_requirement(ID, "node", A1). satisfied(trigger(PackageNode), condition_requirement("virtual_node", A1)) :- trigger_real_node(ID, PackageNode), reused_provider(PackageNode, node(_, A1)), condition_requirement(ID, "virtual_node", A1). satisfied(trigger(PackageNode), condition_requirement("virtual_on_incoming_edges", A1, A2)) :- trigger_real_node(ID, PackageNode), reused_provider(node(Hash, A1), node(Hash, A2)), condition_requirement(ID, "virtual_on_incoming_edges", A1, A2). satisfied(trigger(node(Hash, Package)), condition_requirement(Name, Package, A1)) :- trigger_real_node(ID, node(Hash, Package)), reused_provider(node(Hash, Package), node(Hash, Language)), hash_attr(Hash, Name, Package, A1), condition_requirement(ID, Name, Package, A1). satisfied(trigger(node(Hash, Package)), condition_requirement("node_version_satisfies", Package, VersionConstraint)) :- trigger_real_node(ID, node(Hash, Package)), reused_provider(node(Hash, Package), node(Hash, Language)), hash_attr(Hash, "version", Package, Version), condition_requirement(ID, "node_version_satisfies", Package, VersionConstraint), pkg_fact(Package, version_order(Version, VersionIdx)), pkg_fact(Package, version_range(VersionConstraint, MinIdx, MaxIdx)), VersionIdx >= MinIdx, VersionIdx <= MaxIdx. satisfied(trigger(node(Hash, Package)), condition_requirement(Name, Package, A1, A2)) :- trigger_real_node(ID, node(Hash, Package)), reused_provider(node(Hash, Package), node(Hash, Language)), hash_attr(Hash, Name, Package, A1, A2), condition_requirement(ID, Name, Package, A1, A2). condition_with_concrete_variant(ID, Package, Variant) :- condition_requirement(ID, "concrete_variant_request", Package, Variant, _). cannot_hold(ID, PackageNode) :- not attr("variant_value", node(X, A1), Variant, Value), condition_with_concrete_variant(ID, A1, Variant), condition_requirement(ID, "concrete_variant_request", A1, Variant, Value), condition_nodes(PackageNode, node(X, A1)). cannot_hold(ID, PackageNode) :- attr("variant_value", node(X, A1), Variant, Value), condition_with_concrete_variant(ID, A1, Variant), not condition_requirement(ID, "concrete_variant_request", A1, Variant, Value), condition_nodes(PackageNode, node(X, A1)). condition_holds(ConditionID, node(X, Package)) :- pkg_fact(Package, condition_trigger(ConditionID, TriggerID)), trigger_condition_holds(TriggerID, node(X, Package)). trigger_and_effect(Package, ID, TriggerID, EffectID) :- pkg_fact(Package, condition_trigger(ID, TriggerID)), pkg_fact(Package, condition_effect(ID, EffectID)). trigger_and_effect(Package, TriggerID, EffectID) :- trigger_and_effect(Package, ID, TriggerID, EffectID). % condition_holds(ID, node(ID, Package)) implies all imposed_constraints, unless do_not_impose(ID, node(ID, Package)) % is derived. This allows imposed constraints to be canceled in special cases. % Effects of direct conditions hold if the trigger holds impose(EffectID, node(X, Package)) :- not subcondition(ConditionID, _), trigger_and_effect(Package, ConditionID, TriggerID, EffectID), trigger_requestor_node(TriggerID, node(X, Package)), trigger_condition_holds(TriggerID, node(X, Package)), not do_not_impose(EffectID, node(X, Package)). % Effects of subconditions hold if the trigger holds and the % primary condition holds impose(EffectID, node(X, Package)) :- subcondition(SubconditionID, ConditionID), condition_holds(ConditionID, ConditionNode), condition_set(ConditionNode, node(X, Package)), trigger_and_effect(Package, SubconditionID, TriggerID, EffectID), trigger_requestor_node(TriggerID, node(X, Package)), trigger_condition_holds(TriggerID, node(X, Package)), not do_not_impose(EffectID, node(X, Package)). imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1). imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _). imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _, _). imposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _, _, _). imposed_packages(ID, A1) :- imposed_constraint(ID, "depends_on", _, A1, _). imposed_nodes(node(ID, Package), node(X, A1)) :- condition_set(node(ID, Package), node(X, A1)), % We don't want to add build requirements to imposed nodes, to avoid % unsat problems when we deal with self-dependencies: gcc@14 %gcc@10 not self_build_requirement(node(ID, Package), node(X, A1)). self_build_requirement(node(X, Package), node(Y, Package)) :- build_requirement(node(X, Package), node(Y, Package)). imposed_nodes(PackageNode, node(X, A1)) :- imposed_packages(ConditionID, A1), condition_set(PackageNode, node(X, A1)), attr("hash", PackageNode, ConditionID). :- imposed_packages(ID, A1), impose(ID, PackageNode), not condition_set(PackageNode, node(_, A1)). :- imposed_packages(ID, A1), impose(ID, PackageNode), not imposed_nodes(PackageNode, node(_, A1)). % Conditions that hold impose may impose constraints on other specs attr(Name, node(X, A1)) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1), imposed_nodes(PackageNode, node(X, A1)). attr(Name, node(X, A1), A2) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2), imposed_nodes(PackageNode, node(X, A1)), not node_attributes_with_custom_rules(Name). attr(Name, node(X, A1), A2, A3) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2, A3), imposed_nodes(PackageNode, node(X, A1)), not node_attributes_with_custom_rules(Name). attr(Name, node(X, A1), A2, A3, A4) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2, A3, A4), imposed_nodes(PackageNode, node(X, A1)). % Provider set is relevant only for literals, since it's the only place where `^[virtuals=foo] bar` % might appear in the HEAD of a rule 1 { attr("provider_set", node(0..MaxProvider-1, Provider), node(0..MaxVirtual-1, Virtual)): max_dupes(Provider, MaxProvider), max_dupes(Virtual, MaxVirtual) } 1 :- solve_literal(TriggerID), trigger_and_effect(_, TriggerID, EffectID), impose(EffectID, _), imposed_constraint(EffectID, "provider_set", Provider, Virtual). provider(ProviderNode, VirtualNode) :- attr("provider_set", ProviderNode, VirtualNode). % Here we can't use the condition set because it's a recursive definition, that doesn't define the % node index, and leads to unsatisfiability. Hence we say that one and only one node index must % satisfy the dependency. 1 { attr("depends_on", node(X, A1), node(0..Y-1, A2), A3) : max_dupes(A2, Y) } 1 :- impose(ID, node(X, A1)), imposed_constraint(ID, "depends_on", A1, A2, A3). % For := we must keep track of the origin of the fact, since we need to check % each condition separately, i.e. foo:=a,b in one place and foo:=c in another % should not make foo:=a,b,c possible attr("concrete_variant_set", node(X, A1), Variant, Value, ID) :- impose(ID, PackageNode), imposed_nodes(PackageNode, node(X, A1)), imposed_constraint(ID, "concrete_variant_set", A1, Variant, Value). % The rule below accounts for expressions like: % % root ^dep %compiler % % where "compiler" is a dependency of "dep", but is enforced by a condition imposed by "root" 1 { attr("depends_on", node(A1_DUPE_ID, A1), node(0..Y-1, A2), A3) : max_dupes(A2, Y) } 1 :- impose(ID, RootNode), unification_set("root", RootNode), condition_set(RootNode, node(A1_DUPE_ID, A1)), not self_build_requirement(RootNode, node(A1_DUPE_ID, A1)), imposed_constraint(ID, "depends_on", A1, A2, A3). :- attr("direct_dependency", ParentNode, node_requirement("virtual_on_incoming_edges", ChildPkg, Virtual)), not attr("virtual_on_edge", ParentNode, node(_, ChildPkg), Virtual). % If the parent is built, then we have a build_requirement on another node. For concrete nodes, % we don't since we are trimming their build dependencies. % Concrete nodes :- attr("direct_dependency", ParentNode, node_requirement("node", BuildDependency)), concrete_build_requirement(ParentNode, BuildDependency), concrete(ParentNode), not attr("concrete_build_dependency", ParentNode, BuildDependency, _). :- attr("direct_dependency", ParentNode, node_requirement("node_version_satisfies", BuildDependency, Constraint)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), not hash_attr(BuildDependencyHash, "node_version_satisfies", BuildDependency, Constraint). :- attr("direct_dependency", ParentNode, node_requirement("provider_set", BuildDependency, Virtual)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), not attr("virtual_on_build_edge", ParentNode, BuildDependency, Virtual). % Give a penalty if reuse introduces a node compiled with a compiler that is not used otherwise. % The only exception is if the current node is a compiler itself. compiler_from_reuse(Hash, DependencyPackage) :- attr("concrete_build_dependency", ParentNode, DependencyPackage, Hash), attr("virtual_on_build_edge", ParentNode, DependencyPackage, Virtual), not node_compiler(_, ParentNode), language(Virtual). compiler_penalty_from_reuse(Hash) :- compiler_from_reuse(Hash, DependencyPackage), not node_compiler(_, node(_, DependencyPackage)), % We don't want to give penalties if we're just installing binaries will_build_packages(). compiler_penalty_from_reuse(Hash) :- compiler_from_reuse(Hash, DependencyPackage), not 1 { attr("hash", node(X, DependencyPackage), Hash) : node_compiler(_, node(X, DependencyPackage)) }, % We don't want to give penalties if we're just installing binaries will_build_packages(). error(100, "Cannot satisfy the request on {0} to have {1}={2}", BuildDependency, Variant, Value) :- attr("direct_dependency", ParentNode, node_requirement("variant_set", BuildDependency, Variant, Value)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), not hash_attr(BuildDependencyHash, "variant_value", BuildDependency, Variant, Value). error(100, "Cannot satisfy the request on {0} to have the target set to {1}", BuildDependency, Target) :- attr("direct_dependency", ParentNode, node_requirement("node_target_set", BuildDependency, Target)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), not hash_attr(BuildDependencyHash, "node_target", BuildDependency, Target). error(100, "Cannot satisfy the request on {0} to have the os set to {1}", BuildDependency, NodeOS) :- attr("direct_dependency", ParentNode, node_requirement("node_os_set", BuildDependency, NodeOS)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), not hash_attr(BuildDependencyHash, "node_os", BuildDependency, NodeOS). error(100, "Cannot satisfy the request on {0} to have the platform set to {1}", BuildDependency, Platform) :- attr("direct_dependency", ParentNode, node_requirement("node_platform_set", BuildDependency, Platform)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), not hash_attr(BuildDependencyHash, "node_platform", BuildDependency, Platform). error(100, "Cannot satisfy the request on {0} to have the following hash {1}", BuildDependency, BuildHash) :- attr("direct_dependency", ParentNode, node_requirement("node_target_set", BuildDependency, Target)), concrete_build_requirement(ParentNode, BuildDependency), attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), attr("direct_dependency", ParentNode, node_requirement("hash", BuildDependency, BuildHash)), BuildHash != BuildDependencyHash. % Asking for gcc@10 %gcc@9 shouldn't give us back an external gcc@10, just because of the hack % we have on externals :- attr("direct_dependency", node(X, Parent), node_requirement("node", BuildDependency)), Parent == BuildDependency, external(node(X, Parent)). % If a virtual is needed only to build a package, its provider can be considered a build requirement 1 { virtual_build_requirement(ParentNode, node(0..Y-1, Virtual)) : max_dupes(Virtual, Y) } 1 :- attr("dependency_holds", ParentNode, Virtual, "build"), not attr("dependency_holds", ParentNode, Virtual,"link"), not attr("dependency_holds", ParentNode, Virtual,"run"), virtual(Virtual). % The virtual build dependency must be on the correct duplicate :- virtual_build_requirement(ParentNode, node(X, Virtual)), provider(ProviderNode, node(X, Virtual)), not depends_on(ParentNode, ProviderNode). attr("virtual_node", VirtualNode) :- virtual_build_requirement(ParentNode, VirtualNode). build_requirement(ParentNode, ProviderNode) :- virtual_build_requirement(ParentNode, VirtualNode), provider(ProviderNode, VirtualNode), not attr("depends_on", ParentNode, ProviderNode, "link"), not attr("depends_on", ParentNode, ProviderNode, "run"), attr("depends_on", ParentNode, ProviderNode, "build"). % From cli we can have literal expressions like: % % root %gcc@12.0 ^dep %gcc@11.2 % % Adding a "direct_dependency" is a way to discriminate between the incompatible % version constraints on "gcc" in the "imposed_constraint". attr("node_version_satisfies", node(X, BuildDependency), Constraint) :- attr("direct_dependency", ParentNode, node_requirement("node_version_satisfies", BuildDependency, Constraint)), build_requirement(ParentNode, node(X, BuildDependency)). % Account for properties on the build requirements % % root %gcc@12.0 ^dep % attr("variant_set", node(X, BuildDependency), Variant, Value) :- attr("direct_dependency", ParentNode, node_requirement("variant_set", BuildDependency, Variant, Value)), build_requirement(ParentNode, node(X, BuildDependency)). attr("depends_on", node(X, Parent), node(Y, BuildDependency), "build") :- build_requirement(node(X, Parent), node(Y, BuildDependency)). attr("node_target_set", node(X, BuildDependency), Target) :- attr("direct_dependency", ParentNode, node_requirement("node_target_set", BuildDependency, Target)), build_requirement(ParentNode, node(X, BuildDependency)). attr("node_os_set", node(X, BuildDependency), NodeOS) :- attr("direct_dependency", ParentNode, node_requirement("node_os_set", BuildDependency, NodeOS)), build_requirement(ParentNode, node(X, BuildDependency)). attr("node_platform_set", node(X, BuildDependency), NodePlatform) :- attr("direct_dependency", ParentNode, node_requirement("node_platform_set", BuildDependency, NodePlatform)), build_requirement(ParentNode, node(X, BuildDependency)). attr("node_flag_set", node(X, BuildDependency), NodeFlag) :- attr("direct_dependency", ParentNode, node_requirement("node_flag_set", BuildDependency, NodeFlag)), build_requirement(ParentNode, node(X, BuildDependency)). attr("hash", node(X, BuildDependency), BuildHash) :- attr("direct_dependency", ParentNode, node_requirement("hash", BuildDependency, BuildHash)), build_requirement(ParentNode, node(X, BuildDependency)). % For a spec like `hdf5 %cxx=gcc` we need to ensure that % 1. gcc is a provider for cxx % 2. hdf5 depends on that provider for cxx 1 { attr("provider_set", node(X, BuildDependency), node(0..Y-1, Virtual)) : max_dupes(Virtual, Y) } 1 :- attr("direct_dependency", ParentNode, node_requirement("provider_set", BuildDependency, Virtual)), direct_dependency(ParentNode, node(X, BuildDependency)). error(10, "{0} cannot have a dependency on {1}", Package, Virtual) :- attr("direct_dependency", node(ID, Package), node_requirement("provider_set", BuildDependency, Virtual)), direct_dependency(node(ID, Package), node(X, BuildDependency)), not attr("virtual_on_edge", node(ID, Package), node(X, BuildDependency), Virtual). % For a spec like `hdf5 %cxx` we need to ensure that the virtual is needed on a direct edge error(10, "{0} cannot have a dependency on {1}", Package, Virtual) :- attr("direct_dependency", node(ID, Package), node_requirement("virtual_node", Virtual)), not attr("virtual_on_edge", node(ID, Package), _, Virtual). % Reconstruct virtual dependencies for reused specs attr("virtual_on_edge", node(X, A1), node(Y, A2), Virtual) :- impose(ID, node(X, A1)), depends_on(node(X, A1), node(Y, A2)), imposed_constraint(ID, "virtual_on_edge", A1, A2, Virtual), concrete(node(X, A1)). virtual_condition_holds(node(Y, A2), Virtual) :- impose(ID, node(X, A1)), attr("virtual_on_edge", node(X, A1), node(Y, A2), Virtual), concrete(node(X, A1)). % Simplified virtual information for conditionl requirements in % conditional dependencies % Most specs track virtuals on edges attr("uses_virtual", PackageNode, Virtual) :- attr("virtual_on_edge", PackageNode, _, Virtual). % Reused specs don't track a real edge to build-only deps attr("uses_virtual", PackageNode, Virtual) :- attr("virtual_on_build_edge", PackageNode, _, Virtual). % we cannot have additional variant values when we are working with concrete specs :- attr("node", node(ID, Package)), attr("hash", node(ID, Package), Hash), attr("variant_value", node(ID, Package), Variant, Value), not imposed_constraint(Hash, "variant_value", Package, Variant, Value). % we cannot have additional flag values when we are working with concrete specs :- attr("node", node(ID, Package)), attr("hash", node(ID, Package), Hash), attr("node_flag", node(ID, Package), node_flag(FlagType, Flag, _, _)), not imposed_constraint(Hash, "node_flag", Package, node_flag(FlagType, Flag, _, _)). % we cannot have two nodes with the same hash :- attr("hash", PackageNode1, Hash), attr("hash", PackageNode2, Hash), PackageNode1 < PackageNode2. #defined condition/1. #defined subcondition/2. #defined condition_requirement/3. #defined condition_requirement/4. #defined condition_requirement/5. #defined condition_requirement/6. #defined imposed_constraint/3. #defined imposed_constraint/4. #defined imposed_constraint/5. #defined imposed_constraint/6. %----------------------------------------------------------------------------- % Concrete specs %----------------------------------------------------------------------------- % if a package is assigned a hash, it's concrete. concrete(PackageNode) :- attr("hash", PackageNode, _), attr("node", PackageNode). :- concrete(PackageNode), depends_on(PackageNode, DependencyNode), not concrete(DependencyNode), not abi_splice_conditions_hold(_, DependencyNode, _, _). %----------------------------------------------------------------------------- % Dependency semantics %----------------------------------------------------------------------------- % Dependencies of any type imply that one package "depends on" another depends_on(PackageNode, DependencyNode) :- attr("depends_on", PackageNode, DependencyNode, _). % Dependencies of concrete specs don't need to be resolved -- they arise from the concrete specs themselves. do_not_impose(EffectID, node(X, Package)) :- trigger_and_effect(Package, _, TriggerID, EffectID), trigger_condition_holds(TriggerID, node(X, Package)), imposed_constraint(EffectID, "dependency_holds", Package, _, _), concrete(node(X, Package)). % If a dependency holds on a package node, there must be one and only one dependency node satisfying it 1 { attr("depends_on", PackageNode, node(0..Y-1, Dependency), Type) : max_dupes(Dependency, Y) } 1 :- attr("dependency_holds", PackageNode, Dependency, Type), not virtual(Dependency). % all nodes in the graph must be reachable from some root % this ensures a user can't say `zlib ^libiconv` (neither of which have any % dependencies) and get a two-node unconnected graph needed(PackageNode) :- attr("root", PackageNode). needed(ChildNode) :- edge_needed(ParentNode, ChildNode). edge_needed(ParentNode, node(X, Child)) :- depends_on(ParentNode, node(X, Child)), runtime(Child). edge_needed(ParentNode, ChildNode) :- depends_on(ParentNode, ChildNode) , concrete(ParentNode). edge_needed(ParentNode, node(X, Child)) :- depends_on(ParentNode, node(X, Child)), build(ParentNode), attr("dependency_holds", ParentNode, Child, _). virtual_edge_needed(ParentNode, ChildNode, node(X, Virtual)) :- build(ParentNode), attr("virtual_on_edge", ParentNode, ChildNode, Virtual), provider(ChildNode, node(X, Virtual)). virtual_edge_needed(ParentNode, ChildNode, node(X, Virtual)) :- concrete(ParentNode), concrete(ChildNode), provider(ChildNode, node(X, Virtual)), attr("virtual_on_edge", ParentNode, ChildNode, Virtual). virtual_edge_needed(ParentNode, ChildNode, node(X, Virtual)) :- concrete(ParentNode), abi_splice_conditions_hold(_, ChildNode, _, _), provider(ChildNode, node(X, Virtual)), attr("virtual_on_edge", ParentNode, ChildNode, Virtual). edge_needed(ParentNode, ChildNode) :- virtual_edge_needed(ParentNode, ChildNode, _). provider_needed(ChildNode, VirtualNode) :- virtual_edge_needed(_, ChildNode, VirtualNode). provider_needed(ChildNode, VirtualNode) :- attr("virtual_root", VirtualNode), provider(ChildNode, VirtualNode). error(10, "'{0}' is not a valid dependency for any package in the DAG", Package) :- attr("node", node(ID, Package)), not needed(node(ID, Package)). :- depends_on(ParentNode, ChildNode), not edge_needed(ParentNode, ChildNode), build(ParentNode). :- provider(PackageNode, VirtualNode), not provider_needed(PackageNode, VirtualNode), not attr("virtual_root", VirtualNode). % Extensions depending on each other must all extend the same node (e.g. all Python packages % depending on each other must depend on the same Python interpreter) error(100, "{0} and {1} must depend on the same {2}", ExtensionParent, ExtensionChild, ExtendeePackage) :- depends_on(ExtensionParent, ExtensionChild), attr("extends", ExtensionParent, ExtendeePackage), depends_on(ExtensionParent, node(X, ExtendeePackage)), depends_on(ExtensionChild, node(Y, ExtendeePackage)), X != Y. %----------------------------------------------------------------------------- % Conflicts %----------------------------------------------------------------------------- % Most conflicts are internal to the same package conflict_is_cross_package(Package, TriggerID) :- pkg_fact(Package, conflict(TriggerID, _, _)), pkg_fact(TriggerPackage, condition_trigger(TriggerID, _)), TriggerPackage != Package. conflict_internal(Package, TriggerID, ConstraintID, Msg) :- pkg_fact(Package, conflict(TriggerID, ConstraintID, Msg)), not conflict_is_cross_package(Package, TriggerID). % Case 1: conflict is within the same package error(1, Msg) :- conflict_internal(Package, TriggerID, ConstraintID, Msg), condition_holds(TriggerID, node(ID, Package)), condition_holds(ConstraintID, node(ID, Package)), build(node(ID, Package)). % ignore conflicts for installed packages % Case 2: Cross-package conflicts (Rare case - slower) error(1, Msg) :- build(node(ID, Package)), conflict_is_cross_package(Package, TriggerID), pkg_fact(Package, conflict(TriggerID, ConstraintID, Msg)), condition_holds(TriggerID, node(ID1, TriggerPackage)), condition_holds(ConstraintID, node(ID, Package)), unification_set(X, node(ID, Package)), unification_set(X, node(ID1, TriggerPackage)). %----------------------------------------------------------------------------- % Virtual dependencies %----------------------------------------------------------------------------- % Package provides to this client at least one virtual from those that need to be provided together node_uses_provider_with_constraints(ClientNode, node(X, Package), ID, SetID) :- condition_holds(ID, node(X, Package)), pkg_fact(Package, provided_together(ID, SetID, V)), attr("virtual_on_edge", ClientNode, node(X, Package), V). % This error is triggered if the package provides some but not all required virtuals from % the set that needs to be provided together error(100, "Package '{0}' needs to also provide '{1}' (provided_together constraint)", Package, Virtual) :- node_uses_provider_with_constraints(ClientNode, node(X, Package), ID, SetID), pkg_fact(Package, provided_together(ID, SetID, Virtual)), node_depends_on_virtual(ClientNode, Virtual), not attr("virtual_on_edge", ClientNode, node(X, Package), Virtual). % if a package depends on a virtual, it's not external and we have a % provider for that virtual then it depends on the provider node_depends_on_virtual(PackageNode, Virtual, Type) :- attr("dependency_holds", PackageNode, Virtual, Type), virtual(Virtual). node_depends_on_virtual(PackageNode, Virtual) :- node_depends_on_virtual(PackageNode, Virtual, Type). virtual_is_needed(Virtual) :- node_depends_on_virtual(PackageNode, Virtual). 1 { virtual_on_edge(PackageNode, ProviderNode, node(VirtualID, Virtual), Type) : provider(ProviderNode, node(VirtualID, Virtual)) } 1 :- node_depends_on_virtual(PackageNode, Virtual, Type). % A package that depends on a virtual with type ("build", "link") cannot have two providers % (one for the "build" and one for the "link") :- node_depends_on_virtual(PackageNode, Virtual), M = #count { VirtualID : virtual_on_edge(PackageNode, ProviderNode, node(VirtualID, Virtual), _) }, M > 1. attr("virtual_on_edge", PackageNode, ProviderNode, Virtual) :- virtual_on_edge(PackageNode, ProviderNode, node(_, Virtual), _). attr("depends_on", PackageNode, ProviderNode, Type) :- virtual_on_edge(PackageNode, ProviderNode, _, Type). % If a virtual node is in the answer set, it must be either a virtual root, % or used somewhere :- attr("virtual_node", node(_, Virtual)), not attr("virtual_on_incoming_edges", _, Virtual), not attr("virtual_root", node(_, Virtual)). attr("virtual_on_incoming_edges", ProviderNode, Virtual) :- attr("virtual_on_edge", _, ProviderNode, Virtual). % This is needed to allow requirement on virtuals, % when a virtual root is requested attr("virtual_on_incoming_edges", ProviderNode, Virtual) :- attr("virtual_root", node(min_dupe_id, Virtual)), attr("root", ProviderNode), provider(ProviderNode, node(min_dupe_id, Virtual)). % If a virtual is needed on an edge, at least one virtual node must exist :- virtual_is_needed(Virtual), not 1 { attr("virtual_node", node(0..X-1, Virtual)) : max_dupes(Virtual, X) }. % If there's a virtual node, we must select one and only one provider. % The provider must be selected among the possible providers. error(100, "'{0}' cannot be a provider for the '{1}' virtual", Package, Virtual) :- attr("provider_set", node(X, Package), node(Y, Virtual)), not virtual_condition_holds( node(X, Package), Virtual). error(100, "Cannot find valid provider for virtual {0}", Virtual) :- attr("virtual_node", node(X, Virtual)), not provider(_, node(X, Virtual)). error(100, "Cannot select a single provider for virtual '{0}'", Virtual) :- attr("virtual_node", node(X, Virtual)), 2 { provider(P, node(X, Virtual)) }. % virtual roots imply virtual nodes, and that one provider is a root attr("virtual_node", VirtualNode) :- attr("virtual_root", VirtualNode). % If we asked for a virtual root and we have a provider for that, % then the provider is the root package. attr("root", PackageNode) :- attr("virtual_root", VirtualNode), provider(PackageNode, VirtualNode). % The provider is selected among the nodes for which the virtual condition holds 1 { provider(PackageNode, node(X, Virtual)) : attr("node", PackageNode), virtual_condition_holds(PackageNode, Virtual) } 1 :- attr("virtual_node", node(X, Virtual)). % The provider provides the virtual if some provider condition holds. virtual_condition_holds(node(ProviderID, Provider), Virtual) :- virtual_condition_holds(ID, node(ProviderID, Provider), Virtual). virtual_condition_holds(ID, node(ProviderID, Provider), Virtual) :- pkg_fact(Provider, provider_condition(ID, Virtual)), condition_holds(ID, node(ProviderID, Provider)), virtual(Virtual). % If a "provider" condition holds, but this package is not a provider, do not impose the "provider" condition do_not_impose(EffectID, node(X, Package)) :- virtual_condition_holds(ID, node(X, Package), Virtual), pkg_fact(Package, condition_effect(ID, EffectID)), not provider(node(X, Package), node(_, Virtual)). % Choose the provider among root specs, if possible :- provider(ProviderNode, node(min_dupe_id, Virtual)), virtual_condition_holds(_, PossibleProvider, Virtual), PossibleProvider != ProviderNode, explicitly_requested_root(PossibleProvider), not self_build_requirement(PossibleProvider, ProviderNode), not explicitly_requested_root(ProviderNode), not language(Virtual). % A package cannot be the actual provider for a virtual if it does not % fulfill the conditions to provide that virtual :- provider(PackageNode, node(VirtualID, Virtual)), not virtual_condition_holds(PackageNode, Virtual). #defined provided_together/3. %----------------------------------------------------------------------------- % Virtual dependency weights %----------------------------------------------------------------------------- % Any configured provider has a weight based on index in the preference list provider_weight(node(ProviderID, Provider), node(VirtualID, Virtual), Weight) :- provider(node(ProviderID, Provider), node(VirtualID, Virtual)), provider_weight_from_config(Virtual, Provider, Weight). % Any non-configured provider has a default weight of 100 provider_weight(node(ProviderID, Provider), node(VirtualID, Virtual), 100) :- provider(node(ProviderID, Provider), node(VirtualID, Virtual)), not provider_weight_from_config(Virtual, Provider, _). % do not warn if generated program contains none of these. #defined virtual/1. #defined buildable_false/1. #defined provider_weight_from_config/3. %----------------------------------------------------------------------------- % External semantics %----------------------------------------------------------------------------- external(PackageNode) :- attr("external", PackageNode). % if a package is not buildable, only concrete specs are allowed error(1000, "Cannot build {0}, since it is configured `buildable:false` and no externals satisfy the request", Package) :- buildable_false(Package), attr("node", node(ID, Package)), build(node(ID, Package)). % Account for compiler annotation on externals :- not attr("root", ExternalNode), attr("external_build_requirement", ExternalNode, node_requirement("node", Compiler)), not node_compiler(_, node(_, Compiler)). 1 { attr("node_version_satisfies", node(X, Compiler), Constraint) : node_compiler(_, node(X, Compiler)) } :- not attr("root", ExternalNode), attr("external_build_requirement", ExternalNode, node_requirement("node", Compiler)), attr("external_build_requirement", ExternalNode, node_requirement("node_version_satisfies", Compiler, Constraint)). %----------------------------------------------------------------------------- % Config required semantics %----------------------------------------------------------------------------- package_in_dag(Node) :- attr("node", Node). package_in_dag(Node) :- attr("virtual_node", Node). package_in_dag(Node) :- attr("reused_virtual_node", Node). reused_provider(node(CompilerHash, CompilerName), node(CompilerHash, Virtual)) :- language(Virtual), attr("hash", PackageNode, Hash), attr("concrete_build_dependency", PackageNode, CompilerName, CompilerHash), attr("virtual_on_build_edge", PackageNode, CompilerName, Virtual). attr("reused_virtual_node", VirtualNode) :- reused_provider(_, VirtualNode). trigger_node(ID, node(PackageID, Package), node(VirtualID, Virtual)) :- pkg_fact(Virtual, trigger_id(ID)), reused_provider(node(PackageID, Package), node(VirtualID, Virtual)). activate_requirement(node(ID, Package), X) :- package_in_dag(node(ID, Package)), requirement_group(Package, X), not requirement_conditional(Package, X, _). activate_requirement(node(ID, Package), X) :- package_in_dag(node(ID, Package)), requirement_group(Package, X), condition_holds(Y, node(ID, Package)), requirement_conditional(Package, X, Y). activate_requirement(node(ID, Package), X) :- package_in_dag(node(ID, Package)), package_in_dag(node(CID, ConditionPackage)), requirement_group(Package, X), pkg_fact(ConditionPackage, condition(Y)), related(node(CID, ConditionPackage), node(ID, Package)), condition_holds(Y, node(CID, ConditionPackage)), requirement_conditional(Package, X, Y), ConditionPackage != Package. requirement_group_satisfied(node(ID, Package), GroupID) :- 1 { requirement_is_met(GroupID, ConditionID, node(ID, Package)) } 1, requirement_policy(Package, GroupID, "one_of"), activate_requirement(node(ID, Package), GroupID), requirement_group(Package, GroupID). requirement_weight(node(ID, Package), Group, W) :- condition_holds(Y, node(ID, Package)), requirement_has_weight(Y, W), requirement_group_member(Y, Package, Group), requirement_policy(Package, Group, "one_of"), requirement_group_satisfied(node(ID, Package), Group). { attr("direct_dependency", node(ID, Package), BuildRequirement) : condition_requirement(TriggerID, "direct_dependency", Package, BuildRequirement) } :- pkg_fact(Package, condition_trigger(ConditionID, TriggerID)), requirement_group_member(ConditionID, Package, Group), activate_requirement(node(ID, Package), Group), requirement_group(Package, Group). requirement_group_satisfied(node(ID, Package), GroupID) :- 1 { requirement_is_met(GroupID, ConditionID, node(ID, Package)) } , requirement_policy(Package, GroupID, "any_of"), activate_requirement(node(ID, Package), GroupID), requirement_group(Package, GroupID). % Derive a fact to represent that the entire requirement, including possible subconditions, is met requirement_is_met(GroupID, ConditionID, node(X, Package)) :- requirement_group_member(ConditionID, Package, GroupID), condition_holds(ConditionID, node(X, Package)), not subcondition(_, ConditionID). requirement_is_met(GroupID, ConditionID, node(X, Package)) :- requirement_group_member(ConditionID, Package, GroupID), condition_holds(ConditionID, node(X, Package)), impose(EffectID, node(X, Package)) : subcondition(SubconditionID, ConditionID), pkg_fact(Package, condition_effect(SubconditionID, EffectID)), condition_holds(SubconditionID, node(X, Package)); subcondition(_, ConditionID). { do_not_impose(EffectID, node(X, Package)) } :- requirement_group_member(ConditionID, Package, GroupID), condition_holds(ConditionID, node(X, Package)), subcondition(SubconditionID, ConditionID), pkg_fact(Package, condition_effect(SubconditionID, EffectID)). % clingo decided not to impose a condition for a subcondition that holds exclude_requirement_weight(ConditionID, Package, GroupID) :- requirement_group_member(ConditionID, Package, GroupID), condition_holds(ConditionID, node(X, Package)), subcondition(SubconditionID, ConditionID), condition_holds(SubconditionID, node(X, Package)), do_not_impose(EffectID, node(X, Package)), pkg_fact(Package, condition_effect(SubconditionID, EffectID)). % Do not impose requirements, if the conditional requirement is not active do_not_impose(EffectID, node(ID, Package)) :- trigger_condition_holds(TriggerID, node(ID, Package)), pkg_fact(Package, condition_trigger(ConditionID, TriggerID)), pkg_fact(Package, condition_effect(ConditionID, EffectID)), requirement_group_member(ConditionID , Package, RequirementID), not activate_requirement(node(ID, Package), RequirementID). do_not_impose(EffectID, node(Hash, Virtual)) :- trigger_condition_holds(TriggerID, node(Hash, Virtual)), pkg_fact(Virtual, condition_trigger(ConditionID, TriggerID)), pkg_fact(Virtual, condition_effect(ConditionID, EffectID)), requirement_group_member(ConditionID , Virtual, RequirementID), reused_provider(_, node(Hash, Virtual)), activate_requirement(node(Hash, Virtual), RequirementID). % When we have a required provider, we need to ensure that the provider/2 facts respect % the requirement. This is particularly important for packages that could provide multiple % virtuals independently required_provider(Provider, Virtual) :- requirement_group_member(ConditionID, Virtual, RequirementID), condition_holds(ConditionID, _), virtual(Virtual), pkg_fact(Virtual, condition_effect(ConditionID, EffectID)), imposed_constraint(EffectID, "node", Provider). error(1, "Cannot use {1} for the {0} virtual, but that is required", Virtual, Provider) :- required_provider(Provider, Virtual), not reused_provider(node(_, Provider), node(_, Virtual)), not provider(node(_, Provider), node(_, Virtual)). % TODO: the following choice rule allows the solver to add compiler % flags if their only source is from a requirement. This is overly-specific % and should use a more-generic approach like in https://github.com/spack/spack/pull/37180 { attr("node_flag", node(ID, Package), NodeFlag) } :- requirement_group_member(ConditionID, Package, RequirementID), activate_requirement(node(ID, Package), RequirementID), pkg_fact(Package, condition_effect(ConditionID, EffectID)), imposed_constraint(EffectID, "node_flag_set", Package, NodeFlag). { attr("node_flag", node(ID, Package), NodeFlag) } :- requirement_group_member(ConditionID, Virtual, RequirementID), activate_requirement(node(VirtualID, Virtual), RequirementID), provider(node(ID, Package), VirtualNode), pkg_fact(Virtual, condition_effect(ConditionID, EffectID)), imposed_constraint(EffectID, "node_flag_set", Package, NodeFlag). requirement_weight(node(ID, Package), Group, W) :- W = #min { Z : requirement_has_weight(Y, Z), condition_holds(Y, node(ID, Package)), requirement_group_member(Y, Package, Group), not exclude_requirement_weight(Y, Package, Group); % We need this to avoid an annoying warning during the solve % concretize.lp:1151:5-11: info: tuple ignored: % #sup@73 10000 }, requirement_policy(Package, Group, "any_of"), requirement_group_satisfied(node(ID, Package), Group). requirement_penalty(node(ID, Package), Group, W) :- requirement_weight(node(ID, Package), Group, W), not language(Package). requirement_penalty(PackageNode, Language, Group, W) :- requirement_weight(node(ID, Language), Group, W), language(Language), provider(ProviderNode, node(ID, Language)), attr("virtual_on_edge", PackageNode, ProviderNode, Language). requirement_penalty(PackageNode, Language, Group, W) :- requirement_weight(node(CompilerHash, Language), Group, W), language(Language), reused_provider(node(CompilerHash, CompilerName), node(CompilerHash, Language)), attr("concrete_build_dependency", PackageNode, CompilerName, CompilerHash), attr("virtual_on_build_edge", PackageNode, CompilerName, Language). error(60000, "cannot satisfy a requirement for package '{0}'.", Package) :- activate_requirement(node(ID, Package), X), requirement_group(Package, X), not requirement_message(Package, X, _), not requirement_group_satisfied(node(ID, Package), X). error(50000, Message) :- activate_requirement(node(ID, Package), X), requirement_group(Package, X), requirement_message(Package, X, Message), not requirement_group_satisfied(node(ID, Package), X). #defined requirement_group/2. #defined requirement_conditional/3. #defined requirement_message/3. #defined requirement_group_member/3. #defined requirement_has_weight/2. #defined requirement_policy/3. %----------------------------------------------------------------------------- % Variant semantics %----------------------------------------------------------------------------- % Packages define potentially several definitions for each variant, and depending % on their attributes, duplicate nodes for the same package may use different % definitions. So the variant logic has several jobs: % A. Associate a variant definition with a node, by VariantID % B. Associate defaults and attributes (sticky, etc.) for the selected variant ID with the node. % C. Once these rules are established for a node, select variant value(s) based on them. % A: Selecting a variant definition % Variant definitions come from package facts in two ways: % 1. unconditional variants are always defined on all nodes for a given package variant_definition(node(ID, Package), Name, VariantID) :- pkg_fact(Package, variant_definition(Name, VariantID)), attr("node", node(ID, Package)). % 2. conditional variants are only defined if the conditions hold for the node variant_definition(node(ID, Package), Name, VariantID) :- pkg_fact(Package, variant_condition(Name, VariantID, ConditionID)), condition_holds(ConditionID, node(ID, Package)). % If there are any definitions for a variant on a node, the variant is "defined". variant_defined(PackageNode, Name) :- variant_definition(PackageNode, Name, _). % Solver must pick the variant definition with the highest id. When conditions hold % for two or more variant definitions, this prefers the last one defined. node_has_variant(PackageNode, Name, SelectedVariantID) :- SelectedVariantID = #max { VariantID : variant_definition(PackageNode, Name, VariantID) }, variant_defined(PackageNode, Name). % B: Associating applicable package rules with nodes % The default value for a variant in a package is what is prescribed: % 1. On the command line % 2. In packages.yaml (if there's no command line settings) % 3. In the package.py file (if there are no settings in packages.yaml and the command line) % -- Associate the definition's default values with the node % note that the package.py variant defaults are associated with a particular definition, but % packages.yaml and CLI are associated with just the variant name. % Also, settings specified on the CLI apply to all duplicates, but always have % `min_dupe_id` as their node id. variant_default_value(node(ID, Package), VariantName, Value) :- node_has_variant(node(ID, Package), VariantName, VariantID), pkg_fact(Package, variant_default_value_from_package_py(VariantID, Value)), not variant_default_value_from_packages_yaml(Package, VariantName, _), not attr("variant_default_value_from_cli", node(min_dupe_id, Package), VariantName, _). variant_default_value(node(ID, Package), VariantName, Value) :- node_has_variant(node(ID, Package), VariantName, _), variant_default_value_from_packages_yaml(Package, VariantName, Value), not attr("variant_default_value_from_cli", node(min_dupe_id, Package), VariantName, _). variant_default_value(node(ID, Package), VariantName, Value) :- node_has_variant(node(ID, Package), VariantName, _), attr("variant_default_value_from_cli", node(min_dupe_id, Package), VariantName, Value). % Penalty from the variant definition possible_variant_penalty(VariantID, Value, Penalty) :- pkg_fact(Package, variant_penalty(VariantID, Value, Penalty)). % Use a very high penalty for variant values that are not defined in the package, % for instance those defined implicitly by a validator. possible_variant_penalty(VariantID, Value, 100) :- pkg_fact(Package, variant_possible_value(VariantID, Value)), not pkg_fact(Package, variant_penalty(VariantID, Value, _)). variant_penalty(node(NodeID, Package), Variant, Value, Penalty) :- node_has_variant(node(NodeID, Package), Variant, VariantID), attr("variant_value", node(NodeID, Package), Variant, Value), possible_variant_penalty(VariantID, Value, Penalty), not variant_default_value(node(NodeID, Package), Variant, Value), % variants set explicitly from a directive don't count as non-default not attr("variant_set", node(NodeID, Package), Variant, Value), % variant values forced by propagation don't count as non-default not propagate(node(NodeID, Package), variant_value(Variant, Value, _)). % -- Associate the definition's possible values with the node variant_possible_value(node(ID, Package), VariantName, Value) :- node_has_variant(node(ID, Package), VariantName, VariantID), pkg_fact(Package, variant_possible_value(VariantID, Value)). variant_possible_value(node(ID, Package), VariantName, Value) :- node_has_variant(node(ID, Package), VariantName, VariantID), pkg_fact(Package, variant_penalty(VariantID, Value, _)). variant_value_from_disjoint_sets(node(ID, Package), VariantName, Value1, Set1) :- node_has_variant(node(ID, Package), VariantName, VariantID), pkg_fact(Package, variant_value_from_disjoint_sets(VariantID, Value1, Set1)). % -- Associate definition's arity with the node variant_single_value(node(ID, Package), VariantName) :- node_has_variant(node(ID, Package), VariantName, VariantID), variant_type(VariantID, VariantType), VariantType != "multi". % C: Determining variant values on each node % if a variant is sticky, but not set, its value is the default value attr("variant_value", node(ID, Package), Variant, Value) :- node_has_variant(node(ID, Package), Variant, VariantID), variant_default_value(node(ID, Package), Variant, Value), pkg_fact(Package, variant_sticky(VariantID)), not attr("variant_set", node(ID, Package), Variant), build(node(ID, Package)). % we can choose variant values from all the possible values for the node 1 { attr("variant_value", PackageNode, Variant, Value) : variant_possible_value(PackageNode, Variant, Value) } :- node_has_variant(PackageNode, Variant, _), build(PackageNode). % variant_selected is only needed for reconstruction on the python side attr("variant_selected", PackageNode, Variant, Value, VariantType, VariantID) :- attr("variant_value", PackageNode, Variant, Value), node_has_variant(PackageNode, Variant, VariantID), variant_type(VariantID, VariantType). % a variant cannot be set if it is not a variant on the package error(100, "Cannot set variant '{0}' for package '{1}' because the variant condition cannot be satisfied for the given spec", Variant, Package) :- attr("variant_set", node(ID, Package), Variant), not node_has_variant(node(ID, Package), Variant, _), build(node(ID, Package)). % a variant cannot take on a value if it is not a variant of the package error(100, "Cannot set variant '{0}' for package '{1}' because the variant condition cannot be satisfied for the given spec", Variant, Package) :- attr("variant_value", node(ID, Package), Variant, _), not node_has_variant(node(ID, Package), Variant, _), build(node(ID, Package)). % at most one variant value for single-valued variants. error(1000, "'{0}' requires conflicting variant values 'Spec({1}={2})' and 'Spec({1}={3})'", Package, Variant, Value1, Value2) :- attr("node", node(ID, Package)), node_has_variant(node(ID, Package), Variant, _), variant_single_value(node(ID, Package), Variant), attr("variant_value", node(ID, Package), Variant, Value1), attr("variant_value", node(ID, Package), Variant, Value2), Value1 < Value2, build(node(ID, Package)). error(100, "No valid value for variant '{1}' of package '{0}'", Package, Variant) :- attr("node", node(ID, Package)), node_has_variant(node(ID, Package), Variant, _), build(node(ID, Package)), not attr("variant_value", node(ID, Package), Variant, _). % if a variant is set to anything, it is considered 'set'. attr("variant_set", PackageNode, Variant) :- attr("variant_set", PackageNode, Variant, _). % Setting a concrete variant implies setting a variant concrete_variant_value(PackageNode, Variant, Value, Origin) :- attr("concrete_variant_set", PackageNode, Variant, Value, Origin). attr("variant_set", PackageNode, Variant, Value) :- attr("concrete_variant_set", PackageNode, Variant, Value, _). % Concrete variant values must be in the answer set :- concrete_variant_value(PackageNode, Variant, Value, _), not attr("variant_value", PackageNode, Variant, Value). % Extra variant values are not allowed, if the variant is concrete variant_is_concrete(PackageNode, Variant, Origin) :- concrete_variant_value(PackageNode, Variant, _, Origin). error(100, "The variant {0} in package {1} specified as := has the extra value {2}", Variant, PackageNode, Value) :- variant_is_concrete(PackageNode, Variant, Origin), attr("variant_value", PackageNode, Variant, Value), not concrete_variant_value(PackageNode, Variant, Value, Origin). % A variant cannot have a value that is not also a possible value % This only applies to packages we need to build -- concrete packages may % have been built w/different variants from older/different package versions. error(10, "'Spec({1}={2})' is not a valid value for '{0}' variant '{1}'", Package, Variant, Value) :- attr("variant_value", node(ID, Package), Variant, Value), not variant_possible_value(node(ID, Package), Variant, Value), build(node(ID, Package)). % Some multi valued variants accept multiple values from disjoint sets. Ensure that we % respect that constraint and we don't pick values from more than one set at once error(100, "{0} variant '{1}' cannot have values '{2}' and '{3}' as they come from disjoint value sets", Package, Variant, Value1, Value2) :- attr("variant_value", node(ID, Package), Variant, Value1), attr("variant_value", node(ID, Package), Variant, Value2), variant_value_from_disjoint_sets(node(ID, Package), Variant, Value1, Set1), variant_value_from_disjoint_sets(node(ID, Package), Variant, Value2, Set2), Set1 < Set2, % see[1] build(node(ID, Package)). :- attr("variant_set", node(ID, Package), Variant, Value), not attr("variant_value", node(ID, Package), Variant, Value). % In a case with `variant("foo", when="+bar")` and a user request for +foo or ~foo, % force +bar to be set too. This gives no penalty if `+bar` is not the default value, and % optimizes a long chain of deductions that may cause clingo to hang. attr("variant_set", node(ID, Package), AnotherVariant, AnotherValue) :- attr("variant_set", node(ID, Package), Variant, Value), attr("variant_selected", node(ID, Package), Variant, _, _, VariantID), pkg_fact(Package, variant_condition(Variant, VariantID, ConditionID)), pkg_fact(Package, condition_trigger(ConditionID, TriggerID)), condition_requirement(TriggerID,"variant_value",Package, AnotherVariant, AnotherValue), build(node(ID, Package)). % A default variant value that is not used, makes sense only for multi valued variants variant_default_not_used(node(ID, Package), Variant, Value) :- variant_default_value(node(ID, Package), Variant, Value), node_has_variant(node(ID, Package), Variant, VariantID), variant_type(VariantID, VariantType), VariantType == "multi", not attr("variant_value", node(ID, Package), Variant, Value), not propagate(node(ID, Package), variant_value(Variant, Value, _)), % variant set explicitly don't count for this metric not attr("variant_set", node(ID, Package), Variant, Value), attr("node", node(ID, Package)). % Treat 'none' in a special way - it cannot be combined with other % values even if the variant is multi-valued error(100, "{0} variant '{1}' cannot have values '{2}' and 'none'", Package, Variant, Value) :- attr("variant_value", node(X, Package), Variant, Value), attr("variant_value", node(X, Package), Variant, "none"), Value != "none", build(node(X, Package)). % -- Auto variants % These don't have to be declared in the package. We allow them to spring into % existence when assigned a value. variant_possible_value(PackageNode, Variant, Value) :- attr("variant_set", PackageNode, Variant, Value), auto_variant(Variant, _). node_has_variant(PackageNode, Variant, VariantID) :- attr("variant_set", PackageNode, Variant, _), auto_variant(Variant, VariantID). variant_single_value(PackageNode, Variant) :- node_has_variant(PackageNode, Variant, VariantID), auto_variant(Variant, VariantID), variant_type(VariantID, VariantType), VariantType != "multi". % to respect requirements/preferences we need to define that an auto_variant is set { attr("variant_set", node(ID, Package), Variant, VariantValue)} :- auto_variant(Variant, _ ), condition_requirement(TriggerID, "variant_value", Package, Variant, VariantValue), pkg_fact(Package, condition_trigger(ConditionID, TriggerID)), requirement_group_member(ConditionID , Package, RequirementID), activate_requirement(node(ID, Package), RequirementID). % suppress warnings about this atom being unset. It's only set if some % spec or some package sets it, and without this, clingo will give % warnings like 'info: atom does not occur in any rule head'. #defined variant_default_value_from_packages_yaml/3. #defined variant_default_value_from_package_py/2. %----------------------------------------------------------------------------- % Propagation semantics %----------------------------------------------------------------------------- non_default_propagation(variant_value(Name, Value)) :- attr("propagate", RootNode, variant_value(Name, Value)). % Propagation roots have a corresponding attr("propagate", ...) propagate(RootNode, PropagatedAttribute) :- attr("propagate", RootNode, PropagatedAttribute), not non_default_propagation(PropagatedAttribute). propagate(RootNode, PropagatedAttribute, EdgeTypes) :- attr("propagate", RootNode, PropagatedAttribute, EdgeTypes). % Special case variants, to inject the source node in the propagated attribute propagate(RootNode, variant_value(Name, Value, RootNode)) :- attr("propagate", RootNode, variant_value(Name, Value)). % Propagate an attribute along edges to child nodes propagate(ChildNode, PropagatedAttribute) :- propagate(ParentNode, PropagatedAttribute), depends_on(ParentNode, ChildNode). propagate(ChildNode, PropagatedAttribute, edge_types(DepType1, DepType2)) :- propagate(ParentNode, PropagatedAttribute, edge_types(DepType1, DepType2)), depends_on(ParentNode, ChildNode), 1 { attr("depends_on", ParentNode, ChildNode, DepType1); attr("depends_on", ParentNode, ChildNode, DepType2) }. %----------------------------------------------------------------------------- % Activation of propagated values %----------------------------------------------------------------------------- %---- % Variants %---- % If a variant is propagated, and can be accepted, set its value attr("variant_value", PackageNode, Variant, Value) :- propagate(PackageNode, variant_value(Variant, Value, _)), node_has_variant(PackageNode, Variant, _), variant_possible_value(PackageNode, Variant, Value). % If a variant is propagated, we cannot have extraneous values variant_is_propagated(PackageNode, Variant) :- attr("variant_value", PackageNode, Variant, Value), propagate(PackageNode, variant_value(Variant, Value, _)), not attr("variant_set", PackageNode, Variant). :- variant_is_propagated(PackageNode, Variant), attr("variant_value", PackageNode, Variant, Value), not propagate(PackageNode, variant_value(Variant, Value, _)). error(100, "{0} and {1} cannot both propagate variant '{2}' to the shared dependency: {3}", Package1, Package2, Variant, Dependency) :- % The variant is a singlevalued variant variant_single_value(node(X, Package1), Variant), % Dependency is trying to propagate Variant with different values and is not the source package propagate(node(Z, Dependency), variant_value(Variant, Value1, node(X, Package1))), propagate(node(Z, Dependency), variant_value(Variant, Value2, node(Y, Package2))), % Package1 and Package2 and their values are different Package1 > Package2, Value1 != Value2, not propagate(node(Z, Dependency), variant_value(Variant, _, node(Z, Dependency))). % Cannot propagate the same variant from two different packages if one is a dependency of the other error(100, "{0} and {1} cannot both propagate variant '{2}'", Package1, Package2, Variant) :- % The variant is a single-valued variant variant_single_value(node(X, Package1), Variant), % Package1 and Package2 and their values are different Package1 != Package2, Value1 != Value2, % Package2 is set to propagate the value from Package1 propagate(node(Y, Package2), variant_value(Variant, Value2, node(X, Package2))), propagate(node(Y, Package2), variant_value(Variant, Value1, node(X, Package1))), variant_is_propagated(node(Y, Package2), Variant). % Cannot propagate a variant if a different value was set for it in a dependency error(100, "Cannot propagate the variant '{0}' from the package: {1} because package: {2} is set to exclude it", Variant, Source, Package) :- % Package has a Variant and Source is propagating Variant attr("variant_set", node(X, Package), Variant, Value1), % The packages and values are different Source != Package, Value1 != Value2, % The variant is a single-valued variant variant_single_value(node(X, Package1), Variant), % A different value is being propagated from somewhere else propagate(node(X, Package), variant_value(Variant, Value2, node(Y, Source))). %---- % Flags %---- % A propagated flag implies: % 1. The same flag type is not set on this node % 2. This node has the same compilers as the propagation source node_compiler(node(X, Package), node(Y, Compiler)) :- node_compiler(node(X, Package), node(Y, Compiler), Language). node_compiler(node(X, Package), node(Y, Compiler), Language) :- attr("virtual_on_edge", node(X, Package), node(Y, Compiler), Language), compiler(Compiler), language(Language). propagated_flag(node(PackageID, Package), node_flag(FlagType, Flag, FlagGroup, Source), SourceNode) :- propagate(node(PackageID, Package), node_flag(FlagType, Flag, FlagGroup, Source), _), not attr("node_flag_set", node(PackageID, Package), node_flag(FlagType, _, _, "literal")), % Same compilers as propagation source node_compiler(node(PackageID, Package), CompilerNode, Language) : node_compiler(SourceNode, CompilerNode, Language); attr("propagate", SourceNode, node_flag(FlagType, Flag, FlagGroup, Source), _), node(PackageID, Package) != SourceNode, not runtime(Package). attr("node_flag", PackageNode, NodeFlag) :- propagated_flag(PackageNode, NodeFlag, _). % Cannot propagate the same flag from two distinct sources error(100, "{0} and {1} cannot both propagate compiler flags '{2}' to {3}", Source1, Source2, FlagType, Package) :- propagated_flag(node(ID, Package), node_flag(FlagType, _, _, _), node(_, Source1)), propagated_flag(node(ID, Package), node_flag(FlagType, _, _, _), node(_, Source2)), Source1 < Source2. %---- % Compiler constraints %---- % If a node is built, impose constraints on the compiler coming from dependents attr("node_version_satisfies", node(Y, Compiler), VersionRange) :- propagate(node(X, Package), node_version_satisfies(Compiler, VersionRange)), attr("depends_on", node(X, Package), node(Y, Compiler), "build"), not runtime(Package). attr("node_version_satisfies", node(X, Runtime), VersionRange) :- attr("node", node(X, Runtime)), attr("compatible_runtime", PackageNode, Runtime, VersionRange), concrete(PackageNode). % If a compiler package is depended on with type link, it's used as a library compiler_used_as_a_library(node(X, Child), Hash) :- concrete(node(X, Child)), attr("hash", node(X, Child), Hash), compiler_package(Child), % Used to restrict grounding for this rule attr("depends_on", _, node(X, Child), "link"). % If a compiler is used for C on a package, it must provide C++ too, if need be, and vice-versa :- attr("virtual_on_edge", PackageNode, CompilerNode1, "c"), attr("virtual_on_edge", PackageNode, CompilerNode2, "cxx"), CompilerNode1 != CompilerNode2. % Compiler-unmixing: 1st rule unification_set_compiler("root", CompilerNode, Language) :- node_compiler(node(ID, Package), CompilerNode, Language), no_compiler_mixing(Language), not allow_mixing(Package), unification_set("root", node(ID, Package)). % Compiler for a reused node % This differs from compiler_from_reuse in that this is about x->y % where y is a compiler and x is reused (compiler_from_reuse is % is concerned with reuse of the compiler itself) reused_node_compiler(PackageNode, node(CompilerHash, Compiler), Language) :- concrete(PackageNode), attr("concrete_build_dependency", PackageNode, Compiler, CompilerHash), attr("virtual_on_build_edge", PackageNode, Compiler, Language), language(Language). % Compiler-unmixing: 2nd rule % The compiler appears on a reused node as well as a built node. In % that case there will be a generated node() with an ID. % While easier to understand than rule 3, in fact this rule addresses % a small set of use cases beyond rules 1 and 3: generally speaking % rule 1 ensures that all non-reused nodes get a consistent compiler. % Rule 3 generates compiler IDs that almost always fail the count % rule, but does not "activate" when in combination with rule 1 when % it is possible to propagate a compiler to another built node in the % unification set. This in fact is only really used when the reused % node compiler has a node(), but associated with a different % unification set. unification_set_compiler("root", CompilerNode, Language) :- reused_node_compiler(node(ID, Package), node(CompilerHash, Compiler), Language), attr("hash", CompilerNode, CompilerHash), no_compiler_mixing(Language), not allow_mixing(Package), unification_set("root", node(ID, Package)). % Compiler-unmixing: 3rd rule % If the compiler only appears in reused nodes, then there is no node() % for it; this will always generate an error unless all nodes in the % root unification set are reused. unification_set_compiler("root", node(CompilerHash, Compiler), Language) :- reused_node_compiler(node(ID, Package), node(CompilerHash, Compiler), Language), not attr("hash", _, CompilerHash), no_compiler_mixing(Language), not allow_mixing(Package), unification_set("root", node(ID, Package)). #defined no_compiler_mixing/1. #defined allow_mixing/1. % You can't have >1 compiler for a given language if mixing is disabled error(100, "Compiler mixing is disabled for the {0} language", Language) :- language(Language), #count { CompilerNode : unification_set_compiler("root", CompilerNode, Language) } > 1. %----------------------------------------------------------------------------- % Runtimes %----------------------------------------------------------------------------- % Check whether the DAG has any built package will_build_packages() :- build(X). % "gcc-runtime" is always built :- concrete(node(X, "gcc-runtime")), will_build_packages(). % The "gcc" linked to "gcc-runtime" must be used by at least another package :- attr("depends_on", node(X, "gcc-runtime"), node(Y, "gcc"), "build"), node_compiler(_, node(_, "gcc")), not 2 { attr("depends_on", PackageNode, node(Y, "gcc"), "build") : attr("node", PackageNode) }. %----------------------------------------------------------------------------- % Platform semantics %----------------------------------------------------------------------------- % NOTE: Currently we have a single allowed platform per DAG, therefore there is no % need to have additional optimization criteria. If we ever add cross-platform dags, % this needs to be changed. :- 2 { allowed_platform(Platform) }. 1 { attr("node_platform", PackageNode, Platform) : allowed_platform(Platform) } 1 :- attr("node", PackageNode). % setting platform on a node is a hard constraint attr("node_platform", PackageNode, Platform) :- attr("node", PackageNode), attr("node_platform_set", PackageNode, Platform). % platform is set if set to anything attr("node_platform_set", PackageNode) :- attr("node_platform_set", PackageNode, _). %----------------------------------------------------------------------------- % OS semantics %----------------------------------------------------------------------------- % convert weighted OS declarations to simple one os(OS) :- os(OS, _). % one os per node { attr("node_os", PackageNode, OS) : os(OS) } :- attr("node", PackageNode). % can't have a non-buildable OS on a node we need to build error(100, "Cannot select '{0} os={1}' (operating system '{1}' is not buildable)", Package, OS) :- build(node(X, Package)), attr("node_os", node(X, Package), OS), not buildable_os(OS). % give OS choice weights according to os declarations node_os_weight(PackageNode, Weight) :- attr("node", PackageNode), attr("node_os", PackageNode, OS), os(OS, Weight). % every OS is compatible with itself. We can use `os_compatible` to declare os_compatible(OS, OS) :- os(OS). % Transitive compatibility among operating systems os_compatible(OS1, OS3) :- os_compatible(OS1, OS2), os_compatible(OS2, OS3). % If an OS is set explicitly respect the value attr("node_os", PackageNode, OS) :- attr("node_os_set", PackageNode, OS), attr("node", PackageNode). #defined os_compatible/2. %----------------------------------------------------------------------------- % Target semantics %----------------------------------------------------------------------------- % Each node has only one target chosen among the known targets 1 { attr("node_target", PackageNode, Target) : target(Target) } 1 :- attr("node", PackageNode). % If a node must satisfy a target constraint, enforce it error(10, "'{0} target={1}' cannot satisfy constraint 'target={2}'", Package, Target, Constraint) :- attr("node_target", node(X, Package), Target), attr("node_target_satisfies", node(X, Package), Constraint), not target_satisfies(Constraint, Target). % If a node has a target and the target satisfies a constraint, then the target % associated with the node satisfies the same constraint attr("node_target_satisfies", PackageNode, Constraint) :- attr("node_target", PackageNode, Target), target_satisfies(Constraint, Target). % If a node has a target, all of its dependencies must be compatible with that target error(100, "Cannot find compatible targets for {0} and {1}", Package, Dependency) :- attr("depends_on", node(X, Package), node(Y, Dependency), Type), Type != "build", attr("node_target", node(X, Package), Target), not node_target_compatible(node(Y, Dependency), Target). % Intermediate step for performance reasons % When the integrity constraint above was formulated including this logic % we suffered a substantial performance penalty node_target_compatible(ChildNode, ParentTarget) :- attr("node_target", ChildNode, ChildTarget), target_compatible(ParentTarget, ChildTarget). #defined target_satisfies/2. compiler(Compiler) :- target_supported(Compiler, _, _). % Can't use targets on node if the compiler for the node doesn't support them language("c"). language("cuda-lang"). language("cxx"). language("fortran"). language("hip-lang"). language_runtime("fortran-rt"). error(10, "Only external, or concrete, compilers are allowed for the {0} language", Language) :- provider(ProviderNode, node(_, Language)), language(Language), build(ProviderNode). error(10, "{0} compiler '{2}@{3}' incompatible with 'target={1}'", Package, Target, Compiler, Version) :- attr("node_target", node(X, Package), Target), node_compiler(node(X, Package), node(Y, Compiler)), attr("version", node(Y, Compiler), Version), target_not_supported(Compiler, Version, Target), build(node(X, Package)). #defined target_supported/3. #defined target_not_supported/3. % if a target is set explicitly, respect it attr("node_target", PackageNode, Target) :- attr("node", PackageNode), attr("node_target_set", PackageNode, Target). node_target_weight(PackageNode, MinWeight) :- attr("node", PackageNode), attr("node_target", PackageNode, Target), target(Target), MinWeight = #min { Weight : target_weight(Target, Weight) }. :- attr("node_target", PackageNode, Target), not node_target_weight(PackageNode, _). % compatibility rules for targets among nodes node_target_match(ParentNode, DependencyNode) :- attr("depends_on", ParentNode, DependencyNode, Type), Type != "build", attr("node_target", ParentNode, Target), attr("node_target", DependencyNode, Target). node_target_mismatch(ParentNode, DependencyNode) :- attr("depends_on", ParentNode, DependencyNode, Type), Type != "build", not node_target_match(ParentNode, DependencyNode). % disallow reusing concrete specs that don't have a compatible target error(100, "'{0} target={1}' is not compatible with this machine", Package, Target) :- attr("node", node(X, Package)), attr("node_target", node(X, Package), Target), not target(Target). %----------------------------------------------------------------------------- % Compiler flags %----------------------------------------------------------------------------- attr("node_flag", PackageNode, NodeFlag) :- attr("node_flag_set", PackageNode, NodeFlag). % If we set "foo %bar cflags=A ^fee %bar cflags=B" we want two nodes for "bar" error(100, "Cannot set multiple {0} values for {1} from cli", FlagType, Package) :- attr("node_flag_set", node(X, Package), node_flag(FlagType, _, FlagGroup1, "literal")), attr("node_flag_set", node(X, Package), node_flag(FlagType, _, FlagGroup2, "literal")), FlagGroup1 < FlagGroup2. %----------------------------------------------------------------------------- % Installed Packages %----------------------------------------------------------------------------- #defined installed_hash/2. #defined abi_splice_conditions_hold/4. % These are the previously concretized attributes of the installed package as % a hash. It has the general form: % hash_attr(Hash, Attribute, PackageName, Args*) #defined hash_attr/3. #defined hash_attr/4. #defined hash_attr/5. #defined hash_attr/6. #defined hash_attr/7. { attr("hash", node(ID, PackageName), Hash): installed_hash(PackageName, Hash) } 1 :- attr("node", node(ID, PackageName)). % you can't choose an installed hash for a dev spec :- attr("hash", PackageNode, Hash), attr("variant_value", PackageNode, "dev_path", _). % You can't install a hash, if it is not installed :- attr("hash", node(ID, Package), Hash), not installed_hash(Package, Hash). % hash_attrs are versions, but can_splice_attr are usually node_version_satisfies hash_attr(Hash, "node_version_satisfies", PackageName, Constraint) :- hash_attr(Hash, "version", PackageName, Version), pkg_fact(PackageName, version_order(Version, VersionIdx)), pkg_fact(PackageName, version_range(Constraint, MinIdx, MaxIdx)), VersionIdx >= MinIdx, VersionIdx <= MaxIdx. % This recovers the exact semantics for hash reuse hash and depends_on are where % splices are decided, and virtual_on_edge can result in name-changes, which is % why they are all treated separately. imposed_constraint(Hash, Attr, PackageName) :- hash_attr(Hash, Attr, PackageName), Attr != "virtual_node". imposed_constraint(Hash, Attr, PackageName, A1) :- hash_attr(Hash, Attr, PackageName, A1), Attr != "hash". imposed_constraint(Hash, Attr, PackageName, A1, A2) :- hash_attr(Hash, Attr, PackageName, A1, A2), Attr != "depends_on", Attr != "virtual_on_edge". imposed_constraint(Hash, Attr, PackageName, A1, A2, A3) :- hash_attr(Hash, Attr, PackageName, A1, A2, A3). imposed_constraint(Hash, "hash", PackageName, Hash) :- installed_hash(PackageName, Hash). % If a compiler is not used as a library, we just enforce "run" dependency, so we % can get by with a much smaller search space. avoid_link_dependency(Hash, DepName) :- hash_attr(Hash, "depends_on", PackageName, DepName, "link"), not hash_attr(Hash, "depends_on", PackageName, DepName, "run"), hash_attr(Hash, "hash", DepName, DepHash), compiler_package(PackageName), not compiler_used_as_a_library(node(_, PackageName), Hash). % When a compiler is not used as a library, its transitive run-type dependencies are unified in the % build environment (they appear in PATH etc.), but their pure link-type dependencies are NOT. This % is different from unification sets that include all link/run deps: the goal is to avoid imposing % transitive link constraints from the toolchain, which would add to max_dupes and require us to % increase the max_dupes threshold for many packages. % Base case: direct run dep of a non-library compiler compiler_non_lib_run_dep(DepHash, DepName) :- compiler_package(CompilerName), not compiler_used_as_a_library(node(_, CompilerName), CompilerHash), hash_attr(CompilerHash, "depends_on", CompilerName, DepName, "run"), hash_attr(CompilerHash, "hash", DepName, DepHash). % Recursive case: run deps of run deps (full transitive closure) compiler_non_lib_run_dep(TransDepHash, TransDepName) :- compiler_non_lib_run_dep(DepHash, DepName), hash_attr(DepHash, "depends_on", DepName, TransDepName, "run"), hash_attr(DepHash, "hash", TransDepName, TransDepHash). % Pure link deps (link but not run) of any node in that closure are avoided avoid_link_dependency(DepHash, LinkDepName) :- compiler_non_lib_run_dep(DepHash, DepName), hash_attr(DepHash, "depends_on", DepName, LinkDepName, "link"), not hash_attr(DepHash, "depends_on", DepName, LinkDepName, "run"). % Without splicing, we simply recover the exact semantics imposed_constraint(ParentHash, "hash", ChildName, ChildHash) :- hash_attr(ParentHash, "hash", ChildName, ChildHash), ChildHash != ParentHash, not avoid_link_dependency(ParentHash, ChildName), not abi_splice_conditions_hold(_, _, ChildName, ChildHash). imposed_constraint(Hash, "depends_on", PackageName, DepName, Type) :- hash_attr(Hash, "depends_on", PackageName, DepName, Type), hash_attr(Hash, "hash", DepName, DepHash), not avoid_link_dependency(Hash, DepName), not attr("splice_at_hash", _, _, DepName, DepHash). imposed_constraint(Hash, "virtual_on_edge", PackageName, DepName, VirtName) :- hash_attr(Hash, "virtual_on_edge", PackageName, DepName, VirtName), not avoid_link_dependency(Hash, DepName), not attr("splice_at_hash", _, _, DepName,_). imposed_constraint(Hash, "virtual_node", VirtName) :- hash_attr(Hash, "virtual_on_edge", PackageName, DepName, VirtName), hash_attr(Hash, "virtual_node", VirtName), not avoid_link_dependency(Hash, DepName), not attr("splice_at_hash", _, _, DepName,_). % Rules pertaining to attr("splice_at_hash") and abi_splice_conditions_hold will % be conditionally loaded from splices.lp impose(Hash, PackageNode) :- attr("hash", PackageNode, Hash), attr("node", PackageNode). % If there is not a hash for a package, we build it. build(PackageNode) :- attr("node", PackageNode), not concrete(PackageNode). % Minimizing builds is tricky. We want a minimizing criterion % because we want to reuse what is available, but % we also want things that are built to stick to *default preferences* from % the package and from the user. We therefore treat built specs differently and apply % a different set of optimization criteria to them. Spack's *first* priority is to % reuse what it *can*, but if it builds something, the built specs will respect % defaults and preferences. This is implemented by bumping the priority of optimization % criteria for built specs -- so that they take precedence over the otherwise % topmost-priority criterion to reuse what is installed. % % The priority ranges are: % 1000+ Optimizations for concretization errors % 300 - 1000 Highest priority optimizations for valid solutions % 200 - 299 Shifted priorities for build nodes; correspond to priorities 0 - 99. % 100 - 199 Unshifted priorities. Currently only includes minimizing #builds and minimizing dupes. % 0 - 99 Priorities for non-built nodes. treat_node_as_concrete(node(X, Package)) :- attr("node", node(X, Package)), runtime(Package). build_priority(PackageNode, 200) :- build(PackageNode), attr("node", PackageNode), not treat_node_as_concrete(PackageNode). build_priority(PackageNode, 0) :- build(PackageNode), attr("node", PackageNode), treat_node_as_concrete(PackageNode). build_priority(PackageNode, 0) :- concrete(PackageNode), attr("node", PackageNode). % This statement, which is a hidden feature of clingo, let us avoid cycles in the DAG #edge (A, B) : depends_on(A, B). %----------------------------------------------------------------- % Optimization to avoid errors %----------------------------------------------------------------- % Some errors are handled as rules instead of constraints because % it allows us to explain why something failed. #minimize{ 0@1000: #true}. #minimize{ Weight@1000,Msg: error(Weight, Msg) }. #minimize{ Weight@1000,Msg,Arg1: error(Weight, Msg, Arg1) }. #minimize{ Weight@1000,Msg,Arg1,Arg2: error(Weight, Msg, Arg1, Arg2) }. #minimize{ Weight@1000,Msg,Arg1,Arg2,Arg3: error(Weight, Msg, Arg1, Arg2, Arg3) }. #minimize{ Weight@1000,Msg,Arg1,Arg2,Arg3,Arg4: error(Weight, Msg, Arg1, Arg2, Arg3, Arg4) }. %----------------------------------------------------------------------------- % How to optimize the spec (high to low priority) %----------------------------------------------------------------------------- % Each criterion below has: % 1. an opt_criterion(ID, Name) fact that describes the criterion, and % 2. a `#minimize{ 0@2 : #true }.` statement that ensures the criterion % is displayed (clingo doesn't display sums over empty sets by default) % A condition group specifies one or more specs that must be satisfied. % Specs declared first are preferred, so we assign increasing weights and % minimize the weights. opt_criterion(310, "requirement weight"). #minimize{ 0@310: #true }. #minimize { Weight@310,PackageNode,Group : requirement_penalty(PackageNode, Group, Weight) }. #minimize { Weight@310,PackageNode,Language,Group : requirement_penalty(PackageNode, Language, Group, Weight) }. % Try hard to reuse installed packages (i.e., minimize the number built) opt_criterion(120, "number of packages to build (vs. reuse)"). #minimize { 0@120: #true }. #minimize { 1@120,PackageNode : build(PackageNode), not treat_node_as_concrete(PackageNode) }. opt_criterion(110, "number of nodes from the same package"). #minimize { 0@110: #true }. #minimize { ID@110,Package : attr("node", node(ID, Package)), not self_build_requirement(_, node(ID, Package)) }. #minimize { ID@110,Package : attr("virtual_node", node(ID, Package)) }. #defined optimize_for_reuse/0. % Minimize the unification set ID used for build dependencies. This reduces the number of optimal % solutions that differ only by which node belongs to which unification set. opt_criterion(100, "build unification sets"). #minimize{ 0@100: #true }. #minimize{ ID@100,ParentNode : build_set_id(ParentNode, ID) }. % Minimize the number of deprecated versions being used opt_criterion(73, "deprecated versions used"). #minimize{ 0@273: #true }. #minimize{ 0@73: #true }. #minimize{ 1@73+Priority,PackageNode : attr("deprecated", PackageNode, _), not external(PackageNode), build_priority(PackageNode, Priority) }. % Minimize the: % 1. Version weight % 2. Number of variants with a non default value, if not set % for the root package. opt_criterion(70, "version badness (roots)"). #minimize{ 0@270: #true }. #minimize{ 0@70: #true }. #minimize { Weight@70+Priority,PackageNode : attr("root", PackageNode), version_weight(PackageNode, Weight), build_priority(PackageNode, Priority) }. #minimize { Penalty@70+Priority,PackageNode : attr("root", PackageNode), version_deprecation_penalty(PackageNode, Penalty), build_priority(PackageNode, Priority) }. opt_criterion(65, "variant penalty (roots)"). #minimize{ 0@265: #true }. #minimize{ 0@65: #true }. #minimize { Penalty@65+Priority,PackageNode,Variant,Value : variant_penalty(PackageNode, Variant, Value, Penalty), attr("root", PackageNode), build_priority(PackageNode, Priority) }. opt_criterion(55, "default values of variants not being used (roots)"). #minimize{ 0@255: #true }. #minimize{ 0@55: #true }. #minimize{ 1@55+Priority,PackageNode,Variant,Value : variant_default_not_used(PackageNode, Variant, Value), attr("root", PackageNode), build_priority(PackageNode, Priority) }. % Choose the preferred compiler before penalizing variants, to avoid that a variant penalty % on e.g. gcc causes clingo to pick another compiler e.g. llvm opt_criterion(48, "preferred compilers"). #minimize{ 0@248: #true }. #minimize{ 0@48: #true }. #minimize{ Weight@48+Priority,ProviderNode,X,Virtual : provider_weight(ProviderNode, node(X, Virtual), Weight), language(Virtual), build_priority(ProviderNode, Priority) }. opt_criterion(41, "compiler penalty from reuse"). #minimize{ 0@241: #true }. #minimize{ 0@41: #true }. #minimize{1@41,Hash : compiler_penalty_from_reuse(Hash)}. % Try to use default variants or variants that have been set opt_criterion(40, "variant penalty (non-roots)"). #minimize{ 0@240: #true }. #minimize{ 0@40: #true }. #minimize { Penalty@40+Priority,PackageNode,Variant,Value : variant_penalty(PackageNode, Variant, Value, Penalty), not attr("root", PackageNode), build_priority(PackageNode, Priority) }. % Minimize the weights of all the other providers (mpi, lapack, etc.) opt_criterion(38, "preferred providers (excluded compilers and language runtimes)"). #minimize{ 0@238: #true }. #minimize{ 0@38: #true }. #minimize{ Weight@38+Priority,ProviderNode,X,Virtual : provider_weight(ProviderNode, node(X, Virtual), Weight), not language(Virtual), not language_runtime(Virtual), build_priority(ProviderNode, Priority) }. % Minimize the number of compilers used on nodes compiler_penalty(PackageNode, C-1) :- C = #count { CompilerNode : node_compiler(PackageNode, CompilerNode) }, node_compiler(PackageNode, _), C > 0. opt_criterion(36, "number of compilers used on the same node"). #minimize{ 0@236: #true }. #minimize{ 0@36: #true }. #minimize{ Penalty@36+Priority,PackageNode : compiler_penalty(PackageNode, Penalty), build_priority(PackageNode, Priority) }. opt_criterion(30, "non-preferred OS's"). #minimize{ 0@230: #true }. #minimize{ 0@30: #true }. #minimize{ Weight@30+Priority,PackageNode : node_os_weight(PackageNode, Weight), build_priority(PackageNode, Priority) }. % Choose more recent versions for nodes opt_criterion(25, "version badness (non roots)"). #minimize{ 0@225: #true }. #minimize{ 0@25: #true }. #minimize{ Weight@25+Priority,node(X, Package) : version_weight(node(X, Package), Weight), build_priority(node(X, Package), Priority), not attr("root", node(X, Package)), not runtime(Package) }. #minimize { Penalty@25+Priority,node(X, Package) : version_deprecation_penalty(node(X, Package), Penalty), build_priority(node(X, Package), Priority), not attr("root", node(X, Package)), not runtime(Package) }. % Try to use all the default values of variants opt_criterion(20, "default values of variants not being used (non-roots)"). #minimize{ 0@220: #true }. #minimize{ 0@20: #true }. #minimize{ 1@20+Priority,PackageNode,Variant,Value : variant_default_not_used(PackageNode, Variant, Value), not attr("root", PackageNode), build_priority(PackageNode, Priority) }. % Minimize the number of mismatches for targets in the DAG, try % to select the preferred target. opt_criterion(10, "target mismatches"). #minimize{ 0@210: #true }. #minimize{ 0@10: #true }. #minimize{ 1@10+Priority,PackageNode,node(ID, Dependency) : node_target_mismatch(PackageNode, node(ID, Dependency)), build_priority(node(ID, Dependency), Priority), not runtime(Dependency) }. opt_criterion(7, "non-preferred targets"). #minimize{ 0@207: #true }. #minimize{ 0@7: #true }. #minimize{ Weight@7+Priority,node(X, Package) : node_target_weight(node(X, Package), Weight), build_priority(node(X, Package), Priority), not runtime(Package) }. opt_criterion(5, "preferred providers (language runtimes)"). #minimize{ 0@205: #true }. #minimize{ 0@5: #true }. #minimize{ Weight@5+Priority,ProviderNode,X,Virtual : provider_weight(ProviderNode, node(X, Virtual), Weight), language_runtime(Virtual), build_priority(ProviderNode, Priority) }. % Choose more recent versions for runtimes opt_criterion(4, "version badness (runtimes)"). #minimize{ 0@204: #true }. #minimize{ 0@4: #true }. #minimize{ Weight@4,node(X, Package) : version_weight(node(X, Package), Weight), runtime(Package) }. % Choose best target for runtimes opt_criterion(3, "non-preferred targets (runtimes)"). #minimize{ 0@203: #true }. #minimize{ 0@3: #true }. #minimize{ Weight@3,node(X, Package) : node_target_weight(node(X, Package), Weight), runtime(Package) }. % Try to use the most optimal providers as much as possible opt_criterion(2, "providers on edges"). #minimize{ 0@202: #true }. #minimize{ 0@2: #true }. #minimize{ Weight@2,ParentNode,ProviderNode,node(X, Virtual) : provider_weight(ProviderNode, node(X, Virtual), Weight), max_dupes(Virtual, MaxDupes), MaxDupes > 1, not attr("root", ProviderNode), language(Virtual), depends_on(ParentNode, ProviderNode) }. % Try to use latest versions of nodes as much as possible opt_criterion(1, "version badness on edges"). #minimize{ 0@201: #true }. #minimize{ 0@1: #true }. #minimize{ Weight@1,ParentNode,node(X, Package) : version_weight(node(X, Package), Weight), multiple_unification_sets(Package), not attr("root", node(X, Package)), depends_on(ParentNode, node(X, Package)) }. % Reduce symmetry on duplicates opt_criterion(0, "penalty on symmetric duplicates"). #minimize{ 0@200: #true }. #minimize{ 0@0: #true }. #minimize{ Weight@1,PackageNode,Reason : duplicate_penalty(PackageNode, Weight, Reason) }. %----------- % Notes %----------- % [1] Clingo ensures a total ordering among all atoms. We rely on that total ordering % to reduce symmetry in the solution by checking `<` instead of `!=` in symmetric % cases. These choices are made without loss of generality. ================================================ FILE: lib/spack/spack/solver/core.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Low-level wrappers around clingo API and other basic functionality related to ASP""" import importlib import pathlib from types import ModuleType from typing import Any, Callable, NamedTuple, Optional, Tuple import spack.platforms from spack.llnl.util import lang def _ast_getter(*names: str) -> Callable[[Any], Any]: """Helper to retrieve AST attributes from different versions of the clingo API""" def getter(node): for name in names: result = getattr(node, name, None) if result: return result raise KeyError(f"node has no such keys: {names}") return getter ast_type = _ast_getter("ast_type", "type") ast_sym = _ast_getter("symbol", "term") class AspVar: """Represents a variable in an ASP rule, allows for conditionally generating rules""" __slots__ = ("name",) def __init__(self, name: str) -> None: self.name = name def __str__(self) -> str: return str(self.name) @lang.key_ordering class AspFunction: """A term in the ASP logic program""" __slots__ = ("name", "args") def __init__(self, name: str, args: Tuple[Any, ...] = ()) -> None: self.name = name self.args = args def _cmp_key(self) -> Tuple[str, Tuple[Any, ...]]: return self.name, self.args def __call__(self, *args: Any) -> "AspFunction": """Return a new instance of this function with added arguments. Note that calls are additive, so you can do things like:: >>> attr = AspFunction("attr") attr() >>> attr("version") attr("version") >>> attr("version")("foo") attr("version", "foo") >>> v = AspFunction("attr", "version") attr("version") >>> v("foo", "bar") attr("version", "foo", "bar") """ return AspFunction(self.name, self.args + args) def __str__(self) -> str: parts = [] for arg in self.args: if type(arg) is str: arg = arg.replace("\\", r"\\").replace("\n", r"\n").replace('"', r"\"") parts.append(f'"{arg}"') elif type(arg) is AspFunction or type(arg) is int or type(arg) is AspVar: parts.append(str(arg)) else: parts.append(f'"{arg}"') return f"{self.name}({','.join(parts)})" def __repr__(self) -> str: return str(self) class _AspFunctionBuilder: def __getattr__(self, name: str) -> AspFunction: return AspFunction(name) #: Global AspFunction builder fn = _AspFunctionBuilder() _CLINGO_MODULE: Optional[ModuleType] = None def clingo() -> ModuleType: """Lazy imports the Python module for clingo, and returns it.""" if _CLINGO_MODULE is not None: return _CLINGO_MODULE try: clingo_mod = importlib.import_module("clingo") # Make sure we didn't import an empty module _ensure_clingo_or_raise(clingo_mod) except ImportError: clingo_mod = None if clingo_mod is not None: return _set_clingo_module_cache(clingo_mod) clingo_mod = _bootstrap_clingo() return _set_clingo_module_cache(clingo_mod) def _set_clingo_module_cache(clingo_mod: ModuleType) -> ModuleType: """Sets the global cache to the lazy imported clingo module""" global _CLINGO_MODULE importlib.import_module("clingo.ast") _CLINGO_MODULE = clingo_mod return clingo_mod def _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None: """Ensures the clingo module can access expected attributes, otherwise raises an error.""" # These are imports that may be problematic at top level (circular imports). They are used # only to provide exhaustive details when erroring due to a broken clingo module. import spack.config import spack.paths as sp import spack.util.path as sup try: clingo_mod.Symbol except AttributeError: assert clingo_mod.__file__ is not None, "clingo installation is incomplete or invalid" # Reaching this point indicates a broken clingo installation # If Spack derived clingo, suggest user re-run bootstrap # if non-spack, suggest user investigate installation # assume Spack is not responsible for broken clingo msg = ( f"Clingo installation at {clingo_mod.__file__} is incomplete or invalid." "Please repair installation or re-install. " "Alternatively, consider installing clingo via Spack." ) # check whether Spack is responsible if ( pathlib.Path( sup.canonicalize_path( spack.config.CONFIG.get("bootstrap:root", sp.default_user_bootstrap_path) ) ) in pathlib.Path(clingo_mod.__file__).parents ): # Spack is responsible for the broken clingo msg = ( "Spack bootstrapped copy of Clingo is broken, " "please re-run the bootstrapping process via command `spack bootstrap now`." " If this issue persists, please file a bug at: github.com/spack/spack" ) raise RuntimeError( "Clingo installation may be broken or incomplete, " "please verify clingo has been installed correctly" "\n\nClingo does not provide symbol clingo.Symbol" f"{msg}" ) def clingo_cffi() -> bool: """Returns True if clingo uses the CFFI interface""" return hasattr(clingo().Symbol, "_rep") def _bootstrap_clingo() -> ModuleType: """Bootstraps the clingo module and returns it""" import spack.bootstrap with spack.bootstrap.ensure_bootstrap_configuration(): spack.bootstrap.ensure_clingo_importable_or_raise() clingo_mod = importlib.import_module("clingo") return clingo_mod def parse_files(*args, **kwargs): """Wrapper around clingo parse_files, that dispatches the function according to clingo API version. """ clingo() try: return importlib.import_module("clingo.ast").parse_files(*args, **kwargs) except (ImportError, AttributeError): return clingo().parse_files(*args, **kwargs) def parse_term(*args, **kwargs): """Wrapper around clingo parse_term, that dispatches the function according to clingo API version. """ clingo() try: return importlib.import_module("clingo.symbol").parse_term(*args, **kwargs) except (ImportError, AttributeError): return clingo().parse_term(*args, **kwargs) class NodeId(NamedTuple): """Represents a node in the DAG""" id: str pkg: str class NodeFlag(NamedTuple): flag_type: str flag: str flag_group: str source: str def intermediate_repr(sym): """Returns an intermediate representation of clingo models for Spack's spec builder. Currently, transforms symbols from clingo models either to strings or to NodeId objects. Returns: This will turn a ``clingo.Symbol`` into a string or NodeId, or a sequence of ``clingo.Symbol`` objects into a tuple of those objects. """ # TODO: simplify this when we no longer have to support older clingo versions. if isinstance(sym, (list, tuple)): return tuple(intermediate_repr(a) for a in sym) try: if sym.name == "node": return NodeId( id=intermediate_repr(sym.arguments[0]), pkg=intermediate_repr(sym.arguments[1]) ) elif sym.name == "node_flag": return NodeFlag( flag_type=intermediate_repr(sym.arguments[0]), flag=intermediate_repr(sym.arguments[1]), flag_group=intermediate_repr(sym.arguments[2]), source=intermediate_repr(sym.arguments[3]), ) except RuntimeError: # This happens when using clingo w/ CFFI and trying to access ".name" for symbols # that are not functions pass if clingo_cffi(): # Clingo w/ CFFI will throw an exception on failure try: return sym.string except RuntimeError: return str(sym) else: return sym.string or str(sym) def extract_args(model, predicate_name): """Extract the arguments to predicates with the provided name from a model. Pull out all the predicates with name ``predicate_name`` from the model, and return their intermediate representation. """ return [intermediate_repr(sym.arguments) for sym in model if sym.name == predicate_name] class SourceContext: """Tracks context in which a Spec's clause-set is generated (i.e. with ``SpackSolverSetup.spec_clauses``). Facts generated for the spec may include this context. """ def __init__(self, *, source: Optional[str] = None): # This can be "literal" for constraints that come from a user # spec (e.g. from the command line); it can be the output of # `ConstraintOrigin.append_type_suffix`; the default is "none" # (which means it isn't important to keep track of the source # in that case). self.source = "none" if source is None else source self.wrap_node_requirement: Optional[bool] = None def using_libc_compatibility() -> bool: """Returns True if we are currently using libc compatibility""" return spack.platforms.host().name == "linux" ================================================ FILE: lib/spack/spack/solver/direct_dependency.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) % A direct dependency is either a runtime requirement, or a build requirement 1 { build_requirement(PackageNode, node(0..X-1, DirectDependency)) : max_dupes(DirectDependency, X); runtime_requirement(PackageNode, node(0..X-1, DirectDependency)) : max_dupes(DirectDependency, X) } 1 :- attr("direct_dependency", PackageNode, node_requirement("node", DirectDependency)), build(PackageNode). 1 { concrete_build_requirement(PackageNode, DirectDependency); runtime_requirement(PackageNode, node(0..X-1, DirectDependency)) : max_dupes(DirectDependency, X) } 1 :- attr("direct_dependency", PackageNode, node_requirement("node", DirectDependency)), concrete(PackageNode). direct_dependency(ParentNode, ChildNode) :- build_requirement(ParentNode, ChildNode). direct_dependency(ParentNode, ChildNode) :- runtime_requirement(ParentNode, ChildNode). %%%% % Build requirement %%%% % A build requirement that is not concrete has a "build" only dependency attr("depends_on", node(X, Parent), node(Y, BuildDependency), "build") :- build_requirement(node(X, Parent), node(Y, BuildDependency)), build(node(X, Parent)). % Any other dependency type is forbidden :- build_requirement(ParentNode, ChildNode), build(ParentNode), attr("depends_on", ParentNode, ChildNode, Type), Type != "build". :- concrete_build_requirement(ParentNode, ChildPackage), concrete(ParentNode), attr("depends_on", ParentNode, node(_, ChildPackage), _). %%%% % Runtime requirement %%%% :- runtime_requirement(ParentNode, ChildNode), not 1 { attr("depends_on", ParentNode, ChildNode, "link"); attr("depends_on", ParentNode, ChildNode, "run") }. attr(AttributeName, node(X, ChildPackage), A1) :- runtime_requirement(ParentNode, node(X, ChildPackage)), attr("direct_dependency", ParentNode, node_requirement(AttributeName, ChildPackage, A1)), AttributeName != "provider_set". attr(AttributeName, node(X, ChildPackage), A1, A2) :- runtime_requirement(ParentNode, node(X, ChildPackage)), attr("direct_dependency", ParentNode, node_requirement(AttributeName, ChildPackage, A1, A2)). attr(AttributeName, node(X, ChildPackage), A1, A2, A3) :- runtime_requirement(ParentNode, node(X, ChildPackage)), attr("direct_dependency", ParentNode, node_requirement(AttributeName, ChildPackage, A1, A2, A3)). attr(AttributeName, node(X, ChildPackage), A1, A2, A3, A4) :- runtime_requirement(ParentNode, node(X, ChildPackage)), attr("direct_dependency", ParentNode, node_requirement(AttributeName, ChildPackage, A1, A2, A3, A4)). ================================================ FILE: lib/spack/spack/solver/display.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) %=============================================================================- % Display Results % % This section determines what parts of the model are printed at the end %============================================================================== % Spec attributes #show attr/2. #show attr/3. #show attr/4. #show attr/5. #show attr/6. % names of optimization criteria #show opt_criterion/2. % error types #show error/2. #show error/3. #show error/4. #show error/5. #show error/6. % for error causation #show condition_reason/2. % For error messages to use later #show pkg_fact/2. #show condition_holds/2. #show imposed_constraint/3. #show imposed_constraint/4. #show imposed_constraint/5. #show imposed_constraint/6. #show condition_requirement/3. #show condition_requirement/4. #show condition_requirement/5. #show condition_requirement/6. #show node_has_variant/3. #show build/1. #show external/1. #show trigger_and_effect/3. #show unification_set/2. #show provider/2. #show condition_nodes/2. #show trigger_node/3. #show imposed_nodes/2. #show variant_single_value/2. % debug ================================================ FILE: lib/spack/spack/solver/error_messages.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) %============================================================================= % This logic program adds detailed error messages to Spack's concretizer % % Note that functions used in rule bodies here need to have a corresponding % #show line in display.lp, otherwise they won't be passed through to the % error solve. %============================================================================= #program error_messages. % The following "condition_cause" rules create a causal tree between trigger % conditions by locating the effect conditions that are triggers for another % condition. In all these rules, Condition2 is caused by Condition1 % For condition_cause rules, it is not necessary to confirm that the attr is present % on the node because `condition_holds` and `impose_constraint` collectively % guarantee it. % We omit those facts to reduce the burden on the grounder/solver. condition_cause(Condition2, ID2, Condition1, ID1) :- condition_holds(Condition2, node(ID2, Package2)), pkg_fact(Package2, condition_trigger(Condition2, Trigger)), condition_requirement(Trigger, Name, Package), condition_nodes(TriggerNode, node(ID, Package)), trigger_node(Trigger, TriggerNode, node(ID2, Package2)), condition_holds(Condition1, node(ID1, Package1)), pkg_fact(Package1, condition_effect(Condition1, Effect)), imposed_constraint(Effect, Name, Package), imposed_nodes(node(ID1, Package1), node(ID, Package)). condition_cause(Condition2, ID2, Condition1, ID1) :- condition_holds(Condition2, node(ID2, Package2)), pkg_fact(Package2, condition_trigger(Condition2, Trigger)), condition_requirement(Trigger, Name, Package, A1), condition_nodes(TriggerNode, node(ID, Package)), trigger_node(Trigger, TriggerNode, node(ID2, Package2)), condition_holds(Condition1, node(ID1, Package1)), pkg_fact(Package1, condition_effect(Condition1, Effect)), imposed_constraint(Effect, Name, Package, A1), imposed_nodes(node(ID1, Package1), node(ID, Package)). condition_cause(Condition2, ID2, Condition1, ID1) :- condition_holds(Condition2, node(ID2, Package2)), pkg_fact(Package2, condition_trigger(Condition2, Trigger)), condition_requirement(Trigger, Name, Package, A1, A2), condition_nodes(TriggerNode, node(ID, Package)), trigger_node(Trigger, TriggerNode, node(ID2, Package2)), condition_holds(Condition1, node(ID1, Package1)), pkg_fact(Package1, condition_effect(Condition1, Effect)), imposed_constraint(Effect, Name, Package, A1, A2), imposed_nodes(node(ID1, Package1), node(ID, Package)). condition_cause(Condition2, ID2, Condition1, ID1) :- condition_holds(Condition2, node(ID2, Package2)), pkg_fact(Package2, condition_trigger(Condition2, Trigger)), condition_requirement(Trigger, Name, Package, A1, A2, A3), condition_nodes(TriggerNode, node(ID, Package)), trigger_node(Trigger, TriggerNode, node(ID2, Package2)), condition_holds(Condition1, node(ID1, Package1)), pkg_fact(Package1, condition_effect(Condition1, Effect)), imposed_constraint(Effect, Name, Package, A1, A2, A3), imposed_nodes(node(ID1, Package1), node(ID, Package)). % special condition cause for dependency conditions % we can't simply impose the existence of the node for dependency conditions % because we need to allow for the choice of which dupe ID the node gets condition_cause(Condition2, ID2, Condition1, ID1) :- condition_holds(Condition2, node(ID2, Package2)), pkg_fact(Package2, condition_trigger(Condition2, Trigger)), condition_requirement(Trigger, "node", Package), condition_nodes(TriggerNode, node(ID, Package)), trigger_node(Trigger, TriggerNode, node(ID2, Package2)), condition_holds(Condition1, node(ID1, Package1)), pkg_fact(Package1, condition_effect(Condition1, Effect)), imposed_constraint(Effect, "dependency_holds", Parent, Package, Type), imposed_nodes(node(ID1, Package1), node(ID, Package)), attr("depends_on", node(X, Parent), node(ID, Package), Type). % The literal startcauses is used to separate the variables that are part of the error from the % ones describing the causal tree of the error. After startcauses, each successive pair must be % a condition and a condition_set id for which it holds. #defined choose_version/2. % More specific error message if the version cannot satisfy some constraint % Otherwise covered by `no_version_error` and `versions_conflict_error`. error(10000, "Cannot satisfy '{0}@{1}' (selected version {2} does not match)", Package, Constraint, Version, startcauses, ConstraintCause, CauseID) :- attr("node_version_satisfies", node(ID, Package), Constraint), pkg_fact(TriggerPkg, condition_effect(ConstraintCause, EffectID)), imposed_constraint(EffectID, "node_version_satisfies", Package, Constraint), condition_holds(ConstraintCause, node(CauseID, TriggerPkg)), attr("version", node(ID, Package), Version), not pkg_fact(Package, version_satisfies(Constraint, Version)), choose_version(node(ID, Package), Version). error(100, "Cannot satisfy '{0}@{1}' (version {2} does not match)", Package, Constraint, Version, startcauses, ConstraintCause, CauseID) :- attr("node_version_satisfies", node(ID, Package), Constraint), pkg_fact(TriggerPkg, condition_effect(ConstraintCause, EffectID)), imposed_constraint(EffectID, "node_version_satisfies", Package, Constraint), condition_holds(ConstraintCause, node(CauseID, TriggerPkg)), attr("version", node(ID, Package), Version), not pkg_fact(Package, version_satisfies(Constraint, Version)), not choose_version(node(ID, Package), Version). error(0, "Cannot satisfy '{0}@{1}' and '{0}@{2}'", Package, Constraint1, Constraint2, startcauses, Cause1, C1ID, Cause2, C2ID) :- attr("node_version_satisfies", node(ID, Package), Constraint1), pkg_fact(TriggerPkg1, condition_effect(Cause1, EffectID1)), imposed_constraint(EffectID1, "node_version_satisfies", Package, Constraint1), condition_holds(Cause1, node(C1ID, TriggerPkg1)), % two constraints attr("node_version_satisfies", node(ID, Package), Constraint2), pkg_fact(TriggerPkg2, condition_effect(Cause2, EffectID2)), imposed_constraint(EffectID2, "node_version_satisfies", Package, Constraint2), condition_holds(Cause2, node(C2ID, TriggerPkg2)), % version chosen attr("version", node(ID, Package), Version), % version satisfies one but not the other pkg_fact(Package, version_satisfies(Constraint1, Version)), not pkg_fact(Package, version_satisfies(Constraint2, Version)), Cause1 < Cause2. % causation tracking error for no or multiple virtual providers error(0, "Cannot find a valid provider for virtual {0}", Virtual, startcauses, Cause, CID) :- attr("virtual_node", node(X, Virtual)), not provider(_, node(X, Virtual)), imposed_constraint(EID, "dependency_holds", Parent, Virtual, Type), pkg_fact(TriggerPkg, condition_effect(Cause, EID)), condition_holds(Cause, node(CID, TriggerPkg)). % At most one variant value for single-valued variants error(0, "'{0}' requires conflicting variant values 'Spec({1}={2})' and 'Spec({1}={3})'", Package, Variant, Value1, Value2, startcauses, Cause1, X, Cause2, X) :- attr("node", node(X, Package)), node_has_variant(node(X, Package), Variant, VariantID), variant_single_value(node(X, Package), Variant), build(node(X, Package)), attr("variant_value", node(X, Package), Variant, Value1), imposed_constraint(EID1, "variant_set", Package, Variant, Value1), pkg_fact(TriggerPkg1, condition_effect(Cause1, EID1)), condition_holds(Cause1, node(X, TriggerPkg1)), attr("variant_value", node(X, Package), Variant, Value2), imposed_constraint(EID2, "variant_set", Package, Variant, Value2), pkg_fact(TriggerPkg2, condition_effect(Cause2, EID2)), condition_holds(Cause2, node(X, TriggerPkg2)), Value1 < Value2. % see[1] in concretize.lp % error message with causes for conflicts error(0, Msg, startcauses, TriggerID, ID1, ConstraintID, ID2) :- attr("node", node(ID, Package)), pkg_fact(Package, conflict(TriggerID, ConstraintID, Msg)), % node(ID1, TriggerPackage) is node(ID2, Package) in most, but not all, cases condition_holds(TriggerID, node(ID1, TriggerPackage)), condition_holds(ConstraintID, node(ID2, Package)), unification_set(X, node(ID2, Package)), unification_set(X, node(ID1, TriggerPackage)), build(node(ID, Package)). % ignore conflicts for concrete packages pkg_fact(Package, version_satisfies(Constraint, Version)) :- pkg_fact(Package, version_order(Version, VersionIdx)), pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)), VersionIdx >= MinIdx, VersionIdx <= MaxIdx. % variables to show #show error/2. #show error/3. #show error/4. #show error/5. #show error/6. #show error/7. #show error/8. #show error/9. #show error/10. #show error/11. #show condition_cause/4. #show condition_reason/2. % Define all variables used to avoid warnings at runtime when the model doesn't happen to have one #defined error/2. #defined error/3. #defined error/4. #defined error/5. #defined error/6. #defined error/7. #defined error/8. #defined error/9. #defined error/10. #defined error/11. #defined attr/2. #defined attr/3. #defined attr/4. #defined attr/5. #defined pkg_fact/2. #defined imposed_constraint/3. #defined imposed_constraint/4. #defined imposed_constraint/5. #defined imposed_constraint/6. #defined condition_cause/4. #defined condition_nodes/2. #defined condition_requirement/3. #defined condition_requirement/4. #defined condition_requirement/5. #defined condition_requirement/6. #defined condition_holds/2. #defined unification_set/2. #defined external/1. #defined trigger_and_effect/3. #defined build/1. #defined node_has_variant/3. #defined provider/2. #defined variant_single_value/2. ================================================ FILE: lib/spack/spack/solver/heuristic.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) %============================================================================= % Heuristic to speed-up solves %============================================================================= % Decide about DAG atoms, before trying to guess facts used only in % the internal representation #heuristic attr("virtual_node", node(X, Virtual)). [80, level] #heuristic attr("node", PackageNode). [80, level] #heuristic attr("version", node(PackageID, Package), Version). [80, level] #heuristic attr("variant_value", PackageNode, Variant, Value). [80, level] #heuristic attr("node_target", node(PackageID, Package), Target). [80, level] #heuristic virtual_on_edge(PackageNode, ProviderNode, Virtual, Type). [80, level] #heuristic attr("virtual_node", node(X, Virtual)). [600, init] #heuristic attr("virtual_node", node(X, Virtual)). [-1, sign] #heuristic attr("virtual_node", node(0, Virtual)) : node_depends_on_virtual(PackageNode, Virtual). [1@2, sign] #heuristic attr("virtual_node", node(0, "c")). [1@3, sign] #heuristic attr("virtual_node", node(0, "cxx")). [1@3, sign] #heuristic attr("virtual_node", node(0, "libc")). [1@3, sign] #heuristic attr("node", PackageNode). [300, init] #heuristic attr("node", PackageNode). [ 4, factor] #heuristic attr("node", PackageNode). [ -1, sign] #heuristic attr("node", node(0, Dependency)) : attr("dependency_holds", ParentNode, Dependency, Type), not virtual(Dependency). [1@2, sign] #heuristic attr("version", node(PackageID, Package), Version). [30, init] #heuristic attr("version", node(PackageID, Package), Version). [-1, sign] #heuristic attr("version", node(PackageID, Package), Version) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(PackageID, Package)). [ 1@2, sign] % Use default targets #heuristic attr("node_target", node(PackageID, Package), Target). [-1, sign] #heuristic attr("node_target", node(PackageID, Package), Target) : target_weight(Target, 0), attr("node", node(PackageID, Package)). [1@2, sign] % Use default variants #heuristic attr("variant_value", PackageNode, Variant, Value). [30, init] #heuristic attr("variant_value", PackageNode, Variant, Value). [-1, sign] #heuristic attr("variant_value", PackageNode, Variant, Value) : variant_default_value(PackageNode, Variant, Value), attr("node", PackageNode). [1@2, sign] ================================================ FILE: lib/spack/spack/solver/input_analysis.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes to analyze the input of a solve, and provide information to set up the ASP problem""" import collections from typing import Dict, List, NamedTuple, Set, Tuple, Union import spack.vendor.archspec.cpu import spack.binary_distribution import spack.concretize import spack.config import spack.deptypes as dt import spack.platforms import spack.repo import spack.spec import spack.store from spack.error import SpackError from spack.llnl.util import lang, tty from spack.spec import EMPTY_SPEC class PossibleGraph(NamedTuple): real_pkgs: Set[str] virtuals: Set[str] edges: Dict[str, Set[str]] class PossibleDependencyGraph: """Returns information needed to set up an ASP problem""" def unreachable(self, *, pkg_name: str, when_spec: spack.spec.Spec) -> bool: """Returns true if the context can determine that the condition cannot ever be met on pkg_name. """ raise NotImplementedError def candidate_targets(self) -> List[spack.vendor.archspec.cpu.Microarchitecture]: """Returns a list of targets that are candidate for concretization""" raise NotImplementedError def possible_dependencies( self, *specs: Union[spack.spec.Spec, str], allowed_deps: dt.DepFlag, transitive: bool = True, strict_depflag: bool = False, expand_virtuals: bool = True, ) -> PossibleGraph: """Returns the set of possible dependencies, and the set of possible virtuals. Runtime packages, which may be injected by compilers, needs to be added to specs if the dependency is not explicit in the package.py recipe. Args: transitive: return transitive dependencies if True, only direct dependencies if False allowed_deps: dependency types to consider strict_depflag: if True, only the specific dep type is considered, if False any deptype that intersects with allowed deptype is considered expand_virtuals: expand virtual dependencies into all possible implementations """ raise NotImplementedError class NoStaticAnalysis(PossibleDependencyGraph): """Implementation that tries to minimize the setup time (i.e. defaults to give fast answers), rather than trying to reduce the ASP problem size with more complex analysis. """ def __init__(self, *, configuration: spack.config.Configuration, repo: spack.repo.RepoPath): self.configuration = configuration self.repo = repo self._platform_condition = spack.spec.Spec( f"platform={spack.platforms.host()} target={spack.vendor.archspec.cpu.host().family}:" ) try: self.libc_pkgs = [x.name for x in self.providers_for("libc")] except spack.repo.UnknownPackageError: self.libc_pkgs = [] def is_virtual(self, name: str) -> bool: return self.repo.is_virtual(name) @lang.memoized def is_allowed_on_this_platform(self, *, pkg_name: str) -> bool: """Returns true if a package is allowed on the current host""" pkg_cls = self.repo.get_pkg_class(pkg_name) for when_spec, conditions in pkg_cls.requirements.items(): # Restrict analysis to unconditional requirements if when_spec != EMPTY_SPEC: continue for requirements, _, _ in conditions: if not any(x.intersects(self._platform_condition) for x in requirements): tty.debug(f"[{__name__}] {pkg_name} is not for this platform") return False return True def providers_for(self, virtual_str: str) -> List[spack.spec.Spec]: """Returns a list of possible providers for the virtual string in input.""" return self.repo.providers_for(virtual_str) def can_be_installed(self, *, pkg_name) -> bool: """Returns True if a package can be installed, False otherwise.""" return True def unreachable(self, *, pkg_name: str, when_spec: spack.spec.Spec) -> bool: """Returns true if the context can determine that the condition cannot ever be met on pkg_name. """ return False def candidate_targets(self) -> List[spack.vendor.archspec.cpu.Microarchitecture]: """Returns a list of targets that are candidate for concretization""" platform = spack.platforms.host() default_target = spack.vendor.archspec.cpu.TARGETS[platform.default] # Construct the list of targets which are compatible with the host candidate_targets = [default_target] + default_target.ancestors granularity = self.configuration.get("concretizer:targets:granularity") host_compatible = self.configuration.get("concretizer:targets:host_compatible") # Add targets which are not compatible with the current host if not host_compatible: additional_targets_in_family = sorted( [ t for t in spack.vendor.archspec.cpu.TARGETS.values() if (t.family.name == default_target.family.name and t not in candidate_targets) ], key=lambda x: len(x.ancestors), reverse=True, ) candidate_targets += additional_targets_in_family # Check if we want only generic architecture if granularity == "generic": candidate_targets = [t for t in candidate_targets if t.vendor == "generic"] return candidate_targets def possible_dependencies( self, *specs: Union[spack.spec.Spec, str], allowed_deps: dt.DepFlag, transitive: bool = True, strict_depflag: bool = False, expand_virtuals: bool = True, ) -> PossibleGraph: stack = [x for x in self._package_list(specs)] virtuals: Set[str] = set() edges: Dict[str, Set[str]] = {} while stack: pkg_name = stack.pop() if pkg_name in edges: continue edges[pkg_name] = set() # Since libc is not buildable, there is no need to extend the # search space with libc dependencies. if pkg_name in self.libc_pkgs: continue pkg_cls = self.repo.get_pkg_class(pkg_name=pkg_name) for when_spec, dependencies in pkg_cls.dependencies.items(): # Check if we need to process this condition at all. We can skip the unreachable # check if all dependencies in this condition are already accounted for. new_dependencies: List[str] = [] for name, dep in dependencies.items(): if strict_depflag: if dep.depflag != allowed_deps: continue elif not (dep.depflag & allowed_deps): continue if name in edges[pkg_name] or name in virtuals: continue new_dependencies.append(name) if not new_dependencies: continue if self.unreachable(pkg_name=pkg_name, when_spec=when_spec): tty.debug( f"[{__name__}] Skipping {', '.join(new_dependencies)} dependencies of " f"{pkg_name}, because {when_spec} is not met" ) continue for name in new_dependencies: dep_names: Set[str] = set() if self.is_virtual(name): virtuals.add(name) if expand_virtuals: providers = self.providers_for(name) dep_names = {spec.name for spec in providers} else: dep_names = {name} edges[pkg_name].update(dep_names) if not transitive: continue for dep_name in dep_names: if dep_name in edges: continue if not self._is_possible(pkg_name=dep_name): continue stack.append(dep_name) real_packages = set(edges) if not transitive: # We exit early, so add children from the edges information for root, children in edges.items(): real_packages.update(x for x in children if self._is_possible(pkg_name=x)) return PossibleGraph(real_pkgs=real_packages, virtuals=virtuals, edges=edges) def _package_list(self, specs: Tuple[Union[spack.spec.Spec, str], ...]) -> List[str]: stack = [] for current_spec in specs: if isinstance(current_spec, str): current_spec = spack.spec.Spec(current_spec) if self.repo.is_virtual(current_spec.name): stack.extend([p.name for p in self.providers_for(current_spec.name)]) continue stack.append(current_spec.name) return sorted(set(stack)) def _has_deptypes(self, dependencies, *, allowed_deps: dt.DepFlag, strict: bool) -> bool: if strict is True: return any( dep.depflag == allowed_deps for deplist in dependencies.values() for dep in deplist ) return any( dep.depflag & allowed_deps for deplist in dependencies.values() for dep in deplist ) def _is_possible(self, *, pkg_name): try: return self.is_allowed_on_this_platform(pkg_name=pkg_name) and self.can_be_installed( pkg_name=pkg_name ) except spack.repo.UnknownPackageError: return False class StaticAnalysis(NoStaticAnalysis): """Performs some static analysis of the configuration, store, etc. to provide more precise answers on whether some packages can be installed, or used as a provider. It increases the setup time, but might decrease the grounding and solve time considerably, especially when requirements restrict the possible choices for providers. """ def __init__( self, *, configuration: spack.config.Configuration, repo: spack.repo.RepoPath, store: spack.store.Store, binary_index: spack.binary_distribution.BinaryCacheIndex, ): self.store = store self.binary_index = binary_index super().__init__(configuration=configuration, repo=repo) @lang.memoized def providers_for(self, virtual_str: str) -> List[spack.spec.Spec]: candidates = super().providers_for(virtual_str) result = [] for spec in candidates: if not self._is_provider_candidate(pkg_name=spec.name, virtual=virtual_str): continue result.append(spec) return result @lang.memoized def buildcache_specs(self) -> List[spack.spec.Spec]: self.binary_index.update() return self.binary_index.get_all_built_specs() @lang.memoized def can_be_installed(self, *, pkg_name) -> bool: if self.configuration.get(f"packages:{pkg_name}:buildable", True): return True if self.configuration.get(f"packages:{pkg_name}:externals", []): return True reuse = self.configuration.get("concretizer:reuse") if reuse is not False and self.store.db.query(pkg_name): return True if reuse is not False and any(x.name == pkg_name for x in self.buildcache_specs()): return True tty.debug(f"[{__name__}] {pkg_name} cannot be installed") return False @lang.memoized def _is_provider_candidate(self, *, pkg_name: str, virtual: str) -> bool: if not self.is_allowed_on_this_platform(pkg_name=pkg_name): return False if not self.can_be_installed(pkg_name=pkg_name): return False virtual_spec = spack.spec.Spec(virtual) if self.unreachable(pkg_name=virtual_spec.name, when_spec=pkg_name): tty.debug(f"[{__name__}] {pkg_name} cannot be a provider for {virtual}") return False return True @lang.memoized def unreachable(self, *, pkg_name: str, when_spec: spack.spec.Spec) -> bool: """Returns true if the context can determine that the condition cannot ever be met on pkg_name. """ candidates = self.configuration.get(f"packages:{pkg_name}:require", []) if not candidates and pkg_name != "all": return self.unreachable(pkg_name="all", when_spec=when_spec) if not candidates: return False if isinstance(candidates, str): candidates = [candidates] union_requirement = spack.spec.Spec() for c in candidates: if not isinstance(c, str): continue try: union_requirement.constrain(c) except SpackError: # Less optimized, but shouldn't fail pass if not union_requirement.intersects(when_spec): return True return False def create_graph_analyzer() -> PossibleDependencyGraph: static_analysis = spack.config.CONFIG.get("concretizer:static_analysis", False) if static_analysis: return StaticAnalysis( configuration=spack.config.CONFIG, repo=spack.repo.PATH, store=spack.store.STORE, binary_index=spack.binary_distribution.BINARY_INDEX, ) return NoStaticAnalysis(configuration=spack.config.CONFIG, repo=spack.repo.PATH) class Counter: """Computes the possible packages and the maximum number of duplicates allowed for each of them. Args: specs: abstract specs to concretize tests: if True, add test dependencies to the list of possible packages """ def __init__( self, specs: List[spack.spec.Spec], tests: spack.concretize.TestsType, possible_graph: PossibleDependencyGraph, ) -> None: self.possible_graph = possible_graph self.specs = specs self.link_run_types: dt.DepFlag = dt.LINK | dt.RUN | dt.TEST self.all_types: dt.DepFlag = dt.ALL if not tests: self.link_run_types = dt.LINK | dt.RUN self.all_types = dt.LINK | dt.RUN | dt.BUILD self._possible_dependencies: Set[str] = set() self._possible_virtuals: Set[str] = { x.name for x in specs if spack.repo.PATH.is_virtual(x.name) } def possible_dependencies(self) -> Set[str]: """Returns the list of possible dependencies""" self.ensure_cache_values() return self._possible_dependencies def possible_virtuals(self) -> Set[str]: """Returns the list of possible virtuals""" self.ensure_cache_values() return self._possible_virtuals def ensure_cache_values(self) -> None: """Ensure the cache values have been computed""" if self._possible_dependencies: return self._compute_cache_values() def possible_packages_facts(self, gen: "spack.solver.asp.ProblemInstanceBuilder", fn) -> None: """Emit facts associated with the possible packages""" raise NotImplementedError("must be implemented by derived classes") def _compute_cache_values(self) -> None: raise NotImplementedError("must be implemented by derived classes") class NoDuplicatesCounter(Counter): def _compute_cache_values(self) -> None: self._possible_dependencies, virtuals, _ = self.possible_graph.possible_dependencies( *self.specs, allowed_deps=self.all_types ) self._possible_virtuals.update(virtuals) def possible_packages_facts(self, gen: "spack.solver.asp.ProblemInstanceBuilder", fn) -> None: gen.h2("Maximum number of nodes (packages)") for package_name in sorted(self.possible_dependencies()): gen.fact(fn.max_dupes(package_name, 1)) gen.newline() gen.h2("Maximum number of nodes (virtual packages)") for package_name in sorted(self.possible_virtuals()): gen.fact(fn.max_dupes(package_name, 1)) gen.newline() gen.h2("Possible package in link-run subDAG") for name in sorted(self.possible_dependencies()): gen.fact(fn.possible_in_link_run(name)) gen.newline() class MinimalDuplicatesCounter(NoDuplicatesCounter): def __init__( self, specs: List[spack.spec.Spec], tests: spack.concretize.TestsType, possible_graph: PossibleDependencyGraph, ) -> None: super().__init__(specs, tests, possible_graph) self._link_run: Set[str] = set() self._direct_build: Set[str] = set() self._total_build: Set[str] = set() self._link_run_virtuals: Set[str] = set() def _compute_cache_values(self) -> None: self._link_run, virtuals, _ = self.possible_graph.possible_dependencies( *self.specs, allowed_deps=self.link_run_types ) self._possible_virtuals.update(virtuals) self._link_run_virtuals.update(virtuals) for x in self._link_run: reals, virtuals, _ = self.possible_graph.possible_dependencies( x, allowed_deps=dt.BUILD, transitive=False, strict_depflag=True ) self._possible_virtuals.update(virtuals) self._direct_build.update(reals) self._total_build, virtuals, _ = self.possible_graph.possible_dependencies( *self._direct_build, allowed_deps=self.all_types ) self._possible_virtuals.update(virtuals) self._possible_dependencies = set(self._link_run) | set(self._total_build) def possible_packages_facts(self, gen, fn): build_tools = set() for current_tag in ("build-tools", "compiler"): build_tools.update(spack.repo.PATH.packages_with_tags(current_tag)) gen.h2("Packages with at most a single node") for package_name in sorted(self.possible_dependencies() - build_tools): gen.fact(fn.max_dupes(package_name, 1)) gen.newline() gen.h2("Packages with multiple possible nodes (build-tools)") default = spack.config.CONFIG.get("concretizer:duplicates:max_dupes:default", 1) duplicates = spack.config.CONFIG.get("concretizer:duplicates:max_dupes", {}) for package_name in sorted(self.possible_dependencies() & build_tools): max_dupes = duplicates.get(package_name, default) gen.fact(fn.max_dupes(package_name, max_dupes)) if max_dupes > 1: gen.fact(fn.multiple_unification_sets(package_name)) gen.newline() gen.h2("Maximum number of nodes (virtuals)") for package_name in sorted(self.possible_virtuals()): max_dupes = duplicates.get(package_name, default) gen.fact(fn.max_dupes(package_name, max_dupes)) gen.newline() gen.h2("Possible package in link-run subDAG") for name in sorted(self._link_run): gen.fact(fn.possible_in_link_run(name)) gen.newline() class FullDuplicatesCounter(MinimalDuplicatesCounter): def possible_packages_facts(self, gen, fn): counter = collections.Counter( list(self._link_run) + list(self._total_build) + list(self._direct_build) ) gen.h2("Maximum number of nodes") for pkg, count in sorted(counter.items(), key=lambda x: (x[1], x[0])): count = min(count, 2) gen.fact(fn.max_dupes(pkg, count)) gen.newline() gen.h2("Build unification sets ") build_tools = set() for current_tag in ("build-tools", "compiler"): build_tools.update(spack.repo.PATH.packages_with_tags(current_tag)) for name in sorted(self.possible_dependencies() & build_tools): gen.fact(fn.multiple_unification_sets(name)) gen.newline() gen.h2("Possible package in link-run subDAG") for name in sorted(self._link_run): gen.fact(fn.possible_in_link_run(name)) gen.newline() counter = collections.Counter( list(self._link_run_virtuals) + list(self._possible_virtuals) ) gen.h2("Maximum number of virtual nodes") for pkg, count in sorted(counter.items(), key=lambda x: (x[1], x[0])): gen.fact(fn.max_dupes(pkg, count)) gen.newline() def create_counter( specs: List[spack.spec.Spec], tests: spack.concretize.TestsType, possible_graph: PossibleDependencyGraph, ) -> Counter: strategy = spack.config.CONFIG.get("concretizer:duplicates:strategy", "none") if strategy == "full": return FullDuplicatesCounter(specs, tests=tests, possible_graph=possible_graph) if strategy == "minimal": return MinimalDuplicatesCounter(specs, tests=tests, possible_graph=possible_graph) return NoDuplicatesCounter(specs, tests=tests, possible_graph=possible_graph) ================================================ FILE: lib/spack/spack/solver/libc_compatibility.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) %============================================================================= % Libc compatibility rules for reusing solves. % % These rules are used on Linux %============================================================================= % Non-libc reused specs must be host libc compatible. In case we build packages, we get a % host compatible libc provider from other rules. If nothing is built, there is no libc provider, % since it's pruned from reusable specs, meaning we have to explicitly impose reused specs are host % compatible. % A package cannot be reused if it needs a libc that is not compatible with the current one error(100, "Cannot reuse {0} since we cannot determine libc compatibility", ReusedPackage) :- provider(node(X, LibcPackage), node(0, "libc")), attr("version", node(X, LibcPackage), LibcVersion), concrete(node(R, ReusedPackage)), attr("needs_libc", node(R, ReusedPackage)), not attr("compatible_libc", node(R, ReusedPackage), LibcPackage, LibcVersion). % In case we don't need a provider for libc, ensure there's at least one compatible libc on the host error(100, "Cannot reuse {0} since we cannot determine libc compatibility", ReusedPackage) :- not provider(_, node(0, "libc")), concrete(node(R, ReusedPackage)), attr("needs_libc", node(R, ReusedPackage)), not attr("compatible_libc", node(R, ReusedPackage), _, _). % The libc provider must be one that a compiler can target :- will_build_packages(), provider(node(X, LibcPackage), node(0, "libc")), attr("node", node(X, LibcPackage)), attr("version", node(X, LibcPackage), LibcVersion), not host_libc(LibcPackage, LibcVersion). ================================================ FILE: lib/spack/spack/solver/os_compatibility.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) %============================================================================= % OS compatibility rules for reusing solves. % os_compatible(RecentOS, OlderOS) % OlderOS binaries can be used on RecentOS % % These rules are used on every platform, but Linux %============================================================================= % macOS os_compatible("tahoe", "sequoia"). os_compatible("sequoia", "sonoma"). os_compatible("sonoma", "ventura"). os_compatible("ventura", "monterey"). os_compatible("monterey", "bigsur"). os_compatible("bigsur", "catalina"). % can't have dependencies on incompatible OS's error(100, "{0} and dependency {1} have incompatible operating systems 'os={2}' and 'os={3}'", Package, Dependency, PackageNodeOS, DependencyOS) :- depends_on(node(X, Package), node(Y, Dependency)), attr("node_os", node(X, Package), PackageNodeOS), attr("node_os", node(Y, Dependency), DependencyOS), not os_compatible(PackageNodeOS, DependencyOS), build(node(X, Package)). % We can select only operating systems compatible with the ones % for which we can build software. We need a cardinality constraint % since we might have more than one "buildable_os(OS)" fact. :- not 1 { os_compatible(CurrentOS, ReusedOS) : buildable_os(CurrentOS) }, attr("node_os", Package, ReusedOS). ================================================ FILE: lib/spack/spack/solver/requirements.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import enum import warnings from typing import List, NamedTuple, Optional, Sequence, Tuple, Union import spack.vendor.archspec.cpu import spack.config import spack.error import spack.package_base import spack.repo import spack.spec import spack.spec_parser import spack.traverse import spack.util.spack_yaml from spack.enums import PropagationPolicy from spack.llnl.util import tty from spack.util.spack_yaml import get_mark_from_yaml_data def _mark_str(raw) -> str: """Return a 'file:line: ' prefix from the YAML mark on *raw*, or empty string.""" mark = get_mark_from_yaml_data(raw) return f"{mark.name}:{mark.line + 1}: " if mark else "" def _check_unknown_targets( raw_strs: List[str], specs: List["spack.spec.Spec"], *, always_warn: bool = False ) -> None: """Either warns or raises for unknown concrete target names in a set of specs. UserWarnings are emitted if *always_warn* is True or if there is at least one spec without unknown targets. If all the specs have unknown targets raises an error. """ specs_with_unknown_targets = [ (raw, spec) for raw, spec in zip(raw_strs, specs) if spec.architecture and spec.architecture.target_concrete and spec.target.name not in spack.vendor.archspec.cpu.TARGETS ] if not specs_with_unknown_targets: return errors = [ f"{_mark_str(raw)}'{spec}' contains unknown targets" for raw, spec in specs_with_unknown_targets ] if len(errors) == 1: msg = f"{errors[0]}. Run 'spack arch --known-targets' to see valid targets." else: details = "\n".join([f"{idx}. {part}" for idx, part in enumerate(errors, 1)]) msg = ( f"unknown targets have been detected in requirements\n{details}\n" f"Run 'spack arch --known-targets' to see valid targets." ) if not always_warn and len(specs_with_unknown_targets) == len(specs): raise spack.error.SpecError(msg) warnings.warn(msg) class RequirementKind(enum.Enum): """Purpose / provenance of a requirement""" #: Default requirement expressed under the 'all' attribute of packages.yaml DEFAULT = enum.auto() #: Requirement expressed on a virtual package VIRTUAL = enum.auto() #: Requirement expressed on a specific package PACKAGE = enum.auto() class RequirementOrigin(enum.Enum): """Origin of a requirement""" REQUIRE_YAML = enum.auto() PREFER_YAML = enum.auto() CONFLICT_YAML = enum.auto() DIRECTIVE = enum.auto() INPUT_SPECS = enum.auto() class RequirementRule(NamedTuple): """Data class to collect information on a requirement""" pkg_name: str policy: str origin: RequirementOrigin requirements: Sequence[spack.spec.Spec] condition: spack.spec.Spec kind: RequirementKind message: Optional[str] def preference( pkg_name: str, constraint: spack.spec.Spec, condition: spack.spec.Spec = spack.spec.Spec(), origin: RequirementOrigin = RequirementOrigin.PREFER_YAML, kind: RequirementKind = RequirementKind.PACKAGE, message: Optional[str] = None, ) -> RequirementRule: """Returns a preference rule""" # A strong preference is defined as: # # require: # - any_of: [spec_str, "@:"] return RequirementRule( pkg_name=pkg_name, policy="any_of", requirements=[constraint, spack.spec.Spec("@:")], kind=kind, condition=condition, origin=origin, message=message, ) def conflict( pkg_name: str, constraint: spack.spec.Spec, condition: spack.spec.Spec = spack.spec.Spec(), origin: RequirementOrigin = RequirementOrigin.CONFLICT_YAML, kind: RequirementKind = RequirementKind.PACKAGE, message: Optional[str] = None, ) -> RequirementRule: """Returns a conflict rule""" # A conflict is defined as: # # require: # - one_of: [spec_str, "@:"] return RequirementRule( pkg_name=pkg_name, policy="one_of", requirements=[constraint, spack.spec.Spec("@:")], kind=kind, condition=condition, origin=origin, message=message, ) class RequirementParser: """Parses requirements from package.py files and configuration, and returns rules.""" def __init__(self, configuration: spack.config.Configuration): self.config = configuration self.runtime_pkgs = spack.repo.PATH.packages_with_tags("runtime") self.compiler_pkgs = spack.repo.PATH.packages_with_tags("compiler") self.preferences_from_input: List[Tuple[spack.spec.Spec, str]] = [] self.toolchains = configuration.get_config("toolchains") self._warned_compiler_all: set = set() def _parse_and_expand(self, string: str, *, named: bool = False) -> spack.spec.Spec: result = parse_spec_from_yaml_string(string, named=named) if self.toolchains: spack.spec_parser.expand_toolchains(result, self.toolchains) return result def rules(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]: result = [] result.extend(self.rules_from_input_specs(pkg)) result.extend(self.rules_from_package_py(pkg)) result.extend(self.rules_from_require(pkg)) result.extend(self.rules_from_prefer(pkg)) result.extend(self.rules_from_conflict(pkg)) return result def parse_rules_from_input_specs(self, specs: Sequence[spack.spec.Spec]): self.preferences_from_input.clear() for edge in spack.traverse.traverse_edges(specs, root=False): if edge.propagation == PropagationPolicy.PREFERENCE: for constraint in _split_edge_on_virtuals(edge): root_name = edge.parent.name self.preferences_from_input.append((constraint, root_name)) def rules_from_input_specs(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]: return [ preference( pkg.name, constraint=s, condition=spack.spec.Spec(f"{root_name} ^[deptypes=link,run]{pkg.name}"), origin=RequirementOrigin.INPUT_SPECS, ) for s, root_name in self.preferences_from_input ] def rules_from_package_py(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]: rules = [] for when_spec, requirement_list in pkg.requirements.items(): for requirements, policy, message in requirement_list: rules.append( RequirementRule( pkg_name=pkg.name, policy=policy, requirements=requirements, kind=RequirementKind.PACKAGE, condition=when_spec, message=message, origin=RequirementOrigin.DIRECTIVE, ) ) return rules def rules_from_virtual(self, virtual_str: str) -> List[RequirementRule]: kind, requests = self._raw_yaml_data(virtual_str, section="require", virtual=True) result = self._rules_from_requirements(virtual_str, requests, kind=kind) kind, requests = self._raw_yaml_data(virtual_str, section="prefer", virtual=True) result.extend(self._rules_from_preferences(virtual_str, preferences=requests, kind=kind)) kind, requests = self._raw_yaml_data(virtual_str, section="conflict", virtual=True) result.extend(self._rules_from_conflicts(virtual_str, conflicts=requests, kind=kind)) return result def rules_from_require(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]: kind, requirements = self._raw_yaml_data(pkg.name, section="require") return self._rules_from_requirements(pkg.name, requirements, kind=kind) def rules_from_prefer(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]: kind, preferences = self._raw_yaml_data(pkg.name, section="prefer") return self._rules_from_preferences(pkg.name, preferences=preferences, kind=kind) def _rules_from_preferences( self, pkg_name: str, *, preferences, kind: RequirementKind ) -> List[RequirementRule]: result = [] for item in preferences: if kind == RequirementKind.DEFAULT: # Warn about %gcc type of preferences under `all`. self._maybe_warn_compiler_in_all(item, "prefer") spec, condition, msg = self._parse_prefer_conflict_item(item) result.append( preference(pkg_name, constraint=spec, condition=condition, kind=kind, message=msg) ) return result def rules_from_conflict(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]: kind, conflicts = self._raw_yaml_data(pkg.name, section="conflict") return self._rules_from_conflicts(pkg.name, conflicts=conflicts, kind=kind) def _rules_from_conflicts( self, pkg_name: str, *, conflicts, kind: RequirementKind ) -> List[RequirementRule]: result = [] for item in conflicts: spec, condition, msg = self._parse_prefer_conflict_item(item) result.append( conflict(pkg_name, constraint=spec, condition=condition, kind=kind, message=msg) ) return result def _parse_prefer_conflict_item(self, item): # The item is either a string or an object with at least a "spec" attribute if isinstance(item, str): spec = self._parse_and_expand(item) condition = spack.spec.Spec() message = None else: spec = self._parse_and_expand(item["spec"]) condition = spack.spec.Spec(item.get("when")) message = item.get("message") raw_key = item if isinstance(item, str) else item.get("spec", item) _check_unknown_targets([raw_key], [spec], always_warn=True) return spec, condition, message def _raw_yaml_data(self, pkg_name: str, *, section: str, virtual: bool = False): config = self.config.get_config("packages") data = config.get(pkg_name, {}).get(section, []) kind = RequirementKind.PACKAGE if virtual: return RequirementKind.VIRTUAL, data if not data: data = config.get("all", {}).get(section, []) kind = RequirementKind.DEFAULT return kind, data def _rules_from_requirements( self, pkg_name: str, requirements, *, kind: RequirementKind ) -> List[RequirementRule]: """Manipulate requirements from packages.yaml, and return a list of tuples with a uniform structure (name, policy, requirements). """ if isinstance(requirements, str): requirements = [requirements] rules = [] for requirement in requirements: # A string is equivalent to a one_of group with a single element if isinstance(requirement, str): requirement = {"one_of": [requirement]} for policy in ("spec", "one_of", "any_of"): if policy not in requirement: continue constraints = requirement[policy] # "spec" is for specifying a single spec if policy == "spec": constraints = [constraints] policy = "one_of" if kind == RequirementKind.DEFAULT: # Warn about %gcc type of requirements under `all`. self._maybe_warn_compiler_in_all(constraints, "require") # validate specs from YAML first, and fail with line numbers if parsing fails. raw_strs = list(constraints) constraints = [ self._parse_and_expand(constraint, named=kind == RequirementKind.VIRTUAL) for constraint in raw_strs ] _check_unknown_targets(raw_strs, constraints) when_str = requirement.get("when") when = self._parse_and_expand(when_str) if when_str else spack.spec.Spec() constraints = [ x for x in constraints if not self.reject_requirement_constraint(pkg_name, constraint=x, kind=kind) ] if not constraints: continue rules.append( RequirementRule( pkg_name=pkg_name, policy=policy, requirements=constraints, kind=kind, message=requirement.get("message"), condition=when, origin=RequirementOrigin.REQUIRE_YAML, ) ) return rules def reject_requirement_constraint( self, pkg_name: str, *, constraint: spack.spec.Spec, kind: RequirementKind ) -> bool: """Returns True if a requirement constraint should be rejected""" # If it's a specific package requirement, it's never rejected if kind != RequirementKind.DEFAULT: return False # Reject requirements with dependencies for runtimes and compilers # These are usually requests on compilers, in the form of % involves_dependencies = bool(constraint.dependencies()) if involves_dependencies and ( pkg_name in self.runtime_pkgs or pkg_name in self.compiler_pkgs ): tty.debug(f"[{__name__}] Rejecting '{constraint}' for compiler package {pkg_name}") return True # Requirements under all: are applied only if they are satisfiable considering only # package rules, so e.g. variants must exist etc. Otherwise, they are rejected. try: s = spack.spec.Spec(pkg_name) s.constrain(constraint) s.validate_or_raise() except spack.error.SpackError as e: tty.debug( f"[{__name__}] Rejecting the default '{constraint}' requirement " f"on '{pkg_name}': {str(e)}" ) return True return False def _maybe_warn_compiler_in_all(self, items: Union[str, list, dict], section: str) -> None: """Warn once if a packages:all: prefer/require entry has compiler dependencies.""" # Stick to single items, not complex one_of / any_of groups to keep things simple. if isinstance(items, str): spec_str = items elif isinstance(items, dict) and "spec" in items and isinstance(items["spec"], str): spec_str = items["spec"] elif isinstance(items, list) and len(items) == 1 and isinstance(items[0], str): spec_str = items[0] else: return if spec_str in self._warned_compiler_all: return self._warned_compiler_all.add(spec_str) suggestions = [] for edge in self._parse_and_expand(spec_str).edges_to_dependencies(): if edge.when != spack.spec.EMPTY_SPEC: # Conditional dependencies are fine (includes toolchains after expansion). continue elif edge.virtuals: # The case `%c,cxx=gcc` or similar. keys = edge.virtuals comment = "" elif edge.spec.name in self.compiler_pkgs: # Just a package `%gcc`. keys = ("c",) comment = "# For each language virtual (c, cxx, fortran, ...):\n" else: # Maybe %mpich or so? Just give a generic suggestion. keys = ("",) comment = "# For each virtual:\n" data = {"packages": {k: {section: [str(edge.spec)]} for k in keys}} suggestion = spack.util.spack_yaml.dump(data).rstrip() suggestions.append(f"{comment}{suggestion}") if suggestions: mark = get_mark_from_yaml_data(spec_str) location = f"{mark.name}:{mark.line + 1}: " if mark else "" prefix = ( f"{location}'packages: all: {section}: [\"{spec_str}\"]' applies a dependency " f"constraint to all packages" ) suffix = "Consider instead:\n" + "\n".join(suggestions) if section == "prefer": warnings.warn( f"{prefix}. This can lead to unexpected concretizations. This was likely " f"intended as a preference for a provider of a (language) virtual. {suffix}" ) else: warnings.warn( f"{prefix}. This often leads to concretization errors. This was likely " f"intended as a requirement for a provider of a (language) virtual. {suffix}" ) def _split_edge_on_virtuals(edge: spack.spec.DependencySpec) -> List[spack.spec.Spec]: """Split the edge on virtuals and removes the parent.""" if not edge.virtuals: return [spack.spec.Spec(str(edge.copy(keep_parent=False)))] result = [] # We split on virtuals so that "%%c,cxx=gcc" enforces "%%c=gcc" and "%%cxx=gcc" separately for v in edge.virtuals: t = edge.copy(keep_parent=False, keep_virtuals=False) t.update_virtuals(v) t.when = spack.spec.Spec(f"%{v}") result.append(spack.spec.Spec(str(t))) return result def parse_spec_from_yaml_string(string: str, *, named: bool = False) -> spack.spec.Spec: """Parse a spec from YAML and add file/line info to errors, if it's available. Parse a ``Spec`` from the supplied string, but also intercept any syntax errors and add file/line information for debugging using file/line annotations from the string. Args: string: a string representing a ``Spec`` from config YAML. named: if True, the spec must have a name """ try: result = spack.spec.Spec(string) except spack.error.SpecSyntaxError as e: mark = get_mark_from_yaml_data(string) if mark: msg = f"{mark.name}:{mark.line + 1}: {str(e)}" raise spack.error.SpecSyntaxError(msg) from e raise e if named is True and not result.name: msg = f"expected a named spec, but got '{string}' instead" mark = get_mark_from_yaml_data(string) # Add a hint in case it's dependencies deps = result.dependencies() if len(deps) == 1: msg = f"{msg}. Did you mean '{deps[0]}'?" if mark: msg = f"{mark.name}:{mark.line + 1}: {msg}" raise spack.error.SpackError(msg) return result ================================================ FILE: lib/spack/spack/solver/reuse.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import enum import functools import typing from typing import Any, Callable, List, Mapping, Optional import spack.binary_distribution import spack.config import spack.llnl.path import spack.repo import spack.spec import spack.store import spack.traverse from spack.externals import ( ExternalSpecsParser, complete_architecture, complete_variants_and_architecture, extract_dicts_from_configuration, ) from spack.spec_filter import SpecFilter from .runtimes import all_libcs if typing.TYPE_CHECKING: import spack.environment def spec_filter_from_store( configuration, *, packages_with_externals, include, exclude ) -> SpecFilter: """Constructs a filter that takes the specs from the current store.""" is_reusable = functools.partial( _is_reusable, packages_with_externals=packages_with_externals, local=True ) factory = functools.partial(_specs_from_store, configuration=configuration) return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude) def spec_filter_from_buildcache(*, packages_with_externals, include, exclude) -> SpecFilter: """Constructs a filter that takes the specs from the configured buildcaches.""" is_reusable = functools.partial( _is_reusable, packages_with_externals=packages_with_externals, local=False ) return SpecFilter( factory=_specs_from_mirror, is_usable=is_reusable, include=include, exclude=exclude ) def spec_filter_from_environment(*, packages_with_externals, include, exclude, env) -> SpecFilter: is_reusable = functools.partial( _is_reusable, packages_with_externals=packages_with_externals, local=True ) factory = functools.partial(_specs_from_environment, env=env) return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude) def spec_filter_from_packages_yaml( *, external_parser: ExternalSpecsParser, packages_with_externals, include, exclude ) -> SpecFilter: is_reusable = functools.partial( _is_reusable, packages_with_externals=packages_with_externals, local=True ) return SpecFilter( external_parser.all_specs, is_usable=is_reusable, include=include, exclude=exclude ) def _has_runtime_dependencies(spec: spack.spec.Spec) -> bool: # Spack v1.0 specs and later return spec.original_spec_format() >= 5 def _is_reusable(spec: spack.spec.Spec, packages_with_externals, local: bool) -> bool: """A spec is reusable if it's not a dev spec, it's imported from the cray manifest, it's not external, or it's external with matching packages.yaml entry. The latter prevents two issues: 1. Externals in build caches: avoid installing an external on the build machine not available on the target machine 2. Local externals: avoid reusing an external if the local config changes. This helps in particular when a user removes an external from packages.yaml, and expects that that takes effect immediately. Arguments: spec: the spec to check packages_with_externals: the pre-processed packages configuration """ if "dev_path" in spec.variants: return False if spec.name == "compiler-wrapper": return False if not spec.external: return _has_runtime_dependencies(spec) # Cray external manifest externals are always reusable if local: _, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash()) if record and record.origin == "external-db": return True try: provided = spack.repo.PATH.get(spec).provided_virtual_names() except spack.repo.RepoError: provided = [] for name in {spec.name, *provided}: for entry in packages_with_externals.get(name, {}).get("externals", []): expected_prefix = entry.get("prefix") if expected_prefix is not None: expected_prefix = spack.llnl.path.path_to_os_path(expected_prefix)[0] if ( spec.satisfies(entry["spec"]) and spec.external_path == expected_prefix and spec.external_modules == entry.get("modules") ): return True return False def _specs_from_store(configuration): store = spack.store.create(configuration) with store.db.read_transaction(): return store.db.query(installed=True) def _specs_from_mirror(): try: return spack.binary_distribution.update_cache_and_get_specs() except (spack.binary_distribution.FetchCacheError, IndexError): # this is raised when no mirrors had indices. # TODO: update mirror configuration so it can indicate that the # TODO: source cache (or any mirror really) doesn't have binaries. return [] def _specs_from_environment(env): """Return all concrete specs from the environment. This includes all included concrete""" if env: return list(spack.traverse.traverse_nodes([s for _, s in env.concretized_specs()])) else: return [] class ReuseStrategy(enum.Enum): ROOTS = enum.auto() DEPENDENCIES = enum.auto() NONE = enum.auto() def create_external_parser( packages_with_externals: Any, completion_mode: str ) -> ExternalSpecsParser: """Get externals from a pre-processed packages.yaml (with implicit externals).""" external_dicts = extract_dicts_from_configuration(packages_with_externals) if completion_mode == "default_variants": complete_fn = complete_variants_and_architecture elif completion_mode == "architecture_only": complete_fn = complete_architecture else: raise ValueError( f"Unknown value for concretizer:externals:completion: {completion_mode!r}" ) return ExternalSpecsParser(external_dicts, complete_node=complete_fn) SpecFiltersFactory = Callable[ [Callable[[spack.spec.Spec], bool], spack.config.Configuration], List[SpecFilter] ] class ReusableSpecsSelector: """Selects specs that can be reused during concretization.""" def __init__( self, *, configuration: spack.config.Configuration, external_parser: ExternalSpecsParser, packages_with_externals: Any, factory: Optional[SpecFiltersFactory] = None, ) -> None: # Local import to break circular dependencies import spack.environment self.configuration = configuration self.store = spack.store.create(configuration) self.reuse_strategy = ReuseStrategy.ROOTS reuse_yaml = self.configuration.get("concretizer:reuse", False) self.reuse_sources = [] if factory is not None: is_reusable = functools.partial( _is_reusable, packages_with_externals=packages_with_externals, local=True ) self.reuse_sources.extend(factory(is_reusable, configuration)) if not isinstance(reuse_yaml, Mapping): self.reuse_sources.append( spec_filter_from_packages_yaml( external_parser=external_parser, packages_with_externals=packages_with_externals, include=[], exclude=[], ) ) if reuse_yaml is False: self.reuse_strategy = ReuseStrategy.NONE return if reuse_yaml == "dependencies": self.reuse_strategy = ReuseStrategy.DEPENDENCIES self.reuse_sources.extend( [ spec_filter_from_store( configuration=self.configuration, packages_with_externals=packages_with_externals, include=[], exclude=[], ), spec_filter_from_buildcache( packages_with_externals=packages_with_externals, include=[], exclude=[] ), ] ) else: has_external_source = False roots = reuse_yaml.get("roots", True) if roots is True: self.reuse_strategy = ReuseStrategy.ROOTS else: self.reuse_strategy = ReuseStrategy.DEPENDENCIES default_include = reuse_yaml.get("include", []) default_exclude = reuse_yaml.get("exclude", []) default_sources = [{"type": "local"}, {"type": "buildcache"}] for source in reuse_yaml.get("from", default_sources): include = source.get("include", default_include) exclude = source.get("exclude", default_exclude) if source["type"] == "environment" and "path" in source: env_dir = spack.environment.as_env_dir(source["path"]) active_env = spack.environment.active_environment() if not active_env or env_dir not in active_env.included_concrete_env_root_dirs: # If the environment is not included as a concrete environment, use the # current specs from its lockfile. self.reuse_sources.append( spec_filter_from_environment( packages_with_externals=packages_with_externals, include=include, exclude=exclude, env=spack.environment.environment_from_name_or_dir(env_dir), ) ) elif source["type"] == "local": self.reuse_sources.append( spec_filter_from_store( self.configuration, packages_with_externals=packages_with_externals, include=include, exclude=exclude, ) ) elif source["type"] == "buildcache": self.reuse_sources.append( spec_filter_from_buildcache( packages_with_externals=packages_with_externals, include=include, exclude=exclude, ) ) elif source["type"] == "external": has_external_source = True if include: # Since libcs are implicit externals, we need to implicitly include them include = include + sorted(all_libcs()) # type: ignore[type-var] self.reuse_sources.append( spec_filter_from_packages_yaml( external_parser=external_parser, packages_with_externals=packages_with_externals, include=include, exclude=exclude, ) ) # If "external" is not specified, we assume that all externals have to be included if not has_external_source: self.reuse_sources.append( spec_filter_from_packages_yaml( external_parser=external_parser, packages_with_externals=packages_with_externals, include=[], exclude=[], ) ) def reusable_specs(self, specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]: result = [] for reuse_source in self.reuse_sources: result.extend(reuse_source.selected_specs()) # If we only want to reuse dependencies, remove the root specs if self.reuse_strategy == ReuseStrategy.DEPENDENCIES: result = [spec for spec in result if not any(root in spec for root in specs)] return result ================================================ FILE: lib/spack/spack/solver/runtimes.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import itertools from typing import Any, Dict, Set, Tuple import spack.compilers.config import spack.compilers.libraries import spack.config import spack.repo import spack.spec import spack.util.libc import spack.version from .core import SourceContext, fn, using_libc_compatibility from .versions import Provenance class RuntimePropertyRecorder: """An object of this class is injected in callbacks to compilers, to let them declare properties of the runtimes they support and of the runtimes they provide, and to add runtime dependencies to the nodes using said compiler. The usage of the object is the following. First, a runtime package name or the wildcard "*" are passed as an argument to __call__, to set which kind of package we are referring to. Then we can call one method with a directive-like API. Examples: >>> pkg = RuntimePropertyRecorder(setup) >>> # Every package compiled with %gcc has a link dependency on 'gcc-runtime' >>> pkg("*").depends_on( ... "gcc-runtime", ... when="%gcc", ... type="link", ... description="If any package uses %gcc, it depends on gcc-runtime" ... ) >>> # The version of gcc-runtime is the same as the %gcc used to "compile" it >>> pkg("gcc-runtime").requires("@=9.4.0", when="%gcc@=9.4.0") """ def __init__(self, setup): self._setup = setup self.rules = [] self.runtime_conditions = set() self.injected_dependencies = set() # State of this object set in the __call__ method, and reset after # each directive-like method self.current_package = None def __call__(self, package_name: str) -> "RuntimePropertyRecorder": """Sets a package name for the next directive-like method call""" assert self.current_package is None, f"state was already set to '{self.current_package}'" self.current_package = package_name return self def reset(self): """Resets the current state.""" self.current_package = None def depends_on(self, dependency_str: str, *, when: str, type: str, description: str) -> None: """Injects conditional dependencies on packages. Conditional dependencies can be either "real" packages or virtual dependencies. Args: dependency_str: the dependency spec to inject when: anonymous condition to be met on a package to have the dependency type: dependency type description: human-readable description of the rule for adding the dependency """ # TODO: The API for this function is not final, and is still subject to change. At # TODO: the moment, we implemented only the features strictly needed for the # TODO: functionality currently provided by Spack, and we assert nothing else is required. msg = "the 'depends_on' method can be called only with pkg('*')" assert self.current_package == "*", msg when_spec = spack.spec.Spec(when) assert not when_spec.name, "only anonymous when specs are accepted" dependency_spec = spack.spec.Spec(dependency_str) if dependency_spec.versions != spack.version.any_version: self._setup.version_constraints[dependency_spec.name].add(dependency_spec.versions) self.injected_dependencies.add(dependency_spec) body_str, node_variable = self.rule_body_from(when_spec) head_clauses = self._setup.spec_clauses(dependency_spec, body=False) runtime_pkg = dependency_spec.name is_virtual = head_clauses[0].args[0] == "virtual_node" main_rule = ( f"% {description}\n" f'1 {{ attr("depends_on", {node_variable}, node(0..X-1, "{runtime_pkg}"), "{type}") :' f' max_dupes("{runtime_pkg}", X)}} 1:-\n' f"{body_str}." ) if is_virtual: main_rule = ( f"% {description}\n" f'attr("dependency_holds", {node_variable}, "{runtime_pkg}", "{type}") :-\n' f"{body_str}." ) self.rules.append(main_rule) for clause in head_clauses: if clause.args[0] == "node": continue runtime_node = f'node(RuntimeID, "{runtime_pkg}")' head_str = str(clause).replace(f'"{runtime_pkg}"', runtime_node) depends_on_constraint = ( f' attr("depends_on", {node_variable}, {runtime_node}, "{type}"),\n' ) if is_virtual: depends_on_constraint = ( f' attr("depends_on", {node_variable}, ProviderNode, "{type}"),\n' f" provider(ProviderNode, {runtime_node}),\n" ) rule = f"{head_str} :-\n{depends_on_constraint}{body_str}." self.rules.append(rule) self.reset() @staticmethod def node_for(name: str) -> str: return f'node(ID{name.replace("-", "_")}, "{name}")' def rule_body_from(self, when_spec: "spack.spec.Spec") -> Tuple[str, str]: """Computes the rule body from a "when" spec, and returns it, along with the node variable. """ node_placeholder = "XXX" node_variable = "node(ID, Package)" when_substitutions = {} for s in when_spec.traverse(root=False): when_substitutions[f'"{s.name}"'] = self.node_for(s.name) body_clauses = self._setup.spec_clauses(when_spec, name=node_placeholder, body=True) for clause in body_clauses: if clause.args[0] == "virtual_on_incoming_edges": # Substitute: attr("virtual_on_incoming_edges", ProviderNode, Virtual) # with: attr("virtual_on_edge", ParentNode, ProviderNode, Virtual) # (avoid adding virtuals everywhere, if a single edge needs it) _, provider, virtual = clause.args clause.args = "virtual_on_edge", node_placeholder, provider, virtual # Check for abstract hashes in the body for s in when_spec.traverse(root=False): if s.abstract_hash: body_clauses.append(fn.attr("hash", s.name, s.abstract_hash)) body_str = ",\n".join(f" {x}" for x in body_clauses) body_str = body_str.replace(f'"{node_placeholder}"', f"{node_variable}") for old, replacement in when_substitutions.items(): body_str = body_str.replace(old, replacement) return body_str, node_variable def requires(self, impose: str, *, when: str): """Injects conditional requirements on a given package. Args: impose: constraint to be imposed when: condition triggering the constraint """ msg = "the 'requires' method cannot be called with pkg('*') or without setting the package" assert self.current_package is not None and self.current_package != "*", msg imposed_spec = spack.spec.Spec(f"{self.current_package}{impose}") when_spec = spack.spec.Spec(f"{self.current_package}{when}") assert imposed_spec.versions.concrete, f"{impose} must have a concrete version" # Add versions to possible versions for s in (imposed_spec, when_spec): if not s.versions.concrete: continue self._setup.possible_versions[s.name][s.version].append(Provenance.RUNTIME) self.runtime_conditions.add((imposed_spec, when_spec)) self.reset() def propagate(self, constraint_str: str, *, when: str): msg = "the 'propagate' method can be called only with pkg('*')" assert self.current_package == "*", msg when_spec = spack.spec.Spec(when) assert not when_spec.name, "only anonymous when specs are accepted" when_substitutions = {} for s in when_spec.traverse(root=False): when_substitutions[f'"{s.name}"'] = self.node_for(s.name) body_str, node_variable = self.rule_body_from(when_spec) constraint_spec = spack.spec.Spec(constraint_str) constraint_clauses = self._setup.spec_clauses(constraint_spec, body=False) for clause in constraint_clauses: if clause.args[0] == "node_version_satisfies": self._setup.version_constraints[constraint_spec.name].add(constraint_spec.versions) args = f'"{constraint_spec.name}", "{constraint_spec.versions}"' head_str = f"propagate({node_variable}, node_version_satisfies({args}))" rule = f"{head_str} :-\n{body_str}." self.rules.append(rule) self.reset() def default_flags(self, spec: "spack.spec.Spec"): if not spec.external or "flags" not in spec.extra_attributes: self.reset() return when_spec = spack.spec.Spec(f"%[deptypes=build] {spec}") body_str, node_variable = self.rule_body_from(when_spec) node_placeholder = "XXX" flags = spec.extra_attributes["flags"] root_spec_str = f"{node_placeholder}" for flag_type, default_values in flags.items(): root_spec_str = f"{root_spec_str} {flag_type}='{default_values}'" root_spec = spack.spec.Spec(root_spec_str) head_clauses = self._setup.spec_clauses( root_spec, body=False, context=SourceContext(source="compiler") ) self.rules.append(f"% Default compiler flags for {spec}\n") for clause in head_clauses: if clause.args[0] == "node": continue head_str = str(clause).replace(f'"{node_placeholder}"', f"{node_variable}") rule = f"{head_str} :-\n{body_str}." self.rules.append(rule) self.reset() def consume_facts(self): """Consume the facts collected by this object, and emits rules and facts for the runtimes. """ self._setup.gen.h2("Runtimes: declarations") runtime_pkgs = sorted( {x.name for x in self.injected_dependencies if not spack.repo.PATH.is_virtual(x.name)} ) for runtime_pkg in runtime_pkgs: self._setup.gen.fact(fn.runtime(runtime_pkg)) self._setup.gen.newline() self._setup.gen.h2("Runtimes: rules") self._setup.gen.newline() for rule in self.rules: self._setup.gen.append(rule) self._setup.gen.newline() self._setup.gen.h2("Runtimes: requirements") for imposed_spec, when_spec in sorted(self.runtime_conditions): msg = f"{when_spec} requires {imposed_spec} at runtime" _ = self._setup.condition(when_spec, imposed_spec=imposed_spec, msg=msg) self._setup.trigger_rules() self._setup.effect_rules() def _normalize_packages_yaml(packages_yaml: Dict[str, Any]) -> None: for pkg_name in list(packages_yaml.keys()): is_virtual = spack.repo.PATH.is_virtual(pkg_name) if pkg_name == "all" or not is_virtual: continue # Remove the virtual entry from the normalized configuration data = packages_yaml.pop(pkg_name) is_buildable = data.get("buildable", True) if not is_buildable: for provider in spack.repo.PATH.providers_for(pkg_name): entry = packages_yaml.setdefault(provider.name, {}) entry["buildable"] = False externals = data.get("externals", []) def keyfn(x): return spack.spec.Spec(x["spec"]).name for provider, specs in itertools.groupby(externals, key=keyfn): entry = packages_yaml.setdefault(provider, {}) entry.setdefault("externals", []).extend(specs) def external_config_with_implicit_externals( configuration: spack.config.Configuration, ) -> Dict[str, Any]: # Read packages.yaml and normalize it so that it will not contain entries referring to # virtual packages. packages_yaml = configuration.deepcopy_as_builtin("packages", line_info=True) _normalize_packages_yaml(packages_yaml) # Add externals for libc from compilers on Linux if not using_libc_compatibility(): return packages_yaml seen = set() for compiler in spack.compilers.config.all_compilers_from(configuration): libc = spack.compilers.libraries.CompilerPropertyDetector(compiler).default_libc() if libc and libc not in seen: seen.add(libc) entry = {"spec": f"{libc}", "prefix": libc.external_path} packages_yaml.setdefault(libc.name, {}).setdefault("externals", []).append(entry) return packages_yaml def all_libcs() -> Set[spack.spec.Spec]: """Return a set of all libc specs targeted by any configured compiler. If none, fall back to libc determined from the current Python process if dynamically linked.""" libcs = set() for c in spack.compilers.config.all_compilers_from(spack.config.CONFIG): candidate = spack.compilers.libraries.CompilerPropertyDetector(c).default_libc() if candidate is not None: libcs.add(candidate) if libcs: return libcs libc = spack.util.libc.libc_from_current_python_process() return {libc} if libc else set() ================================================ FILE: lib/spack/spack/solver/splices.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) %============================================================================= % These rules are conditionally loaded to handle the synthesis of spliced % packages. % ============================================================================= % Consider the concrete spec: % foo@2.72%gcc@11.4 arch=linux-ubuntu22.04-icelake build_system=autotools ^bar ... % It will emit the following facts for reuse (below is a subset) % installed_hash("foo", "xxxyyy") % hash_attr("xxxyyy", "hash", "foo", "xxxyyy") % hash_attr("xxxyyy", "version", "foo", "2.72") % hash_attr("xxxyyy", "node_os", "ubuntu22.04") % hash_attr("xxxyyy", "hash", "bar", "zzzqqq") % hash_attr("xxxyyy", "depends_on", "foo", "bar", "link") % Rules that derive abi_splice_conditions_hold will be generated from % use of the `can_splice` directive. The will have the following form: % can_splice("foo@1.0.0+a", when="@1.0.1+a", match_variants=["b"]) ---> % abi_splice_conditions_hold(0, node(SID, "foo"), "foo", BashHash) :- % installed_hash("foo", BaseHash), % attr("node", node(SID, SpliceName)), % attr("node_version_satisfies", node(SID, "foo"), "1.0.1"), % hash_attr("hash", "node_version_satisfies", "foo", "1.0.1"), % attr("variant_value", node(SID, "foo"), "a", "True"), % hash_attr("hash", "variant_value", "foo", "a", "True"), % attr("variant_value", node(SID, "foo"), "b", VariVar0), % hash_attr("hash", "variant_value", "foo", "b", VariVar0), % If the splice is valid (i.e. abi_splice_conditions_hold is derived) in the % dependency of a concrete spec the solver free to choose whether to continue % with the exact hash semantics by simply imposing the child hash, or introducing % a spliced node as the dependency instead { imposed_constraint(ParentHash, "hash", ChildName, ChildHash) } :- hash_attr(ParentHash, "hash", ChildName, ChildHash), abi_splice_conditions_hold(_, node(SID, SpliceName), ChildName, ChildHash). attr("splice_at_hash", ParentNode, node(SID, SpliceName), ChildName, ChildHash) :- attr("hash", ParentNode, ParentHash), hash_attr(ParentHash, "hash", ChildName, ChildHash), abi_splice_conditions_hold(_, node(SID, SpliceName), ChildName, ChildHash), ParentHash != ChildHash, not imposed_constraint(ParentHash, "hash", ChildName, ChildHash). % Names and virtual providers may change when a dependency is spliced in imposed_constraint(Hash, "dependency_holds", ParentName, SpliceName, Type) :- hash_attr(Hash, "depends_on", ParentName, DepName, Type), hash_attr(Hash, "hash", DepName, DepHash), attr("splice_at_hash", node(ID, ParentName), node(SID, SpliceName), DepName, DepHash). imposed_constraint(Hash, "virtual_on_edge", ParentName, SpliceName, VirtName) :- hash_attr(Hash, "virtual_on_edge", ParentName, DepName, VirtName), attr("splice_at_hash", node(ID, ParentName), node(SID, SpliceName), DepName, DepHash). ================================================ FILE: lib/spack/spack/solver/splicing.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from functools import cmp_to_key from typing import Dict, List, NamedTuple import spack.deptypes as dt from spack.spec import Spec from spack.traverse import by_dag_hash, traverse_nodes class Splice(NamedTuple): #: The spec being spliced into a parent splice_spec: Spec #: The name of the child that splice spec is replacing child_name: str #: The hash of the child that ``splice_spec`` is replacing child_hash: str def _resolve_collected_splices( specs: List[Spec], splices: Dict[Spec, List[Splice]] ) -> Dict[Spec, Spec]: """After all of the specs have been concretized, apply all immediate splices. Returns a dict mapping original specs to their resolved counterparts """ def splice_cmp(s1: Spec, s2: Spec): """This function can be used to sort a list of specs such that that any spec which will be spliced into a parent comes after the parent it will be spliced into. This order ensures that transitive splices will be executed in the correct order. """ s1_splices = splices.get(s1, []) s2_splices = splices.get(s2, []) if any([s2.dag_hash() == splice.splice_spec.dag_hash() for splice in s1_splices]): return -1 elif any([s1.dag_hash() == splice.splice_spec.dag_hash() for splice in s2_splices]): return 1 else: return 0 splice_order = sorted(specs, key=cmp_to_key(splice_cmp)) reverse_topo_order = reversed( [x for x in traverse_nodes(splice_order, order="topo", key=by_dag_hash) if x in specs] ) already_resolved: Dict[Spec, Spec] = {} for spec in reverse_topo_order: immediate = splices.get(spec, []) if not immediate and not any( edge.spec in already_resolved for edge in spec.edges_to_dependencies() ): continue new_spec = spec.copy(deps=False) new_spec.clear_caches(ignore=("package_hash",)) new_spec.build_spec = spec for edge in spec.edges_to_dependencies(): depflag = edge.depflag & ~dt.BUILD if any(edge.spec.dag_hash() == splice.child_hash for splice in immediate): splice = [s for s in immediate if s.child_hash == edge.spec.dag_hash()][0] # If the spec being splice in is also spliced splice_spec = already_resolved.get(splice.splice_spec, splice.splice_spec) new_spec.add_dependency_edge(splice_spec, depflag=depflag, virtuals=edge.virtuals) elif edge.spec in already_resolved: new_spec.add_dependency_edge( already_resolved[edge.spec], depflag=depflag, virtuals=edge.virtuals ) else: new_spec.add_dependency_edge(edge.spec, depflag=depflag, virtuals=edge.virtuals) already_resolved[spec] = new_spec return already_resolved ================================================ FILE: lib/spack/spack/solver/versions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import enum class Provenance(enum.IntEnum): """Enumeration of the possible provenances of a version.""" # A spec literal SPEC = enum.auto() # A dev spec literal DEV_SPEC = enum.auto() # The 'packages' section of the configuration PACKAGES_YAML = enum.auto() # A git version in the 'packages' section of the configuration PACKAGES_YAML_GIT_VERSION = enum.auto() # A package requirement PACKAGE_REQUIREMENT = enum.auto() # A 'package.py' file PACKAGE_PY = enum.auto() # An installed spec INSTALLED = enum.auto() # lower provenance for installed git refs so concretizer prefers StandardVersion installs INSTALLED_GIT_VERSION = enum.auto() # Synthetic versions for virtual packages VIRTUAL_CONSTRAINT = enum.auto() # A runtime injected from another package (e.g. a compiler) RUNTIME = enum.auto() def __str__(self): return f"{self._name_.lower()}" ================================================ FILE: lib/spack/spack/solver/when_possible.lp ================================================ % Copyright Spack Project Developers. See COPYRIGHT file for details. % % SPDX-License-Identifier: (Apache-2.0 OR MIT) %============================================================================= % Minimize the number of literals that are not solved % % This minimization is used for the "when_possible" concretization mode, % otherwise we assume that all literals must be solved. %============================================================================= % Give clingo the choice to solve an input spec or not { solve_literal(ID) } :- literal(ID). literal_not_solved(ID) :- not solve_literal(ID), literal(ID). % Make a problem with "zero literals solved" unsat. This is to trigger % looking for solutions to the ASP problem with "errors", which results % in better reporting for users. See #30669 for details. 1 { solve_literal(ID) : literal(ID) }. % If a literal is not solved, and has subconditions, then the subconditions % should not be imposed even if their trigger condition holds do_not_impose(EffectID, node(X, Package)) :- literal_not_solved(LiteralID), pkg_fact(Package, condition_trigger(ParentConditionID, LiteralID)), subcondition(SubconditionID, ParentConditionID), pkg_fact(Package, condition_effect(SubconditionID, EffectID)), trigger_and_effect(_, TriggerID, EffectID), trigger_requestor_node(TriggerID, node(X, Package)). opt_criterion(320, "number of input specs not concretized"). #minimize{ 0@320: #true }. #minimize { 1@320,ID : literal_not_solved(ID) }. #heuristic solve_literal(ID) : literal(ID). [1, sign] #heuristic solve_literal(ID) : literal(ID). [1000, init] ================================================ FILE: lib/spack/spack/spec.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ Spack allows very fine-grained control over how packages are installed and over how they are built and configured. To make this easy, it has its own syntax for declaring a dependence. We call a descriptor of a particular package configuration a "spec". The syntax looks like this: .. code-block:: sh $ spack install mpileaks ^openmpi @1.2:1.4 +debug %intel @12.1 target=zen 0 1 2 3 4 5 6 The first part of this is the command, ``spack install``. The rest of the line is a spec for a particular installation of the mpileaks package. 0. The package to install 1. A dependency of the package, prefixed by ``^`` 2. A version descriptor for the package. This can either be a specific version, like ``1.2``, or it can be a range of versions, e.g. ``1.2:1.4``. If multiple specific versions or multiple ranges are acceptable, they can be separated by commas, e.g. if a package will only build with versions 1.0, 1.2-1.4, and 1.6-1.8 of mvapich, you could say:: depends_on("mvapich@1.0,1.2:1.4,1.6:1.8") 3. A compile-time variant of the package. If you need openmpi to be built in debug mode for your package to work, you can require it by adding ``+debug`` to the openmpi spec when you depend on it. If you do NOT want the debug option to be enabled, then replace this with ``-debug``. If you would like for the variant to be propagated through all your package's dependencies use ``++`` for enabling and ``--`` or ``~~`` for disabling. 4. The name of the compiler to build with. 5. The versions of the compiler to build with. Note that the identifier for a compiler version is the same ``@`` that is used for a package version. A version list denoted by ``@`` is associated with the compiler only if if it comes immediately after the compiler name. Otherwise it will be associated with the current package spec. 6. The architecture to build with. """ import collections import collections.abc import enum import io import itertools import json import os import pathlib import platform import re import socket import warnings from typing import ( Any, Callable, Dict, Iterable, List, Match, Optional, Sequence, Set, Tuple, Union, overload, ) import spack.vendor.archspec.cpu from spack.vendor.typing_extensions import Literal import spack import spack.aliases import spack.compilers.flags import spack.deptypes as dt import spack.error import spack.hash_types as ht import spack.llnl.path import spack.llnl.string import spack.llnl.util.filesystem as fs import spack.llnl.util.lang as lang import spack.llnl.util.tty as tty import spack.llnl.util.tty.color as clr import spack.patch import spack.paths import spack.platforms import spack.provider_index import spack.repo import spack.spec_parser import spack.traverse import spack.util.hash import spack.util.prefix import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml import spack.variant as vt import spack.version import spack.version as vn import spack.version.git_ref_lookup from .enums import InstallRecordStatus, PropagationPolicy __all__ = [ "CompilerSpec", "Spec", "UnsupportedPropagationError", "DuplicateDependencyError", "UnsupportedCompilerError", "DuplicateArchitectureError", "InvalidDependencyError", "UnsatisfiableSpecNameError", "UnsatisfiableVersionSpecError", "UnsatisfiableArchitectureSpecError", "UnsatisfiableDependencySpecError", "AmbiguousHashError", "InvalidHashError", "SpecDeprecatedError", ] SPEC_FORMAT_RE = re.compile( r"(?:" # this is one big or, with matches ordered by priority # OPTION 1: escaped character (needs to be first to catch opening \{) # Note that an unterminated \ at the end of a string is left untouched r"(?:\\(.))" r"|" # or # OPTION 2: an actual format string r"{" # non-escaped open brace { r"( ?[%@/]|[\w ][\w -]*=)?" # optional sigil (or identifier or space) to print sigil in color r"(?:\^([^}\.]+)\.)?" # optional ^depname. (to get attr from dependency) # after the sigil or depname, we can have a hash expression or another attribute r"(?:" # one of r"(hash\b)(?:\:(\d+))?" # hash followed by : r"|" # or r"([^}]*)" # another attribute to format r")" # end one of r"(})?" # finish format string with non-escaped close brace }, or missing if not present r"|" # OPTION 3: mismatched close brace (option 2 would consume a matched open brace) r"(})" # brace r")", re.IGNORECASE, ) #: Valid pattern for an identifier in Spack IDENTIFIER_RE = r"\w[\w-]*" # Coloring of specs when using color output. Fields are printed with # different colors to enhance readability. # See spack.llnl.util.tty.color for descriptions of the color codes. COMPILER_COLOR = "@g" #: color for highlighting compilers VERSION_COLOR = "@c" #: color for highlighting versions ARCHITECTURE_COLOR = "@m" #: color for highlighting architectures VARIANT_COLOR = "@B" #: color for highlighting variants HASH_COLOR = "@K" #: color for highlighting package hashes HIGHLIGHT_COLOR = "@_R" #: color for highlighting spec parts on demand #: Default format for Spec.format(). This format can be round-tripped, so that: #: Spec(Spec("string").format()) == Spec("string)" DEFAULT_FORMAT = ( "{name}{@versions}{compiler_flags}" "{variants}{ namespace=namespace_if_anonymous}" "{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}" "{/abstract_hash}" ) #: Display format, which eliminates extra `@=` in the output, for readability. DISPLAY_FORMAT = ( "{name}{@version}{compiler_flags}" "{variants}{ namespace=namespace_if_anonymous}" "{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}" "{/abstract_hash}" "{compilers}" ) #: Regular expression to pull spec contents out of clearsigned signature #: file. CLEARSIGN_FILE_REGEX = re.compile( ( r"^-----BEGIN PGP SIGNED MESSAGE-----" r"\s+Hash:\s+[^\s]+\s+(.+)-----BEGIN PGP SIGNATURE-----" ), re.MULTILINE | re.DOTALL, ) #: specfile format version. Must increase monotonically SPECFILE_FORMAT_VERSION = 5 class InstallStatus(enum.Enum): """Maps install statuses to symbols for display. Options are artificially disjoint for display purposes """ installed = "@g{[+]} " upstream = "@g{[^]} " external = "@M{[e]} " absent = "@K{ - } " missing = "@r{[-]} " # regexes used in spec formatting OLD_STYLE_FMT_RE = re.compile(r"\${[A-Z]+}") def ensure_modern_format_string(fmt: str) -> None: """Ensure that the format string does not contain old ${...} syntax.""" result = OLD_STYLE_FMT_RE.search(fmt) if result: raise SpecFormatStringError( f"Format string `{fmt}` contains old syntax `{result.group(0)}`. " "This is no longer supported." ) def _make_microarchitecture(name: str) -> spack.vendor.archspec.cpu.Microarchitecture: if isinstance(name, spack.vendor.archspec.cpu.Microarchitecture): return name return spack.vendor.archspec.cpu.TARGETS.get( name, spack.vendor.archspec.cpu.generic_microarchitecture(name) ) @lang.lazy_lexicographic_ordering class ArchSpec: """Aggregate the target platform, the operating system and the target microarchitecture.""" ANY_TARGET = _make_microarchitecture("*") @staticmethod def default_arch(): """Return the default architecture""" platform = spack.platforms.host() default_os = platform.default_operating_system() default_target = platform.default_target() arch_tuple = str(platform), str(default_os), str(default_target) return ArchSpec(arch_tuple) __slots__ = "_platform", "_os", "_target" def __init__(self, spec_or_platform_tuple=(None, None, None)): """Architecture specification a package should be built with. Each ArchSpec is comprised of three elements: a platform (e.g. Linux), an OS (e.g. RHEL6), and a target (e.g. x86_64). Args: spec_or_platform_tuple (ArchSpec or str or tuple): if an ArchSpec is passed it will be duplicated into the new instance. Otherwise information on platform, OS and target should be passed in either as a spec string or as a tuple. """ # If the argument to __init__ is a spec string, parse it # and construct an ArchSpec def _string_or_none(s): if s and s != "None": return str(s) return None # If another instance of ArchSpec was passed, duplicate it if isinstance(spec_or_platform_tuple, ArchSpec): other = spec_or_platform_tuple platform_tuple = other.platform, other.os, other.target elif isinstance(spec_or_platform_tuple, (str, tuple)): spec_fields = spec_or_platform_tuple # Normalize the string to a tuple if isinstance(spec_or_platform_tuple, str): spec_fields = spec_or_platform_tuple.split("-") if len(spec_fields) != 3: msg = "cannot construct an ArchSpec from {0!s}" raise ValueError(msg.format(spec_or_platform_tuple)) platform, operating_system, target = spec_fields platform_tuple = (_string_or_none(platform), _string_or_none(operating_system), target) self.platform, self.os, self.target = platform_tuple @staticmethod def override(init_spec, change_spec): if init_spec: new_spec = init_spec.copy() else: new_spec = ArchSpec() if change_spec.platform: new_spec.platform = change_spec.platform # TODO: if the platform is changed to something that is incompatible # with the current os, we should implicitly remove it if change_spec.os: new_spec.os = change_spec.os if change_spec.target: new_spec.target = change_spec.target return new_spec def _autospec(self, spec_like): if isinstance(spec_like, ArchSpec): return spec_like return ArchSpec(spec_like) def _cmp_iter(self): yield self.platform yield self.os if self.target is None: yield self.target else: yield self.target.name @property def platform(self): """The platform of the architecture.""" return self._platform @platform.setter def platform(self, value): # The platform of the architecture spec will be verified as a # supported Spack platform before it's set to ensure all specs # refer to valid platforms. value = str(value) if value is not None else None self._platform = value @property def os(self): """The OS of this ArchSpec.""" return self._os @os.setter def os(self, value): # The OS of the architecture spec will update the platform field # if the OS is set to one of the reserved OS types so that the # default OS type can be resolved. Since the reserved OS # information is only available for the host machine, the platform # will assumed to be the host machine's platform. value = str(value) if value is not None else None if value in spack.platforms.Platform.reserved_oss: curr_platform = str(spack.platforms.host()) self.platform = self.platform or curr_platform if self.platform != curr_platform: raise ValueError( "Can't set arch spec OS to reserved value '%s' when the " "arch platform (%s) isn't the current platform (%s)" % (value, self.platform, curr_platform) ) spec_platform = spack.platforms.by_name(self.platform) value = str(spec_platform.operating_system(value)) self._os = value @property def target(self): """The target of the architecture.""" return self._target @target.setter def target(self, value): # The target of the architecture spec will update the platform field # if the target is set to one of the reserved target types so that # the default target type can be resolved. Since the reserved target # information is only available for the host machine, the platform # will assumed to be the host machine's platform. def target_or_none(t): if isinstance(t, spack.vendor.archspec.cpu.Microarchitecture): return t if t and t != "None": return _make_microarchitecture(t) return None value = target_or_none(value) if str(value) in spack.platforms.Platform.reserved_targets: curr_platform = str(spack.platforms.host()) self.platform = self.platform or curr_platform if self.platform != curr_platform: raise ValueError( "Can't set arch spec target to reserved value '%s' when " "the arch platform (%s) isn't the current platform (%s)" % (value, self.platform, curr_platform) ) spec_platform = spack.platforms.by_name(self.platform) value = spec_platform.target(value) self._target = value def satisfies(self, other: "ArchSpec") -> bool: """Return True if all concrete specs matching self also match other, otherwise False. Args: other: spec to be satisfied """ other = self._autospec(other) # Check platform and os for attribute in ("platform", "os"): other_attribute = getattr(other, attribute) self_attribute = getattr(self, attribute) # platform=* or os=* if self_attribute and other_attribute == "*": return True if other_attribute and self_attribute != other_attribute: return False return self._target_satisfies(other, strict=True) def intersects(self, other: "ArchSpec") -> bool: """Return True if there exists at least one concrete spec that matches both self and other, otherwise False. This operation is commutative, and if two specs intersect it means that one can constrain the other. Args: other: spec to be checked for compatibility """ other = self._autospec(other) # Check platform and os for attribute in ("platform", "os"): other_attribute = getattr(other, attribute) self_attribute = getattr(self, attribute) if other_attribute and self_attribute and self_attribute != other_attribute: return False return self._target_satisfies(other, strict=False) def _target_satisfies(self, other: "ArchSpec", strict: bool) -> bool: if strict is True: need_to_check = bool(other.target) else: need_to_check = bool(other.target and self.target) if not need_to_check: return True # other_target is there and strict=True if self.target is None: return False # self.target is not None, and other is target=* if other.target == ArchSpec.ANY_TARGET: return True return bool(self._target_intersection(other)) def _target_constrain(self, other: "ArchSpec") -> bool: if self.target is None and other.target is None: return False if not other._target_satisfies(self, strict=False): raise UnsatisfiableArchitectureSpecError(self, other) if self.target_concrete: return False elif other.target_concrete: self.target = other.target return True # Compute the intersection of every combination of ranges in the lists results = self._target_intersection(other) attribute_str = ",".join(results) intersection_target = _make_microarchitecture(attribute_str) if self.target == intersection_target: return False self.target = intersection_target return True def _target_intersection(self, other): results = [] if not self.target or not other.target: return results for s_target_range in str(self.target).split(","): s_min, s_sep, s_max = s_target_range.partition(":") for o_target_range in str(other.target).split(","): o_min, o_sep, o_max = o_target_range.partition(":") if not s_sep: # s_target_range is a concrete target # get a microarchitecture reference for at least one side # of each comparison so we can use archspec comparators s_comp = _make_microarchitecture(s_min) if not o_sep: if s_min == o_min: results.append(s_min) elif (not o_min or s_comp >= o_min) and (not o_max or s_comp <= o_max): results.append(s_min) elif not o_sep: # "cast" to microarchitecture o_comp = _make_microarchitecture(o_min) if (not s_min or o_comp >= s_min) and (not s_max or o_comp <= s_max): results.append(o_min) else: # Take the "min" of the two max, if there is a partial ordering. n_max = "" if s_max and o_max: _s_max = _make_microarchitecture(s_max) _o_max = _make_microarchitecture(o_max) if _s_max.family != _o_max.family: continue if _s_max <= _o_max: n_max = s_max elif _o_max < _s_max: n_max = o_max else: continue elif s_max: n_max = s_max elif o_max: n_max = o_max # Take the "max" of the two min. n_min = "" if s_min and o_min: _s_min = _make_microarchitecture(s_min) _o_min = _make_microarchitecture(o_min) if _s_min.family != _o_min.family: continue if _s_min >= _o_min: n_min = s_min elif _o_min > _s_min: n_min = o_min else: continue elif s_min: n_min = s_min elif o_min: n_min = o_min if n_min and n_max: _n_min = _make_microarchitecture(n_min) _n_max = _make_microarchitecture(n_max) if _n_min.family != _n_max.family or not _n_min <= _n_max: continue if n_min == n_max: results.append(n_min) else: results.append(f"{n_min}:{n_max}") elif n_min: results.append(f"{n_min}:") elif n_max: results.append(f":{n_max}") return results def constrain(self, other: "ArchSpec") -> bool: """Projects all architecture fields that are specified in the given spec onto the instance spec if they're missing from the instance spec. This will only work if the two specs are compatible. Args: other (ArchSpec or str): constraints to be added Returns: True if the current instance was constrained, False otherwise. """ other = self._autospec(other) if not other.intersects(self): raise UnsatisfiableArchitectureSpecError(other, self) constrained = False for attr in ("platform", "os"): svalue, ovalue = getattr(self, attr), getattr(other, attr) if svalue is None and ovalue is not None: setattr(self, attr, ovalue) constrained = True constrained |= self._target_constrain(other) return constrained def copy(self): """Copy the current instance and returns the clone.""" return ArchSpec(self) @property def concrete(self): """True if the spec is concrete, False otherwise""" return self.platform and self.os and self.target and self.target_concrete @property def target_concrete(self): """True if the target is not a range or list.""" return ( self.target is not None and ":" not in str(self.target) and "," not in str(self.target) ) def to_dict(self): # Generic targets represent either an architecture family (like x86_64) # or a custom micro-architecture if self.target.vendor == "generic": target_data = str(self.target) else: # Get rid of compiler flag information before turning the uarch into a dict target_data = self.target.to_dict() target_data.pop("compilers", None) return {"arch": {"platform": self.platform, "platform_os": self.os, "target": target_data}} @staticmethod def from_dict(d): """Import an ArchSpec from raw YAML/JSON data""" arch = d["arch"] target_name = arch["target"] if not isinstance(target_name, str): target_name = target_name["name"] target = _make_microarchitecture(target_name) return ArchSpec((arch["platform"], arch["platform_os"], target)) def __str__(self): return "%s-%s-%s" % (self.platform, self.os, self.target) def __repr__(self): fmt = "ArchSpec(({0.platform!r}, {0.os!r}, {1!r}))" return fmt.format(self, str(self.target)) def __contains__(self, string): return string in str(self) or string in self.target def complete_with_defaults(self) -> None: default_architecture = ArchSpec.default_arch() if not self.platform: self.platform = default_architecture.platform if not self.os: self.os = default_architecture.os if not self.target: self.target = default_architecture.target class CompilerSpec: """Adaptor to the old compiler spec interface. Exposes just a few attributes""" def __init__(self, spec): self.spec = spec @property def name(self): return self.spec.name @property def version(self): return self.spec.version @property def versions(self): return self.spec.versions @property def display_str(self): """Equivalent to ``{compiler.name}{@compiler.version}`` for Specs, without extra ``@=`` for readability.""" if self.versions != vn.any_version: return self.spec.format("{name}{@version}") return self.spec.format("{name}") def __lt__(self, other): if not isinstance(other, CompilerSpec): return self.spec < other return self.spec < other.spec def __eq__(self, other): if not isinstance(other, CompilerSpec): return self.spec == other return self.spec == other.spec def __hash__(self): return hash(self.spec) def __str__(self): return str(self.spec) def _cmp_iter(self): return self.spec._cmp_iter() def __bool__(self): if self.spec == Spec(): return False return bool(self.spec) class DeprecatedCompilerSpec(lang.DeprecatedProperty): def __init__(self): super().__init__(name="compiler") def factory(self, instance, owner): if instance.original_spec_format() < 5: compiler = instance.annotations.compiler_node_attribute assert compiler is not None, "a compiler spec is expected" return CompilerSpec(compiler) for language in ("c", "cxx", "fortran"): deps = instance.dependencies(virtuals=language) if deps: return CompilerSpec(deps[0]) raise AttributeError(f"{instance} has no C, C++, or Fortran compiler") @lang.lazy_lexicographic_ordering class DependencySpec: """DependencySpecs represent an edge in the DAG, and contain dependency types and information on the virtuals being provided. Dependencies can be one (or more) of several types: - build: needs to be in the PATH at build time. - link: is linked to and added to compiler flags. - run: needs to be in the PATH for the package to run. Args: parent: starting node of the edge spec: ending node of the edge. depflag: represents dependency relationships. virtuals: virtual packages provided from child to parent node. """ __slots__ = "parent", "spec", "depflag", "virtuals", "direct", "when", "propagation" def __init__( self, parent: "Spec", spec: "Spec", *, depflag: dt.DepFlag, virtuals: Tuple[str, ...], direct: bool = False, propagation: PropagationPolicy = PropagationPolicy.NONE, when: Optional["Spec"] = None, ): if direct is False and propagation != PropagationPolicy.NONE: raise InvalidEdgeError("only direct dependencies can be propagated") self.parent = parent self.spec = spec self.depflag = depflag self.virtuals = tuple(sorted(set(virtuals))) self.direct = direct self.propagation = propagation self.when = when or EMPTY_SPEC def update_deptypes(self, depflag: dt.DepFlag) -> bool: """Update the current dependency types""" old = self.depflag new = depflag | old if new == old: return False self.depflag = new return True def update_virtuals(self, virtuals: Union[str, Iterable[str]]) -> bool: """Update the list of provided virtuals""" old = self.virtuals if isinstance(virtuals, str): union = {virtuals, *self.virtuals} else: union = {*virtuals, *self.virtuals} if len(union) == len(old): return False self.virtuals = tuple(sorted(union)) return True def copy(self, *, keep_virtuals: bool = True, keep_parent: bool = True) -> "DependencySpec": """Return a copy of this edge""" parent = self.parent if keep_parent else Spec() virtuals = self.virtuals if keep_virtuals else () return DependencySpec( parent, self.spec, depflag=self.depflag, virtuals=virtuals, propagation=self.propagation, direct=self.direct, when=self.when, ) def _constrain(self, other: "DependencySpec") -> bool: """Constrain this edge with another edge. Precondition: parent and child of self and other are compatible, and both edges have the same when condition. Used as an internal helper function in Spec.constrain. Args: other: edge to use as constraint Returns: True if the current edge was changed, False otherwise. """ changed = False changed |= self.spec.constrain(other.spec) changed |= self.update_deptypes(other.depflag) changed |= self.update_virtuals(other.virtuals) if not self.direct and other.direct: changed = True self.direct = True return changed def _cmp_iter(self): yield self.parent.name if self.parent else None yield self.spec.name if self.spec else None yield self.depflag yield self.virtuals yield self.direct yield self.propagation yield self.when def __str__(self) -> str: return self.format() def __repr__(self) -> str: keywords = [f"depflag={self.depflag}", f"virtuals={self.virtuals}"] if self.direct: keywords.append(f"direct={self.direct}") if self.when != Spec(): keywords.append(f"when={self.when}") if self.propagation != PropagationPolicy.NONE: keywords.append(f"propagation={self.propagation}") keywords_str = ", ".join(keywords) return f"DependencySpec({self.parent.format()!r}, {self.spec.format()!r}, {keywords_str})" def format(self, *, unconditional: bool = False) -> str: """Returns a string, using the spec syntax, representing this edge Args: unconditional: if True, removes any condition statement from the representation """ parent_str, child_str = self.parent.format(), self.spec.format() virtuals_str = f"virtuals={','.join(self.virtuals)}" if self.virtuals else "" when_str = "" if not unconditional and self.when != Spec(): when_str = f"when='{self.when}'" dep_sigil = "%" if self.direct else "^" if self.propagation == PropagationPolicy.PREFERENCE: dep_sigil = "%%" edge_attrs = [x for x in (virtuals_str, when_str) if x] if edge_attrs: return f"{parent_str} {dep_sigil}[{' '.join(edge_attrs)}] {child_str}" return f"{parent_str} {dep_sigil}{child_str}" def flip(self) -> "DependencySpec": """Flips the dependency and keeps its type. Drops all other information.""" return DependencySpec( parent=self.spec, spec=self.parent, depflag=self.depflag, virtuals=() ) class CompilerFlag(str): """Will store a flag value and it's propagation value Args: value (str): the flag's value propagate (bool): if ``True`` the flag value will be passed to the package's dependencies. If ``False`` it will not flag_group (str): if this flag was introduced along with several flags via a single source, then this will store all such flags source (str): identifies the type of constraint that introduced this flag (e.g. if a package has ``depends_on(... cflags=-g)``, then the ``source`` for "-g" would indicate ``depends_on``. """ propagate: bool flag_group: str source: str def __new__(cls, value, **kwargs): obj = str.__new__(cls, value) obj.propagate = kwargs.pop("propagate", False) obj.flag_group = kwargs.pop("flag_group", value) obj.source = kwargs.pop("source", None) return obj _valid_compiler_flags = ["cflags", "cxxflags", "fflags", "ldflags", "ldlibs", "cppflags"] def _shared_subset_pair_iterate(container1, container2): """ [0, a, c, d, f] [a, d, e, f] yields [(a, a), (d, d), (f, f)] no repeated elements """ a_idx, b_idx = 0, 0 max_a, max_b = len(container1), len(container2) while a_idx < max_a and b_idx < max_b: if container1[a_idx] == container2[b_idx]: yield (container1[a_idx], container2[b_idx]) a_idx += 1 b_idx += 1 else: while container1[a_idx] < container2[b_idx]: a_idx += 1 while container1[a_idx] > container2[b_idx]: b_idx += 1 class FlagMap(lang.HashableMap[str, List[CompilerFlag]]): def satisfies(self, other): return all(f in self and set(self[f]) >= set(other[f]) for f in other) def intersects(self, other): return True def constrain(self, other): """Add all flags in other that aren't in self to self. Return whether the spec changed. """ changed = False for flag_type in other: if flag_type not in self: self[flag_type] = other[flag_type] changed = True else: extra_other = set(other[flag_type]) - set(self[flag_type]) if extra_other: self[flag_type] = list(self[flag_type]) + list( x for x in other[flag_type] if x in extra_other ) changed = True # Next, if any flags in other propagate, we force them to propagate in our case shared = list(sorted(set(other[flag_type]) - extra_other)) for x, y in _shared_subset_pair_iterate(shared, sorted(self[flag_type])): if y.propagate is True and x.propagate is False: changed = True y.propagate = False # TODO: what happens if flag groups with a partial (but not complete) # intersection specify different behaviors for flag propagation? return changed @staticmethod def valid_compiler_flags(): return _valid_compiler_flags def copy(self): clone = FlagMap() for name, compiler_flag in self.items(): clone[name] = compiler_flag return clone def add_flag(self, flag_type, value, propagation, flag_group=None, source=None): """Stores the flag's value in CompilerFlag and adds it to the FlagMap Args: flag_type (str): the type of flag value (str): the flag's value that will be added to the flag_type's corresponding list propagation (bool): if ``True`` the flag value will be passed to the packages' dependencies. If``False`` it will not be passed """ flag_group = flag_group or value flag = CompilerFlag(value, propagate=propagation, flag_group=flag_group, source=source) if flag_type not in self: self[flag_type] = [flag] else: self[flag_type].append(flag) def yaml_entry(self, flag_type): """Returns the flag type and a list of the flag values since the propagation values aren't needed when writing to yaml Args: flag_type (str): the type of flag to get values from Returns the flag_type and a list of the corresponding flags in string format """ return flag_type, [str(flag) for flag in self[flag_type]] def _cmp_iter(self): for k, v in sorted(self.dict.items()): yield k def flags(): for flag in v: yield flag yield flag.propagate yield flags def __str__(self): if not self: return "" sorted_items = sorted((k, v) for k, v in self.items() if v) result = "" for flag_type, flags in sorted_items: normal = [f for f in flags if not f.propagate] if normal: value = spack.spec_parser.quote_if_needed(" ".join(normal)) result += f" {flag_type}={value}" propagated = [f for f in flags if f.propagate] if propagated: value = spack.spec_parser.quote_if_needed(" ".join(propagated)) result += f" {flag_type}=={value}" # TODO: somehow add this space only if something follows in Spec.format() if sorted_items: result += " " return result def _sort_by_dep_types(dspec: DependencySpec): return dspec.depflag EdgeMap = Dict[str, List[DependencySpec]] def _add_edge_to_map(edge_map: EdgeMap, key: str, edge: DependencySpec) -> None: if key in edge_map: lst = edge_map[key] lst.append(edge) lst.sort(key=_sort_by_dep_types) else: edge_map[key] = [edge] def _select_edges( edge_map: EdgeMap, *, parent: Optional[str] = None, child: Optional[str] = None, depflag: dt.DepFlag = dt.ALL, virtuals: Optional[Union[str, Sequence[str]]] = None, ) -> List[DependencySpec]: """Selects a list of edges and returns them. If an edge: - Has *any* of the dependency types passed as argument, - Matches the parent and/or child name - Provides *any* of the virtuals passed as argument then it is selected. Args: edge_map: map of edges to select from parent: name of the parent package child: name of the child package depflag: allowed dependency types in flag form virtuals: list of virtuals or specific virtual on the edge """ if not depflag: return [] # Start from all the edges we store selected: Iterable[DependencySpec] = itertools.chain.from_iterable(edge_map.values()) # Filter by parent name if parent: selected = (d for d in selected if d.parent.name == parent) # Filter by child name if child: selected = (d for d in selected if d.spec.name == child) # Filter by allowed dependency types if depflag != dt.ALL: selected = (dep for dep in selected if not dep.depflag or (depflag & dep.depflag)) # Filter by virtuals if virtuals is not None: if isinstance(virtuals, str): selected = (dep for dep in selected if virtuals in dep.virtuals) else: selected = (dep for dep in selected if any(v in dep.virtuals for v in virtuals)) return list(selected) def _headers_default_handler(spec: "Spec"): """Default handler when looking for the 'headers' attribute. Tries to search for ``*.h`` files recursively starting from ``spec.package.home.include``. Parameters: spec: spec that is being queried Returns: HeaderList: The headers in ``prefix.include`` Raises: NoHeadersError: If no headers are found """ home = getattr(spec.package, "home") headers = fs.find_headers("*", root=home.include, recursive=True) if headers: return headers raise spack.error.NoHeadersError(f"Unable to locate {spec.name} headers in {home}") def _libs_default_handler(spec: "Spec"): """Default handler when looking for the 'libs' attribute. Tries to search for ``lib{spec.name}`` recursively starting from ``spec.package.home``. If ``spec.name`` starts with ``lib``, searches for ``{spec.name}`` instead. Parameters: spec: spec that is being queried Returns: LibraryList: The libraries found Raises: NoLibrariesError: If no libraries are found """ # Variable 'name' is passed to function 'find_libraries', which supports # glob characters. For example, we have a package with a name 'abc-abc'. # Now, we don't know if the original name of the package is 'abc_abc' # (and it generates a library 'libabc_abc.so') or 'abc-abc' (and it # generates a library 'libabc-abc.so'). So, we tell the function # 'find_libraries' to give us anything that matches 'libabc?abc' and it # gives us either 'libabc-abc.so' or 'libabc_abc.so' (or an error) # depending on which one exists (there is a possibility, of course, to # get something like 'libabcXabc.so, but for now we consider this # unlikely). name = spec.name.replace("-", "?") home = getattr(spec.package, "home") # Avoid double 'lib' for packages whose names already start with lib if not name.startswith("lib") and not spec.satisfies("platform=windows"): name = "lib" + name # If '+shared' search only for shared library; if '~shared' search only for # static library; otherwise, first search for shared and then for static. search_shared = ( [True] if ("+shared" in spec) else ([False] if ("~shared" in spec) else [True, False]) ) for shared in search_shared: # Since we are searching for link libraries, on Windows search only for # ".Lib" extensions by default as those represent import libraries for implicit links. libs = fs.find_libraries(name, home, shared=shared, recursive=True, runtime=False) if libs: return libs raise spack.error.NoLibrariesError( f"Unable to recursively locate {spec.name} libraries in {home}" ) class ForwardQueryToPackage: """Descriptor used to forward queries from Spec to Package""" def __init__( self, attribute_name: str, default_handler: Optional[Callable[["Spec"], Any]] = None, _indirect: bool = False, ) -> None: """Create a new descriptor. Parameters: attribute_name: name of the attribute to be searched for in the Package instance default_handler: default function to be called if the attribute was not found in the Package instance _indirect: temporarily added to redirect a query to another package. """ self.attribute_name = attribute_name self.default = default_handler self.indirect = _indirect def __get__(self, instance: "SpecBuildInterface", cls): """Retrieves the property from Package using a well defined chain of responsibility. The order of call is: 1. if the query was through the name of a virtual package try to search for the attribute ``{virtual_name}_{attribute_name}`` in Package 2. try to search for attribute ``{attribute_name}`` in Package 3. try to call the default handler The first call that produces a value will stop the chain. If no call can handle the request then AttributeError is raised with a message indicating that no relevant attribute exists. If a call returns None, an AttributeError is raised with a message indicating a query failure, e.g. that library files were not found in a 'libs' query. """ # TODO: this indirection exist solely for `spec["python"].command` to actually return # spec["python-venv"].command. It should be removed when `python` is a virtual. if self.indirect and instance.indirect_spec: pkg = instance.indirect_spec.package else: pkg = instance.wrapped_obj.package try: query = instance.last_query except AttributeError: # There has been no query yet: this means # a spec is trying to access its own attributes _ = instance.wrapped_obj[instance.wrapped_obj.name] # NOQA: ignore=F841 query = instance.last_query callbacks_chain = [] # First in the chain : specialized attribute for virtual packages if query.isvirtual: specialized_name = "{0}_{1}".format(query.name, self.attribute_name) callbacks_chain.append(lambda: getattr(pkg, specialized_name)) # Try to get the generic method from Package callbacks_chain.append(lambda: getattr(pkg, self.attribute_name)) # Final resort : default callback if self.default is not None: _default = self.default # make mypy happy callbacks_chain.append(lambda: _default(instance.wrapped_obj)) # Trigger the callbacks in order, the first one producing a # value wins value = None message = None for f in callbacks_chain: try: value = f() # A callback can return None to trigger an error indicating # that the query failed. if value is None: msg = "Query of package '{name}' for '{attrib}' failed\n" msg += "\tprefix : {spec.prefix}\n" msg += "\tspec : {spec}\n" msg += "\tqueried as : {query.name}\n" msg += "\textra parameters : {query.extra_parameters}" message = msg.format( name=pkg.name, attrib=self.attribute_name, spec=instance, query=instance.last_query, ) else: return value break except AttributeError: pass # value is 'None' if message is not None: # Here we can use another type of exception. If we do that, the # unit test 'test_getitem_exceptional_paths' in the file # lib/spack/spack/test/spec_dag.py will need to be updated to match # the type. raise AttributeError(message) # 'None' value at this point means that there are no appropriate # properties defined and no default handler, or that all callbacks # raised AttributeError. In this case, we raise AttributeError with an # appropriate message. fmt = "'{name}' package has no relevant attribute '{query}'\n" fmt += "\tspec : '{spec}'\n" fmt += "\tqueried as : '{spec.last_query.name}'\n" fmt += "\textra parameters : '{spec.last_query.extra_parameters}'\n" message = fmt.format(name=pkg.name, query=self.attribute_name, spec=instance) raise AttributeError(message) def __set__(self, instance, value): cls_name = type(instance).__name__ msg = "'{0}' object attribute '{1}' is read-only" raise AttributeError(msg.format(cls_name, self.attribute_name)) # Represents a query state in a BuildInterface object QueryState = collections.namedtuple("QueryState", ["name", "extra_parameters", "isvirtual"]) def tree( specs: List["Spec"], *, color: Optional[bool] = None, depth: bool = False, hashes: bool = False, hashlen: Optional[int] = None, cover: spack.traverse.CoverType = "nodes", indent: int = 0, format: str = DEFAULT_FORMAT, deptypes: Union[dt.DepFlag, dt.DepTypes] = dt.ALL, show_types: bool = False, depth_first: bool = False, recurse_dependencies: bool = True, status_fn: Optional[Callable[["Spec"], InstallStatus]] = None, prefix: Optional[Callable[["Spec"], str]] = None, key: Callable[["Spec"], Any] = id, highlight_version_fn: Optional[Callable[["Spec"], bool]] = None, highlight_variant_fn: Optional[Callable[["Spec", str], bool]] = None, ) -> str: """Prints out specs and their dependencies, tree-formatted with indentation. Status function may either output a boolean or an InstallStatus Args: color: if True, always colorize the tree. If False, don't colorize the tree. If None, use the default from spack.llnl.tty.color depth: print the depth from the root hashes: if True, print the hash of each node hashlen: length of the hash to be printed cover: either ``"nodes"`` or ``"edges"`` indent: extra indentation for the tree being printed format: format to be used to print each node deptypes: dependency types to be represented in the tree show_types: if True, show the (merged) dependency type of a node depth_first: if True, traverse the DAG depth first when representing it as a tree recurse_dependencies: if True, recurse on dependencies status_fn: optional callable that takes a node as an argument and return its installation status prefix: optional callable that takes a node as an argument and return its installation prefix highlight_version_fn: optional callable that returns true on nodes where the version needs to be highlighted highlight_variant_fn: optional callable that returns true on variants that need to be highlighted """ out = "" if color is None: color = clr.get_color_when() # reduce deptypes over all in-edges when covering nodes if show_types and cover == "nodes": deptype_lookup: Dict[str, dt.DepFlag] = collections.defaultdict(dt.DepFlag) for edge in spack.traverse.traverse_edges( specs, cover="edges", deptype=deptypes, root=False ): deptype_lookup[edge.spec.dag_hash()] |= edge.depflag # SupportsRichComparisonT issue with List[Spec] sorted_specs: List["Spec"] = sorted(specs) # type: ignore[type-var] for d, dep_spec in spack.traverse.traverse_tree( sorted_specs, cover=cover, deptype=deptypes, depth_first=depth_first, key=key ): node = dep_spec.spec if prefix is not None: out += prefix(node) out += " " * indent if depth: out += "%-4d" % d if status_fn: status = status_fn(node) if status in list(InstallStatus): out += clr.colorize(status.value, color=color) elif status: out += clr.colorize("@g{[+]} ", color=color) else: out += clr.colorize("@r{[-]} ", color=color) if hashes: out += clr.colorize("@K{%s} ", color=color) % node.dag_hash(hashlen) if show_types: if cover == "nodes": depflag = deptype_lookup[dep_spec.spec.dag_hash()] else: # when covering edges or paths, we show dependency # types only for the edge through which we visited depflag = dep_spec.depflag type_chars = dt.flag_to_chars(depflag) out += "[%s] " % type_chars out += " " * d if d > 0: out += "^" out += ( node.format( format, color=color, highlight_version_fn=highlight_version_fn, highlight_variant_fn=highlight_variant_fn, ) + "\n" ) # Check if we wanted just the first line if not recurse_dependencies: break return out class SpecAnnotations: def __init__(self) -> None: self.original_spec_format = SPECFILE_FORMAT_VERSION self.compiler_node_attribute: Optional["Spec"] = None def with_spec_format(self, spec_format: int) -> "SpecAnnotations": self.original_spec_format = spec_format return self def with_compiler(self, compiler: "Spec") -> "SpecAnnotations": self.compiler_node_attribute = compiler return self def __repr__(self) -> str: result = f"SpecAnnotations().with_spec_format({self.original_spec_format})" if self.compiler_node_attribute: result += f".with_compiler({str(self.compiler_node_attribute)})" return result def _anonymous_star(dep: DependencySpec, dep_format: str) -> str: """Determine if a spec needs a star to disambiguate it from an anonymous spec w/variants. Returns: "*" if a star is needed, "" otherwise """ # named spec never needs star if dep.spec.name: return "" # virtuals without a name always need *: %c=* @4.0 foo=bar if dep.virtuals: return "*" # versions are first so checking for @ is faster than != VersionList(':') if dep_format.startswith("@"): return "" # compiler flags are key-value pairs and can be ambiguous with virtual assignment if dep.spec.compiler_flags: return "*" # booleans come first, and they don't need a star. key-value pairs do. If there are # no key value pairs, we're left with either an empty spec, which needs * as in # '^*', or we're left with arch, which is a key value pair, and needs a star. if not any(v.type == vt.VariantType.BOOL for v in dep.spec.variants.values()): return "*" return "*" if dep.spec.architecture else "" def _get_satisfying_edge( lhs_node: "Spec", rhs_edge: DependencySpec, *, resolve_virtuals: bool ) -> Optional[DependencySpec]: """Search for an edge in ``lhs_node`` that satisfies ``rhs_edge``.""" # First check direct deps of all types. for lhs_edge in lhs_node.edges_to_dependencies(): if _satisfies_edge(lhs_edge, rhs_edge, resolve_virtuals): return lhs_edge # Include the historical compiler node if available as an ad-hoc edge. compiler_spec = lhs_node.annotations.compiler_node_attribute if compiler_spec is not None: compiler_edge = DependencySpec( lhs_node, compiler_spec, depflag=dt.BUILD, virtuals=("c", "cxx", "fortran"), direct=True, ) if _satisfies_edge(compiler_edge, rhs_edge, resolve_virtuals): return compiler_edge if rhs_edge.direct: return None # BFS through link/run transitive deps (skip depth 1, already checked). depflag = dt.LINK | dt.RUN queue = collections.deque(lhs_node.edges_to_dependencies(depflag=depflag)) seen = {id(lhs_edge.spec) for lhs_edge in queue} while queue: lhs_edge = queue.popleft() if _satisfies_edge(lhs_edge, rhs_edge, resolve_virtuals): return lhs_edge for lhs_edge in lhs_edge.spec.edges_to_dependencies(depflag=depflag): if id(lhs_edge.spec) not in seen: seen.add(id(lhs_edge.spec)) queue.append(lhs_edge) return None def _satisfies_edge(lhs: "DependencySpec", rhs: "DependencySpec", resolve_virtuals: bool) -> bool: """Helper function for satisfaction tests, which checks edge attributes and the target node. It skips verification of the parent node.""" name_mismatch = rhs.spec.name and lhs.spec.name != rhs.spec.name if name_mismatch and rhs.spec.name not in lhs.virtuals: return False if not rhs.when._satisfies(lhs.when, resolve_virtuals=resolve_virtuals): return False # Subset semantics for virtuals for v in rhs.virtuals: if v not in lhs.virtuals: return False # Subset semantics for dependency types if (lhs.depflag & rhs.depflag) != rhs.depflag: return False if not name_mismatch: return lhs.spec._satisfies_node(rhs.spec, resolve_virtuals=resolve_virtuals) # Right-hand side is virtual provided by left-hand side. The only node attribute supported is # the version of the virtual. Avoid expensive lookups for provider metadata if there's no # version constraint to check. if rhs.spec.versions == spack.version.any_version: return True if not resolve_virtuals: return False return lhs.spec._provides_virtual(rhs.spec) @lang.lazy_lexicographic_ordering(set_hash=False) class Spec: compiler = DeprecatedCompilerSpec() @staticmethod def default_arch(): """Return an anonymous spec for the default architecture""" s = Spec() s.architecture = ArchSpec.default_arch() return s def __init__(self, spec_like=None, *, external_path=None, external_modules=None): """Create a new Spec. Arguments: spec_like: if not provided, we initialize an anonymous Spec that matches any Spec; if provided we parse this as a Spec string, or we copy the provided Spec. Keyword arguments: external_path: prefix, if this is a spec for an external package external_modules: list of external modules, for an external package using modules """ # Copy if spec_like is a Spec. if isinstance(spec_like, Spec): self._dup(spec_like) return # init an empty spec that matches anything. self.name: str = "" self.versions = vn.VersionList.any() self.variants = VariantMap() self.architecture = None self.compiler_flags = FlagMap() self._dependents = {} self._dependencies = {} self.namespace = None self.abstract_hash = None # initial values for all spec hash types for h in ht.HASHES: setattr(self, h.attr, None) # cache for spec's prefix, computed lazily by prefix property self._prefix = None # Python __hash__ is handled separately from the cached spec hashes self._dunder_hash = None # cache of package for this spec self._package = None # whether the spec is concrete or not; set at the end of concretization self._concrete = False # External detection details that can be set by internal Spack calls # in the constructor. self._external_path = external_path self.external_modules = Spec._format_module_list(external_modules) # This attribute is used to store custom information for external specs. self.extra_attributes: Dict[str, Any] = {} # This attribute holds the original build copy of the spec if it is # deployed differently than it was built. None signals that the spec # is deployed "as built." # Build spec should be the actual build spec unless marked dirty. self._build_spec = None self.annotations = SpecAnnotations() if isinstance(spec_like, str): spack.spec_parser.parse_one_or_raise(spec_like, self) elif spec_like is not None: raise TypeError(f"Can't make spec out of {type(spec_like)}") @staticmethod def _format_module_list(modules): """Return a module list that is suitable for YAML serialization and hash computation. Given a module list, possibly read from a configuration file, return an object that serializes to a consistent YAML string before/after round-trip serialization to/from a Spec dictionary (stored in JSON format): when read in, the module list may contain YAML formatting that is discarded (non-essential) when stored as a Spec dictionary; we take care in this function to discard such formatting such that the Spec hash does not change before/after storage in JSON. """ if modules: modules = list(modules) return modules @property def external_path(self): return spack.llnl.path.path_to_os_path(self._external_path)[0] @external_path.setter def external_path(self, ext_path): self._external_path = ext_path @property def external(self): return bool(self.external_path) or bool(self.external_modules) @property def is_develop(self): """Return whether the Spec represents a user-developed package in a Spack Environment (i.e. using ``spack develop``). """ return bool(self.variants.get("dev_path", False)) def clear_dependencies(self): """Trim the dependencies of this spec.""" self._dependencies.clear() def clear_edges(self): """Trim the dependencies and dependents of this spec.""" self._dependencies.clear() self._dependents.clear() def detach(self, deptype="all"): """Remove any reference that dependencies have of this node. Args: deptype (str or tuple): dependency types tracked by the current spec """ key = self.dag_hash() # Go through the dependencies for dep in self.dependencies(deptype=deptype): # Remove the spec from dependents if self.name in dep._dependents: dependents_copy = dep._dependents[self.name] del dep._dependents[self.name] for edge in dependents_copy: if edge.parent.dag_hash() == key: continue _add_edge_to_map(dep._dependents, edge.parent.name, edge) def _get_dependency(self, name): # WARNING: This function is an implementation detail of the # WARNING: original concretizer. Since with that greedy # WARNING: algorithm we don't allow multiple nodes from # WARNING: the same package in a DAG, here we hard-code # WARNING: using index 0 i.e. we assume that we have only # WARNING: one edge from package "name" deps = self.edges_to_dependencies(name=name) if len(deps) != 1: err_msg = 'expected only 1 "{0}" dependency, but got {1}' raise spack.error.SpecError(err_msg.format(name, len(deps))) return deps[0] def edges_from_dependents( self, name: Optional[str] = None, depflag: dt.DepFlag = dt.ALL, *, virtuals: Optional[Union[str, Sequence[str]]] = None, ) -> List[DependencySpec]: """Return a list of edges connecting this node in the DAG to parents. Args: name: filter dependents by package name depflag: allowed dependency types virtuals: allowed virtuals """ return _select_edges(self._dependents, parent=name, depflag=depflag, virtuals=virtuals) def edges_to_dependencies( self, name: Optional[str] = None, depflag: dt.DepFlag = dt.ALL, *, virtuals: Optional[Union[str, Sequence[str]]] = None, ) -> List[DependencySpec]: """Returns a list of edges connecting this node in the DAG to children. Args: name: filter dependencies by package name depflag: allowed dependency types virtuals: allowed virtuals """ return _select_edges(self._dependencies, child=name, depflag=depflag, virtuals=virtuals) @property def edge_attributes(self) -> str: """Helper method to print edge attributes in spec strings.""" edges = self.edges_from_dependents() if not edges: return "" union = DependencySpec(parent=Spec(), spec=self, depflag=0, virtuals=()) all_direct_edges = all(x.direct for x in edges) dep_conditions = set() for edge in edges: union.update_deptypes(edge.depflag) union.update_virtuals(edge.virtuals) dep_conditions.add(edge.when) deptypes_str = "" if not all_direct_edges and union.depflag: deptypes_str = f"deptypes={','.join(dt.flag_to_tuple(union.depflag))}" virtuals_str = f"virtuals={','.join(union.virtuals)}" if union.virtuals else "" conditions = [str(c) for c in dep_conditions if c != Spec()] when_str = f"when='{','.join(conditions)}'" if conditions else "" result = " ".join(filter(lambda x: bool(x), (when_str, deptypes_str, virtuals_str))) if result: result = f"[{result}]" return result def dependencies( self, name: Optional[str] = None, deptype: Union[dt.DepTypes, dt.DepFlag] = dt.ALL, *, virtuals: Optional[Union[str, Sequence[str]]] = None, ) -> List["Spec"]: """Returns a list of direct dependencies (nodes in the DAG) Args: name: filter dependencies by package name deptype: allowed dependency types virtuals: allowed virtuals """ if not isinstance(deptype, dt.DepFlag): deptype = dt.canonicalize(deptype) return [ d.spec for d in self.edges_to_dependencies(name, depflag=deptype, virtuals=virtuals) ] def dependents( self, name: Optional[str] = None, deptype: Union[dt.DepTypes, dt.DepFlag] = dt.ALL ) -> List["Spec"]: """Return a list of direct dependents (nodes in the DAG). Args: name: filter dependents by package name deptype: allowed dependency types """ if not isinstance(deptype, dt.DepFlag): deptype = dt.canonicalize(deptype) return [d.parent for d in self.edges_from_dependents(name, depflag=deptype)] def _dependencies_dict(self, depflag: dt.DepFlag = dt.ALL): """Return a dictionary, keyed by package name, of the direct dependencies. Each value in the dictionary is a list of edges. Args: deptype: allowed dependency types """ _sort_fn = lambda x: (x.spec.name, _sort_by_dep_types(x)) _group_fn = lambda x: x.spec.name selected_edges = _select_edges(self._dependencies, depflag=depflag) result = {} for key, group in itertools.groupby(sorted(selected_edges, key=_sort_fn), key=_group_fn): result[key] = list(group) return result def _add_flag( self, name: str, value: Union[str, bool], propagate: bool, concrete: bool ) -> None: """Called by the parser to add a known flag""" if propagate and name in vt.RESERVED_NAMES: raise UnsupportedPropagationError( f"Propagation with '==' is not supported for '{name}'." ) valid_flags = FlagMap.valid_compiler_flags() if name == "arch" or name == "architecture": assert type(value) is str, "architecture have a string value" parts = tuple(value.split("-")) plat, os, tgt = parts if len(parts) == 3 else (None, None, value) self._set_architecture(platform=plat, os=os, target=tgt) elif name == "platform": self._set_architecture(platform=value) elif name == "os" or name == "operating_system": self._set_architecture(os=value) elif name == "target": self._set_architecture(target=value) elif name == "namespace": self.namespace = value elif name in valid_flags: assert self.compiler_flags is not None assert type(value) is str, f"{name} must have a string value" flags_and_propagation = spack.compilers.flags.tokenize_flags(value, propagate) flag_group = " ".join(x for (x, y) in flags_and_propagation) for flag, propagation in flags_and_propagation: self.compiler_flags.add_flag(name, flag, propagation, flag_group) else: self.variants[name] = vt.VariantValue.from_string_or_bool( name, value, propagate=propagate, concrete=concrete ) def _set_architecture(self, **kwargs): """Called by the parser to set the architecture.""" arch_attrs = ["platform", "os", "target"] if self.architecture and self.architecture.concrete: raise DuplicateArchitectureError("Spec cannot have two architectures.") if not self.architecture: new_vals = tuple(kwargs.get(arg, None) for arg in arch_attrs) self.architecture = ArchSpec(new_vals) else: new_attrvals = [(a, v) for a, v in kwargs.items() if a in arch_attrs] for new_attr, new_value in new_attrvals: if getattr(self.architecture, new_attr): raise DuplicateArchitectureError(f"Cannot specify '{new_attr}' twice") else: setattr(self.architecture, new_attr, new_value) def _add_dependency( self, spec: "Spec", *, depflag: dt.DepFlag, virtuals: Tuple[str, ...], direct: bool = False, propagation: PropagationPolicy = PropagationPolicy.NONE, when: Optional["Spec"] = None, ): """Called by the parser to add another spec as a dependency. Args: depflag: dependency type for this edge virtuals: virtuals on this edge direct: if True denotes a direct dependency (associated with the % sigil) propagation: propagation policy for this edge when: optional condition under which dependency holds """ if when is None: when = EMPTY_SPEC if spec.name not in self._dependencies or not spec.name: self.add_dependency_edge( spec, depflag=depflag, virtuals=virtuals, direct=direct, when=when, propagation=propagation, ) return # Keep the intersection of constraints when a dependency is added multiple times with # the same deptype. Add a new dependency if it is added with a compatible deptype # (for example, a build-only dependency is compatible with a link-only dependency). # The only restrictions, currently, are that we cannot add edges with overlapping # dependency types and we cannot add multiple edges that have link/run dependency types. # See ``spack.deptypes.compatible``. orig = self._dependencies[spec.name] try: dspec = next( dspec for dspec in orig if depflag == dspec.depflag and when == dspec.when ) except StopIteration: # Error if we have overlapping or incompatible deptypes if any(not dt.compatible(dspec.depflag, depflag) for dspec in orig) and all( dspec.when == when for dspec in orig ): edge_attrs = f"deptypes={dt.flag_to_chars(depflag).strip()}" required_dep_str = f"^[{edge_attrs}] {str(spec)}" raise DuplicateDependencyError( f"{spec.name} is a duplicate dependency, with conflicting dependency types\n" f"\t'{str(self)}' cannot depend on '{required_dep_str}'" ) self.add_dependency_edge( spec, depflag=depflag, virtuals=virtuals, direct=direct, when=when ) return try: dspec.spec.constrain(spec) dspec.update_virtuals(virtuals=virtuals) except spack.error.UnsatisfiableSpecError: raise DuplicateDependencyError( f"Cannot depend on incompatible specs '{dspec.spec}' and '{spec}'" ) def add_dependency_edge( self, dependency_spec: "Spec", *, depflag: dt.DepFlag, virtuals: Tuple[str, ...], direct: bool = False, propagation: PropagationPolicy = PropagationPolicy.NONE, when: Optional["Spec"] = None, ): """Add a dependency edge to this spec. Args: dependency_spec: spec of the dependency depflag: dependency type for this edge virtuals: virtuals provided by this edge direct: if True denotes a direct dependency propagation: propagation policy for this edge when: if non-None, condition under which dependency holds """ if when is None: when = EMPTY_SPEC # Check if we need to update edges that are already present selected = self._dependencies.get(dependency_spec.name, []) for edge in selected: has_errors, details = False, [] msg = f"cannot update the edge from {edge.parent.name} to {edge.spec.name}" if edge.when != when: continue # If the dependency is to an existing spec, we can update dependency # types. If it is to a new object, check deptype compatibility. if id(edge.spec) != id(dependency_spec) and not dt.compatible(edge.depflag, depflag): has_errors = True details.append( ( f"{edge.parent.name} has already an edge matching any" f" of these types {depflag}" ) ) if any(v in edge.virtuals for v in virtuals): details.append( ( f"{edge.parent.name} has already an edge matching any" f" of these virtuals {virtuals}" ) ) if has_errors: raise spack.error.SpecError(msg, "\n".join(details)) for edge in selected: if id(dependency_spec) == id(edge.spec) and edge.when == when: # If we are here, it means the edge object was previously added to # both the parent and the child. When we update this object they'll # both see the deptype modification. edge.update_deptypes(depflag=depflag) edge.update_virtuals(virtuals=virtuals) return edge = DependencySpec( self, dependency_spec, depflag=depflag, virtuals=virtuals, direct=direct, propagation=propagation, when=when, ) _add_edge_to_map(self._dependencies, edge.spec.name, edge) _add_edge_to_map(dependency_spec._dependents, edge.parent.name, edge) # # Public interface # @property def fullname(self): return ( f"{self.namespace}.{self.name}" if self.namespace else (self.name if self.name else "") ) @property def anonymous(self): return not self.name and not self.abstract_hash @property def root(self): """Follow dependent links and find the root of this spec's DAG. Spack specs have a single root (the package being installed). """ # FIXME: In the case of multiple parents this property does not # FIXME: make sense. Should we revisit the semantics? if not self._dependents: return self edges_by_package = next(iter(self._dependents.values())) return edges_by_package[0].parent.root @property def package(self): assert self.concrete, "{0}: Spec.package can only be called on concrete specs".format( self.name ) if not self._package: self._package = spack.repo.PATH.get(self) return self._package @property def concrete(self): """A spec is concrete if it describes a single build of a package. More formally, a spec is concrete if concretize() has been called on it and it has been marked ``_concrete``. Concrete specs either can be or have been built. All constraints have been resolved, optional dependencies have been added or removed, a compiler has been chosen, and all variants have values. """ return self._concrete @property def spliced(self): """Returns whether or not this Spec is being deployed as built i.e. whether or not this Spec has ever been spliced. """ return any(s.build_spec is not s for s in self.traverse(root=True)) @property def installed(self): """Installation status of a package. Returns: True if the package has been installed, False otherwise. """ if not self.concrete: return False try: # If the spec is in the DB, check the installed # attribute of the record from spack.store import STORE return STORE.db.get_record(self).installed except KeyError: # If the spec is not in the DB, the method # above raises a Key error return False @property def installed_upstream(self): """Whether the spec is installed in an upstream repository. Returns: True if the package is installed in an upstream, False otherwise. """ if not self.concrete: return False from spack.store import STORE upstream, record = STORE.db.query_by_spec_hash(self.dag_hash()) return upstream and record and record.installed @overload def traverse( self, *, root: bool = ..., order: spack.traverse.OrderType = ..., cover: spack.traverse.CoverType = ..., direction: spack.traverse.DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: Literal[False] = False, key: Callable[["Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable["Spec"]: ... @overload def traverse( self, *, root: bool = ..., order: spack.traverse.OrderType = ..., cover: spack.traverse.CoverType = ..., direction: spack.traverse.DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: Literal[True], key: Callable[["Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable[Tuple[int, "Spec"]]: ... def traverse( self, *, root: bool = True, order: spack.traverse.OrderType = "pre", cover: spack.traverse.CoverType = "nodes", direction: spack.traverse.DirectionType = "children", deptype: Union[dt.DepFlag, dt.DepTypes] = "all", depth: bool = False, key: Callable[["Spec"], Any] = id, visited: Optional[Set[Any]] = None, ) -> Iterable[Union["Spec", Tuple[int, "Spec"]]]: """Shorthand for :meth:`~spack.traverse.traverse_nodes`""" return spack.traverse.traverse_nodes( [self], root=root, order=order, cover=cover, direction=direction, deptype=deptype, depth=depth, key=key, visited=visited, ) @overload def traverse_edges( self, *, root: bool = ..., order: spack.traverse.OrderType = ..., cover: spack.traverse.CoverType = ..., direction: spack.traverse.DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: Literal[False] = False, key: Callable[["Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable[DependencySpec]: ... @overload def traverse_edges( self, *, root: bool = ..., order: spack.traverse.OrderType = ..., cover: spack.traverse.CoverType = ..., direction: spack.traverse.DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: Literal[True], key: Callable[["Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable[Tuple[int, DependencySpec]]: ... def traverse_edges( self, *, root: bool = True, order: spack.traverse.OrderType = "pre", cover: spack.traverse.CoverType = "nodes", direction: spack.traverse.DirectionType = "children", deptype: Union[dt.DepFlag, dt.DepTypes] = "all", depth: bool = False, key: Callable[["Spec"], Any] = id, visited: Optional[Set[Any]] = None, ) -> Iterable[Union[DependencySpec, Tuple[int, DependencySpec]]]: """Shorthand for :meth:`~spack.traverse.traverse_edges`""" return spack.traverse.traverse_edges( [self], root=root, order=order, cover=cover, direction=direction, deptype=deptype, depth=depth, key=key, visited=visited, ) @property def prefix(self) -> spack.util.prefix.Prefix: if not self._concrete: raise spack.error.SpecError(f"Spec is not concrete: {self}") if self._prefix is None: from spack.store import STORE _, record = STORE.db.query_by_spec_hash(self.dag_hash()) if record and record.path: self.set_prefix(record.path) else: self.set_prefix(STORE.layout.path_for_spec(self)) assert self._prefix is not None return self._prefix def set_prefix(self, value: str) -> None: self._prefix = spack.util.prefix.Prefix(spack.llnl.path.convert_to_platform_path(value)) def spec_hash(self, hash: ht.SpecHashDescriptor) -> str: """Utility method for computing different types of Spec hashes. Arguments: hash: type of hash to generate. """ # TODO: currently we strip build dependencies by default. Rethink # this when we move to using package hashing on all specs. if hash.override is not None: return hash.override(self) node_dict = self.to_node_dict(hash=hash) json_text = json.dumps( node_dict, ensure_ascii=True, indent=None, separators=(",", ":"), sort_keys=False ) # This implements "frankenhashes", preserving the last 7 characters of the # original hash when splicing so that we can avoid relocation issues out = spack.util.hash.b32_hash(json_text) if self.build_spec is not self: return out[:-7] + self.build_spec.spec_hash(hash)[-7:] return out def _cached_hash( self, hash: ht.SpecHashDescriptor, length: Optional[int] = None, force: bool = False ) -> str: """Helper function for storing a cached hash on the spec. This will run spec_hash() with the deptype and package_hash parameters, and if this spec is concrete, it will store the value in the supplied attribute on this spec. Arguments: hash: type of hash to generate. length: length of hash prefix to return (default is full hash string) force: cache the hash even if spec is not concrete (default False) """ hash_string = getattr(self, hash.attr, None) if hash_string: return hash_string[:length] hash_string = self.spec_hash(hash) if force or self.concrete: setattr(self, hash.attr, hash_string) return hash_string[:length] def package_hash(self): """Compute the hash of the contents of the package for this node""" # Concrete specs with the old DAG hash did not have the package hash, so we do # not know what the package looked like at concretization time if self.concrete and not self._package_hash: raise ValueError( "Cannot call package_hash() on concrete specs with the old dag_hash()" ) return self._cached_hash(ht.package_hash) def dag_hash(self, length=None): """This is Spack's default hash, used to identify installations. NOTE: Versions of Spack prior to 0.18 only included link and run deps. NOTE: Versions of Spack prior to 1.0 only did not include test deps. """ return self._cached_hash(ht.dag_hash, length) def dag_hash_bit_prefix(self, bits): """Get the first bits of the DAG hash as an integer type.""" return spack.util.hash.base32_prefix_bits(self.dag_hash(), bits) def _lookup_hash(self): """Lookup just one spec with an abstract hash, returning a spec from the the environment, store, or finally, binary caches.""" from spack.binary_distribution import BinaryCacheQuery from spack.environment import active_environment from spack.store import STORE active_env = active_environment() # First env, then store, then binary cache matches = ( (active_env.all_matching_specs(self) if active_env else []) or STORE.db.query(self, installed=InstallRecordStatus.ANY) or BinaryCacheQuery(True)(self) ) if not matches: raise InvalidHashError(self, self.abstract_hash) if len(matches) != 1: raise AmbiguousHashError( f"Multiple packages specify hash beginning '{self.abstract_hash}'.", *matches ) return matches[0] def lookup_hash(self): """Given a spec with an abstract hash, return a copy of the spec with all properties and dependencies by looking up the hash in the environment, store, or finally, binary caches. This is non-destructive.""" if self.concrete or not any(node.abstract_hash for node in self.traverse()): return self spec = self.copy(deps=False) # root spec is replaced if spec.abstract_hash: spec._dup(self._lookup_hash()) return spec # Map the dependencies that need to be replaced node_lookup = { id(node): node._lookup_hash() for node in self.traverse(root=False) if node.abstract_hash } # Reconstruct dependencies for edge in self.traverse_edges(root=False): key = edge.parent.name current_node = spec if key == spec.name else spec[key] child_node = node_lookup.get(id(edge.spec), edge.spec.copy()) current_node._add_dependency( child_node, depflag=edge.depflag, virtuals=edge.virtuals, direct=edge.direct ) return spec def replace_hash(self): """Given a spec with an abstract hash, attempt to populate all properties and dependencies by looking up the hash in the environment, store, or finally, binary caches. This is destructive.""" if not any(node for node in self.traverse(order="post") if node.abstract_hash): return self._dup(self.lookup_hash()) def to_node_dict(self, hash: ht.SpecHashDescriptor = ht.dag_hash) -> Dict[str, Any]: """Create a dictionary representing the state of this Spec. This method creates the content that is eventually hashed by Spack to create identifiers like the DAG hash (see :meth:`dag_hash()`). Example result of this function for the ``sqlite`` package:: { "name": "sqlite", "version": "3.46.0", "arch": {"platform": "linux", "platform_os": "ubuntu24.04", "target": "x86_64_v3"}, "namespace": "builtin", "parameters": { "build_system": "autotools", "column_metadata": True, "dynamic_extensions": True, "fts": True, "functions": False, "rtree": True, "cflags": [], "cppflags": [], "cxxflags": [], "fflags": [], "ldflags": [], "ldlibs": [], }, "package_hash": "umcghjlve5347o3q2odo7vfcso2zhxdzmfdba23nkdhe5jntlhia====", "dependencies": [ { "name": "compiler-wrapper", "hash": "c5bxlim3zge4snwrwtd6rzuvq2unek6s", "parameters": {"deptypes": ("build",), "virtuals": ()}, }, { "name": "gcc", "hash": "6dzveld2rtt2dkhklxfnery5wbtb5uus", "parameters": {"deptypes": ("build",), "virtuals": ("c",)}, }, ... ], "annotations": {"original_specfile_version": 5}, } Note that the dictionary returned does *not* include the hash of the *root* of the spec, though it does include hashes for each dependency and its own package hash. See :meth:`to_dict()` for a "complete" spec hash, with hashes for each node and nodes for each dependency (instead of just their hashes). Arguments: hash: type of hash to generate. """ d: Dict[str, Any] = {"name": self.name} if self.versions: d.update(self.versions.to_dict()) if self.architecture: d.update(self.architecture.to_dict()) if self.namespace: d["namespace"] = self.namespace params: Dict[str, Any] = dict(sorted(v.yaml_entry() for v in self.variants.values())) # Only need the string compiler flag for yaml file params.update( sorted( self.compiler_flags.yaml_entry(flag_type) for flag_type in self.compiler_flags.keys() ) ) if params: d["parameters"] = params if params and not self.concrete: flag_names = [ name for name, flags in self.compiler_flags.items() if any(x.propagate for x in flags) ] d["propagate"] = sorted( itertools.chain( [v.name for v in self.variants.values() if v.propagate], flag_names ) ) d["abstract"] = sorted(v.name for v in self.variants.values() if not v.concrete) if self.external: d["external"] = { "path": self.external_path, "module": self.external_modules or None, "extra_attributes": syaml.sorted_dict(self.extra_attributes), } if not self._concrete: d["concrete"] = False if "patches" in self.variants: variant = self.variants["patches"] if hasattr(variant, "_patches_in_order_of_appearance"): d["patches"] = variant._patches_in_order_of_appearance if ( self._concrete and hash.package_hash and hasattr(self, "_package_hash") and self._package_hash ): # We use the attribute here instead of `self.package_hash()` because this # should *always* be assignhed at concretization time. We don't want to try # to compute a package hash for concrete spec where a) the package might not # exist, or b) the `dag_hash` didn't include the package hash when the spec # was concretized. package_hash = self._package_hash # Full hashes are in bytes if not isinstance(package_hash, str) and isinstance(package_hash, bytes): package_hash = package_hash.decode("utf-8") d["package_hash"] = package_hash # Note: Relies on sorting dict by keys later in algorithm. deps = self._dependencies_dict(depflag=hash.depflag) if deps: dependencies = [] for name, edges_for_name in sorted(deps.items()): for dspec in edges_for_name: dep_attrs = { "name": name, hash.name: dspec.spec._cached_hash(hash), "parameters": { "deptypes": dt.flag_to_tuple(dspec.depflag), "virtuals": dspec.virtuals, }, } if dspec.direct: dep_attrs["parameters"]["direct"] = True dependencies.append(dep_attrs) d["dependencies"] = dependencies # Name is included in case this is replacing a virtual. if self._build_spec: d["build_spec"] = { "name": self.build_spec.name, hash.name: self.build_spec._cached_hash(hash), } # Annotations d["annotations"] = {"original_specfile_version": self.annotations.original_spec_format} if self.annotations.original_spec_format < 5: d["annotations"]["compiler"] = str(self.annotations.compiler_node_attribute) return d def to_dict(self, hash: ht.SpecHashDescriptor = ht.dag_hash) -> Dict[str, Any]: """Create a dictionary suitable for writing this spec to YAML or JSON. This dictionary is like the one that is ultimately written to a ``spec.json`` file in each Spack installation directory. For example, for sqlite:: { "spec": { "_meta": {"version": 5}, "nodes": [ { "name": "sqlite", "version": "3.46.0", "arch": { "platform": "linux", "platform_os": "ubuntu24.04", "target": "x86_64_v3" }, "namespace": "builtin", "parameters": { "build_system": "autotools", "column_metadata": True, "dynamic_extensions": True, "fts": True, "functions": False, "rtree": True, "cflags": [], "cppflags": [], "cxxflags": [], "fflags": [], "ldflags": [], "ldlibs": [], }, "package_hash": "umcghjlve5347o...xdzmfdba23nkdhe5jntlhia====", "dependencies": [ { "name": "compiler-wrapper", "hash": "c5bxlim3zge4snwrwtd6rzuvq2unek6s", "parameters": {"deptypes": ("build",), "virtuals": ()}, }, { "name": "gcc", "hash": "6dzveld2rtt2dkhklxfnery5wbtb5uus", "parameters": {"deptypes": ("build",), "virtuals": ("c",)}, }, ... ], "annotations": {"original_specfile_version": 5}, "hash": "a2ubvvqnula6zdppckwqrjf3zmsdzpoh", }, ... ], } } Note that this dictionary starts with the ``spec`` key, and what follows is a list starting with the root spec, followed by its dependencies in preorder. The method :meth:`from_dict()` can be used to read back in a spec that has been converted to a dictionary, serialized, and read back in. """ node_list = [] # Using a list to preserve preorder traversal for hash. hash_set = set() for s in self.traverse(order="pre", deptype=hash.depflag): spec_hash = s._cached_hash(hash) if spec_hash not in hash_set: node_list.append(s.node_dict_with_hashes(hash)) hash_set.add(spec_hash) if s.build_spec is not s: build_spec_list = s.build_spec.to_dict(hash)["spec"]["nodes"] for node in build_spec_list: node_hash = node[hash.name] if node_hash not in hash_set: node_list.append(node) hash_set.add(node_hash) return {"spec": {"_meta": {"version": SPECFILE_FORMAT_VERSION}, "nodes": node_list}} def node_dict_with_hashes(self, hash: ht.SpecHashDescriptor = ht.dag_hash) -> Dict[str, Any]: """Returns a node dict of this spec with the dag hash, and the provided hash (if not the dag hash).""" node = self.to_node_dict(hash) # All specs have at least a DAG hash node[ht.dag_hash.name] = self.dag_hash() if not self.concrete: node["concrete"] = False # we can also give them other hash types if we want if hash.name != ht.dag_hash.name: node[hash.name] = self._cached_hash(hash) return node def to_yaml(self, stream=None, hash=ht.dag_hash): return syaml.dump(self.to_dict(hash), stream=stream, default_flow_style=False) def to_json(self, stream=None, *, hash=ht.dag_hash, pretty=False): return sjson.dump(self.to_dict(hash), stream=stream, pretty=pretty) @staticmethod def from_specfile(path): """Construct a spec from a JSON or YAML spec file path""" with open(path, "r", encoding="utf-8") as fd: file_content = fd.read() if path.endswith(".json"): return Spec.from_json(file_content) return Spec.from_yaml(file_content) @staticmethod def override(init_spec, change_spec): # TODO: this doesn't account for the case where the changed spec # (and the user spec) have dependencies new_spec = init_spec.copy() package_cls = spack.repo.PATH.get_pkg_class(new_spec.name) if change_spec.versions and not change_spec.versions == vn.any_version: new_spec.versions = change_spec.versions for vname, value in change_spec.variants.items(): if vname in package_cls.variant_names(): if vname in new_spec.variants: new_spec.variants.substitute(value) else: new_spec.variants[vname] = value else: raise ValueError("{0} is not a variant of {1}".format(vname, new_spec.name)) if change_spec.compiler_flags: for flagname, flagvals in change_spec.compiler_flags.items(): new_spec.compiler_flags[flagname] = flagvals if change_spec.architecture: new_spec.architecture = ArchSpec.override( new_spec.architecture, change_spec.architecture ) return new_spec @staticmethod def from_literal(spec_dict: dict, normal: bool = True) -> "Spec": """Builds a Spec from a dictionary containing the spec literal. The dictionary must have a single top level key, representing the root, and as many secondary level keys as needed in the spec. The keys can be either a string or a Spec or a tuple containing the Spec and the dependency types. Args: spec_dict: the dictionary containing the spec literal normal: if :data:`True` the same key appearing at different levels of the ``spec_dict`` will map to the same object in memory. Examples: A simple spec ``foo`` with no dependencies:: {"foo": None} A spec ``foo`` with a ``(build, link)`` dependency ``bar``:: {"foo": {"bar:build,link": None} } A spec with a diamond dependency and various build types:: {"dt-diamond": { "dt-diamond-left:build,link": { "dt-diamond-bottom:build": None }, "dt-diamond-right:build,link": { "dt-diamond-bottom:build,link,run": None } }} The same spec with a double copy of ``dt-diamond-bottom`` and no diamond structure:: Spec.from_literal({"dt-diamond": { "dt-diamond-left:build,link": { "dt-diamond-bottom:build": None }, "dt-diamond-right:build,link": { "dt-diamond-bottom:build,link,run": None } }, normal=False} Constructing a spec using a Spec object as key:: mpich = Spec("mpich") libelf = Spec("libelf@1.8.11") expected_normalized = Spec.from_literal({ "mpileaks": { "callpath": { "dyninst": { "libdwarf": {libelf: None}, libelf: None }, mpich: None }, mpich: None }, }) """ # Maps a literal to a Spec, to be sure we are reusing the same object spec_cache = LazySpecCache() def spec_builder(d): # The invariant is that the top level dictionary must have # only one key assert len(d) == 1 # Construct the top-level spec spec_like, dep_like = next(iter(d.items())) # If the requirements was for unique nodes (default) # then reuse keys from the local cache. Otherwise build # a new node every time. if not isinstance(spec_like, Spec): spec = spec_cache[spec_like] if normal else Spec(spec_like) else: spec = spec_like if dep_like is None: return spec def name_and_dependency_types(s: str) -> Tuple[str, dt.DepFlag]: """Given a key in the dictionary containing the literal, extracts the name of the spec and its dependency types. Args: s: key in the dictionary containing the literal """ t = s.split(":") if len(t) > 2: msg = 'more than one ":" separator in key "{0}"' raise KeyError(msg.format(s)) name = t[0] if len(t) == 2: depflag = dt.flag_from_strings(dep_str.strip() for dep_str in t[1].split(",")) else: depflag = 0 return name, depflag def spec_and_dependency_types( s: Union[Spec, Tuple[Spec, str]], ) -> Tuple[Spec, dt.DepFlag]: """Given a non-string key in the literal, extracts the spec and its dependency types. Args: s: either a Spec object, or a tuple of Spec and string of dependency types """ if isinstance(s, Spec): return s, 0 spec_obj, dtypes = s return spec_obj, dt.flag_from_strings(dt.strip() for dt in dtypes.split(",")) # Recurse on dependencies for s, s_dependencies in dep_like.items(): if isinstance(s, str): dag_node, dep_flag = name_and_dependency_types(s) else: dag_node, dep_flag = spec_and_dependency_types(s) dependency_spec = spec_builder({dag_node: s_dependencies}) spec._add_dependency(dependency_spec, depflag=dep_flag, virtuals=()) return spec return spec_builder(spec_dict) @staticmethod def from_dict(data) -> "Spec": """Construct a spec from JSON/YAML. Args: data: a nested dict/list data structure read from YAML or JSON. """ # Legacy specfile format if isinstance(data["spec"], list): spec = SpecfileV1.load(data) elif int(data["spec"]["_meta"]["version"]) == 2: spec = SpecfileV2.load(data) elif int(data["spec"]["_meta"]["version"]) == 3: spec = SpecfileV3.load(data) elif int(data["spec"]["_meta"]["version"]) == 4: spec = SpecfileV4.load(data) else: spec = SpecfileV5.load(data) # Any git version should for s in spec.traverse(): s.attach_git_version_lookup() return spec @staticmethod def from_yaml(stream) -> "Spec": """Construct a spec from YAML. Args: stream: string or file object to read from. """ data = syaml.load(stream) return Spec.from_dict(data) @staticmethod def from_json(stream) -> "Spec": """Construct a spec from JSON. Args: stream: string or file object to read from. """ try: data = sjson.load(stream) return Spec.from_dict(data) except Exception as e: raise sjson.SpackJSONError("error parsing JSON spec:", e) from e @staticmethod def extract_json_from_clearsig(data): m = CLEARSIGN_FILE_REGEX.search(data) if m: return sjson.load(m.group(1)) return sjson.load(data) @staticmethod def from_signed_json(stream): """Construct a spec from clearsigned json spec file. Args: stream: string or file object to read from. """ data = stream if hasattr(stream, "read"): data = stream.read() extracted_json = Spec.extract_json_from_clearsig(data) return Spec.from_dict(extracted_json) @staticmethod def from_detection( spec_str: str, *, external_path: str, external_modules: Optional[List[str]] = None, extra_attributes: Optional[Dict] = None, ) -> "Spec": """Construct a spec from a spec string determined during external detection and attach extra attributes to it. Args: spec_str: spec string external_path: prefix of the external spec external_modules: optional module files to be loaded when the external spec is used extra_attributes: dictionary containing extra attributes """ s = Spec(spec_str, external_path=external_path, external_modules=external_modules) extra_attributes = syaml.sorted_dict(extra_attributes or {}) # This is needed to be able to validate multi-valued variants, # otherwise they'll still be abstract in the context of detection. substitute_abstract_variants(s) s.extra_attributes = extra_attributes return s def _patches_assigned(self): """Whether patches have been assigned to this spec by the concretizer.""" # FIXME: _patches_in_order_of_appearance is attached after concretization # FIXME: to store the order of patches. # FIXME: Probably needs to be refactored in a cleaner way. if "patches" not in self.variants: return False # ensure that patch state is consistent patch_variant = self.variants["patches"] assert hasattr(patch_variant, "_patches_in_order_of_appearance"), ( "patches should always be assigned with a patch variant." ) return True @staticmethod def ensure_no_deprecated(root: "Spec") -> None: """Raise if a deprecated spec is in the dag of the given root spec. Raises: spack.spec.SpecDeprecatedError: if any deprecated spec is found """ deprecated = [] from spack.store import STORE with STORE.db.read_transaction(): for x in root.traverse(): _, rec = STORE.db.query_by_spec_hash(x.dag_hash()) if rec and rec.deprecated_for: deprecated.append(rec) if deprecated: msg = "\n The following specs have been deprecated" msg += " in favor of specs with the hashes shown:\n" for rec in deprecated: msg += " %s --> %s\n" % (rec.spec, rec.deprecated_for) msg += "\n" msg += " For each package listed, choose another spec\n" raise SpecDeprecatedError(msg) def _mark_root_concrete(self, value=True): """Mark just this spec (not dependencies) concrete.""" if (not value) and self.concrete and self.installed: return self._concrete = value self._validate_version() for variant in self.variants.values(): variant.concrete = True def _validate_version(self): # Specs that were concretized with just a git sha as version, without associated # Spack version, get their Spack version mapped to develop. This should only apply # when reading specs concretized with Spack 0.19 or earlier. Currently Spack always # ensures that GitVersion specs have an associated Spack version. v = self.versions.concrete if not isinstance(v, vn.GitVersion): return try: v.ref_version except vn.VersionLookupError: before = self.cformat("{name}{@version}{/hash:7}") v.std_version = vn.StandardVersion.from_string("develop") tty.debug( f"the git sha of {before} could not be resolved to spack version; " f"it has been replaced by {self.cformat('{name}{@version}{/hash:7}')}." ) def _mark_concrete(self, value=True): """Mark this spec and its dependencies as concrete. Only for internal use -- client code should use "concretize" unless there is a need to force a spec to be concrete. """ # if set to false, clear out all hashes (set to None or remove attr) # may need to change references to respect None for s in self.traverse(): if (not value) and s.concrete and s.installed: continue elif not value: s.clear_caches() s._mark_root_concrete(value) def _finalize_concretization(self): """Assign hashes to this spec, and mark it concrete. There are special semantics to consider for ``package_hash``, because we can't call it on *already* concrete specs, but we need to assign it *at concretization time* to just-concretized specs. So, the concretizer must assign the package hash *before* marking their specs concrete (so that we know which specs were already concrete before this latest concretization). ``dag_hash`` is also tricky, since it cannot compute ``package_hash()`` lazily. Because ``package_hash`` needs to be assigned *at concretization time*, ``to_node_dict()`` can't just assume that it can compute ``package_hash`` itself -- it needs to either see or not see a ``_package_hash`` attribute. Rules of thumb for ``package_hash``: 1. Old-style concrete specs from *before* ``dag_hash`` included ``package_hash`` will not have a ``_package_hash`` attribute at all. 2. New-style concrete specs will have a ``_package_hash`` assigned at concretization time. 3. Abstract specs will not have a ``_package_hash`` attribute at all. """ for spec in self.traverse(): # Already concrete specs either already have a package hash (new dag_hash()) # or they never will b/c we can't know it (old dag_hash()). Skip them. # # We only assign package hash to not-yet-concrete specs, for which we know # we can compute the hash. if not spec.concrete: # we need force=True here because package hash assignment has to happen # before we mark concrete, so that we know what was *already* concrete. spec._cached_hash(ht.package_hash, force=True) # keep this check here to ensure package hash is saved assert getattr(spec, ht.package_hash.attr) # Mark everything in the spec as concrete self._mark_concrete() # Assign dag_hash (this *could* be done lazily, but it's assigned anyway in # ensure_no_deprecated, and it's clearer to see explicitly where it happens). # Any specs that were concrete before finalization will already have a cached # DAG hash. for spec in self.traverse(): spec._cached_hash(ht.dag_hash) def index(self, deptype="all"): """Return a dictionary that points to all the dependencies in this spec. """ dm = collections.defaultdict(list) for spec in self.traverse(deptype=deptype): dm[spec.name].append(spec) return dm def validate_or_raise(self): """Checks that names and values in this spec are real. If they're not, it will raise an appropriate exception. """ # FIXME: this function should be lazy, and collect all the errors # FIXME: before raising the exceptions, instead of being greedy and # FIXME: raise just the first one encountered for spec in self.traverse(): # raise an UnknownPackageError if the spec's package isn't real. if spec.name and not spack.repo.PATH.is_virtual(spec.name): spack.repo.PATH.get_pkg_class(spec.fullname) # FIXME: atm allow '%' on abstract specs only if they depend on C, C++, or Fortran if spec.dependencies(deptype="build"): pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname) pkg_dependencies = pkg_cls.dependency_names() if not any(x in pkg_dependencies for x in ("c", "cxx", "fortran")): raise UnsupportedCompilerError( f"{spec.fullname} does not depend on 'c', 'cxx, or 'fortran'" ) # Ensure correctness of variants (if the spec is not virtual) if not spack.repo.PATH.is_virtual(spec.name): Spec.ensure_valid_variants(spec) substitute_abstract_variants(spec) @staticmethod def ensure_valid_variants(spec: "Spec") -> None: """Ensures that the variant attached to the given spec are valid. Raises: spack.variant.UnknownVariantError: on the first unknown variant found """ # concrete variants are always valid if spec.concrete: return pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname) pkg_variants = pkg_cls.variant_names() # reserved names are variants that may be set on any package # but are not necessarily recorded by the package's class propagate_variants = [name for name, variant in spec.variants.items() if variant.propagate] not_existing = set(spec.variants) not_existing.difference_update(pkg_variants, vt.RESERVED_NAMES, propagate_variants) if not_existing: raise vt.UnknownVariantError( f"No such variant {not_existing} for spec: '{spec}'", list(not_existing) ) def constrain(self, other, deps=True) -> bool: """Constrains self with other, and returns True if self changed, False otherwise. Args: other: constraint to be added to self deps: if False, constrain only the root node, otherwise constrain dependencies as well Raises: spack.error.UnsatisfiableSpecError: when self cannot be constrained """ return self._constrain(other, deps=deps, resolve_virtuals=True) def _constrain_symbolically(self, other, deps=True) -> bool: """Constrains self with other, and returns True if self changed, False otherwise. This function has no notion of virtuals, so it does not need a repository. Args: other: constraint to be added to self deps: if False, constrain only the root node, otherwise constrain dependencies as well Raises: spack.error.UnsatisfiableSpecError: when self cannot be constrained Examples: >>> from spack.spec import Spec, UnsatisfiableDependencySpecError >>> s = Spec("hdf5 ^mpi@4") >>> t = Spec("hdf5 ^mpi=openmpi") >>> try: ... s.constrain(t) ... except UnsatisfiableDependencySpecError as e: ... print(e) ... hdf5 ^mpi=openmpi does not satisfy hdf5 ^mpi@4 >>> s._constrain_symbolically(t) True >>> s hdf5 ^mpi@4 ^mpi=openmpi """ return self._constrain(other, deps=deps, resolve_virtuals=False) def _constrain(self, other, deps=True, *, resolve_virtuals: bool): # If we are trying to constrain a concrete spec, either the spec # already satisfies the constraint (and the method returns False) # or it raises an exception if self.concrete: if self._satisfies(other, resolve_virtuals=resolve_virtuals): return False else: raise spack.error.UnsatisfiableSpecError(self, other, "constrain a concrete spec") other = self._autospec(other) if other.concrete and other._satisfies(self, resolve_virtuals=resolve_virtuals): self._dup(other) return True if other.abstract_hash: if not self.abstract_hash or other.abstract_hash.startswith(self.abstract_hash): self.abstract_hash = other.abstract_hash elif not self.abstract_hash.startswith(other.abstract_hash): raise InvalidHashError(self, other.abstract_hash) if not (self.name == other.name or (not self.name) or (not other.name)): raise UnsatisfiableSpecNameError(self.name, other.name) if ( other.namespace is not None and self.namespace is not None and other.namespace != self.namespace ): raise UnsatisfiableSpecNameError(self.fullname, other.fullname) if not self.versions.overlaps(other.versions): raise UnsatisfiableVersionSpecError(self.versions, other.versions) for v in [x for x in other.variants if x in self.variants]: if not self.variants[v].intersects(other.variants[v]): raise vt.UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) sarch, oarch = self.architecture, other.architecture if ( sarch is not None and oarch is not None and not self.architecture.intersects(other.architecture) ): raise UnsatisfiableArchitectureSpecError(sarch, oarch) changed = False if not self.name and other.name: self.name = other.name changed = True if not self.namespace and other.namespace: self.namespace = other.namespace changed = True changed |= self.versions.intersect(other.versions) changed |= self._constrain_variants(other) changed |= self.compiler_flags.constrain(other.compiler_flags) sarch, oarch = self.architecture, other.architecture if sarch is not None and oarch is not None: changed |= self.architecture.constrain(other.architecture) elif oarch is not None: self.architecture = oarch changed = True if deps: changed |= self._constrain_dependencies(other, resolve_virtuals=resolve_virtuals) if other.concrete and not self.concrete and other.satisfies(self): self._finalize_concretization() return changed def _constrain_dependencies(self, other: "Spec", resolve_virtuals: bool = True) -> bool: """Apply constraints of other spec's dependencies to this spec.""" if not other._dependencies: return False # TODO: might want more detail than this, e.g. specific deps # in violation. if this becomes a priority get rid of this # check and be more specific about what's wrong. if not other._intersects_dependencies(self, resolve_virtuals=resolve_virtuals): raise UnsatisfiableDependencySpecError(other, self) for d in other.traverse(root=False): if not d.name: raise UnconstrainableDependencySpecError(other) changed = False for other_edge in other.edges_to_dependencies(): # Find the first edge in self that matches other_edge by name and when clause. for self_edge in self.edges_to_dependencies(other_edge.spec.name): if self_edge.when == other_edge.when: changed |= self_edge._constrain(other_edge) break else: # Otherwise, a copy of the edge is added as a constraint to self. changed = True self.add_dependency_edge( other_edge.spec.copy(deps=True), depflag=other_edge.depflag, virtuals=other_edge.virtuals, direct=other_edge.direct, propagation=other_edge.propagation, when=other_edge.when, # no need to copy; when conditions are immutable ) return changed def constrained(self, other, deps=True): """Return a constrained copy without modifying this spec.""" clone = self.copy(deps=deps) clone.constrain(other, deps) return clone def _autospec(self, spec_like): """ Used to convert arguments to specs. If spec_like is a spec, returns it. If it's a string, tries to parse a string. If that fails, tries to parse a local spec from it (i.e. name is assumed to be self's name). """ if isinstance(spec_like, Spec): return spec_like return Spec(spec_like) def intersects(self, other: Union[str, "Spec"], deps: bool = True) -> bool: """Return True if there exists at least one concrete spec that matches both self and other, otherwise False. This operation is commutative, and if two specs intersect it means that one can constrain the other. Args: other: spec to be checked for compatibility deps: if True check compatibility of dependency nodes too, if False only check root """ return self._intersects(other=other, deps=deps, resolve_virtuals=True) def _intersects( self, other: Union[str, "Spec"], deps: bool = True, resolve_virtuals: bool = True ) -> bool: if other is EMPTY_SPEC: return True other = self._autospec(other) if other.concrete and self.concrete: return self.dag_hash() == other.dag_hash() elif self.concrete: return self._satisfies(other, resolve_virtuals=resolve_virtuals) elif other.concrete: return other._satisfies(self, resolve_virtuals=resolve_virtuals) # From here we know both self and other are not concrete self_hash = self.abstract_hash other_hash = other.abstract_hash if ( self_hash and other_hash and not (self_hash.startswith(other_hash) or other_hash.startswith(self_hash)) ): return False # If the names are different, we need to consider virtuals if self.name != other.name and self.name and other.name: if not resolve_virtuals: return False self_virtual = spack.repo.PATH.is_virtual(self.name) other_virtual = spack.repo.PATH.is_virtual(other.name) if self_virtual and other_virtual: # Two virtual specs intersect only if there are providers for both lhs = spack.repo.PATH.providers_for(str(self)) rhs = spack.repo.PATH.providers_for(str(other)) intersection = [s for s in lhs if any(s.intersects(z) for z in rhs)] return bool(intersection) # A provider can satisfy a virtual dependency. elif self_virtual or other_virtual: virtual_spec, non_virtual_spec = (self, other) if self_virtual else (other, self) try: # Here we might get an abstract spec pkg_cls = spack.repo.PATH.get_pkg_class(non_virtual_spec.fullname) pkg = pkg_cls(non_virtual_spec) except spack.repo.UnknownEntityError: # If we can't get package info on this spec, don't treat # it as a provider of this vdep. return False if pkg.provides(virtual_spec.name): for when_spec, provided in pkg.provided.items(): if non_virtual_spec.intersects(when_spec, deps=False): if any(vpkg.intersects(virtual_spec) for vpkg in provided): return True return False # namespaces either match, or other doesn't require one. if ( other.namespace is not None and self.namespace is not None and self.namespace != other.namespace ): return False if self.versions and other.versions: if not self.versions.intersects(other.versions): return False if not self._intersects_variants(other): return False if self.architecture and other.architecture: if not self.architecture.intersects(other.architecture): return False if not self.compiler_flags.intersects(other.compiler_flags): return False # If we need to descend into dependencies, do it, otherwise we're done. if deps: return self._intersects_dependencies(other, resolve_virtuals=resolve_virtuals) return True def _intersects_dependencies(self, other, resolve_virtuals: bool = True): if not other._dependencies or not self._dependencies: # one spec *could* eventually satisfy the other return True # Handle first-order constraints directly common_dependencies = {x.name for x in self.dependencies()} common_dependencies &= {x.name for x in other.dependencies()} for name in common_dependencies: if not self[name]._intersects( other[name], deps=True, resolve_virtuals=resolve_virtuals ): return False if not resolve_virtuals: return True # For virtual dependencies, we need to dig a little deeper. self_index = spack.provider_index.ProviderIndex( repository=spack.repo.PATH, specs=self.traverse(), restrict=True ) other_index = spack.provider_index.ProviderIndex( repository=spack.repo.PATH, specs=other.traverse(), restrict=True ) # These two loops handle cases where there is an overly restrictive # vpkg in one spec for a provider in the other (e.g., mpi@3: is not # compatible with mpich2) for spec in self.traverse(): if ( spack.repo.PATH.is_virtual(spec.name) and spec.name in other_index and not other_index.providers_for(spec) ): return False for spec in other.traverse(): if ( spack.repo.PATH.is_virtual(spec.name) and spec.name in self_index and not self_index.providers_for(spec) ): return False return True def satisfies(self, other: Union[str, "Spec"], deps: bool = True) -> bool: """Return True if all concrete specs matching self also match other, otherwise False. Args: other: spec to be satisfied deps: if True, descend to dependencies, otherwise only check root node """ return self._satisfies(other=other, deps=deps, resolve_virtuals=True) def _provides_virtual(self, virtual_spec: "Spec") -> bool: """Return True if this spec provides the given virtual spec. Args: virtual_spec: abstract virtual spec (e.g. ``"mpi"`` or ``"mpi@3:"``) """ if not virtual_spec.name: return False # Get the package instance if self.concrete: try: pkg = self.package except spack.repo.UnknownPackageError: return False else: try: pkg_cls = spack.repo.PATH.get_pkg_class(self.fullname) pkg = pkg_cls(self) except spack.repo.UnknownEntityError: # If we can't get package info on this spec, don't treat # it as a provider of this vdep. return False for when_spec, provided in pkg.provided.items(): # Don't use satisfies for virtuals, because an abstract vs. abstract spec may use the # repo index if self.satisfies(when_spec, deps=False) and any( provided_virtual.name == virtual_spec.name and provided_virtual.versions.intersects(virtual_spec.versions) for provided_virtual in provided ): return True return False def _satisfies( self, other: Union[str, "Spec"], deps: bool = True, resolve_virtuals: bool = True ) -> bool: """Return True if all concrete specs matching self also match other, otherwise False. Args: other: spec to be satisfied deps: if True, descend to dependencies, otherwise only check root node resolve_virtuals: if True, resolve virtuals in self and other. This requires a repository to be available. """ if other is EMPTY_SPEC: return True other = self._autospec(other) if not self._satisfies_node(other, resolve_virtuals=resolve_virtuals): return False # If there are no dependencies on the rhs, or we don't recurse, they are satisfied. if not deps or not other._dependencies: return True stack = [(self, other)] while stack: lhs, rhs = stack.pop() for rhs_edge in rhs.edges_to_dependencies(): # Skip rhs edges whose when condition doesn't apply to the lhs node. if rhs_edge.when is not EMPTY_SPEC and not lhs._intersects( rhs_edge.when, resolve_virtuals=resolve_virtuals ): continue lhs_edge = _get_satisfying_edge(lhs, rhs_edge, resolve_virtuals=resolve_virtuals) if not lhs_edge: return False # Recursive case: `^zlib %gcc` if not rhs_edge.spec.concrete and rhs_edge.spec._dependencies: stack.append((lhs_edge.spec, rhs_edge.spec)) return True def _satisfies_node(self, other: "Spec", resolve_virtuals: bool) -> bool: """Compares self and other without looking at dependencies""" if other.concrete: # The left-hand side must be the same singleton with identical hash. Notice that # package hashes can be different for otherwise indistinguishable concrete Spec # objects. return self.concrete and self.dag_hash() == other.dag_hash() if self.name != other.name and self.name and other.name: # Name mismatch can still be satisfiable if lhs provides the virtual mentioned by rhs. if not resolve_virtuals: return False return self._provides_virtual(other) # If the right-hand side has an abstract hash, make sure it's a prefix of the # left-hand side's (abstract) hash. if other.abstract_hash: compare_hash = self.dag_hash() if self.concrete else self.abstract_hash if not compare_hash or not compare_hash.startswith(other.abstract_hash): return False # namespaces either match, or other doesn't require one. if ( other.namespace is not None and self.namespace is not None and self.namespace != other.namespace ): return False if not self.versions.satisfies(other.versions): return False if not self._satisfies_variants(other): return False if self.architecture and other.architecture: if not self.architecture.satisfies(other.architecture): return False elif other.architecture and not self.architecture: return False if not self.compiler_flags.satisfies(other.compiler_flags): return False return True def _satisfies_variants(self, other: "Spec") -> bool: if self.concrete: return self._satisfies_variants_when_self_concrete(other) return self._satisfies_variants_when_self_abstract(other) def _satisfies_variants_when_self_concrete(self, other: "Spec") -> bool: non_propagating, propagating = other.variants.partition_variants() result = all( name in self.variants and self.variants[name].satisfies(other.variants[name]) for name in non_propagating ) if not propagating: return result for node in self.traverse(): if not all( node.variants[name].satisfies(other.variants[name]) for name in propagating if name in node.variants ): return False return result def _satisfies_variants_when_self_abstract(self, other: "Spec") -> bool: other_non_propagating, other_propagating = other.variants.partition_variants() self_non_propagating, self_propagating = self.variants.partition_variants() # First check variants without propagation set result = all( name in self_non_propagating and ( self.variants[name].propagate or self.variants[name].satisfies(other.variants[name]) ) for name in other_non_propagating ) if result is False or (not other_propagating and not self_propagating): return result # Check that self doesn't contradict variants propagated by other if other_propagating: for node in self.traverse(): if not all( node.variants[name].satisfies(other.variants[name]) for name in other_propagating if name in node.variants ): return False # Check that other doesn't contradict variants propagated by self if self_propagating: for node in other.traverse(): if not all( node.variants[name].satisfies(self.variants[name]) for name in self_propagating if name in node.variants ): return False return result def _intersects_variants(self, other: "Spec") -> bool: self_dict = self.variants.dict other_dict = other.variants.dict return all(self_dict[k].intersects(other_dict[k]) for k in other_dict if k in self_dict) def _constrain_variants(self, other: "Spec") -> bool: """Add all variants in other that aren't in self to self. Also constrain all multi-valued variants that are already present. Return True iff self changed""" if other is not None and other._concrete: for k in self.variants: if k not in other.variants: raise vt.UnsatisfiableVariantSpecError(self.variants[k], "") changed = False for k in other.variants: if k in self.variants: if not self.variants[k].intersects(other.variants[k]): raise vt.UnsatisfiableVariantSpecError(self.variants[k], other.variants[k]) # If they are compatible merge them changed |= self.variants[k].constrain(other.variants[k]) else: # If it is not present copy it straight away self.variants[k] = other.variants[k].copy() changed = True return changed @property # type: ignore[misc] # decorated prop not supported in mypy def patches(self): """Return patch objects for any patch sha256 sums on this Spec. This is for use after concretization to iterate over any patches associated with this spec. TODO: this only checks in the package; it doesn't resurrect old patches from install directories, but it probably should. """ if not hasattr(self, "_patches"): self._patches = [] # translate patch sha256sums to patch objects by consulting the index if self._patches_assigned(): sha256s = list(self.variants["patches"]._patches_in_order_of_appearance) pkg_cls = spack.repo.PATH.get_pkg_class(self.name) try: self._patches = spack.repo.PATH.get_patches_for_package(sha256s, pkg_cls) except spack.error.PatchLookupError as e: raise spack.error.SpecError( f"{e}. This usually means the patch was modified or removed. " "To fix this, either reconcretize or use the original package " "repository" ) from e return self._patches def _dup( self, other: "Spec", deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, *, propagation: Optional[PropagationPolicy] = None, ) -> bool: """Copies "other" into self, by overwriting all attributes. Args: other: spec to be copied onto ``self`` deps: if True copies all the dependencies. If False copies None. If deptype, or depflag, copy matching types. Returns: True if ``self`` changed because of the copy operation, False otherwise. """ # We don't count dependencies as changes here changed = True if hasattr(self, "name"): changed = ( self.name != other.name and self.versions != other.versions and self.architecture != other.architecture and self.variants != other.variants and self.concrete != other.concrete and self.external_path != other.external_path and self.external_modules != other.external_modules and self.compiler_flags != other.compiler_flags and self.abstract_hash != other.abstract_hash ) self._package = None # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() self.architecture = other.architecture.copy() if other.architecture else None self.compiler_flags = other.compiler_flags.copy() self.compiler_flags.spec = self self.variants = other.variants.copy() self._build_spec = other._build_spec # Clear dependencies self._dependents = {} self._dependencies = {} # FIXME: we manage _patches_in_order_of_appearance specially here # to keep it from leaking out of spec.py, but we should figure # out how to handle it more elegantly in the Variant classes. for k, v in other.variants.items(): patches = getattr(v, "_patches_in_order_of_appearance", None) if patches: self.variants[k]._patches_in_order_of_appearance = patches self.variants.spec = self self.external_path = other.external_path self.external_modules = other.external_modules self.extra_attributes = other.extra_attributes self.namespace = other.namespace self.annotations = other.annotations # If we copy dependencies, preserve DAG structure in the new spec if deps: # If caller restricted deptypes to be copied, adjust that here. # By default, just copy all deptypes depflag = dt.ALL if isinstance(deps, (tuple, list, str)): depflag = dt.canonicalize(deps) self._dup_deps(other, depflag, propagation=propagation) self._prefix = other._prefix self._concrete = other._concrete self.abstract_hash = other.abstract_hash if self._concrete: self._dunder_hash = other._dunder_hash for h in ht.HASHES: setattr(self, h.attr, getattr(other, h.attr, None)) else: self._dunder_hash = None for h in ht.HASHES: setattr(self, h.attr, None) return changed def _dup_deps( self, other, depflag: dt.DepFlag, propagation: Optional[PropagationPolicy] = None ): def spid(spec): return id(spec) new_specs = {spid(other): self} for edge in other.traverse_edges(cover="edges", root=False): if edge.depflag and not depflag & edge.depflag: continue if spid(edge.parent) not in new_specs: new_specs[spid(edge.parent)] = edge.parent.copy(deps=False) if spid(edge.spec) not in new_specs: new_specs[spid(edge.spec)] = edge.spec.copy(deps=False) edge_propagation = edge.propagation if propagation is None else propagation new_specs[spid(edge.parent)].add_dependency_edge( new_specs[spid(edge.spec)], depflag=edge.depflag, virtuals=edge.virtuals, propagation=edge_propagation, direct=edge.direct, when=edge.when, ) def copy(self, deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, **kwargs): """Make a copy of this spec. Args: deps: Defaults to :data:`True`. If boolean, controls whether dependencies are copied (copied if :data:`True`). If a DepTypes or DepFlag is provided, *only* matching dependencies are copied. kwargs: additional arguments for internal use (passed to ``_dup``). Returns: A copy of this spec. Examples: Deep copy with dependencies:: spec.copy() spec.copy(deps=True) Shallow copy (no dependencies):: spec.copy(deps=False) Only build and run dependencies:: deps=("build", "run"): """ clone = Spec.__new__(Spec) clone._dup(self, deps=deps, **kwargs) return clone @property def version(self): if not self.versions.concrete: raise spack.error.SpecError("Spec version is not concrete: " + str(self)) return self.versions[0] def __getitem__(self, name: str): """Get a dependency from the spec by its name. This call implicitly sets a query state in the package being retrieved. The behavior of packages may be influenced by additional query parameters that are passed after a colon symbol. Note that if a virtual package is queried a copy of the Spec is returned while for non-virtual a reference is returned. """ query_parameters: List[str] = name.split(":") if len(query_parameters) > 2: raise KeyError("key has more than one ':' symbol. At most one is admitted.") name, query_parameters = query_parameters[0], query_parameters[1:] if query_parameters: # We have extra query parameters, which are comma separated # values csv = query_parameters.pop().strip() query_parameters = re.split(r"\s*,\s*", csv) # Consider all direct dependencies and transitive runtime dependencies order = itertools.chain( self.edges_to_dependencies(depflag=dt.BUILD | dt.TEST), self.traverse_edges(deptype=dt.LINK | dt.RUN, order="breadth", cover="edges"), ) try: edge = next((e for e in order if e.spec.name == name or name in e.virtuals)) except StopIteration as e: raise KeyError(f"No spec with name {name} in {self}") from e if self._concrete: return SpecBuildInterface( edge.spec, name, query_parameters, _parent=self, is_virtual=name in edge.virtuals ) return edge.spec def __contains__(self, spec): """True if this spec or some dependency satisfies the spec. Note: If ``spec`` is anonymous, we ONLY check whether the root satisfies it, NOT dependencies. This is because most anonymous specs (e.g., ``@1.2``) don't make sense when applied across an entire DAG -- we limit them to the root. """ spec = self._autospec(spec) # if anonymous or same name, we only have to look at the root if not spec.name or spec.name == self.name: return self.satisfies(spec) try: dep = self[spec.name] except KeyError: return False return dep.satisfies(spec) def eq_dag(self, other, deptypes=True, vs=None, vo=None): """True if the full dependency DAGs of specs are equal.""" if vs is None: vs = set() if vo is None: vo = set() vs.add(id(self)) vo.add(id(other)) if not self.eq_node(other): return False if len(self._dependencies) != len(other._dependencies): return False ssorted = [self._dependencies[name] for name in sorted(self._dependencies)] osorted = [other._dependencies[name] for name in sorted(other._dependencies)] for s_dspec, o_dspec in zip( itertools.chain.from_iterable(ssorted), itertools.chain.from_iterable(osorted) ): if deptypes and s_dspec.depflag != o_dspec.depflag: return False s, o = s_dspec.spec, o_dspec.spec visited_s = id(s) in vs visited_o = id(o) in vo # Check for duplicate or non-equal dependencies if visited_s != visited_o: return False # Skip visited nodes if visited_s or visited_o: continue # Recursive check for equality if not s.eq_dag(o, deptypes, vs, vo): return False return True def _cmp_node(self): """Yield comparable elements of just *this node* and not its deps.""" yield self.name yield self.namespace yield self.versions yield self.variants yield self.compiler_flags yield self.architecture yield self.abstract_hash # this is not present on older specs yield getattr(self, "_package_hash", None) def eq_node(self, other): """Equality with another spec, not including dependencies.""" return (other is not None) and lang.lazy_eq(self._cmp_node, other._cmp_node) def _cmp_fast_eq(self, other) -> Optional[bool]: """Short-circuit compare with other for equality, for lazy_lexicographic_ordering.""" # If there is ever a breaking change to hash computation, whether accidental or purposeful, # two specs can be identical modulo DAG hash, depending on what time they were concretized # From the perspective of many operation in Spack (database, build cache, etc) a different # DAG hash means a different spec. Here we ensure that two otherwise identical specs, one # serialized before the hash change and one after, are considered different. if self is other: return True if self.concrete and other and other.concrete: return self.dag_hash() == other.dag_hash() return None def _cmp_iter(self): """Lazily yield components of self for comparison.""" # Spec comparison in Spack needs to be fast, so there are several cases here for # performance. The main places we care about this are: # # * Abstract specs: there are lots of abstract specs in package.py files, # which are put into metadata dictionaries and sorted during concretization # setup. We want comparing abstract specs to be fast. # # * Concrete specs: concrete specs are bigger and have lots of nodes and # edges. Because of the graph complexity, we need a full, linear time # traversal to compare them -- that's pretty much is unavoidable. But they # also have precoputed cryptographic hashes (dag_hash()), which we can use # to do fast equality comparison. See _cmp_fast_eq() above for the # short-circuit logic for hashes. # # A full traversal involves constructing data structures, visitor objects, etc., # and it can be expensive if we have to do it to compare a bunch of tiny # abstract specs. Therefore, there are 3 cases below, which avoid calling # `spack.traverse.traverse_edges()` unless necessary. # # WARNING: the cases below need to be consistent, so don't mess with this code # unless you really know what you're doing. Be sure to keep all three consistent. # # All cases lazily yield: # # 1. A generator over nodes # 2. A generator over canonical edges # # Canonical edges have consistent ids defined by breadth-first traversal order. That is, # the root is always 0, dependencies of the root are 1, 2, 3, etc., and so on. # # The three cases are: # # 1. Spec has no dependencies # * We can avoid any traversal logic and just yield this node's _cmp_node generator. # # 2. Spec has dependencies, but dependencies have no dependencies. # * We need to sort edges, but we don't need to track visited nodes, which # can save us the cost of setting up all the tracking data structures # `spack.traverse` uses. # # 3. Spec has dependencies that have dependencies. # * In this case, the spec is *probably* concrete. Equality comparisons # will be short-circuited by dag_hash(), but other comparisons will need # to lazily enumerate components of the spec. The traversal logic is # unavoidable. # # TODO: consider reworking `spack.traverse` to construct fewer data structures # and objects, as this would make all traversals faster and could eliminate the # need for the complexity here. It was not clear at the time of writing that how # much optimization was possible in `spack.traverse`. sorted_l1_edges = None edge_list = None node_ids = None def nodes(): nonlocal sorted_l1_edges nonlocal edge_list nonlocal node_ids # Level 0: root node yield self._cmp_node # always yield the root (this node) if not self._dependencies: # done if there are no dependencies return # Level 1: direct dependencies # we can yield these in sorted order without tracking visited nodes deps_have_deps = False sorted_l1_edges = self.edges_to_dependencies(depflag=dt.ALL) if len(sorted_l1_edges) > 1: sorted_l1_edges = spack.traverse.sort_edges(sorted_l1_edges) for edge in sorted_l1_edges: yield edge.spec._cmp_node if edge.spec._dependencies: deps_have_deps = True if not deps_have_deps: # done if level 1 specs have no dependencies return # Level 2: dependencies of direct dependencies # now it's general; we need full traverse() to track visited nodes l1_specs = [edge.spec for edge in sorted_l1_edges] # the node_ids dict generates consistent ids based on BFS traversal order # these are used to identify edges later node_ids = collections.defaultdict(lambda: len(node_ids)) node_ids[id(self)] # self is 0 for spec in l1_specs: node_ids[id(spec)] # l1 starts at 1 edge_list = [] for edge in spack.traverse.traverse_edges( l1_specs, order="breadth", cover="edges", root=False, visited=set([0]) ): # yield each node only once, and generate a consistent id for it the # first time it's encountered. if id(edge.spec) not in node_ids: yield edge.spec._cmp_node node_ids[id(edge.spec)] if edge.parent is None: # skip fake edge to root continue edge_list.append( ( node_ids[id(edge.parent)], node_ids[id(edge.spec)], edge.depflag, edge.virtuals, edge.direct, edge.when, ) ) def edges(): # no edges in single-node graph if not self._dependencies: return # level 1 edges all start with zero for i, edge in enumerate(sorted_l1_edges, start=1): yield (0, i, edge.depflag, edge.virtuals, edge.direct, edge.when) # yield remaining edges in the order they were encountered during traversal if edge_list: yield from edge_list yield nodes yield edges @property def namespace_if_anonymous(self): return self.namespace if not self.name else None @property def spack_root(self): """Special field for using ``{spack_root}`` in :meth:`format`.""" return spack.paths.spack_root @property def spack_install(self): """Special field for using ``{spack_install}`` in :meth:`format`.""" from spack.store import STORE return STORE.layout.root def _format_default(self) -> str: """Fast path for formatting with DEFAULT_FORMAT and no color. This method manually concatenates the string representation of spec attributes, avoiding the regex parsing overhead of the general format() method. """ parts = [] if self.name: parts.append(self.name) if self.versions: version_str = str(self.versions) if version_str and version_str != ":": # only include if not full range parts.append(f"@{version_str}") compiler_flags_str = str(self.compiler_flags) if compiler_flags_str: parts.append(compiler_flags_str) variants_str = str(self.variants) if variants_str: parts.append(variants_str) if not self.name and self.namespace: parts.append(f" namespace={self.namespace}") if self.architecture: if self.architecture.platform: parts.append(f" platform={self.architecture.platform}") if self.architecture.os: parts.append(f" os={self.architecture.os}") if self.architecture.target: parts.append(f" target={self.architecture.target}") if self.abstract_hash: parts.append(f"/{self.abstract_hash}") return "".join(parts).strip() def format( self, format_string: str = DEFAULT_FORMAT, color: Optional[bool] = False, *, highlight_version_fn: Optional[Callable[["Spec"], bool]] = None, highlight_variant_fn: Optional[Callable[["Spec", str], bool]] = None, ) -> str: r"""Prints out attributes of a spec according to a format string. Using an ``{attribute}`` format specifier, any field of the spec can be selected. Those attributes can be recursive. For example, ``s.format({compiler.version})`` will print the version of the compiler. If the attribute in a format specifier evaluates to ``None``, then the format specifier will evaluate to the empty string, ``""``. Commonly used attributes of the Spec for format strings include: .. code-block:: text name version compiler_flags compilers variants architecture architecture.platform architecture.os architecture.target prefix namespace Some additional special-case properties can be added: .. code-block:: text hash[:len] The DAG hash with optional length argument spack_root The spack root directory spack_install The spack install directory The ``^`` sigil can be used to access dependencies by name. ``s.format({^mpi.name})`` will print the name of the MPI implementation in the spec. The ``@``, ``%``, and ``/`` sigils can be used to include the sigil with the printed string. These sigils may only be used with the appropriate attributes, listed below: * ``@``: ``{@version}``, ``{@compiler.version}`` * ``%``: ``{%compiler}``, ``{%compiler.name}`` * ``/``: ``{/hash}``, ``{/hash:7}``, etc The ``@`` sigil may also be used for any other property named ``version``. Sigils printed with the attribute string are only printed if the attribute string is non-empty, and are colored according to the color of the attribute. Variants listed by name naturally print with their sigil. For example, ``spec.format("{variants.debug}")`` prints either ``+debug`` or ``~debug`` depending on the name of the variant. Non-boolean variants print as ``name=value``. To print variant names or values independently, use ``spec.format("{variants..name}")`` or ``spec.format("{variants..value}")``. There are a few attributes on specs that can be specified as key-value pairs that are *not* variants, e.g.: ``os``, ``arch``, ``architecture``, ``target``, ``namespace``, etc. You can format these with an optional ``key=`` prefix, e.g. ``{namespace=namespace}`` or ``{arch=architecture}``, etc. The ``key=`` prefix will be colorized along with the value. When formatting specs, key-value pairs are separated from preceding parts of the spec by whitespace. To avoid printing extra whitespace when the formatted attribute is not set, you can add whitespace to the key *inside* the braces of the format string, e.g.: .. code-block:: text { namespace=namespace} This evaluates to ``" namespace=builtin"`` if ``namespace`` is set to ``builtin``, and to ``""`` if ``namespace`` is ``None``. Spec format strings use ``\`` as the escape character. Use ``\{`` and ``\}`` for literal braces, and ``\\`` for the literal ``\`` character. Args: format_string: string containing the format to be expanded color: True for colorized result; False for no color; None for auto color. highlight_version_fn: optional callable that returns true on nodes where the version needs to be highlighted highlight_variant_fn: optional callable that returns true on variants that need to be highlighted """ # Fast path for the common case: default format with no color if format_string == DEFAULT_FORMAT and color is False: return self._format_default() ensure_modern_format_string(format_string) def safe_color(sigil: str, string: str, color_fmt: Optional[str]) -> str: # avoid colorizing if there is no color or the string is empty if (color is False) or not color_fmt or not string: return sigil + string # escape and add the sigil here to avoid multiple concatenations if sigil == "@": sigil = "@@" return clr.colorize(f"{color_fmt}{sigil}{clr.cescape(string)}@.", color=color) def format_attribute(match_object: Match) -> str: (esc, sig, dep, hash, hash_len, attribute, close_brace, unmatched_close_brace) = ( match_object.groups() ) if esc: return esc elif unmatched_close_brace: raise SpecFormatStringError(f"Unmatched close brace: '{format_string}'") elif not close_brace: raise SpecFormatStringError(f"Missing close brace: '{format_string}'") current_node = self if dep is None else self[dep] current = current_node # Hash attributes can return early. # NOTE: we currently treat abstract_hash like an attribute and ignore # any length associated with it. We may want to change that. if hash: if sig and sig != "/": raise SpecFormatSigilError(sig, "DAG hashes", hash) try: length = int(hash_len) if hash_len else None except ValueError: raise SpecFormatStringError(f"Invalid hash length: '{hash_len}'") return safe_color(sig or "", current.dag_hash(length), HASH_COLOR) if attribute == "": raise SpecFormatStringError("Format string attributes must be non-empty") attribute = attribute.lower() parts = attribute.split(".") assert parts # check that the sigil is valid for the attribute. if not sig: sig = "" elif sig == "@" and parts[-1] not in ("versions", "version"): raise SpecFormatSigilError(sig, "versions", attribute) elif sig == "%" and attribute not in ("compiler", "compiler.name"): raise SpecFormatSigilError(sig, "compilers", attribute) elif sig == "/" and attribute != "abstract_hash": raise SpecFormatSigilError(sig, "DAG hashes", attribute) # Iterate over components using getattr to get next element for idx, part in enumerate(parts): if not part: raise SpecFormatStringError("Format string attributes must be non-empty") elif part.startswith("_"): raise SpecFormatStringError("Attempted to format private attribute") elif isinstance(current, VariantMap): # subscript instead of getattr for variant names try: current = current[part] except KeyError: raise SpecFormatStringError(f"Variant '{part}' does not exist") else: # aliases if part == "arch": part = "architecture" elif part == "version" and not current.versions.concrete: # version (singular) requires a concrete versions list. Avoid # pedantic errors by using versions (plural) when not concrete. # These two are not entirely equivalent for pkg@=1.2.3: # - version prints '1.2.3' # - versions prints '=1.2.3' part = "versions" try: current = getattr(current, part) except AttributeError: if part == "compiler": return "none" elif part == "specfile_version": return f"v{current.original_spec_format()}" raise SpecFormatStringError( f"Attempted to format attribute {attribute}. " f"Spec {'.'.join(parts[:idx])} has no attribute {part}" ) if isinstance(current, vn.VersionList) and current == vn.any_version: # don't print empty version lists return "" if callable(current): raise SpecFormatStringError("Attempted to format callable object") if current is None: # not printing anything return "" # Set color codes for various attributes color = None if "architecture" in parts: color = ARCHITECTURE_COLOR elif "variants" in parts or sig.endswith("="): color = VARIANT_COLOR elif any(c in parts for c in ("compiler", "compilers", "compiler_flags")): color = COMPILER_COLOR elif "version" in parts or "versions" in parts: color = VERSION_COLOR if highlight_version_fn and highlight_version_fn(current_node): color = HIGHLIGHT_COLOR # return empty string if the value of the attribute is None. if current is None: return "" # Override the color for single variants, if need be if color and highlight_variant_fn and isinstance(current, VariantMap): bool_keys, kv_keys = current.partition_keys() result = "" for key in bool_keys: current_color = color if highlight_variant_fn(current_node, key): current_color = HIGHLIGHT_COLOR result += safe_color(sig, str(current[key]), current_color) for key in kv_keys: current_color = color if highlight_variant_fn(current_node, key): current_color = HIGHLIGHT_COLOR # Don't highlight the space before the key/value pair result += " " + safe_color(sig, f"{current[key]}", current_color) return result # return colored output return safe_color(sig, str(current), color) return SPEC_FORMAT_RE.sub(format_attribute, format_string).strip() def cformat(self, format_string: str = DEFAULT_FORMAT) -> str: """Same as :meth:`format`, but color defaults to auto instead of False.""" return self.format(format_string, color=None) def format_path( # self, format_string: str, _path_ctor: Optional[pathlib.PurePath] = None self, format_string: str, _path_ctor: Optional[Callable[[Any], pathlib.PurePath]] = None, ) -> str: """Given a ``format_string`` that is intended as a path, generate a string like from :meth:`format`, but eliminate extra path separators introduced by formatting of Spec properties. Path separators explicitly added to the string are preserved, so for example ``{name}/{version}`` would generate a directory based on the Spec's name, and a subdirectory based on its version; this function guarantees though that the resulting string would only have two directories (i.e. that if under normal circumstances that ``str(self.version)`` would contain a path separator, it would not in this case). """ format_component_with_sep = r"\{[^}]*[/\\][^}]*}" if re.search(format_component_with_sep, format_string): raise SpecFormatPathError( f"Invalid path format string: cannot contain {{/...}}\n\t{format_string}" ) path_ctor = _path_ctor or pathlib.PurePath format_string_as_path = path_ctor(format_string) if format_string_as_path.is_absolute() or ( # Paths that begin with a single "\" on windows are relative, but we still # want to preserve the initial "\\" to be consistent with PureWindowsPath. # Ensure that this '\' is not passed to polite_filename() so it's not converted to '_' (os.name == "nt" or path_ctor == pathlib.PureWindowsPath) and format_string_as_path.parts[0] == "\\" ): output_path_components = [format_string_as_path.parts[0]] input_path_components = list(format_string_as_path.parts[1:]) else: output_path_components = [] input_path_components = list(format_string_as_path.parts) output_path_components += [ fs.polite_filename(self.format(part)) for part in input_path_components ] return str(path_ctor(*output_path_components)) def _format_edge_attributes(self, dep: DependencySpec, deptypes=True, virtuals=True): deptypes_str = ( f"deptypes={','.join(dt.flag_to_tuple(dep.depflag))}" if deptypes and dep.depflag else "" ) when_str = f"when='{(dep.when)}'" if dep.when != EMPTY_SPEC else "" virtuals_str = f"virtuals={','.join(dep.virtuals)}" if virtuals and dep.virtuals else "" attrs = " ".join(s for s in (when_str, deptypes_str, virtuals_str) if s) if attrs: attrs = f"[{attrs}] " return attrs def _format_dependencies( self, format_string: str = DEFAULT_FORMAT, include: Optional[Callable[[DependencySpec], bool]] = None, deptypes: bool = True, color: Optional[bool] = False, _force_direct: bool = False, ): """Helper for formatting dependencies on specs. Arguments: format_string: format string to use for each dependency include: predicate to select which dependencies to include deptypes: whether to format deptypes color: colorize if True, don't colorize if False, auto-colorize if None _force_direct: if True, print all dependencies as direct dependencies (to be removed when we have this metadata on concrete edges) """ include = include or (lambda dep: True) parts = [] if self.concrete: direct = self.edges_to_dependencies() transitive: List[DependencySpec] = [] else: direct, transitive = lang.stable_partition( self.edges_to_dependencies(), predicate_fn=lambda x: x.direct ) # helper for direct and transitive loops below def format_edge(edge: DependencySpec, sigil: str, dep_spec: Optional[Spec] = None) -> str: dep_spec = dep_spec or edge.spec dep_format = dep_spec.format(format_string, color=color) edge_attributes = ( self._format_edge_attributes(edge, deptypes=deptypes, virtuals=False) if edge.depflag or edge.when != EMPTY_SPEC else "" ) virtuals = f"{','.join(edge.virtuals)}=" if edge.virtuals else "" star = _anonymous_star(edge, dep_format) return f"{sigil}{edge_attributes}{star}{virtuals}{dep_format}" # direct dependencies for edge in sorted(direct, key=lambda x: x.spec.name): if not include(edge): continue # replace legacy compiler names old_name = edge.spec.name new_name = spack.aliases.BUILTIN_TO_LEGACY_COMPILER.get(old_name) try: # this is ugly but copies can be expensive sigil = "%" if new_name: edge.spec.name = new_name if edge.propagation == PropagationPolicy.PREFERENCE: sigil = "%%" parts.append(format_edge(edge, sigil=sigil, dep_spec=edge.spec)) finally: edge.spec.name = old_name if self.concrete: # Concrete specs should go no further, as the complexity # below is O(paths) return " ".join(parts).strip() # transitive dependencies (with any direct dependencies) for edge in sorted(transitive, key=lambda x: x.spec.name): if not include(edge): continue sigil = "%" if _force_direct else "^" # hack til direct deps represented better parts.append(format_edge(edge, sigil, edge.spec)) # also recursively add any direct dependencies of transitive dependencies if edge.spec._dependencies: parts.append( edge.spec._format_dependencies( format_string=format_string, include=include, deptypes=deptypes, _force_direct=_force_direct, ) ) return " ".join(parts).strip() def _long_spec(self, color: Optional[bool] = False) -> str: """Helper for :attr:`long_spec` and :attr:`clong_spec`.""" if self.concrete: return self.tree(format=DISPLAY_FORMAT, color=color) return f"{self.format(color=color)} {self._format_dependencies(color=color)}".strip() def _short_spec(self, color: Optional[bool] = False) -> str: """Helper for :attr:`short_spec` and :attr:`cshort_spec`.""" return self.format( "{name}{@version}{variants}" "{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}" "{/hash:7}", color=color, ) @property def compilers(self): if self.original_spec_format() < 5: # These specs don't have compilers as dependencies, return the compiler node attribute return f" %{self.compiler}" # TODO: get rid of the space here and make formatting smarter return " " + self._format_dependencies( "{name}{@version}", include=lambda dep: any(lang in dep.virtuals for lang in ("c", "cxx", "fortran")), deptypes=False, _force_direct=True, ) @property def long_spec(self): """Long string of the spec, including dependencies.""" return self._long_spec(color=False) @property def clong_spec(self): """Returns an auto-colorized version of :attr:`long_spec`.""" return self._long_spec(color=None) @property def short_spec(self): """Short string of the spec, with hash and without dependencies.""" return self._short_spec(color=False) @property def cshort_spec(self): """Returns an auto-colorized version of :attr:`short_spec`.""" return self._short_spec(color=None) @property def colored_str(self) -> str: """Auto-colorized string representation of this spec.""" return self._str(color=None) def _str(self, color: Optional[bool] = False) -> str: """String representation of this spec. Args: color: colorize if True, don't colorize if False, auto-colorize if None """ if self._concrete: return self.format("{name}{@version}{/hash}", color=color) if not self._dependencies: return self.format(color=color) return self._long_spec(color=color) def __str__(self) -> str: """String representation of this spec.""" return self._str(color=False) def install_status(self) -> InstallStatus: """Helper for tree to print DB install status.""" if not self.concrete: return InstallStatus.absent if self.external: return InstallStatus.external from spack.store import STORE upstream, record = STORE.db.query_by_spec_hash(self.dag_hash()) if not record: return InstallStatus.absent elif upstream and record.installed: return InstallStatus.upstream elif record.installed: return InstallStatus.installed else: return InstallStatus.missing def _installed_explicitly(self): """Helper for tree to print DB install status.""" if not self.concrete: return None try: from spack.store import STORE record = STORE.db.get_record(self) return record.explicit except KeyError: return None def tree( self, *, color: Optional[bool] = None, depth: bool = False, hashes: bool = False, hashlen: Optional[int] = None, cover: spack.traverse.CoverType = "nodes", indent: int = 0, format: str = DEFAULT_FORMAT, deptypes: Union[dt.DepTypes, dt.DepFlag] = dt.ALL, show_types: bool = False, depth_first: bool = False, recurse_dependencies: bool = True, status_fn: Optional[Callable[["Spec"], InstallStatus]] = None, prefix: Optional[Callable[["Spec"], str]] = None, key=id, highlight_version_fn: Optional[Callable[["Spec"], bool]] = None, highlight_variant_fn: Optional[Callable[["Spec", str], bool]] = None, ) -> str: """Prints out this spec and its dependencies, tree-formatted with indentation. See multi-spec ``spack.spec.tree()`` function for details. Args: specs: List of specs to format. color: if True, always colorize the tree. If False, don't colorize the tree. If None, use the default from spack.llnl.tty.color depth: print the depth from the root hashes: if True, print the hash of each node hashlen: length of the hash to be printed cover: either ``"nodes"`` or ``"edges"`` indent: extra indentation for the tree being printed format: format to be used to print each node deptypes: dependency types to be represented in the tree show_types: if True, show the (merged) dependency type of a node depth_first: if True, traverse the DAG depth first when representing it as a tree recurse_dependencies: if True, recurse on dependencies status_fn: optional callable that takes a node as an argument and return its installation status prefix: optional callable that takes a node as an argument and return its installation prefix highlight_version_fn: optional callable that returns true on nodes where the version needs to be highlighted highlight_variant_fn: optional callable that returns true on variants that need to be highlighted """ return tree( [self], color=color, depth=depth, hashes=hashes, hashlen=hashlen, cover=cover, indent=indent, format=format, deptypes=deptypes, show_types=show_types, depth_first=depth_first, recurse_dependencies=recurse_dependencies, status_fn=status_fn, prefix=prefix, key=key, highlight_version_fn=highlight_version_fn, highlight_variant_fn=highlight_variant_fn, ) def __repr__(self): return str(self) @property def platform(self): return self.architecture.platform @property def os(self): return self.architecture.os @property def target(self): return self.architecture.target @property def build_spec(self): return self._build_spec or self @build_spec.setter def build_spec(self, value): self._build_spec = value def trim(self, dep_name): """ Remove any package that is or provides ``dep_name`` transitively from this tree. This can also remove other dependencies if they are only present because of ``dep_name``. """ for spec in list(self.traverse()): new_dependencies = {} for pkg_name, edge_list in spec._dependencies.items(): for edge in edge_list: if (dep_name not in edge.virtuals) and (not dep_name == edge.spec.name): _add_edge_to_map(new_dependencies, edge.spec.name, edge) spec._dependencies = new_dependencies def _virtuals_provided(self, root): """Return set of virtuals provided by self in the context of root""" if root is self: # Could be using any virtual the package can provide return set(v.name for v in self.package.virtuals_provided) hashes = [s.dag_hash() for s in root.traverse()] in_edges = set( [edge for edge in self.edges_from_dependents() if edge.parent.dag_hash() in hashes] ) return set().union(*[edge.virtuals for edge in in_edges]) def _splice_match(self, other, self_root, other_root): """Return True if other is a match for self in a splice of other_root into self_root Other is a splice match for self if it shares a name, or if self is a virtual provider and other provides a superset of the virtuals provided by self. Virtuals provided are evaluated in the context of a root spec (self_root for self, other_root for other). This is a slight oversimplification. Other could be a match for self in the context of one edge in self_root and not in the context of another edge. This method could be expanded in the future to account for these cases. """ if other.name == self.name: return True return bool( bool(self._virtuals_provided(self_root)) and self._virtuals_provided(self_root) <= other._virtuals_provided(other_root) ) def _splice_detach_and_add_dependents(self, replacement, context): """Helper method for Spec._splice_helper. replacement is a node to splice in, context is the scope of dependents to consider relevant to this splice.""" # Update build_spec attributes for all transitive dependents # before we start changing their dependencies ancestors_in_context = [ a for a in self.traverse(root=False, direction="parents") if a in context.traverse(deptype=dt.LINK | dt.RUN) ] for ancestor in ancestors_in_context: # Only set it if it hasn't been spliced before ancestor._build_spec = ancestor._build_spec or ancestor.copy() ancestor.clear_caches(ignore=(ht.package_hash.attr,)) for edge in ancestor.edges_to_dependencies(depflag=dt.BUILD): if edge.depflag & ~dt.BUILD: edge.depflag &= ~dt.BUILD else: ancestor._dependencies[edge.spec.name].remove(edge) edge.spec._dependents[ancestor.name].remove(edge) # For each direct dependent in the link/run graph, replace the dependency on # node with one on replacement for edge in self.edges_from_dependents(): if edge.parent not in ancestors_in_context: continue edge.parent._dependencies[self.name].remove(edge) self._dependents[edge.parent.name].remove(edge) edge.parent._add_dependency(replacement, depflag=edge.depflag, virtuals=edge.virtuals) def _splice_helper(self, replacement): """Main loop of a transitive splice. The while loop around a traversal of self ensures that changes to self from previous iterations are reflected in the traversal. This avoids evaluating irrelevant nodes using topological traversal (all incoming edges traversed before any outgoing edge). If any node will not be in the end result, its parent will be spliced and it will not ever be considered. For each node in self, find any analogous node in replacement and swap it in. We assume all build deps are handled outside of this method Arguments: replacement: The node that will replace any equivalent node in self self_root: The root of the spec that self comes from. This provides the context for evaluating whether ``replacement`` is a match for each node of ``self``. See ``Spec._splice_match`` and ``Spec._virtuals_provided`` for details. other_root: The root of the spec that replacement comes from. This provides the context for evaluating whether ``replacement`` is a match for each node of ``self``. See ``Spec._splice_match`` and ``Spec._virtuals_provided`` for details. """ ids = set(id(s) for s in replacement.traverse()) # Sort all possible replacements by name and virtual for easy access later replacements_by_name = collections.defaultdict(list) for node in replacement.traverse(): replacements_by_name[node.name].append(node) virtuals = node._virtuals_provided(root=replacement) for virtual in virtuals: replacements_by_name[virtual].append(node) changed = True while changed: changed = False # Intentionally allowing traversal to change on each iteration # using breadth-first traversal to ensure we only reach nodes that will # be in final result for node in self.traverse(root=False, order="topo", deptype=dt.ALL & ~dt.BUILD): # If this node has already been swapped in, don't consider it again if id(node) in ids: continue analogs = replacements_by_name[node.name] if not analogs: # If we have to check for matching virtuals, then we need to check that it # matches all virtuals. Use `_splice_match` to validate possible matches for virtual in node._virtuals_provided(root=self): analogs += [ r for r in replacements_by_name[virtual] if node._splice_match(r, self_root=self, other_root=replacement) ] # No match, keep iterating over self if not analogs: continue # If there are multiple analogs, this package must satisfy the constraint # that a newer version can always replace a lesser version. analog = max(analogs, key=lambda s: s.version) # No splice needed here, keep checking if analog == node: continue node._splice_detach_and_add_dependents(analog, context=self) changed = True break def splice(self, other: "Spec", transitive: bool = True) -> "Spec": """Returns a new, spliced concrete :class:`Spec` with the ``other`` dependency and, optionally, its dependencies. Args: other: alternate dependency transitive: include other's dependencies Returns: a concrete, spliced version of the current :class:`Spec` When transitive is :data:`True`, use the dependencies from ``other`` to reconcile conflicting dependencies. When transitive is :data:`False`, use dependencies from self. For example, suppose we have the following dependency graph: .. code-block:: text T | \\ Z<-H Spec ``T`` depends on ``H`` and ``Z``, and ``H`` also depends on ``Z``. Now we want to use a different ``H``, called ``H'``. This function can be used to splice in ``H'`` to create a new spec, called ``T*``. If ``H'`` was built with ``Z'``, then ``transitive=True`` will ensure ``H'`` and ``T*`` both depend on ``Z'``: .. code-block:: text T* | \\ Z'<-H' If ``transitive=False``, then ``H'`` and ``T*`` will both depend on the original ``Z``, resulting in a new ``H'*``: .. code-block:: text T* | \\ Z<-H'* Provenance of the build is tracked through the :attr:`build_spec` property of the spliced spec and any correspondingly modified dependency specs. The build specs are set to that of the original spec, so the original spec's provenance is preserved unchanged.""" assert self.concrete assert other.concrete if self._splice_match(other, self_root=self, other_root=other): return other.copy() if not any( node._splice_match(other, self_root=self, other_root=other) for node in self.traverse(root=False, deptype=dt.LINK | dt.RUN) ): other_str = other.format("{name}/{hash:7}") self_str = self.format("{name}/{hash:7}") msg = f"Cannot splice {other_str} into {self_str}." msg += f" Either {self_str} cannot depend on {other_str}," msg += f" or {other_str} fails to provide a virtual used in {self_str}" raise SpliceError(msg) # Copies of all non-build deps, build deps will get added at the end spec = self.copy(deps=dt.ALL & ~dt.BUILD) replacement = other.copy(deps=dt.ALL & ~dt.BUILD) def make_node_pairs(orig_spec, copied_spec): return list( zip( orig_spec.traverse(deptype=dt.ALL & ~dt.BUILD), copied_spec.traverse(deptype=dt.ALL & ~dt.BUILD), ) ) def mask_build_deps(in_spec): for edge in in_spec.traverse_edges(cover="edges"): edge.depflag &= ~dt.BUILD if transitive: # These pairs will allow us to reattach all direct build deps # We need the list of pairs while the two specs still match node_pairs = make_node_pairs(self, spec) # Ignore build deps in the modified spec while doing the splice # They will be added back in at the end mask_build_deps(spec) # Transitively splice any relevant nodes from new into base # This handles all shared dependencies between self and other spec._splice_helper(replacement) else: # Do the same thing as the transitive splice, but reversed node_pairs = make_node_pairs(other, replacement) mask_build_deps(replacement) replacement._splice_helper(spec) # Intransitively splice replacement into spec # This is very simple now that all shared dependencies have been handled for node in spec.traverse(order="topo", deptype=dt.LINK | dt.RUN): if node._splice_match(other, self_root=spec, other_root=other): node._splice_detach_and_add_dependents(replacement, context=spec) # For nodes that were spliced, modify the build spec to ensure build deps are preserved # For nodes that were not spliced, replace the build deps on the spec itself for orig, copy in node_pairs: if copy._build_spec: copy._build_spec = orig.build_spec.copy() else: for edge in orig.edges_to_dependencies(depflag=dt.BUILD): copy._add_dependency(edge.spec, depflag=dt.BUILD, virtuals=edge.virtuals) return spec def mutate(self, mutator, rehash=True) -> bool: """Mutate concrete spec to match constraints represented by mutator. Mutation can modify the spec version, variants, compiler flags, and architecture. Mutation cannot change the spec name, namespace, dependencies, or abstract_hash. Any attribute which is unset will not be touched. Variant values can be replaced with the literal ``None`` to remove the variant. ``None`` as a variant value is represented by ``VariantValue(..., (None,))``. If ``rehash``, concrete spec and its dependents have hashes updated. Returns whether the spec was modified by the mutation""" assert self.concrete if mutator.name and mutator.name != self.name: raise SpecMutationError(f"Cannot mutate spec name: spec {self} mutator {mutator}") if mutator.namespace and mutator.namespace != self.namespace: raise SpecMutationError(f"Cannot mutate spec namespace: spec {self} mutator {mutator}") if len(mutator.dependencies()) > 0: raise SpecMutationError(f"Cannot mutate dependencies: spec {self} mutator {mutator}") if ( mutator.versions != vn.VersionList(":") and not mutator.versions.concrete_range_as_version ): raise SpecMutationError( f"Cannot mutate abstract version: spec {self} mutator {mutator}" ) if mutator.abstract_hash and mutator.abstract_hash != self.abstract_hash: raise SpecMutationError(f"Cannot mutate abstract_hash: spec {self} mutator {mutator}") changed = False if mutator.versions != vn.VersionList(":") and self.versions != mutator.versions: self.versions = mutator.versions changed = True for name, variant in mutator.variants.items(): if variant == self.variants.get(name, None): continue old_variant = self.variants.pop(name, None) if not isinstance(variant, vt.VariantValueRemoval): # sigil type for removing variant if old_variant: variant.type = old_variant.type # coerce variant type to match self.variants[name] = variant changed = True for name, flags in mutator.compiler_flags.items(): if not flags or flags == self.compiler_flags[name]: continue self.compiler_flags[name] = flags changed = True if mutator.architecture: if mutator.platform and mutator.platform != self.architecture.platform: self.architecture.platform = mutator.platform changed = True if mutator.os and mutator.os != self.architecture.os: self.architecture.os = mutator.os changed = True if mutator.target and mutator.target != self.architecture.target: self.architecture.target = mutator.target changed = True if changed and rehash: roots = [] for parent in spack.traverse.traverse_nodes([self], direction="parents"): if not parent.dependents(): roots.append(parent) # invalidate hashes parent._mark_root_concrete(False) parent.clear_caches() for root in roots: # compute new hashes on full DAGs root._finalize_concretization() return changed def clear_caches(self, ignore: Tuple[str, ...] = ()) -> None: """ Clears all cached hashes in a Spec, while preserving other properties. """ for h in ht.HASHES: if h.attr not in ignore: if hasattr(self, h.attr): setattr(self, h.attr, None) for attr in ("_dunder_hash", "_prefix"): if attr not in ignore: setattr(self, attr, None) def __hash__(self): # If the spec is concrete, we leverage the dag hash and just use a 64-bit prefix of it. # The dag hash has the advantage that it's computed once per concrete spec, and it's saved # -- so if we read concrete specs we don't need to recompute the whole hash. if self.concrete: if not self._dunder_hash: self._dunder_hash = self.dag_hash_bit_prefix(64) return self._dunder_hash if not self._dependencies: return hash( ( self.name, self.namespace, self.versions, (self.variants if self.variants.dict else None), self.architecture, self.abstract_hash, ) ) return hash(lang.tuplify(self._cmp_iter)) def __getstate__(self): state = self.__dict__.copy() # The package is lazily loaded upon demand. state.pop("_package", None) # As with to_dict, do not include dependents. This avoids serializing more than intended. state.pop("_dependents", None) # Do not pickle attributes dynamically set by SpecBuildInterface state.pop("wrapped_obj", None) state.pop("token", None) state.pop("last_query", None) state.pop("indirect_spec", None) # Optimize variants and compiler_flags serialization variants = state.pop("variants", None) if variants: state["_variants_data"] = variants.dict flags = state.pop("compiler_flags", None) if flags: state["_compiler_flags_data"] = flags.dict return state def __setstate__(self, state): variants_data = state.pop("_variants_data", None) compiler_flags_data = state.pop("_compiler_flags_data", None) self.__dict__.update(state) self._package = None # Reconstruct variants and compiler_flags self.variants = VariantMap() self.compiler_flags = FlagMap() if variants_data is not None: self.variants.dict = variants_data if compiler_flags_data is not None: self.compiler_flags.dict = compiler_flags_data # Reconstruct dependents map if not hasattr(self, "_dependents"): self._dependents = {} for edges in self._dependencies.values(): for edge in edges: if not hasattr(edge.spec, "_dependents"): edge.spec._dependents = {} _add_edge_to_map(edge.spec._dependents, edge.parent.name, edge) def attach_git_version_lookup(self): # Add a git lookup method for GitVersions if not self.name: return for v in self.versions: if isinstance(v, vn.GitVersion) and v.std_version is None: v.attach_lookup(spack.version.git_ref_lookup.GitRefLookup(self.fullname)) def original_spec_format(self) -> int: """Returns the spec format originally used for this spec.""" return self.annotations.original_spec_format def has_virtual_dependency(self, virtual: str) -> bool: return bool(self.dependencies(virtuals=(virtual,))) class VariantMap(lang.HashableMap[str, vt.VariantValue]): """Map containing variant instances. New values can be added only if the key is not already present.""" def __setitem__(self, name, vspec): # Raise a TypeError if vspec is not of the right type if not isinstance(vspec, vt.VariantValue): raise TypeError( "VariantMap accepts only values of variant types " f"[got {type(vspec).__name__} instead]" ) # Raise an error if the variant was already in this map if name in self.dict: msg = 'Cannot specify variant "{0}" twice'.format(name) raise vt.DuplicateVariantError(msg) # Raise an error if name and vspec.name don't match if name != vspec.name: raise KeyError( f'Inconsistent key "{name}", must be "{vspec.name}" to match VariantSpec' ) # Set the item super().__setitem__(name, vspec) def substitute(self, vspec): """Substitutes the entry under ``vspec.name`` with ``vspec``. Args: vspec: variant spec to be substituted """ if vspec.name not in self: raise KeyError(f"cannot substitute a key that does not exist [{vspec.name}]") # Set the item super().__setitem__(vspec.name, vspec) def partition_variants(self): non_prop, prop = lang.stable_partition(self.values(), lambda x: not x.propagate) # Just return the names non_prop = [x.name for x in non_prop] prop = [x.name for x in prop] return non_prop, prop def copy(self) -> "VariantMap": clone = VariantMap() for name, variant in self.items(): clone[name] = variant.copy() return clone def __str__(self): if not self: return "" # Separate boolean variants from key-value pairs as they print # differently. All booleans go first to avoid ' ~foo' strings that # break spec reuse in zsh. bool_keys, kv_keys = self.partition_keys() # add spaces before and after key/value variants. string = io.StringIO() for key in bool_keys: string.write(str(self[key])) for key in kv_keys: string.write(" ") string.write(str(self[key])) return string.getvalue() def partition_keys(self) -> Tuple[List[str], List[str]]: """Partition the keys of the map into two lists: booleans and key-value pairs.""" bool_keys, kv_keys = lang.stable_partition( sorted(self.keys()), lambda x: self[x].type == vt.VariantType.BOOL ) return bool_keys, kv_keys class SpecBuildInterface(lang.ObjectWrapper, Spec): # home is available in the base Package so no default is needed home = ForwardQueryToPackage("home", default_handler=None) headers = ForwardQueryToPackage("headers", default_handler=_headers_default_handler) libs = ForwardQueryToPackage("libs", default_handler=_libs_default_handler) command = ForwardQueryToPackage("command", default_handler=None, _indirect=True) def __init__( self, spec: "Spec", name: str, query_parameters: List[str], _parent: "Spec", is_virtual: bool, ): lang.ObjectWrapper.__init__(self, spec) # Adding new attributes goes after ObjectWrapper.__init__ call since the ObjectWrapper # resets __dict__ to behave like the passed object original_spec = getattr(spec, "wrapped_obj", spec) self.wrapped_obj = original_spec self.token = original_spec, name, query_parameters, _parent, is_virtual self.last_query = QueryState( name=name, extra_parameters=query_parameters, isvirtual=is_virtual ) # TODO: this ad-hoc logic makes `spec["python"].command` return # `spec["python-venv"].command` and should be removed when `python` is a virtual. self.indirect_spec = None if spec.name == "python": python_venvs = _parent.dependencies("python-venv") if not python_venvs: return self.indirect_spec = python_venvs[0] def __reduce__(self): return SpecBuildInterface, self.token def copy(self, *args, **kwargs): return self.wrapped_obj.copy(*args, **kwargs) def substitute_abstract_variants(spec: Spec): """Uses the information in ``spec.package`` to turn any variant that needs it into a SingleValuedVariant or BoolValuedVariant. This method is best effort. All variants that can be substituted will be substituted before any error is raised. Args: spec: spec on which to operate the substitution """ # This method needs to be best effort so that it works in matrix exclusion # in $spack/lib/spack/spack/spec_list.py unknown = [] for name, v in spec.variants.items(): if v.concrete and v.type == vt.VariantType.MULTI: continue if name in ("dev_path", "commit"): v.type = vt.VariantType.SINGLE v.concrete = True continue elif name in vt.RESERVED_NAMES: continue variant_defs = spack.repo.PATH.get_pkg_class(spec.fullname).variant_definitions(name) valid_defs = [] for when, vdef in variant_defs: if when.intersects(spec): valid_defs.append(vdef) if not valid_defs: if name not in spack.repo.PATH.get_pkg_class(spec.fullname).variant_names(): unknown.append(name) else: whens = [str(when) for when, _ in variant_defs] raise InvalidVariantForSpecError(v.name, f"({', '.join(whens)})", spec) continue pkg_variant, *rest = valid_defs if rest: continue new_variant = pkg_variant.make_variant(*v.values) pkg_variant.validate_or_raise(new_variant, spec.name) spec.variants.substitute(new_variant) if unknown: variants = spack.llnl.string.plural(len(unknown), "variant") raise vt.UnknownVariantError( f"Tried to set {variants} {spack.llnl.string.comma_and(unknown)}. " f"{spec.name} has no such {variants}", unknown_variants=unknown, ) def parse_with_version_concrete(spec_like: Union[str, Spec]): """Same as Spec(string), but interprets @x as @=x""" s = Spec(spec_like) interpreted_version = s.versions.concrete_range_as_version if interpreted_version: s.versions = vn.VersionList([interpreted_version]) return s def reconstruct_virtuals_on_edges(spec: Spec) -> None: """Reconstruct virtuals on edges. Used to read from old DB and reindex.""" virtuals_needed: Dict[str, Set[str]] = {} virtuals_provided: Dict[str, Set[str]] = {} for edge in spec.traverse_edges(cover="edges", root=False): parent_key = edge.parent.dag_hash() if parent_key not in virtuals_needed: # Construct which virtuals are needed by parent virtuals_needed[parent_key] = set() try: parent_pkg = edge.parent.package except Exception as e: warnings.warn( f"cannot reconstruct virtual dependencies on {edge.parent.name}: {e}" ) continue virtuals_needed[parent_key].update( name for name, when_deps in parent_pkg.dependencies_by_name(when=True).items() if spack.repo.PATH.is_virtual(name) and any(edge.parent.satisfies(x) for x in when_deps) ) if not virtuals_needed[parent_key]: continue child_key = edge.spec.dag_hash() if child_key not in virtuals_provided: virtuals_provided[child_key] = set() try: child_pkg = edge.spec.package except Exception as e: warnings.warn( f"cannot reconstruct virtual dependencies on {edge.parent.name}: {e}" ) continue virtuals_provided[child_key].update(x.name for x in child_pkg.virtuals_provided) if not virtuals_provided[child_key]: continue virtuals_to_add = virtuals_needed[parent_key] & virtuals_provided[child_key] if virtuals_to_add: edge.update_virtuals(virtuals_to_add) class SpecfileReaderBase: SPEC_VERSION: int @classmethod def from_node_dict(cls, node): spec = Spec() name, node = cls.name_and_data(node) for h in ht.HASHES: setattr(spec, h.attr, node.get(h.name, None)) # old anonymous spec files had name=None, we use name="" now spec.name = name if isinstance(name, str) else "" spec.namespace = node.get("namespace", None) if "version" in node or "versions" in node: spec.versions = vn.VersionList.from_dict(node) spec.attach_git_version_lookup() if "arch" in node: spec.architecture = ArchSpec.from_dict(node) propagated_names = node.get("propagate", []) abstract_variants = set(node.get("abstract", ())) for name, values in node.get("parameters", {}).items(): propagate = name in propagated_names if name in _valid_compiler_flags: spec.compiler_flags[name] = [] for val in values: spec.compiler_flags.add_flag(name, val, propagate) else: spec.variants[name] = vt.VariantValue.from_node_dict( name, values, propagate=propagate, abstract=name in abstract_variants ) spec.external_path = None spec.external_modules = None if "external" in node: # This conditional is needed because sometimes this function is # called with a node already constructed that contains a 'versions' # and 'external' field. Related to virtual packages provider # indexes. if node["external"]: spec.external_path = node["external"]["path"] spec.external_modules = node["external"]["module"] if spec.external_modules is False: spec.external_modules = None spec.extra_attributes = node["external"].get("extra_attributes") or {} # specs read in are concrete unless marked abstract if node.get("concrete", True): spec._mark_root_concrete() if "patches" in node: patches = node["patches"] if len(patches) > 0: mvar = spec.variants.setdefault("patches", vt.MultiValuedVariant("patches", ())) mvar.set(*patches) # FIXME: Monkey patches mvar to store patches order mvar._patches_in_order_of_appearance = patches # Annotate the compiler spec, might be used later if "annotations" not in node: # Specfile v4 and earlier spec.annotations.with_spec_format(cls.SPEC_VERSION) if "compiler" in node: spec.annotations.with_compiler(cls.legacy_compiler(node)) else: spec.annotations.with_spec_format(node["annotations"]["original_specfile_version"]) if "compiler" in node["annotations"]: spec.annotations.with_compiler(Spec(f"{node['annotations']['compiler']}")) # Don't read dependencies here; from_dict() is used by # from_yaml() and from_json() to read the root *and* each dependency # spec. return spec @classmethod def legacy_compiler(cls, node): d = node["compiler"] return Spec(f"{d['name']}@{vn.VersionList.from_dict(d)}") @classmethod def _load(cls, data): """Construct a spec from JSON/YAML using the format version 2. This format is used in Spack v0.17, was introduced in https://github.com/spack/spack/pull/22845 Args: data: a nested dict/list data structure read from YAML or JSON. """ # Current specfile format nodes = data["spec"]["nodes"] hash_type = None any_deps = False # Pass 0: Determine hash type for node in nodes: for _, _, _, dhash_type, _, _ in cls.dependencies_from_node_dict(node): any_deps = True if dhash_type: hash_type = dhash_type break if not any_deps: # If we never see a dependency... hash_type = ht.dag_hash.name elif not hash_type: # Seen a dependency, still don't know hash_type raise spack.error.SpecError( "Spec dictionary contains malformed dependencies. Old format?" ) hash_dict = {} root_spec_hash = None # Pass 1: Create a single lookup dictionary by hash for i, node in enumerate(nodes): node_hash = node[hash_type] node_spec = cls.from_node_dict(node) hash_dict[node_hash] = node hash_dict[node_hash]["node_spec"] = node_spec if i == 0: root_spec_hash = node_hash if not root_spec_hash: raise spack.error.SpecError("Spec dictionary contains no nodes.") # Pass 2: Finish construction of all DAG edges (including build specs) for node_hash, node in hash_dict.items(): node_spec = node["node_spec"] for _, dhash, dtype, _, virtuals, direct in cls.dependencies_from_node_dict(node): node_spec._add_dependency( hash_dict[dhash]["node_spec"], depflag=dt.canonicalize(dtype), virtuals=virtuals, direct=direct, ) if "build_spec" in node.keys(): _, bhash, _ = cls.extract_build_spec_info_from_node_dict(node, hash_type=hash_type) node_spec._build_spec = hash_dict[bhash]["node_spec"] return hash_dict[root_spec_hash]["node_spec"] @classmethod def extract_build_spec_info_from_node_dict(cls, node, hash_type=ht.dag_hash.name): raise NotImplementedError("Subclasses must implement this method.") @classmethod def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name): raise NotImplementedError("Subclasses must implement this method.") class SpecfileV1(SpecfileReaderBase): SPEC_VERSION = 1 @classmethod def load(cls, data): """Construct a spec from JSON/YAML using the format version 1. Note: Version 1 format has no notion of a build_spec, and names are guaranteed to be unique. This function is guaranteed to read specs as old as v0.10 - while it was not checked for older formats. Args: data: a nested dict/list data structure read from YAML or JSON. """ nodes = data["spec"] # Read nodes out of list. Root spec is the first element; # dependencies are the following elements. dep_list = [cls.from_node_dict(node) for node in nodes] if not dep_list: raise spack.error.SpecError("specfile contains no nodes.") deps = {spec.name: spec for spec in dep_list} result = dep_list[0] for node in nodes: # get dependency dict from the node. name, data = cls.name_and_data(node) for dname, _, dtypes, _, virtuals, direct in cls.dependencies_from_node_dict(data): deps[name]._add_dependency( deps[dname], depflag=dt.canonicalize(dtypes), virtuals=virtuals, direct=direct ) reconstruct_virtuals_on_edges(result) return result @classmethod def name_and_data(cls, node): name = next(iter(node)) node = node[name] return name, node @classmethod def dependencies_from_node_dict(cls, node): if "dependencies" not in node: return [] for t in cls.read_specfile_dep_specs(node["dependencies"]): yield t @classmethod def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name): """Read the DependencySpec portion of a YAML-formatted Spec. This needs to be backward-compatible with older spack spec formats so that reindex will work on old specs/databases. """ for dep_name, elt in deps.items(): if isinstance(elt, dict): for h in ht.HASHES: if h.name in elt: dep_hash, deptypes = elt[h.name], elt["type"] hash_type = h.name virtuals = [] break else: # We never determined a hash type... raise spack.error.SpecError("Couldn't parse dependency spec.") else: raise spack.error.SpecError("Couldn't parse dependency types in spec.") yield dep_name, dep_hash, list(deptypes), hash_type, list(virtuals), True class SpecfileV2(SpecfileReaderBase): SPEC_VERSION = 2 @classmethod def load(cls, data): result = cls._load(data) reconstruct_virtuals_on_edges(result) return result @classmethod def name_and_data(cls, node): return node["name"], node @classmethod def dependencies_from_node_dict(cls, node): return cls.read_specfile_dep_specs(node.get("dependencies", [])) @classmethod def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name): """Read the DependencySpec portion of a YAML-formatted Spec. This needs to be backward-compatible with older spack spec formats so that reindex will work on old specs/databases. """ if not isinstance(deps, list): raise spack.error.SpecError("Spec dictionary contains malformed dependencies") result = [] for dep in deps: elt = dep dep_name = dep["name"] if isinstance(elt, dict): # new format: elements of dependency spec are keyed. for h in ht.HASHES: if h.name in elt: dep_hash, deptypes, hash_type, virtuals, direct = ( cls.extract_info_from_dep(elt, h) ) break else: # We never determined a hash type... raise spack.error.SpecError("Couldn't parse dependency spec.") else: raise spack.error.SpecError("Couldn't parse dependency types in spec.") result.append((dep_name, dep_hash, list(deptypes), hash_type, list(virtuals), direct)) return result @classmethod def extract_info_from_dep(cls, elt, hash): dep_hash, deptypes = elt[hash.name], elt["type"] hash_type = hash.name virtuals = [] direct = True return dep_hash, deptypes, hash_type, virtuals, direct @classmethod def extract_build_spec_info_from_node_dict(cls, node, hash_type=ht.dag_hash.name): build_spec_dict = node["build_spec"] return build_spec_dict["name"], build_spec_dict[hash_type], hash_type class SpecfileV3(SpecfileV2): SPEC_VERSION = 3 class SpecfileV4(SpecfileV2): SPEC_VERSION = 4 @classmethod def extract_info_from_dep(cls, elt, hash): dep_hash = elt[hash.name] deptypes = elt["parameters"]["deptypes"] hash_type = hash.name virtuals = elt["parameters"]["virtuals"] direct = True return dep_hash, deptypes, hash_type, virtuals, direct @classmethod def load(cls, data): return cls._load(data) class SpecfileV5(SpecfileV4): SPEC_VERSION = 5 @classmethod def legacy_compiler(cls, node): raise RuntimeError("The 'compiler' option is unexpected in specfiles at v5 or greater") @classmethod def extract_info_from_dep(cls, elt, hash): dep_hash = elt[hash.name] deptypes = elt["parameters"]["deptypes"] hash_type = hash.name virtuals = elt["parameters"]["virtuals"] direct = elt["parameters"].get("direct", False) return dep_hash, deptypes, hash_type, virtuals, direct #: Alias to the latest version of specfiles SpecfileLatest = SpecfileV5 class LazySpecCache(collections.defaultdict): """Cache for Specs that uses a spec_like as key, and computes lazily the corresponding value ``Spec(spec_like``. """ def __init__(self): super().__init__(Spec) def __missing__(self, key): value = self.default_factory(key) self[key] = value return value def save_dependency_specfiles(root: Spec, output_directory: str, dependencies: List[Spec]): """Given a root spec (represented as a yaml object), index it with a subset of its dependencies, and write each dependency to a separate yaml file in the output directory. By default, all dependencies will be written out. To choose a smaller subset of dependencies to be written, pass a list of package names in the dependencies parameter. If the format of the incoming spec is not json, that can be specified with the spec_format parameter. This can be used to convert from yaml specfiles to the json format.""" for spec in root.traverse(): if not any(spec.satisfies(dep) for dep in dependencies): continue json_path = os.path.join(output_directory, f"{spec.name}.json") with open(json_path, "w", encoding="utf-8") as fd: fd.write(spec.to_json(hash=ht.dag_hash)) def get_host_environment_metadata() -> Dict[str, str]: """Get the host environment, reduce to a subset that we can store in the install directory, and add the spack version. """ environ = get_host_environment() return { "host_os": environ["os"], "platform": environ["platform"], "host_target": environ["target"], "hostname": environ["hostname"], "spack_version": spack.get_version(), "kernel_version": platform.version(), } def get_host_environment() -> Dict[str, Any]: """Returns a dictionary with host information (not including the os.environ).""" host_platform = spack.platforms.host() host_target = host_platform.default_target() host_os = host_platform.default_operating_system() arch_fmt = "platform={0} os={1} target={2}" arch_spec = Spec(arch_fmt.format(host_platform, host_os, host_target)) return { "target": str(host_target), "os": str(host_os), "platform": str(host_platform), "arch": arch_spec, "architecture": arch_spec, "arch_str": str(arch_spec), "hostname": socket.gethostname(), } def eval_conditional(string): """Evaluate conditional definitions using restricted variable scope.""" valid_variables = get_host_environment() valid_variables.update({"re": re, "env": os.environ}) return eval(string, valid_variables) def _inject_patches_variant(root: Spec) -> None: # This dictionary will store object IDs rather than Specs as keys # since the Spec __hash__ will change as patches are added to them spec_to_patches: Dict[int, Set[spack.patch.Patch]] = {} for s in root.traverse(): assert s.namespace is not None, ( f"internal error: {s.name} has no namespace after concretization. " f"Please report a bug at https://github.com/spack/spack/issues" ) if s.concrete: continue # Add any patches from the package to the spec. node_patches = { patch for cond, patch_list in spack.repo.PATH.get_pkg_class(s.fullname).patches.items() if s.satisfies(cond) for patch in patch_list } if node_patches: spec_to_patches[id(s)] = node_patches # Also record all patches required on dependencies by depends_on(..., patch=...) for dspec in root.traverse_edges(deptype=dt.ALL, cover="edges", root=False): if dspec.spec.concrete: continue pkg_deps = spack.repo.PATH.get_pkg_class(dspec.parent.fullname).dependencies edge_patches: List[spack.patch.Patch] = [] for cond, deps_by_name in pkg_deps.items(): dependency = deps_by_name.get(dspec.spec.name) if not dependency: continue if not dspec.parent.satisfies(cond): continue for pcond, patch_list in dependency.patches.items(): if dspec.spec.satisfies(pcond): edge_patches.extend(patch_list) if edge_patches: spec_to_patches.setdefault(id(dspec.spec), set()).update(edge_patches) for spec in root.traverse(): if id(spec) not in spec_to_patches: continue patches = list(spec_to_patches[id(spec)]) variant: vt.VariantValue = spec.variants.setdefault( "patches", vt.MultiValuedVariant("patches", ()) ) variant.set(*(p.sha256 for p in patches)) # FIXME: Monkey patches variant to store patches order ordered_hashes = [(*p.ordering_key, p.sha256) for p in patches if p.ordering_key] ordered_hashes.sort() tty.debug( f"Ordered hashes [{spec.name}]: " + ", ".join("/".join(str(e) for e in t) for t in ordered_hashes) ) setattr( variant, "_patches_in_order_of_appearance", [sha256 for _, _, sha256 in ordered_hashes] ) class InvalidVariantForSpecError(spack.error.SpecError): """Raised when an invalid conditional variant is specified.""" def __init__(self, variant, when, spec): msg = f"Invalid variant {variant} for spec {spec}.\n" msg += f"{variant} is only available for {spec.name} when satisfying one of {when}." super().__init__(msg) class UnsupportedPropagationError(spack.error.SpecError): """Raised when propagation (==) is used with reserved variant names.""" class DuplicateDependencyError(spack.error.SpecError): """Raised when the same dependency occurs in a spec twice.""" class UnsupportedCompilerError(spack.error.SpecError): """Raised when the user asks for a compiler spack doesn't know about.""" class DuplicateArchitectureError(spack.error.SpecError): """Raised when the same architecture occurs in a spec twice.""" class InvalidDependencyError(spack.error.SpecError): """Raised when a dependency in a spec is not actually a dependency of the package.""" def __init__(self, pkg, deps): self.invalid_deps = deps super().__init__( "Package {0} does not depend on {1}".format(pkg, spack.llnl.string.comma_or(deps)) ) class UnsatisfiableSpecNameError(spack.error.UnsatisfiableSpecError): """Raised when two specs aren't even for the same package.""" def __init__(self, provided, required): super().__init__(provided, required, "name") class UnsatisfiableVersionSpecError(spack.error.UnsatisfiableSpecError): """Raised when a spec version conflicts with package constraints.""" def __init__(self, provided, required): super().__init__(provided, required, "version") class UnsatisfiableArchitectureSpecError(spack.error.UnsatisfiableSpecError): """Raised when a spec architecture conflicts with package constraints.""" def __init__(self, provided, required): super().__init__(provided, required, "architecture") # TODO: get rid of this and be more specific about particular incompatible # dep constraints class UnsatisfiableDependencySpecError(spack.error.UnsatisfiableSpecError): """Raised when some dependency of constrained specs are incompatible""" def __init__(self, provided, required): super().__init__(provided, required, "dependency") class UnconstrainableDependencySpecError(spack.error.SpecError): """Raised when attempting to constrain by an anonymous dependency spec""" def __init__(self, spec): msg = "Cannot constrain by spec '%s'. Cannot constrain by a" % spec msg += " spec containing anonymous dependencies" super().__init__(msg) class AmbiguousHashError(spack.error.SpecError): def __init__(self, msg, *specs): spec_fmt = ( "{namespace}.{name}{@version}{variants}" "{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}" "{/hash:7}" ) specs_str = "\n " + "\n ".join(spec.format(spec_fmt) for spec in specs) super().__init__(msg + specs_str) class InvalidHashError(spack.error.SpecError): def __init__(self, spec, hash): msg = f"No spec with hash {hash} could be found to match {spec}." msg += " Either the hash does not exist, or it does not match other spec constraints." super().__init__(msg) class SpecFormatStringError(spack.error.SpecError): """Called for errors in Spec format strings.""" class SpecFormatPathError(spack.error.SpecError): """Called for errors in Spec path-format strings.""" class SpecFormatSigilError(SpecFormatStringError): """Called for mismatched sigils and attributes in format strings""" def __init__(self, sigil, requirement, used): msg = "The sigil %s may only be used for %s." % (sigil, requirement) msg += " It was used with the attribute %s." % used super().__init__(msg) class ConflictsInSpecError(spack.error.SpecError, RuntimeError): def __init__(self, spec, matches): message = 'Conflicts in concretized spec "{0}"\n'.format(spec.short_spec) visited = set() long_message = "" match_fmt_default = '{0}. "{1}" conflicts with "{2}"\n' match_fmt_custom = '{0}. "{1}" conflicts with "{2}" [{3}]\n' for idx, (s, c, w, msg) in enumerate(matches): if s not in visited: visited.add(s) long_message += "List of matching conflicts for spec:\n\n" long_message += s.tree(indent=4) + "\n" if msg is None: long_message += match_fmt_default.format(idx + 1, c, w) else: long_message += match_fmt_custom.format(idx + 1, c, w, msg) super().__init__(message, long_message) class SpecDeprecatedError(spack.error.SpecError): """Raised when a spec concretizes to a deprecated spec or dependency.""" class InvalidSpecDetected(spack.error.SpecError): """Raised when a detected spec doesn't pass validation checks.""" class SpliceError(spack.error.SpecError): """Raised when a splice is not possible due to dependency or provider satisfaction mismatch. The resulting splice would be unusable.""" class InvalidEdgeError(spack.error.SpecError): """Raised when an edge doesn't pass validation checks.""" class SpecMutationError(spack.error.SpecError): """Raised when a mutation is attempted with invalid attributes.""" class _ImmutableSpec(Spec): """An immutable Spec that prevents a class of accidental mutations.""" _mutable: bool def __init__(self, spec_like: Optional[str] = None) -> None: object.__setattr__(self, "_mutable", True) super().__init__(spec_like) object.__delattr__(self, "_mutable") def __setstate__(self, state) -> None: object.__setattr__(self, "_mutable", True) super().__setstate__(state) object.__delattr__(self, "_mutable") def constrain(self, *args, **kwargs) -> bool: assert self._mutable return super().constrain(*args, **kwargs) def add_dependency_edge(self, *args, **kwargs): assert self._mutable return super().add_dependency_edge(*args, **kwargs) def __setattr__(self, name, value) -> None: assert self._mutable super().__setattr__(name, value) def __delattr__(self, name) -> None: assert self._mutable object.__delattr__(self, name) #: Immutable empty spec, for fast comparisons and reduced memory usage. EMPTY_SPEC = _ImmutableSpec() ================================================ FILE: lib/spack/spack/spec_filter.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from typing import Callable, List import spack.spec class SpecFilter: """Given a method to produce a list of specs, this class can filter them according to different criteria. """ def __init__( self, factory: Callable[[], List[spack.spec.Spec]], is_usable: Callable[[spack.spec.Spec], bool], include: List[str], exclude: List[str], ) -> None: """ Args: factory: factory to produce a list of specs is_usable: predicate that takes a spec in input and returns False if the spec should not be considered for this filter, True otherwise. include: if present, a spec must match at least one entry in the list, to be in the output exclude: if present, a spec must not match any entry in the list to be in the output """ self.factory = factory self.is_usable = is_usable self.include = include self.exclude = exclude def is_selected(self, s: spack.spec.Spec) -> bool: if not self.is_usable(s): return False if self.include and not any(s.satisfies(c) for c in self.include): return False if self.exclude and any(s.satisfies(c) for c in self.exclude): return False return True def selected_specs(self) -> List[spack.spec.Spec]: return [s for s in self.factory() if self.is_selected(s)] ================================================ FILE: lib/spack/spack/spec_parser.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Parser for spec literals Here is the EBNF grammar for a spec:: spec = [name] [node_options] { ^[edge_properties] node } | [name] [node_options] hash | filename node = name [node_options] | [name] [node_options] hash | filename node_options = [@(version_list|version_pair)] [%compiler] { variant } edge_properties = [ { bool_variant | key_value } ] hash = / id filename = (.|/|[a-zA-Z0-9-_]*/)([a-zA-Z0-9-_./]*)(.json|.yaml) name = id | namespace id namespace = { id . } variant = bool_variant | key_value | propagated_bv | propagated_kv bool_variant = +id | ~id | -id propagated_bv = ++id | ~~id | --id key_value = id=id | id=quoted_id propagated_kv = id==id | id==quoted_id compiler = id [@version_list] version_pair = git_version=vid version_list = (version|version_range) [ { , (version|version_range)} ] version_range = vid:vid | vid: | :vid | : version = vid git_version = git.(vid) | git_hash git_hash = [A-Fa-f0-9]{40} quoted_id = " id_with_ws " | ' id_with_ws ' id_with_ws = [a-zA-Z0-9_][a-zA-Z_0-9-.\\s]* vid = [a-zA-Z0-9_][a-zA-Z_0-9-.]* id = [a-zA-Z0-9_][a-zA-Z_0-9-]* Identifiers using the ``=`` command, such as architectures and compiler flags, require a space before the name. There is one context-sensitive part: ids in versions may contain ``.``, while other ids may not. There is one ambiguity: since ``-`` is allowed in an id, you need to put whitespace space before ``-variant`` for it to be tokenized properly. You can either use whitespace, or you can just use ``~variant`` since it means the same thing. Spack uses ``~variant`` in directory names and in the canonical form of specs to avoid ambiguity. Both are provided because ``~`` can cause shell expansion when it is the first character in an id typed on the command line. """ import json import pathlib import re import sys from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Tuple, Union import spack.deptypes import spack.error import spack.version from spack.aliases import LEGACY_COMPILER_TO_BUILTIN from spack.enums import PropagationPolicy from spack.llnl.util.tty import color from spack.tokenize import Token, TokenBase, Tokenizer if TYPE_CHECKING: import spack.spec #: Valid name for specs and variants. Here we are not using #: the previous ``w[\w.-]*`` since that would match most #: characters that can be part of a word in any language IDENTIFIER = r"(?:[a-zA-Z_0-9][a-zA-Z_0-9\-]*)" DOTTED_IDENTIFIER = rf"(?:{IDENTIFIER}(?:\.{IDENTIFIER})+)" GIT_HASH = r"(?:[A-Fa-f0-9]{40})" #: Git refs include branch names, and can contain ``.`` and ``/`` GIT_REF = r"(?:[a-zA-Z_0-9][a-zA-Z_0-9./\-]*)" GIT_VERSION_PATTERN = rf"(?:(?:git\.(?:{GIT_REF}))|(?:{GIT_HASH}))" #: Substitute a package for a virtual, e.g., c,cxx=gcc. #: NOTE: Overlaps w/KVP; this should be first if matched in sequence. VIRTUAL_ASSIGNMENT = ( r"(?:" rf"(?P{IDENTIFIER}(?:,{IDENTIFIER})*)" # comma-separated virtuals rf"=(?P{DOTTED_IDENTIFIER}|{IDENTIFIER})" # package to substitute r")" ) STAR = r"\*" NAME = r"[a-zA-Z_0-9][a-zA-Z_0-9\-.]*" HASH = r"[a-zA-Z_0-9]+" #: These are legal values that *can* be parsed bare, without quotes on the command line. VALUE = r"(?:[a-zA-Z_0-9\-+\*.,:=%^\~\/\\]+)" #: Quoted values can be *anything* in between quotes, including escaped quotes. QUOTED_VALUE = r"(?:'(?:[^']|(?<=\\)')*'|\"(?:[^\"]|(?<=\\)\")*\")" VERSION = r"=?(?:[a-zA-Z0-9_][a-zA-Z_0-9\-\.]*\b)" VERSION_RANGE = rf"(?:(?:{VERSION})?:(?:{VERSION}(?!\s*=))?)" VERSION_LIST = rf"(?:{VERSION_RANGE}|{VERSION})(?:\s*,\s*(?:{VERSION_RANGE}|{VERSION}))*" SPLIT_KVP = re.compile(rf"^({NAME})(:?==?)(.*)$") #: A filename starts either with a ``.`` or a ``/`` or a ``{name}/``, or on Windows, a drive letter #: followed by a colon and ``\`` or ``.`` or ``{name}\`` WINDOWS_FILENAME = r"(?:\.|[a-zA-Z0-9-_]*\\|[a-zA-Z]:\\)(?:[a-zA-Z0-9-_\.\\]*)(?:\.json|\.yaml)" UNIX_FILENAME = r"(?:\.|\/|[a-zA-Z0-9-_]*\/)(?:[a-zA-Z0-9-_\.\/]*)(?:\.json|\.yaml)" FILENAME = WINDOWS_FILENAME if sys.platform == "win32" else UNIX_FILENAME #: Regex to strip quotes. Group 2 will be the unquoted string. STRIP_QUOTES = re.compile(r"^(['\"])(.*)\1$") #: Values that match this (e.g., variants, flags) can be left unquoted in Spack output NO_QUOTES_NEEDED = re.compile(r"^[a-zA-Z0-9,/_.\-\[\]]+$") class SpecTokens(TokenBase): """Enumeration of the different token kinds of tokens in the spec grammar. Order of declaration is extremely important, since text containing specs is parsed with a single regex obtained by ``"|".join(...)`` of all the regex in the order of declaration. """ # Dependency, with optional virtual assignment specifier START_EDGE_PROPERTIES = r"(?:(?:\^|\%\%|\%)\[)" END_EDGE_PROPERTIES = rf"(?:\](?:\s*{VIRTUAL_ASSIGNMENT})?)" DEPENDENCY = rf"(?:(?:\^|\%\%|\%)(?:\s*{VIRTUAL_ASSIGNMENT})?)" # Version VERSION_HASH_PAIR = rf"(?:@(?:{GIT_VERSION_PATTERN})=(?:{VERSION}))" GIT_VERSION = rf"@(?:{GIT_VERSION_PATTERN})" VERSION = rf"(?:@\s*(?:{VERSION_LIST}))" # Variants PROPAGATED_BOOL_VARIANT = rf"(?:(?:\+\+|~~|--)\s*{NAME})" BOOL_VARIANT = rf"(?:[~+-]\s*{NAME})" PROPAGATED_KEY_VALUE_PAIR = rf"(?:{NAME}:?==(?:{VALUE}|{QUOTED_VALUE}))" KEY_VALUE_PAIR = rf"(?:{NAME}:?=(?:{VALUE}|{QUOTED_VALUE}))" # FILENAME FILENAME = rf"(?:{FILENAME})" # Package name FULLY_QUALIFIED_PACKAGE_NAME = rf"(?:{DOTTED_IDENTIFIER})" UNQUALIFIED_PACKAGE_NAME = rf"(?:{IDENTIFIER}|{STAR})" # DAG hash DAG_HASH = rf"(?:/(?:{HASH}))" # White spaces WS = r"(?:\s+)" # Unexpected character(s) UNEXPECTED = r"(?:.[\s]*)" #: Tokenizer that includes all the regexes in the SpecTokens enum SPEC_TOKENIZER = Tokenizer(SpecTokens) def tokenize(text: str) -> Iterator[Token]: """Return a token generator from the text passed as input. Raises: SpecTokenizationError: when unexpected characters are found in the text """ for token in SPEC_TOKENIZER.tokenize(text): if token.kind == SpecTokens.UNEXPECTED: raise SpecTokenizationError(list(SPEC_TOKENIZER.tokenize(text)), text) yield token def parseable_tokens(text: str) -> Iterator[Token]: """Return non-whitespace tokens from the text passed as input Raises: SpecTokenizationError: when unexpected characters are found in the text """ return filter(lambda x: x.kind != SpecTokens.WS, tokenize(text)) class TokenContext: """Token context passed around by parsers""" __slots__ = "token_stream", "current_token", "next_token", "pushed_tokens" def __init__(self, token_stream: Iterator[Token]): self.token_stream = token_stream self.current_token = None self.next_token = None # the next token to be read # if not empty, back of list is front of stream, and we pop from here instead. self.pushed_tokens: List[Token] = [] self.advance() def advance(self): """Advance one token""" self.current_token = self.next_token if self.pushed_tokens: self.next_token = self.pushed_tokens.pop() else: self.next_token = next(self.token_stream, None) def accept(self, kind: SpecTokens): """If the next token is of the specified kind, advance the stream and return True. Otherwise return False. """ if self.next_token and self.next_token.kind == kind: self.advance() return True return False def push_front(self, token=Token): """Push a token onto the front of the stream. Enables a bit of lookahead.""" self.pushed_tokens.append(self.next_token) # back of list is front of stream self.next_token = token def expect(self, *kinds: SpecTokens): return self.next_token and self.next_token.kind in kinds class SpecTokenizationError(spack.error.SpecSyntaxError): """Syntax error in a spec string""" def __init__(self, tokens: List[Token], text: str): message = f"unexpected characters in the spec string\n{text}\n" underline = "" for token in tokens: is_error = token.kind == SpecTokens.UNEXPECTED underline += ("^" if is_error else " ") * (token.end - token.start) message += color.colorize(f"@*r{{{underline}}}") super().__init__(message) def parse_virtual_assignment(context: TokenContext) -> Tuple[str]: """Look at subvalues and, if present, extract virtual and a push a substitute token. This handles things like: * ``^c=gcc`` * ``^c,cxx=gcc`` * ``%[when=+bar] c=gcc`` * ``%[when=+bar] c,cxx=gcc`` Virtual assignment can happen anywhere a dependency node can appear. It is shorthand for ``%[virtuals=c,cxx] gcc``. The ``virtuals=substitute`` key value pair appears in the subvalues of :attr:`~spack.spec_parser.SpecTokens.DEPENDENCY` and :attr:`~spack.spec_parser.SpecTokens.END_EDGE_PROPERTIES` tokens. We extract the virtuals and create a token from the substitute, which is then pushed back on the parser stream so that the head of the stream can be parsed like a regular node. Returns: the virtuals assigned, or None if there aren't any """ assert context.current_token is not None subvalues = context.current_token.subvalues if not subvalues: return () # build a token for the substitute that we can put back on the stream pkg = subvalues["substitute"] token_type = SpecTokens.UNQUALIFIED_PACKAGE_NAME if "." in pkg: token_type = SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME start = context.current_token.value.index(pkg) token = Token(token_type, pkg, start, start + len(pkg)) context.push_front(token) return tuple(subvalues["virtuals"].split(",")) class SpecParser: """Parse text into specs""" __slots__ = "literal_str", "ctx" def __init__(self, literal_str: str): self.literal_str = literal_str self.ctx = TokenContext(parseable_tokens(literal_str)) def tokens(self) -> List[Token]: """Return the entire list of token from the initial text. White spaces are filtered out. """ return list(filter(lambda x: x.kind != SpecTokens.WS, tokenize(self.literal_str))) def next_spec( self, initial_spec: Optional["spack.spec.Spec"] = None ) -> Optional["spack.spec.Spec"]: """Return the next spec parsed from text. Args: initial_spec: object where to parse the spec. If None a new one will be created. Return: The spec that was parsed """ if not self.ctx.next_token: return initial_spec def add_dependency(dep, **edge_properties): """wrapper around root_spec._add_dependency""" try: target_spec._add_dependency(dep, **edge_properties) except spack.error.SpecError as e: raise SpecParsingError(str(e), self.ctx.current_token, self.literal_str) from e if not initial_spec: from spack.spec import Spec initial_spec = Spec() root_spec = SpecNodeParser(self.ctx, self.literal_str).parse(initial_spec) current_spec = root_spec while True: if self.ctx.accept(SpecTokens.START_EDGE_PROPERTIES): has_edge_attrs = True elif self.ctx.accept(SpecTokens.DEPENDENCY): has_edge_attrs = False else: break is_direct = self.ctx.current_token.value[0] == "%" propagation = PropagationPolicy.NONE if is_direct and self.ctx.current_token.value.startswith("%%"): propagation = PropagationPolicy.PREFERENCE if has_edge_attrs: edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse() edge_properties.setdefault("virtuals", ()) edge_properties.setdefault("depflag", 0) else: virtuals = parse_virtual_assignment(self.ctx) edge_properties = {"virtuals": virtuals, "depflag": 0} edge_properties["direct"] = is_direct edge_properties["propagation"] = propagation dependency = self._parse_node(root_spec) if is_direct: target_spec = current_spec if dependency.name in LEGACY_COMPILER_TO_BUILTIN: dependency.name = LEGACY_COMPILER_TO_BUILTIN[dependency.name] else: current_spec = dependency target_spec = root_spec add_dependency(dependency, **edge_properties) return root_spec def _parse_node(self, root_spec: "spack.spec.Spec", root: bool = True): dependency = SpecNodeParser(self.ctx, self.literal_str).parse(root=root) if dependency is None: msg = ( "the dependency sigil and any optional edge attributes must be followed by a " "package name or a node attribute (version, variant, etc.)" ) raise SpecParsingError(msg, self.ctx.current_token, self.literal_str) if root_spec.concrete: raise spack.error.SpecError(str(root_spec), "^" + str(dependency)) return dependency def all_specs(self) -> List["spack.spec.Spec"]: """Return all the specs that remain to be parsed""" return list(iter(self.next_spec, None)) class SpecNodeParser: """Parse a single spec node from a stream of tokens""" __slots__ = "ctx", "has_version", "literal_str" def __init__(self, ctx, literal_str): self.ctx = ctx self.literal_str = literal_str self.has_version = False def parse( self, initial_spec: Optional["spack.spec.Spec"] = None, root: bool = True ) -> "spack.spec.Spec": """Parse a single spec node from a stream of tokens Args: initial_spec: object to be constructed root: True if we're parsing a root, False if dependency after ^ or % Return: The object passed as argument """ if initial_spec is None: from spack.spec import Spec initial_spec = Spec() if not self.ctx.next_token or self.ctx.expect(SpecTokens.DEPENDENCY): return initial_spec # If we start with a package name we have a named spec, we cannot # accept another package name afterwards in a node if self.ctx.accept(SpecTokens.UNQUALIFIED_PACKAGE_NAME): # if name is '*', this is an anonymous spec if self.ctx.current_token.value != "*": initial_spec.name = self.ctx.current_token.value elif self.ctx.accept(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME): parts = self.ctx.current_token.value.split(".") name = parts[-1] namespace = ".".join(parts[:-1]) initial_spec.name = name initial_spec.namespace = namespace elif self.ctx.accept(SpecTokens.FILENAME): return FileParser(self.ctx).parse(initial_spec) def raise_parsing_error(string: str, cause: Optional[Exception] = None): """Raise a spec parsing error with token context.""" raise SpecParsingError(string, self.ctx.current_token, self.literal_str) from cause def add_flag(name: str, value: Union[str, bool], propagate: bool, concrete: bool): """Wrapper around ``Spec._add_flag()`` that adds parser context to errors raised.""" try: initial_spec._add_flag(name, value, propagate, concrete) except Exception as e: raise_parsing_error(str(e), e) while True: if ( self.ctx.accept(SpecTokens.VERSION_HASH_PAIR) or self.ctx.accept(SpecTokens.GIT_VERSION) or self.ctx.accept(SpecTokens.VERSION) ): if self.has_version: raise_parsing_error("Spec cannot have multiple versions") initial_spec.versions = spack.version.VersionList( [spack.version.from_string(self.ctx.current_token.value[1:])] ) initial_spec.attach_git_version_lookup() self.has_version = True elif self.ctx.accept(SpecTokens.BOOL_VARIANT): name = self.ctx.current_token.value[1:].strip() variant_value = self.ctx.current_token.value[0] == "+" add_flag(name, variant_value, propagate=False, concrete=True) elif self.ctx.accept(SpecTokens.PROPAGATED_BOOL_VARIANT): name = self.ctx.current_token.value[2:].strip() variant_value = self.ctx.current_token.value[0:2] == "++" add_flag(name, variant_value, propagate=True, concrete=True) elif self.ctx.accept(SpecTokens.KEY_VALUE_PAIR): name, value = self.ctx.current_token.value.split("=", maxsplit=1) concrete = name.endswith(":") if concrete: name = name[:-1] add_flag( name, strip_quotes_and_unescape(value), propagate=False, concrete=concrete ) elif self.ctx.accept(SpecTokens.PROPAGATED_KEY_VALUE_PAIR): name, value = self.ctx.current_token.value.split("==", maxsplit=1) concrete = name.endswith(":") if concrete: name = name[:-1] add_flag(name, strip_quotes_and_unescape(value), propagate=True, concrete=concrete) elif self.ctx.expect(SpecTokens.DAG_HASH): if initial_spec.abstract_hash: break self.ctx.accept(SpecTokens.DAG_HASH) initial_spec.abstract_hash = self.ctx.current_token.value[1:] else: break return initial_spec class FileParser: """Parse a single spec from a JSON or YAML file""" __slots__ = ("ctx",) def __init__(self, ctx): self.ctx = ctx def parse(self, initial_spec: "spack.spec.Spec") -> "spack.spec.Spec": """Parse a spec tree from a specfile. Args: initial_spec: object where to parse the spec Return: The initial_spec passed as argument, once constructed """ file = pathlib.Path(self.ctx.current_token.value) if not file.exists(): raise spack.error.NoSuchSpecFileError(f"No such spec file: '{file}'") from spack.spec import Spec with file.open("r", encoding="utf-8") as stream: if str(file).endswith(".json"): spec_from_file = Spec.from_json(stream) else: spec_from_file = Spec.from_yaml(stream) initial_spec._dup(spec_from_file) return initial_spec class EdgeAttributeParser: __slots__ = "ctx", "literal_str" def __init__(self, ctx, literal_str): self.ctx = ctx self.literal_str = literal_str def parse(self): attributes = {} while True: if self.ctx.accept(SpecTokens.KEY_VALUE_PAIR): name, value = self.ctx.current_token.value.split("=", maxsplit=1) if name.endswith(":"): name = name[:-1] value = value.strip("'\" ").split(",") attributes[name] = value if name not in ("deptypes", "virtuals", "when"): msg = ( "the only edge attributes that are currently accepted " 'are "deptypes", "virtuals", and "when"' ) raise SpecParsingError(msg, self.ctx.current_token, self.literal_str) # TODO: Add code to accept bool variants here as soon as use variants are implemented elif self.ctx.accept(SpecTokens.END_EDGE_PROPERTIES): virtuals = attributes.get("virtuals", ()) virtuals += parse_virtual_assignment(self.ctx) attributes["virtuals"] = virtuals break else: msg = "unexpected token in edge attributes" raise SpecParsingError(msg, self.ctx.next_token, self.literal_str) # Turn deptypes=... to depflag representation if "deptypes" in attributes: deptype_string = attributes.pop("deptypes") attributes["depflag"] = spack.deptypes.canonicalize(deptype_string) # Turn "when" into a spec if "when" in attributes: attributes["when"] = parse_one_or_raise(attributes["when"][0]) return attributes def parse(text: str, *, toolchains: Optional[Dict] = None) -> List["spack.spec.Spec"]: """Parse text into a list of specs Args: text: text to be parsed toolchains: optional toolchain definitions to expand after parsing Return: List of specs """ specs = SpecParser(text).all_specs() if toolchains: cache: Dict[str, "spack.spec.Spec"] = {} for spec in specs: expand_toolchains(spec, toolchains, _cache=cache) return specs def parse_one_or_raise( text: str, initial_spec: Optional["spack.spec.Spec"] = None, *, toolchains: Optional[Dict] = None, ) -> "spack.spec.Spec": """Parse exactly one spec from text and return it, or raise Args: text: text to be parsed initial_spec: buffer where to parse the spec. If None a new one will be created. toolchains: optional toolchain definitions to expand after parsing """ parser = SpecParser(text) result = parser.next_spec(initial_spec) next_token = parser.ctx.next_token if next_token: message = f"expected a single spec, but got more:\n{text}" underline = f"\n{' ' * next_token.start}{'^' * len(next_token.value)}" message += color.colorize(f"@*r{{{underline}}}") raise ValueError(message) if result is None: raise ValueError("expected a single spec, but got none") if toolchains: expand_toolchains(result, toolchains) return result def _parse_toolchain_config(toolchain_config: Union[str, List[Dict]]) -> "spack.spec.Spec": """Parse a toolchain config entry (string or list) into a Spec.""" if isinstance(toolchain_config, str): toolchain = parse_one_or_raise(toolchain_config) _ensure_all_direct_edges(toolchain) else: from spack.spec import EMPTY_SPEC, Spec toolchain = Spec() for entry in toolchain_config: toolchain_part = parse_one_or_raise(entry["spec"]) when = entry.get("when", "") _ensure_all_direct_edges(toolchain_part) if when: when_spec = Spec(when) for edge in toolchain_part.traverse_edges(): if edge.when is EMPTY_SPEC: edge.when = when_spec.copy() else: edge.when.constrain(when_spec) toolchain.constrain(toolchain_part) return toolchain def _ensure_all_direct_edges(constraint: "spack.spec.Spec") -> None: """Validate that a toolchain spec only has direct (%) edges.""" for edge in constraint.traverse_edges(root=False): if not edge.direct: raise spack.error.SpecError( f"cannot use '^' in toolchain definitions, and the current " f"toolchain contains '{edge.format()}'" ) def expand_toolchains( spec: "spack.spec.Spec", toolchains: Dict, *, _cache: Optional[Dict[str, "spack.spec.Spec"]] = None, ) -> None: """Replace toolchain placeholder deps with expanded toolchain constraints. Walks every node in the spec DAG. For each node, finds direct dependency edges whose child name is a key in ``toolchains``. Removes the placeholder edge, parses the toolchain config, copies with the edge's propagation policy, and constrains the node. """ if _cache is None: _cache = {} for node in list(spec.traverse()): for edge in list(node.edges_to_dependencies()): if not edge.direct: continue name = edge.spec.name if name not in toolchains: continue # Remove the placeholder edge (both directions) node._dependencies[name].remove(edge) if not node._dependencies[name]: del node._dependencies[name] edge.spec._dependents[node.name].remove(edge) if not edge.spec._dependents[node.name]: del edge.spec._dependents[node.name] # Parse and cache toolchain if name not in _cache: _cache[name] = _parse_toolchain_config(toolchains[name]) propagation = edge.propagation propagation_arg = None if propagation != PropagationPolicy.PREFERENCE else propagation # Copy so each usage gets a distinct object (solver depends on this) toolchain = _cache[name].copy(propagation=propagation_arg) node.constrain(toolchain) class SpecParsingError(spack.error.SpecSyntaxError): """Error when parsing tokens""" def __init__(self, message, token, text): message += f"\n{text}" if token: underline = f"\n{' ' * token.start}{'^' * (token.end - token.start)}" message += color.colorize(f"@*r{{{underline}}}") super().__init__(message) def strip_quotes_and_unescape(string: str) -> str: """Remove surrounding single or double quotes from string, if present.""" match = STRIP_QUOTES.match(string) if not match: return string # replace any escaped quotes with bare quotes quote, result = match.groups() return result.replace(rf"\{quote}", quote) def quote_if_needed(value: str) -> str: """Add quotes around the value if it requires quotes. This will add quotes around the value unless it matches :data:`NO_QUOTES_NEEDED`. This adds: * single quotes by default * double quotes around any value that contains single quotes If double quotes are used, we json-escape the string. That is, we escape ``\\``, ``"``, and control codes. """ if NO_QUOTES_NEEDED.match(value): return value return json.dumps(value) if "'" in value else f"'{value}'" ================================================ FILE: lib/spack/spack/stage.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import abc import errno import glob import hashlib import io import os import shutil import stat import sys import tempfile from typing import TYPE_CHECKING, Callable, Dict, Generator, Iterable, List, Optional, Set, Union import spack.caches import spack.config import spack.error import spack.llnl.string import spack.llnl.util.lang import spack.llnl.util.tty as tty import spack.oci.image import spack.resource import spack.spec import spack.util.crypto import spack.util.lock import spack.util.parallel import spack.util.path as sup import spack.util.url as url_util from spack import fetch_strategy as fs # breaks a cycle from spack.llnl.util.filesystem import ( AlreadyExistsError, can_access, get_owner_uid, getuid, install, install_tree, mkdirp, partition_path, remove_linked_tree, symlink, ) from spack.llnl.util.tty.colify import colify from spack.llnl.util.tty.color import colorize from spack.util.crypto import bit_length, prefix_bits from spack.util.editor import editor, executable from spack.version import StandardVersion, VersionList if TYPE_CHECKING: import spack.mirrors.layout import spack.mirrors.mirror import spack.mirrors.utils # The well-known stage source subdirectory name. _source_path_subdir = "spack-src" # The temporary stage name prefix. stage_prefix = "spack-stage-" def compute_stage_name(spec): """Determine stage name given a spec""" spec_stage_structure = stage_prefix if spec.concrete: spec_stage_structure += "{name}-{version}-{hash}" else: spec_stage_structure += "{name}-{version}" # TODO (psakiev, scheibelp) Technically a user could still reintroduce a hash via # config:stage_name. This is a fix for how to handle staging an abstract spec (see #51305) stage_name_structure = spack.config.get("config:stage_name", default=spec_stage_structure) return spec.format_path(format_string=stage_name_structure) def create_stage_root(path: str) -> None: """Create the stage root directory and ensure appropriate access perms.""" assert os.path.isabs(path) and len(path.strip()) > 1 err_msg = "Cannot create stage root {0}: Access to {1} is denied" user_uid = getuid() # Obtain lists of ancestor and descendant paths of the $user node, if any. group_paths, user_node, user_paths = partition_path(path, sup.get_user()) for p in group_paths: if not os.path.exists(p): # Ensure access controls of subdirs created above `$user` inherit # from the parent and share the group. par_stat = os.stat(os.path.dirname(p)) mkdirp(p, group=par_stat.st_gid, mode=par_stat.st_mode) p_stat = os.stat(p) if par_stat.st_gid != p_stat.st_gid: tty.warn( "Expected {0} to have group {1}, but it is {2}".format( p, par_stat.st_gid, p_stat.st_gid ) ) if par_stat.st_mode & p_stat.st_mode != par_stat.st_mode: tty.warn( "Expected {0} to support mode {1}, but it is {2}".format( p, par_stat.st_mode, p_stat.st_mode ) ) if not can_access(p): raise OSError(errno.EACCES, err_msg.format(path, p)) # Add the path ending with the $user node to the user paths to ensure paths # from $user (on down) meet the ownership and permission requirements. if user_node: user_paths.insert(0, user_node) for p in user_paths: # Ensure access controls of subdirs from `$user` on down are # restricted to the user. owner_uid = get_owner_uid(p) if user_uid != owner_uid: tty.warn( "Expected user {0} to own {1}, but it is owned by {2}".format( user_uid, p, owner_uid ) ) spack_src_subdir = os.path.join(path, _source_path_subdir) # When staging into a user-specified directory with `spack stage -p `, we need # to ensure the `spack-src` subdirectory exists, as we can't rely on it being # created automatically by spack. It's not clear why this is the case for `spack # stage -p`, but since `mkdirp()` is idempotent, this should not change the behavior # for any other code paths. if not os.path.isdir(spack_src_subdir): mkdirp(spack_src_subdir, mode=stat.S_IRWXU) def _first_accessible_path(paths): """Find the first path that is accessible, creating it if necessary.""" for path in paths: try: # Ensure the user has access, creating the directory if necessary. if os.path.exists(path): if can_access(path): return path else: # Now create the stage root with the proper group/perms. create_stage_root(path) return path except OSError as e: tty.debug("OSError while checking stage path %s: %s" % (path, str(e))) return None def _resolve_paths(candidates): """ Resolve candidate paths and make user-related adjustments. Adjustments involve removing extra $user from $tempdir if $tempdir includes $user and appending $user if it is not present in the path. """ temp_path = sup.canonicalize_path("$tempdir") user = sup.get_user() tmp_has_usr = user in temp_path.split(os.path.sep) paths = [] for path in candidates: # Remove the extra `$user` node from a `$tempdir/$user` entry for # hosts that automatically append `$user` to `$tempdir`. if path.startswith(os.path.join("$tempdir", "$user")) and tmp_has_usr: path = path.replace("/$user", "", 1) # Ensure the path is unique per user. can_path = sup.canonicalize_path(path) # When multiple users share a stage root, we can avoid conflicts between # them by adding a per-user subdirectory. # Avoid doing this on Windows to keep stage absolute path as short as possible. if user not in can_path and not sys.platform == "win32": can_path = os.path.join(can_path, user) paths.append(can_path) return paths # Cached stage path root _stage_root = None def get_stage_root(): global _stage_root if _stage_root is None: candidates = spack.config.get("config:build_stage") if isinstance(candidates, str): candidates = [candidates] resolved_candidates = _resolve_paths(candidates) path = _first_accessible_path(resolved_candidates) if not path: raise StageError("No accessible stage paths in:", " ".join(resolved_candidates)) _stage_root = path return _stage_root def _mirror_roots(): mirrors = spack.config.get("mirrors") return [ ( sup.substitute_path_variables(root) if root.endswith(os.sep) else sup.substitute_path_variables(root) + os.sep ) for root in mirrors.values() ] class AbstractStage(abc.ABC): """Abstract base class for all stage types. A stage is a directory whose lifetime can be managed with a context manager (but persists if the user requests it). Instances can have a specified name and if they do, then for all instances that have the same name, only one can enter the context manager at a time. This class defines the interface that all stage types must implement. """ #: Set to True to error out if patches fail requires_patch_success = True def __init__(self, name, path, keep, lock): # TODO: This uses a protected member of tempfile, but seemed the only # TODO: way to get a temporary name. It won't be the same as the # TODO: temporary stage area in _stage_root. self.name = name if name is None: self.name = stage_prefix + next(tempfile._get_candidate_names()) # Use the provided path or construct an optionally named stage path. if path is not None: self.path = path else: self.path = os.path.join(get_stage_root(), self.name) # Flag to decide whether to delete the stage folder on exit or not self.keep = keep # File lock for the stage directory. We use one file for all # stage locks. See spack.database.Database.prefix_locker.lock for # details on this approach. self._lock = None self._use_locks = lock # When stages are reused, we need to know whether to re-create # it. This marks whether it has been created/destroyed. self.created = False def _get_lock(self): if not self._lock: sha1 = hashlib.sha1(self.name.encode("utf-8")).digest() lock_id = prefix_bits(sha1, bit_length(sys.maxsize)) stage_lock_path = os.path.join(get_stage_root(), ".lock") self._lock = spack.util.lock.Lock( stage_lock_path, start=lock_id, length=1, desc=self.name ) return self._lock def __enter__(self): """ Entering a stage context will create the stage directory Returns: self """ if self._use_locks: self._get_lock().acquire_write(timeout=60) self.create() return self def __exit__(self, exc_type, exc_val, exc_tb): """ Exiting from a stage context will delete the stage directory unless: - it was explicitly requested not to do so - an exception has been raised Args: exc_type: exception type exc_val: exception value exc_tb: exception traceback Returns: Boolean """ # Delete when there are no exceptions, unless asked to keep. if exc_type is None and not self.keep: self.destroy() if self._use_locks: self._get_lock().release_write() def create(self): """ Ensures the top-level (config:build_stage) directory exists. """ # User has full permissions and group has only read permissions if not os.path.exists(self.path): mkdirp(self.path, mode=stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP) elif not os.path.isdir(self.path): os.remove(self.path) mkdirp(self.path, mode=stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP) # Make sure we can actually do something with the stage we made. ensure_access(self.path) self.created = True @abc.abstractmethod def destroy(self): """Remove the stage directory and its contents.""" ... @abc.abstractmethod def fetch(self, mirror_only: bool = False, err_msg: Optional[str] = None) -> None: """Fetch the source code or resources for this stage.""" ... @abc.abstractmethod def check(self): """Check the integrity of the fetched resources.""" ... @abc.abstractmethod def expand_archive(self): """Expand any downloaded archives.""" ... @abc.abstractmethod def restage(self): """Remove the expanded source and re-expand it.""" ... @abc.abstractmethod def cache_local(self): """Cache the resources locally.""" ... @property @abc.abstractmethod def source_path(self) -> str: """Return the path to the expanded source code.""" ... @property @abc.abstractmethod def expanded(self) -> bool: """Return True if the source has been expanded.""" ... @property @abc.abstractmethod def archive_file(self) -> Optional[str]: """Return the path to the archive file, or None.""" ... def cache_mirror( self, mirror: "spack.caches.MirrorCache", stats: "spack.mirrors.utils.MirrorStatsForOneSpec", ) -> None: """Cache the resources to a mirror (can be no-op).""" pass def steal_source(self, dest: str) -> None: """Copy source to another location (can be no-op).""" pass class Stage(AbstractStage): """Manages a temporary stage directory for building. A Stage object is a context manager that handles a directory where some source code is downloaded and built before being installed. It handles fetching the source code, either as an archive to be expanded or by checking it out of a repository. A stage's lifecycle looks like this:: with Stage() as stage: # Context manager creates and destroys the # stage directory stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the archive into source_path. # Build and install the archive. # (handled by user of Stage) When used as a context manager, the stage is automatically destroyed if no exception is raised by the context. If an exception is raised, the stage is left in the filesystem and NOT destroyed, for potential reuse later. You can also use the stage's create/destroy functions manually, like this:: stage = Stage() try: stage.create() # Explicitly create the stage directory. stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the archive into source_path. # Build and install the archive. # (handled by user of Stage) finally: stage.destroy() # Explicitly destroy the stage directory. There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you fetched a tarball but didn't finish building it, you won't have to fetch it again. Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for only one run of spack. """ requires_patch_success = True def __init__( self, url_or_fetch_strategy, *, name=None, mirror_paths: Optional["spack.mirrors.layout.MirrorLayout"] = None, mirrors: Optional[Iterable["spack.mirrors.mirror.Mirror"]] = None, keep=False, path=None, lock=True, search_fn=None, ): """Create a stage object. Parameters: url_or_fetch_strategy URL of the archive to be downloaded into this stage, OR a valid FetchStrategy. name If a name is provided, then this stage is a named stage and will persist between runs (or if you construct another stage object later). If name is not provided, then this stage will be given a unique name automatically. mirror_paths If provided, Stage will search Spack's mirrors for this archive at each of the provided relative mirror paths before using the default fetch strategy. keep By default, when used as a context manager, the Stage is deleted on exit when no exceptions are raised. Pass True to keep the stage intact even if no exceptions are raised. path If provided, the stage path to use for associated builds. lock True if the stage directory file lock is to be used, False otherwise. search_fn The search function that provides the fetch strategy instance. """ super().__init__(name, path, keep, lock) # TODO: fetch/stage coupling needs to be reworked -- the logic # TODO: here is convoluted and not modular enough. if isinstance(url_or_fetch_strategy, str): self.fetcher = fs.from_url_scheme(url_or_fetch_strategy) elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: raise ValueError("Can't construct Stage without url or fetch strategy") self.fetcher.stage = self # self.fetcher can change with mirrors. self.default_fetcher = self.fetcher self.search_fn = search_fn # If we fetch from a mirror, but the original data is from say git, we can currently not # prove that they are equal (we don't even have a tree hash in package.py). This bool is # used to skip checksum verification and instead warn the user. if isinstance(self.default_fetcher, fs.URLFetchStrategy): self.skip_checksum_for_mirror = not bool(self.default_fetcher.digest) else: self.skip_checksum_for_mirror = True self.srcdir = None self.mirror_layout = mirror_paths self.mirrors = list(mirrors) if mirrors else [] # Allow users the disable both mirrors and download cache self.default_fetcher_only = False @property def expected_archive_files(self) -> List[str]: """Possible archive file paths.""" fnames: List[str] = [] expanded = True if isinstance(self.default_fetcher, fs.URLFetchStrategy): expanded = self.default_fetcher.expand_archive fnames.append(url_util.default_download_filename(self.default_fetcher.url)) if self.mirror_layout: fnames.append(os.path.basename(self.mirror_layout.path)) paths = [os.path.join(self.path, f) for f in fnames] if not expanded: # If the download file is not compressed, the "archive" is a single file placed in # Stage.source_path paths.extend(os.path.join(self.source_path, f) for f in fnames) return paths @property def save_filename(self): possible_filenames = self.expected_archive_files if possible_filenames: # This prefers using the URL associated with the default fetcher if # available, so that the fetched resource name matches the remote # name return possible_filenames[0] @property def archive_file(self) -> Optional[str]: """Path to the source archive within this stage directory.""" for path in self.expected_archive_files: if os.path.exists(path): return path else: return None @property def expanded(self): """Returns True if source path expanded; else False.""" return os.path.exists(self.source_path) @property def source_path(self): """Returns the well-known source directory path.""" return os.path.join(self.path, _source_path_subdir) @property def single_file(self): assert self.expanded, "Must expand stage before calling single_file" files = os.listdir(self.source_path) assert len(files) == 1, f"Expected one file in stage, found {files}" return os.path.join(self.source_path, files[0]) def _generate_fetchers(self, mirror_only=False) -> Generator["fs.FetchStrategy", None, None]: fetchers: List[fs.FetchStrategy] = [] if not mirror_only: fetchers.append(self.default_fetcher) # If this archive is normally fetched from a URL, then use the same digest. if isinstance(self.default_fetcher, fs.URLFetchStrategy): digest = self.default_fetcher.digest expand = self.default_fetcher.expand_archive extension = self.default_fetcher.extension else: digest = None expand = True extension = None # TODO: move mirror logic out of here and clean it up! # TODO: Or @alalazo may have some ideas about how to use a # TODO: CompositeFetchStrategy here. if not self.default_fetcher_only and self.mirror_layout and self.mirrors: # Add URL strategies for all the mirrors with the digest # Insert fetchers in the order that the URLs are provided. fetchers[:0] = ( fs.from_url_scheme( url_util.join(mirror.fetch_url, *self.mirror_layout.path.split(os.sep)), checksum=digest, expand=expand, extension=extension, ) for mirror in self.mirrors if not spack.oci.image.is_oci_url(mirror.fetch_url) # no support for mirrors yet ) if not self.default_fetcher_only and self.mirror_layout and self.default_fetcher.cachable: fetchers.insert( 0, spack.caches.FETCH_CACHE.fetcher( self.mirror_layout.path, digest, expand=expand, extension=extension ), ) yield from fetchers # The search function may be expensive, so wait until now to call it so the user can stop # if a prior fetcher succeeded if self.search_fn and not mirror_only: yield from self.search_fn() def fetch(self, mirror_only: bool = False, err_msg: Optional[str] = None) -> None: """Retrieves the code or archive Args: mirror_only: only fetch from a mirror err_msg: the error message to display if all fetchers fail or ``None`` for the default fetch failure message """ errors: List[str] = [] for fetcher in self._generate_fetchers(mirror_only): try: fetcher.stage = self self.fetcher = fetcher self.fetcher.fetch() break except fs.NoCacheError: # Don't bother reporting when something is not cached. continue except fs.FailedDownloadError as f: errors.extend(f"{fetcher}: {e.__class__.__name__}: {e}" for e in f.exceptions) continue except spack.error.SpackError as e: errors.append(f"{fetcher}: {e.__class__.__name__}: {e}") continue else: self.fetcher = self.default_fetcher if err_msg: raise spack.error.FetchError(err_msg) raise spack.error.FetchError( f"All fetchers failed for {self.name}", "\n".join(f" {e}" for e in errors) ) def steal_source(self, dest): """Copy the source_path directory in its entirety to directory dest This operation creates/fetches/expands the stage if it is not already, and destroys the stage when it is done.""" if not self.created: self.create() if not self.expanded and not self.archive_file: self.fetch() if not self.expanded: self.expand_archive() if not os.path.isdir(dest): mkdirp(dest) # glob all files and directories in the source path hidden_entries = glob.glob(os.path.join(self.source_path, ".*")) entries = glob.glob(os.path.join(self.source_path, "*")) # Move all files from stage to destination directory # Include hidden files for VCS repo history for entry in hidden_entries + entries: if os.path.isdir(entry): d = os.path.join(dest, os.path.basename(entry)) shutil.copytree(entry, d, symlinks=True) else: shutil.copy2(entry, dest) # copy archive file if we downloaded from url -- replaces for vcs if self.archive_file and os.path.exists(self.archive_file): shutil.copy2(self.archive_file, dest) # remove leftover stage self.destroy() def check(self): """Check the downloaded archive against a checksum digest.""" if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: cache = isinstance(self.fetcher, fs.CacheURLFetchStrategy) if cache: secure_msg = "your download cache is in a secure location" else: secure_msg = "you trust this mirror and have a secure connection" tty.warn( f"Using {'download cache' if cache else 'a mirror'} instead of version control", "The required sources are normally checked out from a version control system, " f"but have been archived {'in download cache' if cache else 'on a mirror'}: " f"{self.fetcher}. Spack lacks a tree hash to verify the integrity of this " f"archive. Make sure {secure_msg}.", ) elif spack.config.get("config:checksum"): self.fetcher.check() def cache_local(self): spack.caches.FETCH_CACHE.store(self.fetcher, self.mirror_layout.path) def cache_mirror( self, mirror: "spack.caches.MirrorCache", stats: "spack.mirrors.utils.MirrorStatsForOneSpec", ) -> None: """Perform a fetch if the resource is not already cached Arguments: mirror: the mirror to cache this Stage's resource in stats: this is updated depending on whether the caching operation succeeded or failed """ if isinstance(self.default_fetcher, fs.BundleFetchStrategy): # BundleFetchStrategy has no source to fetch. The associated fetcher does nothing but # the associated stage may still exist. There is currently no method available on the # fetcher to distinguish this ('cachable' refers to whether the fetcher refers to a # resource with a fixed ID, which is not the same concept as whether there is anything # to fetch at all) so we must examine the type of the fetcher. return elif mirror.skip_unstable_versions and not fs.stable_target(self.default_fetcher): return elif not self.mirror_layout: return absolute_storage_path = os.path.join(mirror.root, self.mirror_layout.path) if os.path.exists(absolute_storage_path): stats.already_existed(absolute_storage_path) else: self.fetch() self.check() mirror.store(self.fetcher, self.mirror_layout.path) stats.added(absolute_storage_path) self.mirror_layout.make_alias(mirror.root) def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet downloaded.""" if not self.expanded: self.fetcher.expand() tty.debug(f"Created stage in {self.path}") else: tty.debug(f"Already staged {self.name} in {self.path}") def restage(self): """Removes the expanded archive path if it exists, then re-expands the archive. """ self.fetcher.reset() def destroy(self): """Removes this stage directory.""" remove_linked_tree(self.path) # Make sure we don't end up in a removed directory try: os.getcwd() except OSError as e: tty.debug(e) os.chdir(os.path.dirname(self.path)) # mark as destroyed self.created = False class ResourceStage(Stage): def __init__( self, fetch_strategy: "fs.FetchStrategy", root: Stage, resource: spack.resource.Resource, *, name=None, mirror_paths: Optional["spack.mirrors.layout.MirrorLayout"] = None, mirrors: Optional[Iterable["spack.mirrors.mirror.Mirror"]] = None, keep=False, path=None, lock=True, search_fn=None, ): super().__init__( fetch_strategy, name=name, mirror_paths=mirror_paths, mirrors=mirrors, keep=keep, path=path, lock=lock, search_fn=search_fn, ) self.root_stage = root self.resource = resource def restage(self): super().restage() self._add_to_root_stage() def expand_archive(self): super().expand_archive() self._add_to_root_stage() def _add_to_root_stage(self): """ Move the extracted resource to the root stage (according to placement). """ root_stage = self.root_stage resource = self.resource if resource.placement: placement = resource.placement elif self.srcdir: placement = self.srcdir else: placement = self.source_path if not isinstance(placement, dict): placement = {"": placement} target_path = os.path.join(root_stage.source_path, resource.destination) try: os.makedirs(target_path) except OSError as err: tty.debug(err) if err.errno == errno.EEXIST and os.path.isdir(target_path): pass else: raise for key, value in placement.items(): destination_path = os.path.join(target_path, value) source_path = os.path.join(self.source_path, key) if not os.path.exists(destination_path): tty.info( "Moving resource stage\n\tsource: " "{stage}\n\tdestination: {destination}".format( stage=source_path, destination=destination_path ) ) src = os.path.realpath(source_path) if os.path.isdir(src): install_tree(src, destination_path) else: install(src, destination_path) class StageComposite: """Composite for Stage type objects. The first item in this composite is considered to be the root package, and operations that return a value are forwarded to it.""" def __init__(self): self._stages: List[AbstractStage] = [] @classmethod def from_iterable(cls, iterable: Iterable[AbstractStage]) -> "StageComposite": """Create a new composite from an iterable of stages.""" composite = cls() composite.extend(iterable) return composite def append(self, stage: AbstractStage) -> None: """Add a stage to the composite.""" self._stages.append(stage) def extend(self, stages: Iterable[AbstractStage]) -> None: """Add multiple stages to the composite.""" self._stages.extend(stages) def __iter__(self): """Iterate over stages.""" return iter(self._stages) def __len__(self): """Return the number of stages.""" return len(self._stages) def __getitem__(self, index): """Get a stage by index.""" return self._stages[index] # Context manager methods - delegate to all stages def __enter__(self): for stage in self._stages: stage.__enter__() return self def __exit__(self, exc_type, exc_val, exc_tb): for stage in reversed(self._stages): stage.__exit__(exc_type, exc_val, exc_tb) # Methods that delegate to all stages def fetch(self, mirror_only: bool = False, err_msg: Optional[str] = None) -> None: """Fetch all stages.""" for stage in self._stages: stage.fetch(mirror_only, err_msg) def create(self) -> None: """Create all stages.""" for stage in self._stages: stage.create() def check(self) -> None: """Check all stages.""" for stage in self._stages: stage.check() def expand_archive(self) -> None: """Expand archives for all stages.""" for stage in self._stages: stage.expand_archive() def restage(self) -> None: """Restage all stages.""" for stage in self._stages: stage.restage() def destroy(self) -> None: """Destroy all stages.""" for stage in self._stages: stage.destroy() def cache_local(self) -> None: """Cache all stages locally.""" for stage in self._stages: stage.cache_local() def cache_mirror( self, mirror: "spack.caches.MirrorCache", stats: "spack.mirrors.utils.MirrorStatsForOneSpec", ) -> None: """Cache all stages to mirror.""" for stage in self._stages: stage.cache_mirror(mirror, stats) def steal_source(self, dest: str) -> None: """Steal source from all stages.""" for stage in self._stages: stage.steal_source(dest) def disable_mirrors(self) -> None: """Disable mirrors for all stages that support it.""" for stage in self._stages: if isinstance(stage, Stage): stage.default_fetcher_only = True # Properties that act only on the *first* stage in the composite @property def source_path(self): return self._stages[0].source_path @property def expanded(self): return self._stages[0].expanded @property def path(self): return self._stages[0].path @property def archive_file(self): return self._stages[0].archive_file @property def requires_patch_success(self): return self._stages[0].requires_patch_success @property def keep(self): return self._stages[0].keep @keep.setter def keep(self, value): for stage in self._stages: stage.keep = value class DevelopStage(AbstractStage): requires_patch_success = False def __init__(self, name, dev_path, reference_link): super().__init__(name=name, path=None, keep=False, lock=True) self.dev_path = dev_path self._source_path = dev_path # The path of a link that will point to this stage if reference_link: if os.path.isabs(reference_link): link_path = reference_link else: link_path = os.path.join(self._source_path, reference_link) if not os.path.isdir(os.path.dirname(link_path)): raise StageError(f"The directory containing {link_path} must exist") self.reference_link = link_path else: self.reference_link = None @property def source_path(self): """Returns the development source path.""" return self._source_path @property def archive_file(self): return None def fetch(self, mirror_only: bool = False, err_msg: Optional[str] = None) -> None: tty.debug("No fetching needed for develop stage.") def check(self): tty.debug("No checksum needed for develop stage.") def expand_archive(self): tty.debug("No expansion needed for develop stage.") @property def expanded(self): """Returns True since the source_path must exist.""" return True def create(self): super().create() if self.reference_link: try: symlink(self.path, self.reference_link) except (AlreadyExistsError, FileExistsError): pass def destroy(self): # Destroy all files, but do not follow symlinks try: shutil.rmtree(self.path) except FileNotFoundError: pass if self.reference_link: try: os.remove(self.reference_link) except FileNotFoundError: pass self.created = False def restage(self): self.destroy() self.create() def cache_local(self): tty.debug("Sources for Develop stages are not cached") def ensure_access(file): """Ensure we can access a directory and die with an error if we can't.""" if not can_access(file): tty.die("Insufficient permissions for %s" % file) def purge(): """Remove all build directories in the top-level stage path.""" root = get_stage_root() if os.path.isdir(root): for stage_dir in os.listdir(root): if stage_dir.startswith(stage_prefix) or stage_dir == ".lock": stage_path = os.path.join(root, stage_dir) if os.path.isdir(stage_path): remove_linked_tree(stage_path) else: os.remove(stage_path) def interactive_version_filter( url_dict: Dict[StandardVersion, str], known_versions: Iterable[StandardVersion] = (), *, initial_verion_filter: Optional[VersionList] = None, url_changes: Set[StandardVersion] = set(), input: Callable[..., str] = input, ) -> Optional[Dict[StandardVersion, str]]: """Interactively filter the list of spidered versions. Args: url_dict: Dictionary of versions to URLs known_versions: Versions that can be skipped because they are already known Returns: Filtered dictionary of versions to URLs or None if the user wants to quit """ # Find length of longest string in the list for padding version_filter = initial_verion_filter or VersionList([":"]) max_len = max(len(str(v)) for v in url_dict) if url_dict else 0 sorted_and_filtered = [v for v in url_dict if v.satisfies(version_filter)] sorted_and_filtered.sort(reverse=True) orig_url_dict = url_dict # only copy when using editor to modify print_header = True VERSION_COLOR = spack.spec.VERSION_COLOR while True: if print_header: has_filter = version_filter != VersionList([":"]) header = [] if len(orig_url_dict) > 0 and len(sorted_and_filtered) == len(orig_url_dict): header.append( f"Selected {spack.llnl.string.plural(len(sorted_and_filtered), 'version')}" ) else: header.append( f"Selected {len(sorted_and_filtered)} of " f"{spack.llnl.string.plural(len(orig_url_dict), 'version')}" ) if sorted_and_filtered and known_versions: num_new = sum(1 for v in sorted_and_filtered if v not in known_versions) header.append(f"{spack.llnl.string.plural(num_new, 'new version')}") if has_filter: header.append(colorize(f"Filtered by {VERSION_COLOR}@@{version_filter}@.")) version_with_url = [ colorize( f"{VERSION_COLOR}{str(v):{max_len}}@. {url_dict[v]}" f"{' @K{# NOTE: change of URL}' if v in url_changes else ''}" ) for v in sorted_and_filtered ] tty.msg(". ".join(header), *spack.llnl.util.lang.elide_list(version_with_url)) print() print_header = True tty.info(colorize("Enter @*{number} of versions to take, or use a @*{command}:")) commands = ( "@*b{[c]}hecksum", "@*b{[e]}dit", "@*b{[f]}ilter", "@*b{[a]}sk each", "@*b{[n]}ew only", "@*b{[r]}estart", "@*b{[q]}uit", ) colify(list(map(colorize, commands)), indent=4) try: command = input(colorize("@*g{action>} ")).strip().lower() except EOFError: print() command = "q" if command == "c": break elif command == "e": # Create a temporary file in the stage dir with lines of the form # # which the user can modify. Once the editor is closed, the file is # read back in and the versions to url dict is updated. # Create a temporary file by hashing its contents. buffer = io.StringIO() buffer.write("# Edit this file to change the versions and urls to fetch\n") for v in sorted_and_filtered: buffer.write(f"{str(v):{max_len}} {url_dict[v]}\n") data = buffer.getvalue().encode("utf-8") short_hash = hashlib.sha1(data).hexdigest()[:7] filename = f"{stage_prefix}versions-{short_hash}.txt" filepath = os.path.join(get_stage_root(), filename) # Write contents with open(filepath, "wb") as f: f.write(data) # Open editor editor(filepath, exec_fn=executable) # Read back in with open(filepath, "r", encoding="utf-8") as f: orig_url_dict, url_dict = url_dict, {} for line in f: line = line.strip() # Skip empty lines and comments if not line or line.startswith("#"): continue try: version, url = line.split(None, 1) except ValueError: tty.warn(f"Couldn't parse: {line}") continue try: url_dict[StandardVersion.from_string(version)] = url except ValueError: tty.warn(f"Invalid version: {version}") continue sorted_and_filtered = sorted(url_dict.keys(), reverse=True) os.unlink(filepath) elif command == "f": tty.msg( colorize( f"Examples filters: {VERSION_COLOR}1.2@. " f"or {VERSION_COLOR}1.1:1.3@. " f"or {VERSION_COLOR}=1.2, 1.2.2:@." ) ) try: # Allow a leading @ version specifier filter_spec = input(colorize("@*g{filter>} ")).strip().lstrip("@") except EOFError: print() continue try: version_filter.intersect(VersionList([filter_spec])) except ValueError: tty.warn(f"Invalid version specifier: {filter_spec}") continue # Apply filter sorted_and_filtered = [v for v in sorted_and_filtered if v.satisfies(version_filter)] elif command == "a": i = 0 while i < len(sorted_and_filtered): v = sorted_and_filtered[i] try: answer = input(f" {str(v):{max_len}} {url_dict[v]} [Y/n]? ").strip().lower() except EOFError: # If ^D, don't fully exit, but go back to the command prompt, now with possibly # fewer versions print() break if answer in ("n", "no"): del sorted_and_filtered[i] elif answer in ("y", "yes", ""): i += 1 else: # Went over each version, so go to checksumming break elif command == "n": sorted_and_filtered = [v for v in sorted_and_filtered if v not in known_versions] elif command == "r": url_dict = orig_url_dict sorted_and_filtered = sorted(url_dict.keys(), reverse=True) version_filter = VersionList([":"]) elif command == "q": try: if input("Really quit [y/N]? ").strip().lower() in ("y", "yes"): return None except EOFError: print() return None else: # Last restort: filter the top N versions try: n = int(command) invalid_command = n < 1 except ValueError: invalid_command = True if invalid_command: tty.warn(f"Ignoring invalid command: {command}") print_header = False continue sorted_and_filtered = sorted_and_filtered[:n] return {v: url_dict[v] for v in sorted_and_filtered} def get_checksums_for_versions( url_by_version: Dict[StandardVersion, str], package_name: str, *, first_stage_function: Optional[Callable[[str, str], None]] = None, keep_stage: bool = False, concurrency: Optional[int] = None, fetch_options: Optional[Dict[str, str]] = None, ) -> Dict[StandardVersion, str]: """Computes the checksums for each version passed in input, and returns the results. Archives are fetched according to the usl dictionary passed as input. The ``first_stage_function`` argument allows the caller to inspect the first downloaded archive, e.g., to determine the build system. Args: url_by_version: URL keyed by version package_name: name of the package first_stage_function: function that takes an archive file and a URL; this is run on the stage of the first URL downloaded keep_stage: whether to keep staging area when command completes batch: whether to ask user how many versions to fetch (false) or fetch all versions (true) fetch_options: options used for the fetcher (such as timeout or cookies) concurrency: maximum number of workers to use for retrieving archives Returns: A dictionary mapping each version to the corresponding checksum """ versions = sorted(url_by_version.keys(), reverse=True) search_arguments = [(url_by_version[v], v) for v in versions] version_hashes: Dict[StandardVersion, str] = {} errors: List[str] = [] # Don't spawn 16 processes when we need to fetch 2 urls if concurrency is not None: concurrency = min(concurrency, len(search_arguments)) else: concurrency = min(os.cpu_count() or 1, len(search_arguments)) # The function might have side effects in memory, that would not be reflected in the # parent process, if run in a child process. If this pattern happens frequently, we # can move this function call *after* having distributed the work to executors. if first_stage_function is not None: (url, version), search_arguments = search_arguments[0], search_arguments[1:] result = _fetch_and_checksum(url, fetch_options, keep_stage, first_stage_function) if isinstance(result, Exception): errors.append(str(result)) else: version_hashes[version] = result with spack.util.parallel.make_concurrent_executor(concurrency, require_fork=False) as executor: results = [ (version, executor.submit(_fetch_and_checksum, url, fetch_options, keep_stage)) for url, version in search_arguments ] for version, future in results: result = future.result() if isinstance(result, Exception): errors.append(str(result)) else: version_hashes[version] = result for msg in errors: tty.debug(msg) if not version_hashes: tty.die(f"Could not fetch any versions for {package_name}") num_hash = len(version_hashes) tty.debug(f"Checksummed {num_hash} version{'' if num_hash == 1 else 's'} of {package_name}:") return version_hashes def _fetch_and_checksum( url: str, options: Optional[dict], keep_stage: bool, action_fn: Optional[Callable[[str, str], None]] = None, ) -> Union[str, Exception]: try: with Stage(fs.URLFetchStrategy(url=url, fetch_options=options), keep=keep_stage) as stage: # Fetch the archive stage.fetch() archive = stage.archive_file assert archive is not None, f"Archive not found for {url}" if action_fn is not None and archive: # Only run first_stage_function the first time, # no need to run it every time action_fn(archive, url) # Checksum the archive and add it to the list checksum = spack.util.crypto.checksum(hashlib.sha256, archive) return checksum except Exception as e: return Exception(f"[WORKER] Failed to fetch {url}: {e}") class StageError(spack.error.SpackError): """Superclass for all errors encountered during staging.""" class StagePathError(StageError): """Error encountered with stage path.""" class RestageError(StageError): """Error encountered during restaging.""" class VersionFetchError(StageError): """Raised when we can't determine a URL to fetch a package.""" ================================================ FILE: lib/spack/spack/store.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Components that manage Spack's installation tree. An install tree, or "build store" consists of two parts: 1. A package database that tracks what is installed. 2. A directory layout that determines how the installations are laid out. The store contains all the install prefixes for packages installed by Spack. The simplest store could just contain prefixes named by DAG hash, but we use a fancier directory layout to make browsing the store and debugging easier. """ import contextlib import os import pathlib import re import uuid from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union, cast import spack.config import spack.database import spack.directory_layout import spack.error import spack.llnl.util.lang import spack.paths import spack.spec import spack.util.path from spack.llnl.util import tty #: default installation root, relative to the Spack install path DEFAULT_INSTALL_TREE_ROOT = os.path.join(spack.paths.opt_path, "spack") def parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]: """Parse config settings and return values relevant to the store object. Arguments: config_dict: dictionary of config values, as returned from ``spack.config.get("config")`` Returns: triple of the install tree root, the unpadded install tree root (before padding was applied), and the projections for the install tree Encapsulate backwards compatibility capabilities for install_tree and deprecated values that are now parsed as part of install_tree. """ # The following two configs are equivalent, the first being the old format # and the second the new format. The new format is also more flexible. # config: # install_tree: /path/to/root$padding:128 # install_path_scheme: '{name}-{version}' # config: # install_tree: # root: /path/to/root # padding: 128 # projections: # all: '{name}-{version}' install_tree = config_dict.get("install_tree", {}) padded_length: Union[bool, int] = False if isinstance(install_tree, str): tty.warn("Using deprecated format for configuring install_tree") unpadded_root = install_tree unpadded_root = spack.util.path.canonicalize_path(unpadded_root) # construct projection from previous values for backwards compatibility all_projection = config_dict.get( "install_path_scheme", spack.directory_layout.default_projections["all"] ) projections = {"all": all_projection} else: unpadded_root = install_tree.get("root", DEFAULT_INSTALL_TREE_ROOT) unpadded_root = spack.util.path.canonicalize_path(unpadded_root) padded_length = install_tree.get("padded_length", False) if padded_length is True: padded_length = spack.util.path.get_system_path_max() padded_length -= spack.util.path.SPACK_MAX_INSTALL_PATH_LENGTH projections = install_tree.get("projections", spack.directory_layout.default_projections) path_scheme = config_dict.get("install_path_scheme", None) if path_scheme: tty.warn( "Deprecated config value 'install_path_scheme' ignored" " when using new install_tree syntax" ) # Handle backwards compatibility for padding old_pad = re.search(r"\$padding(:\d+)?|\${padding(:\d+)?}", unpadded_root) if old_pad: if padded_length: msg = "Ignoring deprecated padding option in install_tree root " msg += "because new syntax padding is present." tty.warn(msg) else: unpadded_root = unpadded_root.replace(old_pad.group(0), "") if old_pad.group(1) or old_pad.group(2): length_group = 2 if "{" in old_pad.group(0) else 1 padded_length = int(old_pad.group(length_group)[1:]) else: padded_length = spack.util.path.get_system_path_max() padded_length -= spack.util.path.SPACK_MAX_INSTALL_PATH_LENGTH unpadded_root = unpadded_root.rstrip(os.path.sep) if padded_length: root = spack.util.path.add_padding(unpadded_root, padded_length) if len(root) != padded_length: msg = "Cannot pad %s to %s characters." % (root, padded_length) msg += " It is already %s characters long" % len(root) tty.warn(msg) else: root = unpadded_root return root, unpadded_root, projections class Store: """A store is a path full of installed Spack packages. Stores consist of packages installed according to a ``DirectoryLayout``, along with a database of their contents. The directory layout controls what paths look like and how Spack ensures that each unique spec gets its own unique directory (or not, though we don't recommend that). The database is a single file that caches metadata for the entire Spack installation. It prevents us from having to spider the install tree to figure out what's there. The store is also able to lock installation prefixes, and to mark installation failures. Args: root: path to the root of the install tree unpadded_root: path to the root of the install tree without padding. The sbang script has to be installed here to work with padded roots projections: expression according to guidelines that describes how to construct a path to a package prefix in this store hash_length: length of the hashes used in the directory layout. Spec hash suffixes will be truncated to this length upstreams: optional list of upstream databases lock_cfg: lock configuration for the database """ def __init__( self, root: str, unpadded_root: Optional[str] = None, projections: Optional[Dict[str, str]] = None, hash_length: Optional[int] = None, upstreams: Optional[List[spack.database.Database]] = None, lock_cfg: spack.database.LockConfiguration = spack.database.NO_LOCK, ) -> None: self.root = root self.unpadded_root = unpadded_root or root self.projections = projections self.hash_length = hash_length self.upstreams = upstreams self.lock_cfg = lock_cfg self.layout = spack.directory_layout.DirectoryLayout( root, projections=projections, hash_length=hash_length ) self.db = spack.database.Database( root, upstream_dbs=upstreams, lock_cfg=lock_cfg, layout=self.layout ) timeout_format_str = ( f"{str(lock_cfg.package_timeout)}s" if lock_cfg.package_timeout else "No timeout" ) tty.debug("PACKAGE LOCK TIMEOUT: {0}".format(str(timeout_format_str))) self.prefix_locker = spack.database.SpecLocker( spack.database.prefix_lock_path(root), default_timeout=lock_cfg.package_timeout ) self.failure_tracker = spack.database.FailureTracker( self.root, default_timeout=lock_cfg.package_timeout ) def has_padding(self) -> bool: """Returns True if the store layout includes path padding.""" return self.root != self.unpadded_root def reindex(self) -> None: """Convenience function to reindex the store DB with its own layout.""" return self.db.reindex() def __reduce__(self): return Store, ( self.root, self.unpadded_root, self.projections, self.hash_length, self.upstreams, self.lock_cfg, ) def create(configuration: spack.config.Configuration) -> Store: """Create a store from the configuration passed as input. Args: configuration: configuration to create a store. """ configuration = configuration or spack.config.CONFIG config_dict = configuration.get_config("config") root, unpadded_root, projections = parse_install_tree(config_dict) hash_length = config_dict.get("install_hash_length") install_roots = [ install_properties["install_tree"] for install_properties in configuration.get_config("upstreams").values() ] upstreams = _construct_upstream_dbs_from_install_roots(install_roots) return Store( root=root, unpadded_root=unpadded_root, projections=projections, hash_length=hash_length, upstreams=upstreams, lock_cfg=spack.database.lock_configuration(configuration), ) def _create_global() -> Store: result = create(configuration=spack.config.CONFIG) return result #: Singleton store instance STORE = cast(Store, spack.llnl.util.lang.Singleton(_create_global)) def reinitialize(): """Restore globals to the same state they would have at start-up. Return a token containing the state of the store before reinitialization. """ global STORE token = STORE STORE = cast(Store, spack.llnl.util.lang.Singleton(_create_global)) return token def restore(token): """Restore the environment from a token returned by reinitialize""" global STORE STORE = token def _construct_upstream_dbs_from_install_roots( install_roots: List[str], ) -> List[spack.database.Database]: accumulated_upstream_dbs: List[spack.database.Database] = [] for install_root in reversed(install_roots): upstream_dbs = list(accumulated_upstream_dbs) next_db = spack.database.Database( spack.util.path.canonicalize_path(install_root), is_upstream=True, upstream_dbs=upstream_dbs, ) next_db._read() accumulated_upstream_dbs.insert(0, next_db) return accumulated_upstream_dbs def find( constraints: Union[str, List[str], List["spack.spec.Spec"]], multiple: bool = False, query_fn: Optional[Callable[[Any], List["spack.spec.Spec"]]] = None, **kwargs, ) -> List["spack.spec.Spec"]: """Returns a list of specs matching the constraints passed as inputs. At least one spec per constraint must match, otherwise the function will error with an appropriate message. By default, this function queries the current store, but a custom query function can be passed to hit any other source of concretized specs (e.g. a binary cache). The query function must accept a spec as its first argument. Args: constraints: spec(s) to be matched against installed packages multiple: if True multiple matches per constraint are admitted query_fn (Callable): query function to get matching specs. By default, ``spack.store.STORE.db.query`` **kwargs: keyword arguments forwarded to the query function """ if isinstance(constraints, str): constraints = [spack.spec.Spec(constraints)] matching_specs: List[spack.spec.Spec] = [] errors = [] query_fn = query_fn or STORE.db.query for spec in constraints: current_matches = query_fn(spec, **kwargs) # For each spec provided, make sure it refers to only one package. if not multiple and len(current_matches) > 1: msg_fmt = '"{0}" matches multiple packages: [{1}]' errors.append(msg_fmt.format(spec, ", ".join([m.format() for m in current_matches]))) # No installed package matches the query if len(current_matches) == 0 and spec is not any: msg_fmt = '"{0}" does not match any installed packages' errors.append(msg_fmt.format(spec)) matching_specs.extend(current_matches) if errors: raise MatchError( message="errors occurred when looking for specs in the store", long_message="\n".join(errors), ) return matching_specs def specfile_matches(filename: str, **kwargs) -> List["spack.spec.Spec"]: """Same as find but reads the query from a spec file. Args: filename: YAML or JSON file from which to read the query. **kwargs: keyword arguments forwarded to :func:`find` """ query = [spack.spec.Spec.from_specfile(filename)] return find(query, **kwargs) def ensure_singleton_created() -> None: """Ensures the lazily evaluated singleton is created""" _ = STORE.db @contextlib.contextmanager def use_store( path: Union[str, pathlib.Path], extra_data: Optional[Dict[str, Any]] = None ) -> Generator[Store, None, None]: """Use the store passed as argument within the context manager. Args: path: path to the store. extra_data: extra configuration under ``config:install_tree`` to be taken into account. Yields: Store object associated with the context manager's store """ global STORE assert not isinstance(path, Store), "cannot pass a store anymore" scope_name = "use-store-{}".format(uuid.uuid4()) data = {"root": str(path)} if extra_data: data.update(extra_data) # Swap the store with the one just constructed and return it spack.config.CONFIG.push_scope( spack.config.InternalConfigScope(name=scope_name, data={"config": {"install_tree": data}}) ) temporary_store = create(configuration=spack.config.CONFIG) original_store, STORE = STORE, temporary_store try: yield temporary_store finally: # Restore the original store STORE = original_store spack.config.CONFIG.remove_scope(scope_name=scope_name) class MatchError(spack.error.SpackError): """Error occurring when trying to match specs in store against a constraint""" ================================================ FILE: lib/spack/spack/subprocess_context.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This module handles transmission of Spack state to child processes started using the ``"spawn"`` start method. Notably, installations are performed in a subprocess and require transmitting the Package object (in such a way that the repository is available for importing when it is deserialized); installations performed in Spack unit tests may include additional modifications to global state in memory that must be replicated in the child process. """ import importlib import io import multiprocessing import multiprocessing.context import pickle from types import ModuleType from typing import TYPE_CHECKING, Optional import spack.config import spack.paths import spack.platforms import spack.repo import spack.store if TYPE_CHECKING: import spack.package_base #: Used in tests to track monkeypatches that need to be restored in child processes MONKEYPATCHES: list = [] def serialize(pkg: "spack.package_base.PackageBase") -> io.BytesIO: serialized_pkg = io.BytesIO() pickle.dump(pkg, serialized_pkg) serialized_pkg.seek(0) return serialized_pkg def deserialize(serialized_pkg: io.BytesIO) -> "spack.package_base.PackageBase": pkg = pickle.load(serialized_pkg) pkg.spec._package = pkg # ensure overwritten package class attributes get applied spack.repo.PATH.get_pkg_class(pkg.spec.name) return pkg class SpackTestProcess: def __init__(self, fn): self.fn = fn def _restore_and_run(self, fn, test_state): test_state.restore() fn() def create(self): test_state = GlobalStateMarshaler() return multiprocessing.Process(target=self._restore_and_run, args=(self.fn, test_state)) class PackageInstallContext: """Captures the in-memory process state of a package installation that needs to be transmitted to a child process.""" def __init__( self, pkg: "spack.package_base.PackageBase", *, ctx: Optional[multiprocessing.context.BaseContext] = None, ): ctx = ctx or multiprocessing.get_context() self.global_state = GlobalStateMarshaler(ctx=ctx) self.pkg = pkg if ctx.get_start_method() == "fork" else serialize(pkg) self.spack_working_dir = spack.paths.spack_working_dir def restore(self) -> "spack.package_base.PackageBase": spack.paths.spack_working_dir = self.spack_working_dir self.global_state.restore() return deserialize(self.pkg) if isinstance(self.pkg, io.BytesIO) else self.pkg class GlobalStateMarshaler: """Class to serialize and restore global state for child processes if needed. Spack may modify state that is normally read from disk or command line in memory; this object is responsible for properly serializing that state to be applied to a subprocess. """ def __init__( self, *, ctx: Optional[Optional[multiprocessing.context.BaseContext]] = None ) -> None: ctx = ctx or multiprocessing.get_context() self.is_forked = ctx.get_start_method() == "fork" if self.is_forked: return from spack.environment import active_environment self.config = spack.config.CONFIG.ensure_unwrapped() self.platform = spack.platforms.host self.store = spack.store.STORE self.test_patches = TestPatches.create() self.env = active_environment() def restore(self): if self.is_forked: return spack.config.CONFIG = self.config spack.repo.enable_repo(spack.repo.RepoPath.from_config(self.config)) spack.platforms.host = self.platform spack.store.STORE = self.store self.test_patches.restore() if self.env: from spack.environment import activate activate(self.env) class TestPatches: def __init__(self, module_patches, class_patches): self.module_patches = [(x, y, serialize(z)) for (x, y, z) in module_patches] self.class_patches = [(x, y, serialize(z)) for (x, y, z) in class_patches] def restore(self): if not self.module_patches and not self.class_patches: return # this code path is only followed in tests, so use inline imports from pydoc import locate for module_name, attr_name, value in self.module_patches: value = pickle.load(value) module = importlib.import_module(module_name) setattr(module, attr_name, value) for class_fqn, attr_name, value in self.class_patches: value = pickle.load(value) cls = locate(class_fqn) setattr(cls, attr_name, value) @staticmethod def create(): module_patches = [] class_patches = [] for target, name in MONKEYPATCHES: if isinstance(target, ModuleType): new_val = getattr(target, name) module_name = target.__name__ module_patches.append((module_name, name, new_val)) elif isinstance(target, type): new_val = getattr(target, name) class_fqn = ".".join([target.__module__, target.__name__]) class_patches.append((class_fqn, name, new_val)) return TestPatches(module_patches, class_patches) ================================================ FILE: lib/spack/spack/tag.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes and functions to manage package tags""" from typing import TYPE_CHECKING, Dict, List, Set import spack.error import spack.util.spack_json as sjson if TYPE_CHECKING: import spack.repo class TagIndex: """Maps tags to list of package names.""" def __init__(self) -> None: self.tags: Dict[str, List[str]] = {} def to_json(self, stream) -> None: sjson.dump({"tags": self.tags}, stream) @staticmethod def from_json(stream) -> "TagIndex": d = sjson.load(stream) if not isinstance(d, dict): raise TagIndexError("TagIndex data was not a dict.") if "tags" not in d: raise TagIndexError("TagIndex data does not start with 'tags'") r = TagIndex() for tag, packages in d["tags"].items(): r.tags[tag] = packages return r def get_packages(self, tag: str) -> List[str]: """Returns all packages associated with the tag.""" return self.tags.get(tag, []) def merge(self, other: "TagIndex") -> None: """Merge another tag index into this one. Args: other: tag index to be merged """ for tag, pkgs in other.tags.items(): if tag not in self.tags: self.tags[tag] = pkgs.copy() else: self.tags[tag] = sorted({*self.tags[tag], *pkgs}) def update_packages(self, pkg_names: Set[str], repo: "spack.repo.Repo") -> None: """Updates packages in the tag index. Args: pkg_names: names of the packages to be updated repo: the repository to get package classes from """ # Remove the packages from the list of packages, if present for pkg_list in self.tags.values(): if pkg_names.isdisjoint(pkg_list): continue pkg_list[:] = [pkg for pkg in pkg_list if pkg not in pkg_names] # Add them again under the appropriate tags for pkg_name in pkg_names: pkg_cls = repo.get_pkg_class(pkg_name) for tag in getattr(pkg_cls, "tags", []): tag = tag.lower() if tag not in self.tags: self.tags[tag] = [pkg_cls.name] else: self.tags[tag].append(pkg_cls.name) class TagIndexError(spack.error.SpackError): """Raised when there is a problem with a TagIndex.""" ================================================ FILE: lib/spack/spack/tengine.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import itertools import textwrap from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple import spack.config import spack.extensions import spack.llnl.util.lang from spack.util.path import canonicalize_path if TYPE_CHECKING: import spack.vendor.jinja2 class ContextMeta(type): """Metaclass for Context. It helps reduce the boilerplate in client code.""" #: Keeps track of the context properties that have been added #: by the class that is being defined _new_context_properties: List[str] = [] def __new__(cls, name, bases, attr_dict): # Merge all the context properties that are coming from base classes # into a list without duplicates. context_properties = list(cls._new_context_properties) for x in bases: try: context_properties.extend(x.context_properties) except AttributeError: pass context_properties = list(spack.llnl.util.lang.dedupe(context_properties)) # Flush the list cls._new_context_properties = [] # Attach the list to the class being created attr_dict["context_properties"] = context_properties return super(ContextMeta, cls).__new__(cls, name, bases, attr_dict) @classmethod def context_property(cls, func): """Decorator that adds a function name to the list of new context properties, and then returns a property. """ name = func.__name__ cls._new_context_properties.append(name) return property(func) #: A saner way to use the decorator context_property = ContextMeta.context_property class Context(metaclass=ContextMeta): """Base class for context classes that are used with the template engine.""" context_properties: List[str] def to_dict(self) -> Dict[str, Any]: """Returns a dictionary containing all the context properties.""" return {name: getattr(self, name) for name in self.context_properties} def make_environment(dirs: Optional[Tuple[str, ...]] = None) -> "spack.vendor.jinja2.Environment": """Returns a configured environment for template rendering.""" if dirs is None: # Default directories where to search for templates dirs = default_template_dirs(spack.config.CONFIG) return make_environment_from_dirs(dirs) @spack.llnl.util.lang.memoized def make_environment_from_dirs(dirs: Tuple[str, ...]) -> "spack.vendor.jinja2.Environment": # Import at this scope to avoid slowing Spack startup down import spack.vendor.jinja2 # Loader for the templates loader = spack.vendor.jinja2.FileSystemLoader(dirs) # Environment of the template engine env = spack.vendor.jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True) # Custom filters _set_filters(env) return env def default_template_dirs(configuration: spack.config.Configuration) -> Tuple[str, ...]: config_yaml = configuration.get_config("config") builtins = config_yaml.get("template_dirs", ["$spack/share/spack/templates"]) extensions = spack.extensions.get_template_dirs() return tuple(canonicalize_path(d) for d in itertools.chain(builtins, extensions)) # Extra filters for the template engine environment def prepend_to_line(text, token): """Prepends a token to each line in text""" return [token + line for line in text] def quote(text): """Quotes each line in text""" return ['"{0}"'.format(line) for line in text] def curly_quote(text): """Encloses each line of text in curly braces""" return ["{{{0}}}".format(line) for line in text] def _set_filters(env): """Sets custom filters to the template engine environment""" env.filters["textwrap"] = textwrap.wrap env.filters["prepend_to_line"] = prepend_to_line env.filters["join"] = "\n".join env.filters["quote"] = quote env.filters["curly_quote"] = curly_quote ================================================ FILE: lib/spack/spack/test/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/test/architecture.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform import pytest import spack.vendor.archspec.cpu import spack.concretize import spack.operating_systems import spack.platforms from spack.spec import ArchSpec, Spec @pytest.fixture(scope="module") def current_host_platform(): """Return the platform of the current host as detected by the 'platform' stdlib package. """ current_platform = None if "Linux" in platform.system(): current_platform = spack.platforms.Linux() elif "Darwin" in platform.system(): current_platform = spack.platforms.Darwin() elif "Windows" in platform.system(): current_platform = spack.platforms.Windows() elif "FreeBSD" in platform.system(): current_platform = spack.platforms.FreeBSD() return current_platform # Valid keywords for os=xxx or target=xxx VALID_KEYWORDS = ["fe", "be", "frontend", "backend"] TEST_PLATFORM = spack.platforms.Test() @pytest.fixture(params=([str(x) for x in TEST_PLATFORM.targets] + VALID_KEYWORDS), scope="module") def target_str(request): """All the possible strings that can be used for targets""" return request.param @pytest.fixture( params=([str(x) for x in TEST_PLATFORM.operating_sys] + VALID_KEYWORDS), scope="module" ) def os_str(request): """All the possible strings that can be used for operating systems""" return request.param def test_platform(current_host_platform): """Check that current host detection return the correct platform""" detected_platform = spack.platforms.real_host() assert str(detected_platform) == str(current_host_platform) def test_user_input_combination(config, target_str, os_str): """Test for all the valid user input combinations that both the target and the operating system match. """ spec = Spec(f"libelf os={os_str} target={target_str}") assert spec.architecture.os == str(TEST_PLATFORM.operating_system(os_str)) assert spec.architecture.target == TEST_PLATFORM.target(target_str) def test_default_os_and_target(default_mock_concretization): """Test that is we don't specify `os=` or `target=` we get the default values after concretization. """ spec = default_mock_concretization("libelf") assert spec.architecture.os == str(TEST_PLATFORM.default_operating_system()) assert spec.architecture.target == TEST_PLATFORM.default_target() def test_operating_system_conversion_to_dict(): operating_system = spack.operating_systems.OperatingSystem("os", "1.0") assert operating_system.to_dict() == {"name": "os", "version": "1.0"} @pytest.mark.parametrize( "item,architecture_str", [ # We can search the architecture string representation ("linux", "linux-ubuntu18.04-haswell"), ("ubuntu", "linux-ubuntu18.04-haswell"), ("haswell", "linux-ubuntu18.04-haswell"), # We can also search flags of the target, ("avx512", "linux-ubuntu18.04-icelake"), ], ) def test_arch_spec_container_semantic(item, architecture_str): architecture = ArchSpec(architecture_str) assert item in architecture @pytest.mark.regression("15306") @pytest.mark.parametrize( "architecture_tuple,constraint_tuple", [ (("linux", "ubuntu18.04", None), ("linux", None, "x86_64")), (("linux", "ubuntu18.04", None), ("linux", None, "x86_64:")), ], ) def test_satisfy_strict_constraint_when_not_concrete(architecture_tuple, constraint_tuple): architecture = ArchSpec(architecture_tuple) constraint = ArchSpec(constraint_tuple) assert not architecture.satisfies(constraint) @pytest.mark.parametrize( "root_target_range,dep_target_range,result", [ ("x86_64:nocona", "x86_64:core2", "nocona"), # pref not in intersection ("x86_64:core2", "x86_64:nocona", "nocona"), ("x86_64:haswell", "x86_64:mic_knl", "core2"), # pref in intersection ("ivybridge", "nocona:skylake", "ivybridge"), # one side concrete ("haswell:icelake", "broadwell", "broadwell"), # multiple ranges in lists with multiple overlaps ("x86_64:nocona,haswell:broadwell", "nocona:haswell,skylake:", "nocona"), # lists with concrete targets, lists compared to ranges ("x86_64,haswell", "core2:broadwell", "haswell"), ], ) @pytest.mark.usefixtures("mock_packages", "config") @pytest.mark.skipif( str(spack.vendor.archspec.cpu.host().family) != "x86_64", reason="tests are for x86_64 uarch ranges", ) def test_concretize_target_ranges(root_target_range, dep_target_range, result, monkeypatch): spec = spack.concretize.concretize_one( f"pkg-a foobar=bar target={root_target_range} %gcc@10 ^pkg-b target={dep_target_range}" ) assert spec.target == spec["pkg-b"].target == result ================================================ FILE: lib/spack/spack/test/audit.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.audit import spack.config @pytest.mark.parametrize( # PKG-PROPERTIES are ubiquitous in mock packages, since they don't use sha256 # and they don't change the example.com URL very often. "packages,expected_errors", [ # A non existing variant is used in a conflict directive (["wrong-variant-in-conflicts"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # The package declares a non-existing dependency (["missing-dependency"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # The package use a non existing variant in a depends_on directive (["wrong-variant-in-depends-on"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has a GitHub pull request commit patch URL (["invalid-github-pull-commits-patch-url"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has a GitHub patch URL without full_index=1 (["invalid-github-patch-url"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has invalid GitLab patch URLs (["invalid-gitlab-patch-url"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has invalid GitLab patch URLs (["invalid-selfhosted-gitlab-patch-url"], ["PKG-DIRECTIVES", "PKG-PROPERTIES"]), # This package has a stand-alone test method in build-time callbacks (["fail-test-audit"], ["PKG-PROPERTIES"]), # This package has stand-alone test methods without non-trivial docstrings (["fail-test-audit-docstring"], ["PKG-PROPERTIES"]), # This package has a stand-alone test method without an implementation (["fail-test-audit-impl"], ["PKG-PROPERTIES"]), # This package has no issues (["mpileaks"], None), # This package has a conflict with a trigger which cannot constrain the constraint # Should not raise an error (["unconstrainable-conflict"], None), ], ) def test_package_audits(packages, expected_errors, mock_packages): reports = spack.audit.run_group("packages", pkgs=packages) # Check that errors were reported only for the expected failure actual_errors = [check for check, errors in reports if errors] msg = "\n".join([str(e) for _, errors in reports for e in errors]) if expected_errors: assert expected_errors == actual_errors, msg else: assert not actual_errors, msg # Data used in the test below to audit the double definition of a compiler _double_compiler_definition = [ { "compiler": { "spec": "gcc@9.0.1", "paths": { "cc": "/usr/bin/gcc-9", "cxx": "/usr/bin/g++-9", "f77": "/usr/bin/gfortran-9", "fc": "/usr/bin/gfortran-9", }, "flags": {}, "operating_system": "ubuntu18.04", "target": "x86_64", "modules": [], "environment": {}, "extra_rpaths": [], } }, { "compiler": { "spec": "gcc@9.0.1", "paths": { "cc": "/usr/bin/gcc-9", "cxx": "/usr/bin/g++-9", "f77": "/usr/bin/gfortran-9", "fc": "/usr/bin/gfortran-9", }, "flags": {"cflags": "-O3"}, "operating_system": "ubuntu18.04", "target": "x86_64", "modules": [], "environment": {}, "extra_rpaths": [], } }, ] # TODO/RepoSplit: Should this not rely on mock packages post split? @pytest.mark.parametrize( "config_section,data,failing_check", [ # Double compiler definitions in compilers.yaml ("compilers", _double_compiler_definition, "CFG-COMPILER"), # Multiple definitions of the same external spec in packages.yaml ( "packages", { "mpileaks": { "externals": [ {"spec": "mpileaks@1.0.0", "prefix": "/"}, {"spec": "mpileaks@1.0.0", "prefix": "/usr"}, ] } }, "CFG-PACKAGES", ), ], ) def test_config_audits(config_section, data, failing_check, mock_packages): with spack.config.override(config_section, data): reports = spack.audit.run_group("configs") assert any((check == failing_check) and errors for check, errors in reports) ================================================ FILE: lib/spack/spack/test/binary_distribution.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp import glob import gzip import io import json import os import pathlib import re import tarfile import urllib.error import urllib.request import urllib.response from pathlib import Path, PurePath from typing import Any, Callable, Dict, NamedTuple, Optional import pytest import spack.binary_distribution import spack.concretize import spack.config import spack.environment as ev import spack.hooks.sbang as sbang import spack.main import spack.mirrors.mirror import spack.oci.image import spack.spec import spack.stage import spack.store import spack.url_buildcache import spack.util.gpg import spack.util.spack_yaml as syaml import spack.util.url as url_util import spack.util.web as web_util from spack.binary_distribution import CannotListKeys, GenerateIndexError from spack.database import INDEX_JSON_FILE from spack.installer import PackageInstaller from spack.llnl.util.filesystem import join_path, readlink, working_dir from spack.spec import Spec from spack.url_buildcache import ( INDEX_MANIFEST_FILE, BuildcacheComponent, BuildcacheEntryError, URLBuildcacheEntry, URLBuildcacheEntryV2, compression_writer, get_entries_from_cache, get_url_buildcache_class, get_valid_spec_file, ) pytestmark = pytest.mark.not_on_windows("does not run on windows") mirror_cmd = spack.main.SpackCommand("mirror") install_cmd = spack.main.SpackCommand("install") uninstall_cmd = spack.main.SpackCommand("uninstall") buildcache_cmd = spack.main.SpackCommand("buildcache") @pytest.fixture def dummy_prefix(tmp_path: pathlib.Path): """Dummy prefix used for testing tarball creation, validation, extraction""" p = tmp_path / "prefix" p.mkdir() assert os.path.isabs(str(p)) (p / "bin").mkdir() (p / "share").mkdir() (p / ".spack").mkdir() app = p / "bin" / "app" relative_app_link = p / "bin" / "relative_app_link" absolute_app_link = p / "bin" / "absolute_app_link" data = p / "share" / "file" with open(app, "w", encoding="utf-8") as f: f.write("hello world") with open(data, "w", encoding="utf-8") as f: f.write("hello world") with open(p / ".spack" / "binary_distribution", "w", encoding="utf-8") as f: f.write("{}") os.symlink("app", str(relative_app_link)) os.symlink(str(app), str(absolute_app_link)) return str(p) @pytest.mark.maybeslow def test_buildcache_cmd_smoke_test(tmp_path: pathlib.Path, install_mockery): """ Test the creation and installation of buildcaches with default rpaths into the default directory layout scheme. """ mirror_cmd("add", "--type", "binary", "--unsigned", "test-mirror", str(tmp_path)) # Install 'corge' without using a cache install_cmd("--fake", "--no-cache", "corge") install_cmd("--fake", "--no-cache", "symly") # Create a buildache buildcache_cmd("push", "-u", str(tmp_path), "corge", "symly") # Test force overwrite create buildcache (-f option) buildcache_cmd("push", "-uf", str(tmp_path), "corge") # Create mirror index buildcache_cmd("update-index", str(tmp_path)) # List the buildcaches in the mirror buildcache_cmd("list", "-alv") # Uninstall the package and deps uninstall_cmd("-y", "--dependents", "garply") # Test installing from build caches buildcache_cmd("install", "-uo", "corge", "symly") # This gives warning that spec is already installed buildcache_cmd("install", "-uo", "corge") # Test overwrite install buildcache_cmd("install", "-ufo", "corge") buildcache_cmd("keys", "-f") buildcache_cmd("list") buildcache_cmd("list", "-a") buildcache_cmd("list", "-l", "-v") def test_push_and_fetch_keys(mock_gnupghome, tmp_path: pathlib.Path): testpath = str(mock_gnupghome) mirror = os.path.join(testpath, "mirror") mirrors = {"test-mirror": url_util.path_to_file_url(mirror)} mirrors = spack.mirrors.mirror.MirrorCollection(mirrors) mirror = spack.mirrors.mirror.Mirror(url_util.path_to_file_url(mirror)) gpg_dir1 = os.path.join(testpath, "gpg1") gpg_dir2 = os.path.join(testpath, "gpg2") # dir 1: create a new key, record its fingerprint, and push it to a new # mirror with spack.util.gpg.gnupghome_override(gpg_dir1): spack.util.gpg.create(name="test-key", email="fake@test.key", expires="0", comment=None) keys = spack.util.gpg.public_keys() assert len(keys) == 1 fpr = keys[0] spack.binary_distribution._url_push_keys( mirror, keys=[fpr], tmpdir=str(tmp_path), update_index=True ) # dir 2: import the key from the mirror, and confirm that its fingerprint # matches the one created above with spack.util.gpg.gnupghome_override(gpg_dir2): assert len(spack.util.gpg.public_keys()) == 0 spack.binary_distribution.get_keys(mirrors=mirrors, install=True, trust=True, force=True) new_keys = spack.util.gpg.public_keys() assert len(new_keys) == 1 assert new_keys[0] == fpr @pytest.mark.maybeslow def test_built_spec_cache(install_mockery, tmp_path: pathlib.Path): """Because the buildcache list command fetches the buildcache index and uses it to populate the binary_distribution built spec cache, when this test calls get_mirrors_for_spec, it is testing the popluation of that cache from a buildcache index.""" install_cmd("--fake", "--no-cache", "corge") buildcache_cmd("push", "--unsigned", "--update-index", str(tmp_path), "corge") mirror_cmd("add", "--type", "binary", "--unsigned", "test-mirror", str(tmp_path)) buildcache_cmd("list", "-a", "-l") gspec = spack.concretize.concretize_one("garply") cspec = spack.concretize.concretize_one("corge") for s in [gspec, cspec]: results = spack.binary_distribution.get_mirrors_for_spec(s) assert len(results) == 1 assert results[0].url == url_util.path_to_file_url(str(tmp_path)) def fake_dag_hash(spec, length=None): # Generate an arbitrary hash that is intended to be different than # whatever a Spec reported before (to test actions that trigger when # the hash changes) return "tal4c7h4z0gqmixb1eqa92mjoybxn5l6"[:length] @pytest.mark.usefixtures("install_mockery", "mock_packages", "mock_fetch", "temporary_mirror") def test_spec_needs_rebuild(monkeypatch, tmp_path: pathlib.Path): """Make sure needs_rebuild properly compares remote hash against locally computed one, avoiding unnecessary rebuilds""" # Create a temp mirror directory for buildcache usage mirror_dir = tmp_path / "mirror_dir" mirror_url = url_util.path_to_file_url(str(mirror_dir)) s = spack.concretize.concretize_one("libdwarf") # Install a package install_cmd("--fake", s.name) # Put installed package in the buildcache buildcache_cmd("push", "-u", str(mirror_dir), s.name) rebuild = spack.binary_distribution.needs_rebuild(s, mirror_url) assert not rebuild # Now monkey patch Spec to change the hash on the package monkeypatch.setattr(spack.spec.Spec, "dag_hash", fake_dag_hash) rebuild = spack.binary_distribution.needs_rebuild(s, mirror_url) assert rebuild @pytest.mark.usefixtures("install_mockery", "mock_packages", "mock_fetch") def test_generate_index_missing(monkeypatch, tmp_path: pathlib.Path, mutable_config): """Ensure spack buildcache index only reports available packages""" # Create a temp mirror directory for buildcache usage mirror_dir = tmp_path / "mirror_dir" mirror_url = url_util.path_to_file_url(str(mirror_dir)) spack.config.set("mirrors", {"test": mirror_url}) s = spack.concretize.concretize_one("libdwarf") # Install a package install_cmd("--fake", "--no-cache", s.name) # Create a buildcache and update index buildcache_cmd("push", "-u", str(mirror_dir), s.name) buildcache_cmd("update-index", str(mirror_dir)) # Check package and dependency in buildcache cache_list = buildcache_cmd("list", "--allarch") assert "libdwarf" in cache_list assert "libelf" in cache_list # Remove dependency from cache libelf_files = glob.glob( os.path.join( str(mirror_dir / spack.binary_distribution.buildcache_relative_specs_path()), "libelf", "*libelf*", ) ) os.remove(*libelf_files) # Update index buildcache_cmd("update-index", str(mirror_dir)) with spack.config.override("config:binary_index_ttl", 0): # Check dependency not in buildcache cache_list = buildcache_cmd("list", "--allarch") assert "libdwarf" in cache_list assert "libelf" not in cache_list @pytest.mark.usefixtures("install_mockery", "mock_packages", "mock_fetch") def test_use_bin_index(monkeypatch, tmp_path: pathlib.Path, mutable_config): """Check use of binary cache index: perform an operation that instantiates it, and a second operation that reconstructs it. """ index_cache_root = str(tmp_path / "index_cache") monkeypatch.setattr( spack.binary_distribution, "BINARY_INDEX", spack.binary_distribution.BinaryCacheIndex(index_cache_root), ) # Create a mirror, configure us to point at it, install a spec, and # put it in the mirror mirror_dir = tmp_path / "mirror_dir" mirror_url = url_util.path_to_file_url(str(mirror_dir)) spack.config.set("mirrors", {"test": mirror_url}) s = spack.concretize.concretize_one("libdwarf") install_cmd("--fake", "--no-cache", s.name) buildcache_cmd("push", "-u", str(mirror_dir), s.name) buildcache_cmd("update-index", str(mirror_dir)) # Now the test buildcache_cmd("list", "-al") spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex( index_cache_root ) cache_list = buildcache_cmd("list", "-al") assert "libdwarf" in cache_list @pytest.mark.usefixtures("install_mockery", "mock_packages", "mock_fetch") def test_use_bin_index_active_env_with_view( monkeypatch, tmp_path: pathlib.Path, mutable_config, mutable_mock_env_path ): """Check use of binary cache index: perform an operation that instantiates it, and a second operation that reconstructs it. """ index_cache_root = str(tmp_path / "index_cache") monkeypatch.setattr( spack.binary_distribution, "BINARY_INDEX", spack.binary_distribution.BinaryCacheIndex(index_cache_root), ) # Create a mirror, configure us to point at it, install a spec, and # put it in the mirror mirror_dir = tmp_path / "mirror_dir" mirror_url = url_util.path_to_file_url(str(mirror_dir)) spack.config.set("mirrors", {"test": {"url": mirror_url, "view": "test"}}) s = spack.concretize.concretize_one("libdwarf") # Create an environment and install specs for the view ev.create("testenv") with ev.read("testenv"): install_cmd("--add", "--fake", "--no-cache", s.name) buildcache_cmd("push", "-u", "test", s.name) buildcache_cmd("update-index", "test") # Now the test buildcache_cmd("list", "-al") spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex( index_cache_root ) cache_list = buildcache_cmd("list", "-al") assert "libdwarf" in cache_list @pytest.mark.usefixtures("install_mockery", "mock_packages", "mock_fetch") def test_use_bin_index_with_view( monkeypatch, tmp_path: pathlib.Path, mutable_config, mutable_mock_env_path ): """Check use of binary cache index: perform an operation that instantiates it, and a second operation that reconstructs it. """ index_cache_root = str(tmp_path / "index_cache") monkeypatch.setattr( spack.binary_distribution, "BINARY_INDEX", spack.binary_distribution.BinaryCacheIndex(index_cache_root), ) # Create a mirror, configure us to point at it, install a spec, and # put it in the mirror mirror_dir = tmp_path / "mirror_dir" mirror_url = url_util.path_to_file_url(str(mirror_dir)) spack.config.set("mirrors", {"test": {"url": mirror_url, "view": "test"}}) s = spack.concretize.concretize_one("libdwarf") # Create an environment and install specs for the view ev.create("testenv") with ev.read("testenv"): install_cmd("--add", "--fake", "--no-cache", s.name) buildcache_cmd("push", "-u", "test", s.name) buildcache_cmd("update-index", "test", "testenv") # Now the test buildcache_cmd("list", "-al") spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex( index_cache_root ) cache_list = buildcache_cmd("list", "-al") assert "libdwarf" in cache_list def test_generate_key_index_failure(monkeypatch, tmp_path: pathlib.Path): def list_url(url, recursive=False): if "fails-listing" in url: raise Exception("Couldn't list the directory") return ["first.pub", "second.pub"] def push_to_url(*args, **kwargs): raise Exception("Couldn't upload the file") monkeypatch.setattr(web_util, "list_url", list_url) monkeypatch.setattr(web_util, "push_to_url", push_to_url) with pytest.raises(CannotListKeys, match="Encountered problem listing keys"): spack.binary_distribution.generate_key_index( "s3://non-existent/fails-listing", str(tmp_path) ) with pytest.raises(GenerateIndexError, match="problem pushing .* Couldn't upload"): spack.binary_distribution.generate_key_index( "s3://non-existent/fails-uploading", str(tmp_path) ) def test_generate_package_index_failure(monkeypatch, tmp_path: pathlib.Path, capfd): def mock_list_url(url, recursive=False): raise Exception("Some HTTP error") monkeypatch.setattr(web_util, "list_url", mock_list_url) test_url = "file:///fake/keys/dir" with pytest.raises(GenerateIndexError, match="Unable to generate package index"): spack.binary_distribution._url_generate_package_index(test_url, str(tmp_path)) assert ( "Warning: Encountered problem listing packages at " f"{test_url}: Some HTTP error" in capfd.readouterr().err ) def test_generate_indices_exception(monkeypatch, tmp_path: pathlib.Path, capfd): def mock_list_url(url, recursive=False): raise Exception("Test Exception handling") monkeypatch.setattr(web_util, "list_url", mock_list_url) url = "file:///fake/keys/dir" with pytest.raises(GenerateIndexError, match=f"Encountered problem listing keys at {url}"): spack.binary_distribution.generate_key_index(url, str(tmp_path)) with pytest.raises(GenerateIndexError, match="Unable to generate package index"): spack.binary_distribution._url_generate_package_index(url, str(tmp_path)) assert f"Encountered problem listing packages at {url}" in capfd.readouterr().err def test_update_sbang(tmp_path: pathlib.Path, temporary_mirror, mock_fetch, install_mockery): """Test relocation of the sbang shebang line in a package script""" s = spack.concretize.concretize_one("old-sbang") PackageInstaller([s.package]).install() old_prefix, old_sbang_shebang = s.prefix, sbang.sbang_shebang_line() old_contents = f"""\ {old_sbang_shebang} #!/usr/bin/env python3 {s.prefix.bin} """ with open(os.path.join(s.prefix.bin, "script.sh"), encoding="utf-8") as f: assert f.read() == old_contents # Create a buildcache with the installed spec. buildcache_cmd("push", "--update-index", "--unsigned", temporary_mirror, f"/{s.dag_hash()}") # Switch the store to the new install tree locations with spack.store.use_store(str(tmp_path)): s._prefix = None # clear the cached old prefix new_prefix, new_sbang_shebang = s.prefix, sbang.sbang_shebang_line() assert old_prefix != new_prefix assert old_sbang_shebang != new_sbang_shebang PackageInstaller( [s.package], root_policy="cache_only", dependencies_policy="cache_only", unsigned=True ).install() # Check that the sbang line refers to the new install tree new_contents = f"""\ {sbang.sbang_shebang_line()} #!/usr/bin/env python3 {s.prefix.bin} """ with open(os.path.join(s.prefix.bin, "script.sh"), encoding="utf-8") as f: assert f.read() == new_contents def test_FetchCacheError_only_accepts_lists_of_errors(): with pytest.raises(TypeError, match="list"): spack.binary_distribution.FetchCacheError("error") def test_FetchCacheError_pretty_printing_multiple(): e = spack.binary_distribution.FetchCacheError([RuntimeError("Oops!"), TypeError("Trouble!")]) str_e = str(e) assert "Multiple errors" in str_e assert "Error 1: RuntimeError: Oops!" in str_e assert "Error 2: TypeError: Trouble!" in str_e assert str_e.rstrip() == str_e def test_FetchCacheError_pretty_printing_single(): e = spack.binary_distribution.FetchCacheError([RuntimeError("Oops!")]) str_e = str(e) assert "Multiple errors" not in str_e assert "RuntimeError: Oops!" in str_e assert str_e.rstrip() == str_e def test_text_relocate_if_needed( install_mockery, temporary_store, mock_fetch, tmp_path: pathlib.Path ): install_cmd("needs-text-relocation") spec = temporary_store.db.query_one("needs-text-relocation") tgz_path = tmp_path / "relocatable.tar.gz" spack.binary_distribution.create_tarball(spec, str(tgz_path)) # extract the .spack/binary_distribution file with tarfile.open(tgz_path) as tar: entry_name = next(x for x in tar.getnames() if x.endswith(".spack/binary_distribution")) bd_file = tar.extractfile(entry_name) manifest = syaml.load(bd_file) assert join_path("bin", "exe") in manifest["relocate_textfiles"] assert join_path("bin", "otherexe") not in manifest["relocate_textfiles"] assert join_path("bin", "secretexe") not in manifest["relocate_textfiles"] def test_compression_writer(tmp_path: pathlib.Path): text = "This is some text. We might or might not like to compress it as we write." checksum_algo = "sha256" # Write the data using gzip compression compressed_output_path = str(tmp_path / "compressed_text") with compression_writer(compressed_output_path, "gzip", checksum_algo) as ( compressor, checker, ): compressor.write(text.encode("utf-8")) compressed_size = checker.length compressed_checksum = checker.hexdigest() with open(compressed_output_path, "rb") as f: binary_content = f.read() assert spack.binary_distribution.compute_hash(binary_content) == compressed_checksum assert os.stat(compressed_output_path).st_size == compressed_size assert binary_content[:2] == b"\x1f\x8b" decompressed_content = gzip.decompress(binary_content).decode("utf-8") assert decompressed_content == text # Write the data without compression uncompressed_output_path = str(tmp_path / "uncompressed_text") with compression_writer(uncompressed_output_path, "none", checksum_algo) as ( compressor, checker, ): compressor.write(text.encode("utf-8")) uncompressed_size = checker.length uncompressed_checksum = checker.hexdigest() with open(uncompressed_output_path, "r", encoding="utf-8") as f: content = f.read() assert spack.binary_distribution.compute_hash(content) == uncompressed_checksum assert os.stat(uncompressed_output_path).st_size == uncompressed_size assert content == text # Make sure we raise if requesting unknown compression type nocare_output_path = str(tmp_path / "wontwrite") with pytest.raises(BuildcacheEntryError, match="Unknown compression type"): with compression_writer(nocare_output_path, "gsip", checksum_algo) as ( compressor, checker, ): compressor.write(text) def test_v2_etag_fetching_304(): # Test conditional fetch with etags. If the remote hasn't modified the file # it returns 304, which is an HTTPError in urllib-land. That should be # handled as success, since it means the local cache is up-to-date. def response_304(request: urllib.request.Request): url = request.get_full_url() if url == f"https://www.example.com/build_cache/{INDEX_JSON_FILE}": assert request.get_header("If-none-match") == '"112a8bbc1b3f7f185621c1ee335f0502"' raise urllib.error.HTTPError( url, 304, "Not Modified", hdrs={}, # type: ignore[arg-type] fp=None, # type: ignore[arg-type] ) assert False, "Should not fetch {}".format(url) fetcher = spack.binary_distribution.EtagIndexFetcherV2( url="https://www.example.com", etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_304, ) result = fetcher.conditional_fetch() assert isinstance(result, spack.binary_distribution.FetchIndexResult) assert result.fresh def test_v2_etag_fetching_200(): # Test conditional fetch with etags. The remote has modified the file. def response_200(request: urllib.request.Request): url = request.get_full_url() if url == f"https://www.example.com/build_cache/{INDEX_JSON_FILE}": assert request.get_header("If-none-match") == '"112a8bbc1b3f7f185621c1ee335f0502"' return urllib.response.addinfourl( io.BytesIO(b"Result"), headers={"Etag": '"59bcc3ad6775562f845953cf01624225"'}, # type: ignore[arg-type] url=url, code=200, ) assert False, "Should not fetch {}".format(url) fetcher = spack.binary_distribution.EtagIndexFetcherV2( url="https://www.example.com", etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_200, ) result = fetcher.conditional_fetch() assert isinstance(result, spack.binary_distribution.FetchIndexResult) assert not result.fresh assert result.etag == "59bcc3ad6775562f845953cf01624225" assert result.data == "Result" # decoded utf-8. assert result.hash == spack.binary_distribution.compute_hash("Result") def test_v2_etag_fetching_404(): # Test conditional fetch with etags. The remote has modified the file. def response_404(request: urllib.request.Request): raise urllib.error.HTTPError( request.get_full_url(), 404, "Not found", hdrs={"Etag": '"59bcc3ad6775562f845953cf01624225"'}, # type: ignore[arg-type] fp=None, ) fetcher = spack.binary_distribution.EtagIndexFetcherV2( url="https://www.example.com", etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_404, ) with pytest.raises(spack.binary_distribution.FetchIndexError): fetcher.conditional_fetch() def test_v2_default_index_fetch_200(): index_json = '{"Hello": "World"}' index_json_hash = spack.binary_distribution.compute_hash(index_json) def urlopen(request: urllib.request.Request): url = request.get_full_url() if url.endswith("index.json.hash"): return urllib.response.addinfourl( # type: ignore[arg-type] io.BytesIO(index_json_hash.encode()), headers={}, # type: ignore[arg-type] url=url, code=200, ) elif url.endswith(INDEX_JSON_FILE): return urllib.response.addinfourl( io.BytesIO(index_json.encode()), headers={"Etag": '"59bcc3ad6775562f845953cf01624225"'}, # type: ignore[arg-type] url=url, code=200, ) assert False, "Unexpected request {}".format(url) fetcher = spack.binary_distribution.DefaultIndexFetcherV2( url="https://www.example.com", local_hash="outdated", urlopen=urlopen ) result = fetcher.conditional_fetch() assert isinstance(result, spack.binary_distribution.FetchIndexResult) assert not result.fresh assert result.etag == "59bcc3ad6775562f845953cf01624225" assert result.data == index_json assert result.hash == index_json_hash def test_v2_default_index_dont_fetch_index_json_hash_if_no_local_hash(): # When we don't have local hash, we should not be fetching the # remote index.json.hash file, but only index.json. index_json = '{"Hello": "World"}' index_json_hash = spack.binary_distribution.compute_hash(index_json) def urlopen(request: urllib.request.Request): url = request.get_full_url() if url.endswith(INDEX_JSON_FILE): return urllib.response.addinfourl( io.BytesIO(index_json.encode()), headers={"Etag": '"59bcc3ad6775562f845953cf01624225"'}, # type: ignore[arg-type] url=url, code=200, ) assert False, "Unexpected request {}".format(url) fetcher = spack.binary_distribution.DefaultIndexFetcherV2( url="https://www.example.com", local_hash=None, urlopen=urlopen ) result = fetcher.conditional_fetch() assert isinstance(result, spack.binary_distribution.FetchIndexResult) assert result.data == index_json assert result.hash == index_json_hash assert result.etag == "59bcc3ad6775562f845953cf01624225" assert not result.fresh def test_v2_default_index_not_modified(): index_json = '{"Hello": "World"}' index_json_hash = spack.binary_distribution.compute_hash(index_json) def urlopen(request: urllib.request.Request): url = request.get_full_url() if url.endswith("index.json.hash"): return urllib.response.addinfourl( io.BytesIO(index_json_hash.encode()), headers={}, # type: ignore[arg-type] url=url, code=200, ) # No request to index.json should be made. assert False, "Unexpected request {}".format(url) fetcher = spack.binary_distribution.DefaultIndexFetcherV2( url="https://www.example.com", local_hash=index_json_hash, urlopen=urlopen ) assert fetcher.conditional_fetch().fresh @pytest.mark.parametrize("index_json", [b"\xa9", b"!#%^"]) def test_v2_default_index_invalid_hash_file(index_json): # Test invalid unicode / invalid hash type index_json_hash = spack.binary_distribution.compute_hash(index_json) def urlopen(request: urllib.request.Request): return urllib.response.addinfourl( io.BytesIO(), headers={}, # type: ignore[arg-type] url=request.get_full_url(), code=200, ) fetcher = spack.binary_distribution.DefaultIndexFetcherV2( url="https://www.example.com", local_hash=index_json_hash, urlopen=urlopen ) assert fetcher.get_remote_hash() is None def test_v2_default_index_json_404(): # Test invalid unicode / invalid hash type index_json = '{"Hello": "World"}' index_json_hash = spack.binary_distribution.compute_hash(index_json) def urlopen(request: urllib.request.Request): url = request.get_full_url() if url.endswith("index.json.hash"): return urllib.response.addinfourl( io.BytesIO(index_json_hash.encode()), headers={}, # type: ignore[arg-type] url=url, code=200, ) elif url.endswith(INDEX_JSON_FILE): raise urllib.error.HTTPError( url, code=404, msg="Not Found", hdrs={"Etag": '"59bcc3ad6775562f845953cf01624225"'}, # type: ignore[arg-type] fp=None, ) assert False, "Unexpected fetch {}".format(url) fetcher = spack.binary_distribution.DefaultIndexFetcherV2( url="https://www.example.com", local_hash="invalid", urlopen=urlopen ) with pytest.raises(spack.binary_distribution.FetchIndexError, match="Could not fetch index"): fetcher.conditional_fetch() def _all_parents(prefix): parts = [p for p in prefix.split("/")] return ["/".join(parts[: i + 1]) for i in range(len(parts))] def test_tarball_doesnt_include_buildinfo_twice(tmp_path: Path): """When tarballing a package that was installed from a buildcache, make sure that the buildinfo file is not included twice in the tarball.""" p = tmp_path / "prefix" p.joinpath(".spack").mkdir(parents=True) # Create a binary_distribution file in the .spack folder with open(p / ".spack" / "binary_distribution", "w", encoding="utf-8") as f: f.write(syaml.dump({"metadata", "old"})) # Now create a tarball, which should include a new binary_distribution file tarball = str(tmp_path / "prefix.tar.gz") spack.binary_distribution._do_create_tarball( tarfile_path=tarball, prefix=str(p), buildinfo={"metadata": "new"}, prefixes_to_relocate=[] ) expected_prefix = str(p).lstrip("/") # Verify we don't have a repeated binary_distribution file, # and that the tarball contains the new one, not the old one. with tarfile.open(tarball) as tar: assert syaml.load(tar.extractfile(f"{expected_prefix}/.spack/binary_distribution")) == { "metadata": "new", "relocate_binaries": [], "relocate_textfiles": [], "relocate_links": [], } assert tar.getnames() == [ *_all_parents(expected_prefix), f"{expected_prefix}/.spack", f"{expected_prefix}/.spack/binary_distribution", ] def test_reproducible_tarball_is_reproducible(tmp_path: Path): p = tmp_path / "prefix" p.joinpath("bin").mkdir(parents=True) p.joinpath(".spack").mkdir(parents=True) app = p / "bin" / "app" tarball_1 = str(tmp_path / "prefix-1.tar.gz") tarball_2 = str(tmp_path / "prefix-2.tar.gz") with open(app, "w", encoding="utf-8") as f: f.write("hello world") buildinfo = {"metadata": "yes please"} # Create a tarball with a certain mtime of bin/app os.utime(app, times=(0, 0)) spack.binary_distribution._do_create_tarball( tarball_1, prefix=str(p), buildinfo=buildinfo, prefixes_to_relocate=[] ) # Do it another time with different mtime of bin/app os.utime(app, times=(10, 10)) spack.binary_distribution._do_create_tarball( tarball_2, prefix=str(p), buildinfo=buildinfo, prefixes_to_relocate=[] ) # They should be bitwise identical: assert filecmp.cmp(tarball_1, tarball_2, shallow=False) expected_prefix = str(p).lstrip("/") # Sanity check for contents: with tarfile.open(tarball_1, mode="r") as f: for m in f.getmembers(): assert m.uid == m.gid == m.mtime == 0 assert m.uname == m.gname == "" assert set(f.getnames()) == { *_all_parents(expected_prefix), f"{expected_prefix}/bin", f"{expected_prefix}/bin/app", f"{expected_prefix}/.spack", f"{expected_prefix}/.spack/binary_distribution", } def test_tarball_normalized_permissions(tmp_path: pathlib.Path): p = tmp_path / "prefix" p.mkdir() (p / "bin").mkdir() (p / "share").mkdir() (p / ".spack").mkdir() app = p / "bin" / "app" data = p / "share" / "file" tarball = str(tmp_path / "prefix.tar.gz") # Everyone can write & execute. This should turn into 0o755 when the tarball is # extracted (on a different system). with open( app, "w", opener=lambda path, flags: os.open(path, flags, 0o777), encoding="utf-8" ) as f: f.write("hello world") # User doesn't have execute permissions, but group/world have; this should also # turn into 0o644 (user read/write, group&world only read). with open( data, "w", opener=lambda path, flags: os.open(path, flags, 0o477), encoding="utf-8" ) as f: f.write("hello world") spack.binary_distribution._do_create_tarball( tarball, prefix=str(p), buildinfo={}, prefixes_to_relocate=[] ) expected_prefix = str(p).lstrip("/") with tarfile.open(tarball) as tar: path_to_member = {member.name: member for member in tar.getmembers()} # directories should have 0o755 assert path_to_member[f"{expected_prefix}"].mode == 0o755 assert path_to_member[f"{expected_prefix}/bin"].mode == 0o755 assert path_to_member[f"{expected_prefix}/.spack"].mode == 0o755 # executable-by-user files should be 0o755 assert path_to_member[f"{expected_prefix}/bin/app"].mode == 0o755 # not-executable-by-user files should be 0o644 assert path_to_member[f"{expected_prefix}/share/file"].mode == 0o644 def test_tarball_common_prefix(dummy_prefix, tmp_path: pathlib.Path): """Tests whether Spack can figure out the package directory from the tarball contents, and strip them when extracting. This test creates a CURRENT_BUILD_CACHE_LAYOUT_VERSION=1 type tarball where the parent directories of the package prefix are missing. Spack should be able to figure out the common prefix and extract the files into the correct location.""" # When creating a tarball, Python (and tar) use relative paths, # Absolute paths become relative to `/`, so drop the leading `/`. assert os.path.isabs(dummy_prefix) expected_prefix = PurePath(dummy_prefix).as_posix().lstrip("/") with working_dir(str(tmp_path)): # Create a tarball (using absolute path for prefix dir) with tarfile.open("example.tar", mode="w") as tar: tar.add(name=dummy_prefix) # Open, verify common prefix, and extract it. with tarfile.open("example.tar", mode="r") as tar: common_prefix = spack.binary_distribution._ensure_common_prefix(tar) assert common_prefix == expected_prefix # For consistent behavior across all supported Python versions tar.extraction_filter = lambda member, path: member # Extract into prefix2 tar.extractall( path="prefix2", members=spack.binary_distribution._tar_strip_component(tar, common_prefix), ) # Verify files are all there at the correct level. assert set(os.listdir("prefix2")) == {"bin", "share", ".spack"} assert set(os.listdir(os.path.join("prefix2", "bin"))) == { "app", "relative_app_link", "absolute_app_link", } assert set(os.listdir(os.path.join("prefix2", "share"))) == {"file"} # Relative symlink should still be correct assert readlink(os.path.join("prefix2", "bin", "relative_app_link")) == "app" # Absolute symlink should remain absolute -- this is for relocation to fix up. assert readlink(os.path.join("prefix2", "bin", "absolute_app_link")) == os.path.join( dummy_prefix, "bin", "app" ) def test_tarfile_missing_binary_distribution_file(tmp_path: pathlib.Path): """A tarfile that does not contain a .spack/binary_distribution file cannot be used to install.""" with working_dir(str(tmp_path)): # An empty .spack dir. with tarfile.open("empty.tar", mode="w") as tar: tarinfo = tarfile.TarInfo(name="example/.spack") tarinfo.type = tarfile.DIRTYPE tar.addfile(tarinfo) with pytest.raises(ValueError, match="missing binary_distribution file"): spack.binary_distribution._ensure_common_prefix(tarfile.open("empty.tar", mode="r")) def test_tarfile_without_common_directory_prefix_fails(tmp_path: pathlib.Path): """A tarfile that only contains files without a common package directory should fail to extract, as we won't know where to put the files.""" with working_dir(str(tmp_path)): # Create a broken tarball with just a file, no directories. with tarfile.open("empty.tar", mode="w") as tar: tar.addfile( tarfile.TarInfo(name="example/.spack/binary_distribution"), fileobj=io.BytesIO(b"hello"), ) with pytest.raises(ValueError, match="Tarball does not contain a common prefix"): spack.binary_distribution._ensure_common_prefix(tarfile.open("empty.tar", mode="r")) def test_tarfile_with_files_outside_common_prefix(tmp_path: pathlib.Path, dummy_prefix): """If a file is outside of the common prefix, we should fail.""" with working_dir(str(tmp_path)): with tarfile.open("broken.tar", mode="w") as tar: tar.add(name=dummy_prefix) tar.addfile(tarfile.TarInfo(name="/etc/config_file"), fileobj=io.BytesIO(b"hello")) with pytest.raises( ValueError, match="Tarball contains file /etc/config_file outside of prefix" ): spack.binary_distribution._ensure_common_prefix(tarfile.open("broken.tar", mode="r")) def test_tarfile_of_spec_prefix(tmp_path: pathlib.Path): """Tests whether hardlinks, symlinks, files and dirs are added correctly, and that the order of entries is correct.""" prefix = tmp_path / "prefix" prefix.mkdir() (prefix / "a_directory").mkdir() (prefix / "a_directory" / "file").write_text("hello") (prefix / "c_directory").mkdir() (prefix / "c_directory" / "file").write_text("hello") (prefix / "b_directory").mkdir() (prefix / "b_directory" / "file").write_text("hello") (prefix / "file").write_text("hello") os.symlink(str(prefix / "file"), str(prefix / "symlink")) os.link(str(prefix / "file"), str(prefix / "hardlink")) file = tmp_path / "example.tar" with tarfile.open(str(file), mode="w") as tar: spack.binary_distribution.tarfile_of_spec_prefix(tar, str(prefix), prefixes_to_relocate=[]) expected_prefix = str(prefix).lstrip("/") with tarfile.open(str(file), mode="r") as tar: # Verify that entries are added in depth-first pre-order, files preceding dirs, # entries ordered alphabetically assert tar.getnames() == [ *_all_parents(expected_prefix), f"{expected_prefix}/file", f"{expected_prefix}/hardlink", f"{expected_prefix}/symlink", f"{expected_prefix}/a_directory", f"{expected_prefix}/a_directory/file", f"{expected_prefix}/b_directory", f"{expected_prefix}/b_directory/file", f"{expected_prefix}/c_directory", f"{expected_prefix}/c_directory/file", ] # Check that the types are all correct assert tar.getmember(f"{expected_prefix}").isdir() assert tar.getmember(f"{expected_prefix}/file").isreg() assert tar.getmember(f"{expected_prefix}/hardlink").islnk() assert tar.getmember(f"{expected_prefix}/symlink").issym() assert tar.getmember(f"{expected_prefix}/a_directory").isdir() assert tar.getmember(f"{expected_prefix}/a_directory/file").isreg() assert tar.getmember(f"{expected_prefix}/b_directory").isdir() assert tar.getmember(f"{expected_prefix}/b_directory/file").isreg() assert tar.getmember(f"{expected_prefix}/c_directory").isdir() assert tar.getmember(f"{expected_prefix}/c_directory/file").isreg() @pytest.mark.parametrize("layout,expect_success", [(None, True), (1, True), (2, False)]) def test_get_valid_spec_file(tmp_path: pathlib.Path, layout, expect_success): # Test reading a spec.json file that does not specify a layout version. spec_dict = Spec("example").to_dict() path = tmp_path / "spec.json" effective_layout = layout or 0 # If not specified it should be 0 # Add a layout version if layout is not None: spec_dict["buildcache_layout_version"] = layout # Save to file with open(path, "w", encoding="utf-8") as f: json.dump(spec_dict, f) try: spec_dict_disk, layout_disk = get_valid_spec_file(str(path), max_supported_layout=1) assert expect_success assert spec_dict_disk == spec_dict assert layout_disk == effective_layout except spack.binary_distribution.InvalidMetadataFile: assert not expect_success def test_get_valid_spec_file_doesnt_exist(tmp_path: pathlib.Path): with pytest.raises(spack.binary_distribution.InvalidMetadataFile, match="No such file"): get_valid_spec_file(str(tmp_path / "no-such-file"), max_supported_layout=1) @pytest.mark.parametrize("filename", ["spec.json", "spec.json.sig"]) def test_get_valid_spec_file_no_json(tmp_path: pathlib.Path, filename): tmp_path.joinpath(filename).write_text("not json") with pytest.raises(spack.binary_distribution.InvalidMetadataFile): get_valid_spec_file(str(tmp_path / filename), max_supported_layout=1) @pytest.mark.usefixtures("install_mockery", "mock_packages", "mock_fetch", "temporary_mirror") def test_url_buildcache_entry_v3(monkeypatch, tmp_path: pathlib.Path): """Make sure URLBuildcacheEntry behaves as expected""" # Create a temp mirror directory for buildcache usage mirror_dir = tmp_path / "mirror_dir" mirror_url = url_util.path_to_file_url(str(mirror_dir)) s = spack.concretize.concretize_one("libdwarf") # Install libdwarf install_cmd("--fake", s.name) # Push libdwarf to buildcache buildcache_cmd("push", "-u", str(mirror_dir), s.name) cache_class = get_url_buildcache_class( spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ) build_cache = cache_class(mirror_url, s, allow_unsigned=True) manifest = build_cache.read_manifest() spec_dict = build_cache.fetch_metadata() local_tarball_path = build_cache.fetch_archive() assert "spec" in spec_dict for blob_record in manifest.data: blob_path = build_cache.get_staged_blob_path(blob_record) assert os.path.exists(blob_path) actual_blob_size = os.stat(blob_path).st_size assert blob_record.content_length == actual_blob_size build_cache.destroy() assert not os.path.exists(local_tarball_path) def test_relative_path_components(): blobs_v3 = URLBuildcacheEntry.get_relative_path_components(BuildcacheComponent.BLOB) assert len(blobs_v3) == 1 assert "blobs" in blobs_v3 blobs_v2 = URLBuildcacheEntryV2.get_relative_path_components(BuildcacheComponent.BLOB) assert len(blobs_v2) == 1 assert "build_cache" in blobs_v2 v2_spec_url = "file:///home/me/mymirror/build_cache/linux-ubuntu22.04-sapphirerapids-gcc-12.3.0-gmake-4.4.1-5pddli3htvfe6svs7nbrqmwi5735agi3.spec.json.sig" assert URLBuildcacheEntryV2.get_base_url(v2_spec_url) == "file:///home/me/mymirror" v3_manifest_url = "file:///home/me/mymirror/v3/manifests/gmake-4.4.1-5pddli3htvfe6svs7nbrqmwi5735agi3.spec.manifest.json" assert URLBuildcacheEntry.get_base_url(v3_manifest_url) == "file:///home/me/mymirror" @pytest.mark.parametrize( "spec", [ # Standard case "short-name@=1.2.3", # Unsupported characters in git version f"git-version@{1:040x}=develop", # Too long of a name f"{'too-long':x<256}@=1.2.3", ], ) def test_default_tag(spec: str): """Make sure that computed image tags are valid.""" assert re.fullmatch( spack.oci.image.tag, spack.binary_distribution._oci_default_tag(spack.spec.Spec(spec)) ) class IndexInformation(NamedTuple): manifest_contents: Dict[str, Any] index_contents: str index_hash: str manifest_path: str index_path: str manifest_etag: str fetched_blob: Callable[[], bool] @pytest.fixture def mock_index(tmp_path: pathlib.Path, monkeypatch) -> IndexInformation: mirror_root = tmp_path / "mymirror" index_json = '{"Hello": "World"}' index_json_hash = spack.binary_distribution.compute_hash(index_json) fetched = False cache_class = get_url_buildcache_class( layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ) index_blob_path = os.path.join( str(mirror_root), *cache_class.get_relative_path_components(BuildcacheComponent.BLOB), "sha256", index_json_hash[:2], index_json_hash, ) os.makedirs(os.path.dirname(index_blob_path)) with open(index_blob_path, "w", encoding="utf-8") as fd: fd.write(index_json) index_blob_record = spack.binary_distribution.BlobRecord( os.stat(index_blob_path).st_size, cache_class.BUILDCACHE_INDEX_MEDIATYPE, "none", "sha256", index_json_hash, ) index_manifest = { "version": cache_class.get_layout_version(), "data": [index_blob_record.to_dict()], } manifest_json_path = cache_class.get_index_url(str(mirror_root)) os.makedirs(os.path.dirname(manifest_json_path)) with open(manifest_json_path, "w", encoding="utf-8") as f: json.dump(index_manifest, f) def fetch_patch(stage, mirror_only: bool = False, err_msg: Optional[str] = None): nonlocal fetched fetched = True @property # type: ignore def save_filename_patch(stage): return str(index_blob_path) monkeypatch.setattr(spack.stage.Stage, "fetch", fetch_patch) monkeypatch.setattr(spack.stage.Stage, "save_filename", save_filename_patch) def get_did_fetch(): # nonlocal fetched return fetched return IndexInformation( index_manifest, index_json, index_json_hash, manifest_json_path, index_blob_path, "59bcc3ad6775562f845953cf01624225", get_did_fetch, ) def test_etag_fetching_304(): # Test conditional fetch with etags. If the remote hasn't modified the file # it returns 304, which is an HTTPError in urllib-land. That should be # handled as success, since it means the local cache is up-to-date. def response_304(request: urllib.request.Request): url = request.get_full_url() if url.endswith(INDEX_MANIFEST_FILE): assert request.get_header("If-none-match") == '"112a8bbc1b3f7f185621c1ee335f0502"' raise urllib.error.HTTPError( url, 304, "Not Modified", hdrs={}, # type: ignore[arg-type] fp=None, # type: ignore[arg-type] ) assert False, "Unexpected request {}".format(url) fetcher = spack.binary_distribution.EtagIndexFetcher( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_304, ) result = fetcher.conditional_fetch() assert isinstance(result, spack.binary_distribution.FetchIndexResult) assert result.fresh def test_etag_fetching_200(mock_index): # Test conditional fetch with etags. The remote has modified the file. def response_200(request: urllib.request.Request): url = request.get_full_url() if url.endswith(INDEX_MANIFEST_FILE): assert request.get_header("If-none-match") == '"112a8bbc1b3f7f185621c1ee335f0502"' return urllib.response.addinfourl( io.BytesIO(json.dumps(mock_index.manifest_contents).encode()), headers={"Etag": f'"{mock_index.manifest_etag}"'}, # type: ignore[arg-type] url=url, code=200, ) assert False, "Unexpected request {}".format(url) fetcher = spack.binary_distribution.EtagIndexFetcher( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_200, ) result = fetcher.conditional_fetch() assert isinstance(result, spack.binary_distribution.FetchIndexResult) assert not result.fresh assert mock_index.fetched_blob() assert result.etag == mock_index.manifest_etag assert result.data == mock_index.index_contents assert result.hash == mock_index.index_hash def test_etag_fetching_404(): # Test conditional fetch with etags. The remote has modified the file. def response_404(request: urllib.request.Request): raise urllib.error.HTTPError( request.get_full_url(), 404, "Not found", hdrs={"Etag": '"59bcc3ad6775562f845953cf01624225"'}, # type: ignore[arg-type] fp=None, ) fetcher = spack.binary_distribution.EtagIndexFetcher( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), etag="112a8bbc1b3f7f185621c1ee335f0502", urlopen=response_404, ) with pytest.raises(spack.binary_distribution.FetchIndexError): fetcher.conditional_fetch() def test_default_index_fetch_200(mock_index): # We fetch the manifest and then the index blob if the hash is outdated def urlopen(request: urllib.request.Request): url = request.get_full_url() if url.endswith(INDEX_MANIFEST_FILE): return urllib.response.addinfourl( # type: ignore[arg-type] io.BytesIO(json.dumps(mock_index.manifest_contents).encode()), headers={"Etag": f'"{mock_index.manifest_etag}"'}, # type: ignore[arg-type] url=url, code=200, ) assert False, "Unexpected request {}".format(url) fetcher = spack.binary_distribution.DefaultIndexFetcher( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), local_hash="outdated", urlopen=urlopen, ) result = fetcher.conditional_fetch() assert isinstance(result, spack.binary_distribution.FetchIndexResult) assert not result.fresh assert mock_index.fetched_blob() assert result.etag == mock_index.manifest_etag assert result.data == mock_index.index_contents assert result.hash == mock_index.index_hash def test_default_index_404(): # We get a fetch error if the index can't be fetched def urlopen(request: urllib.request.Request): raise urllib.error.HTTPError( request.get_full_url(), 404, "Not found", hdrs={"Etag": '"59bcc3ad6775562f845953cf01624225"'}, # type: ignore[arg-type] fp=None, ) fetcher = spack.binary_distribution.DefaultIndexFetcher( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), local_hash=None, urlopen=urlopen, ) with pytest.raises(spack.binary_distribution.FetchIndexError): fetcher.conditional_fetch() def test_default_index_not_modified(mock_index): # We don't fetch the index blob if hash didn't change def urlopen(request: urllib.request.Request): url = request.get_full_url() if url.endswith(INDEX_MANIFEST_FILE): return urllib.response.addinfourl( io.BytesIO(json.dumps(mock_index.manifest_contents).encode()), headers={}, # type: ignore[arg-type] url=url, code=200, ) # No other request should be made. assert False, "Unexpected request {}".format(url) fetcher = spack.binary_distribution.DefaultIndexFetcher( spack.binary_distribution.MirrorMetadata( "https://www.example.com", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ), local_hash=mock_index.index_hash, urlopen=urlopen, ) assert fetcher.conditional_fetch().fresh assert not mock_index.fetched_blob() @pytest.mark.usefixtures("install_mockery", "mock_packages") def test_get_entries_from_cache_nested_mirrors(monkeypatch, tmp_path: pathlib.Path): """Make sure URLBuildcacheEntry behaves as expected""" # Create a temp mirror directory for buildcache usage mirror_dir = tmp_path / "mirror_dir" mirror_url = url_util.path_to_file_url(str(mirror_dir)) # Install and push libdwarf to the root mirror s = spack.concretize.concretize_one("libdwarf") install_cmd("--fake", s.name) buildcache_cmd("push", "-u", str(mirror_dir), s.name) # Install and push libzlib to the nested mirror s = spack.concretize.concretize_one("zlib") install_cmd("--fake", s.name) buildcache_cmd("push", "-u", str(mirror_dir / "nested"), s.name) spec_manifests, _ = get_entries_from_cache( str(mirror_url), str(tmp_path / "stage"), BuildcacheComponent.SPEC ) nested_mirror_url = url_util.path_to_file_url(str(mirror_dir / "nested")) spec_manifests_nested, _ = get_entries_from_cache( str(nested_mirror_url), str(tmp_path / "stage"), BuildcacheComponent.SPEC ) # Expected specs in root mirror # - gcc-runtime # - compiler-wrapper # - libelf # - libdwarf assert len(spec_manifests) == 4 # Expected specs in nested mirror # - zlib assert len(spec_manifests_nested) == 1 def test_mirror_metadata(): mirror_metadata = spack.binary_distribution.MirrorMetadata("https://dummy.io/__v3", 3) as_str = str(mirror_metadata) from_str = spack.binary_distribution.MirrorMetadata.from_string(as_str) # Verify values assert mirror_metadata.url == "https://dummy.io/__v3" assert mirror_metadata.version == 3 assert mirror_metadata.view is None # Verify round trip assert mirror_metadata == from_str assert as_str == str(from_str) with pytest.raises(spack.url_buildcache.MirrorMetadataError, match="Malformed string"): spack.binary_distribution.MirrorMetadata.from_string("https://dummy.io/__v3@@4") def test_mirror_metadata_with_view(): mirror_metadata = spack.binary_distribution.MirrorMetadata( "https://dummy.io/__v3__@aview", 3, "aview" ) as_str = str(mirror_metadata) from_str = spack.binary_distribution.MirrorMetadata.from_string(as_str) # Verify round trip assert mirror_metadata.url == "https://dummy.io/__v3__@aview" assert mirror_metadata.version == 3 assert mirror_metadata.view == "aview" assert mirror_metadata == from_str assert as_str == str(from_str) with pytest.raises(spack.url_buildcache.MirrorMetadataError, match="Malformed string"): spack.binary_distribution.MirrorMetadata.from_string("https://dummy.io/__v3%asdf__@aview") ================================================ FILE: lib/spack/spack/test/bootstrap.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.bootstrap import spack.bootstrap.clingo import spack.bootstrap.config import spack.bootstrap.core import spack.bootstrap.status import spack.compilers.config import spack.config import spack.environment import spack.store import spack.util.executable import spack.util.path from .conftest import _true @pytest.fixture def active_mock_environment(mutable_config, mutable_mock_env_path): with spack.environment.create("bootstrap-test") as env: yield env @pytest.mark.regression("22294") def test_store_is_restored_correctly_after_bootstrap(mutable_config, tmp_path: pathlib.Path): """Tests that the store is correctly swapped during bootstrapping, and restored afterward.""" user_path = str(tmp_path / "store") with spack.store.use_store(user_path): assert spack.store.STORE.root == user_path assert spack.config.CONFIG.get("config:install_tree:root") == user_path with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.store.STORE.root == spack.bootstrap.config.store_path() assert spack.store.STORE.root == user_path assert spack.config.CONFIG.get("config:install_tree:root") == user_path @pytest.mark.regression("38963") def test_store_padding_length_is_zero_during_bootstrapping(mutable_config, tmp_path: pathlib.Path): """Tests that, even though padded length is set in user config, the bootstrap store maintains a padded length of zero. """ user_path = str(tmp_path / "store") with spack.store.use_store(user_path, extra_data={"padded_length": 512}): assert spack.config.CONFIG.get("config:install_tree:padded_length") == 512 with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.store.STORE.root == spack.bootstrap.config.store_path() assert spack.config.CONFIG.get("config:install_tree:padded_length") == 0 assert spack.config.CONFIG.get("config:install_tree:padded_length") == 512 @pytest.mark.regression("38963") def test_install_tree_customization_is_respected(mutable_config, tmp_path: pathlib.Path): """Tests that a custom user store is respected when we exit the bootstrapping environment. """ spack.store.reinitialize() store_dir = tmp_path / "store" spack.config.CONFIG.set("config:install_tree:root", str(store_dir)) with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.store.STORE.root == spack.bootstrap.config.store_path() assert ( spack.config.CONFIG.get("config:install_tree:root") == spack.bootstrap.config.store_path() ) assert spack.config.CONFIG.get("config:install_tree:padded_length") == 0 assert spack.config.CONFIG.get("config:install_tree:root") == str(store_dir) assert spack.store.STORE.root == str(store_dir) @pytest.mark.parametrize( "config_value,expected", [ # Absolute path without expansion ("/opt/spack/bootstrap", "/opt/spack/bootstrap/store"), # Path with placeholder ("$spack/opt/bootstrap", "$spack/opt/bootstrap/store"), ], ) def test_store_path_customization(config_value, expected, mutable_config): # Set the current configuration to a specific value spack.config.set("bootstrap:root", config_value) # Check the store path current = spack.bootstrap.config.store_path() assert current == spack.util.path.canonicalize_path(expected) def test_raising_exception_if_bootstrap_disabled(mutable_config): # Disable bootstrapping in config.yaml spack.config.set("bootstrap:enable", False) # Check the correct exception is raised with pytest.raises(RuntimeError, match="bootstrapping is currently disabled"): spack.bootstrap.config.store_path() def test_raising_exception_module_importable(config, monkeypatch): monkeypatch.setattr(spack.bootstrap.core, "source_is_enabled", _true) with pytest.raises(ImportError, match='cannot bootstrap the "asdf" Python module'): spack.bootstrap.core.ensure_module_importable_or_raise("asdf") def test_raising_exception_executables_in_path(config, monkeypatch): monkeypatch.setattr(spack.bootstrap.core, "source_is_enabled", _true) with pytest.raises(RuntimeError, match="cannot bootstrap any of the asdf, fdsa executables"): spack.bootstrap.core.ensure_executables_in_path_or_raise(["asdf", "fdsa"], "python") @pytest.mark.regression("25603") def test_bootstrap_deactivates_environments(active_mock_environment): assert spack.environment.active_environment() == active_mock_environment with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.environment.active_environment() is None assert spack.environment.active_environment() == active_mock_environment @pytest.mark.regression("25805") def test_bootstrap_disables_modulefile_generation(mutable_config): # Be sure to enable both lmod and tcl in modules.yaml spack.config.set("modules:default:enable", ["tcl", "lmod"]) assert "tcl" in spack.config.get("modules:default:enable") assert "lmod" in spack.config.get("modules:default:enable") with spack.bootstrap.ensure_bootstrap_configuration(): assert "tcl" not in spack.config.get("modules:default:enable") assert "lmod" not in spack.config.get("modules:default:enable") assert "tcl" in spack.config.get("modules:default:enable") assert "lmod" in spack.config.get("modules:default:enable") @pytest.mark.regression("25992") @pytest.mark.requires_executables("gcc") def test_bootstrap_search_for_compilers_with_no_environment(no_packages_yaml, mock_packages): assert not spack.compilers.config.all_compilers(init_config=False) with spack.bootstrap.ensure_bootstrap_configuration(): spack.bootstrap.clingo._add_compilers_if_missing() assert spack.compilers.config.all_compilers(init_config=False) assert not spack.compilers.config.all_compilers(init_config=False) @pytest.mark.regression("25992") @pytest.mark.requires_executables("gcc") def test_bootstrap_search_for_compilers_with_environment_active( no_packages_yaml, active_mock_environment, mock_packages ): assert not spack.compilers.config.all_compilers(init_config=False) with spack.bootstrap.ensure_bootstrap_configuration(): spack.bootstrap.clingo._add_compilers_if_missing() assert spack.compilers.config.all_compilers(init_config=False) assert not spack.compilers.config.all_compilers(init_config=False) @pytest.mark.regression("26189") def test_config_yaml_is_preserved_during_bootstrap(mutable_config): expected_dir = "/tmp/test" spack.config.set("config:test_stage", expected_dir, scope="command_line") assert spack.config.get("config:test_stage") == expected_dir with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.config.get("config:test_stage") == expected_dir assert spack.config.get("config:test_stage") == expected_dir @pytest.mark.regression("26548") def test_bootstrap_custom_store_in_environment(mutable_config, tmp_path: pathlib.Path): # Test that the custom store in an environment is taken into account # during bootstrapping spack_yaml = tmp_path / "spack.yaml" install_root = tmp_path / "store" spack_yaml.write_text( """ spack: specs: - libelf config: install_tree: root: {0} """.format(install_root) ) with spack.environment.Environment(str(tmp_path)): assert spack.environment.active_environment() assert spack.config.get("config:install_tree:root") == str(install_root) # Don't trigger evaluation here with spack.bootstrap.ensure_bootstrap_configuration(): pass assert str(spack.store.STORE.root) == str(install_root) def test_nested_use_of_context_manager(mutable_config): """Test nested use of the context manager""" user_config = spack.config.CONFIG with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.config.CONFIG != user_config with spack.bootstrap.ensure_bootstrap_configuration(): assert spack.config.CONFIG != user_config assert spack.config.CONFIG == user_config @pytest.mark.parametrize("expected_missing", [False, True]) def test_status_function_find_files( mutable_config, mock_executable, tmp_path: pathlib.Path, monkeypatch, expected_missing ): if not expected_missing: mock_executable("foo", "echo Hello WWorld!") monkeypatch.setattr( spack.bootstrap.status, "_optional_requirements", lambda: [spack.bootstrap.status._required_system_executable("foo", "NOT FOUND")], ) monkeypatch.setenv("PATH", str(tmp_path / "bin")) _, missing = spack.bootstrap.status_message("optional") assert missing is expected_missing @pytest.mark.parametrize( "gpg_in_path,gpg_in_store,expected_missing", [ (True, False, False), # gpg exists in PATH (False, True, False), # gpg exists in bootstrap store (False, False, True), # gpg is missing ], ) def test_gpg_status_check( mutable_config, mock_executable, tmp_path: pathlib.Path, monkeypatch, gpg_in_path, gpg_in_store, expected_missing, ): """Test that gpg/gpg2 status is detected whether it's in PATH or in the bootstrap store.""" # Set up mock PATH with or without gpg path_dir = tmp_path / "bin" path_dir.mkdir(exist_ok=True) monkeypatch.setenv("PATH", str(path_dir)) if gpg_in_path: mock_executable("gpg2", "echo GPG 2.3.4") # Mock the bootstrap store function def mock_executables_in_store(exes, query_spec, query_info=None): if not gpg_in_store: return False # Simulate found gpg in bootstrap store if query_info is not None: query_info["spec"] = "gnupg@2.5.12" query_info["command"] = spack.util.executable.Executable("gpg") return True monkeypatch.setattr(spack.bootstrap.status, "_executables_in_store", mock_executables_in_store) # Call only the buildcache requirements function directly to isolate the test requirements = spack.bootstrap.status._buildcache_requirements() # Find the gpg entry by examining the calls made to set up requirements # We know the first entry in requirements is the gpg entry because of how # _buildcache_requirements is structured: # Make sure we're not out of bounds assert len(requirements) >= 1, "No gpg requirement found" # Check that the gpg requirement matches our expectations gpg_req = requirements[0] assert gpg_req[0] is not expected_missing @pytest.mark.regression("31042") def test_source_is_disabled(mutable_config): # Get the configuration dictionary of the current bootstrapping source conf = next(iter(spack.bootstrap.core.bootstrapping_sources())) # The source is not explicitly enabled or disabled, so the following should return False assert not spack.bootstrap.core.source_is_enabled(conf) # Try to explicitly disable the source and verify that the behavior is the same as above spack.config.add("bootstrap:trusted:{0}:{1}".format(conf["name"], False)) assert not spack.bootstrap.core.source_is_enabled(conf) @pytest.mark.regression("45247") def test_use_store_does_not_try_writing_outside_root( tmp_path: pathlib.Path, monkeypatch, mutable_config ): """Tests that when we use the 'use_store' context manager, there is no attempt at creating a Store outside the given root. """ initial_store = mutable_config.get("config:install_tree:root") user_store = tmp_path / "store" fn = spack.store.Store.__init__ def _checked_init(self, root, *args, **kwargs): fn(self, root, *args, **kwargs) assert self.root == str(user_store) monkeypatch.setattr(spack.store.Store, "__init__", _checked_init) spack.store.reinitialize() with spack.store.use_store(user_store): assert spack.config.CONFIG.get("config:install_tree:root") == str(user_store) assert spack.config.CONFIG.get("config:install_tree:root") == initial_store ================================================ FILE: lib/spack/spack/test/build_distribution.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import shutil import pytest import spack.binary_distribution as bd import spack.concretize import spack.mirrors.mirror from spack.installer import PackageInstaller pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_build_tarball_overwrite(install_mockery, mock_fetch, monkeypatch, tmp_path: pathlib.Path): spec = spack.concretize.concretize_one("trivial-install-test-package") PackageInstaller([spec.package], fake=True).install() specs = [spec] # populate cache, everything is new mirror = spack.mirrors.mirror.Mirror.from_local_path(str(tmp_path)) with bd.make_uploader(mirror) as uploader: skipped = uploader.push_or_raise(specs) assert not skipped # should skip all with bd.make_uploader(mirror) as uploader: skipped = uploader.push_or_raise(specs) assert skipped == specs # with force=True none should be skipped with bd.make_uploader(mirror, force=True) as uploader: skipped = uploader.push_or_raise(specs) assert not skipped # Remove the tarball, which should cause push to push. shutil.rmtree(tmp_path / bd.buildcache_relative_blobs_path()) with bd.make_uploader(mirror) as uploader: skipped = uploader.push_or_raise(specs) assert not skipped ================================================ FILE: lib/spack/spack/test/build_environment.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import multiprocessing import os import pathlib import posixpath import sys from typing import Dict, Optional, Tuple import pytest import spack.vendor.archspec.cpu import spack.build_environment import spack.concretize import spack.config import spack.deptypes as dt import spack.package_base import spack.spec import spack.util.environment import spack.util.module_cmd import spack.util.spack_yaml as syaml from spack.build_environment import UseMode, _static_to_shared_library, dso_suffix from spack.context import Context from spack.installer import PackageInstaller from spack.llnl.path import Path, convert_to_platform_path from spack.llnl.util.filesystem import HeaderList, LibraryList from spack.util.environment import EnvironmentModifications from spack.util.executable import Executable def os_pathsep_join(path, *pths): out_pth = path for pth in pths: out_pth = os.pathsep.join([out_pth, pth]) return out_pth def prep_and_join(path, *pths): return os.path.sep + os.path.join(path, *pths) @pytest.fixture def build_environment(monkeypatch, wrapper_dir, tmp_path: pathlib.Path): realcc = "/bin/mycc" prefix = str(tmp_path) monkeypatch.setenv("SPACK_CC", realcc) monkeypatch.setenv("SPACK_CXX", realcc) monkeypatch.setenv("SPACK_FC", realcc) monkeypatch.setenv("SPACK_PREFIX", prefix) monkeypatch.setenv("SPACK_COMPILER_WRAPPER_PATH", "test") monkeypatch.setenv("SPACK_DEBUG_LOG_DIR", ".") monkeypatch.setenv("SPACK_DEBUG_LOG_ID", "foo-hashabc") monkeypatch.setenv("SPACK_SHORT_SPEC", "foo@1.2 arch=linux-rhel6-x86_64 /hashabc") monkeypatch.setenv("SPACK_CC_RPATH_ARG", "-Wl,-rpath,") monkeypatch.setenv("SPACK_CXX_RPATH_ARG", "-Wl,-rpath,") monkeypatch.setenv("SPACK_F77_RPATH_ARG", "-Wl,-rpath,") monkeypatch.setenv("SPACK_FC_RPATH_ARG", "-Wl,-rpath,") monkeypatch.setenv("SPACK_CC_LINKER_ARG", "-Wl,") monkeypatch.setenv("SPACK_CXX_LINKER_ARG", "-Wl,") monkeypatch.setenv("SPACK_FC_LINKER_ARG", "-Wl,") monkeypatch.setenv("SPACK_F77_LINKER_ARG", "-Wl,") monkeypatch.setenv("SPACK_DTAGS_TO_ADD", "--disable-new-dtags") monkeypatch.setenv("SPACK_DTAGS_TO_STRIP", "--enable-new-dtags") monkeypatch.setenv("SPACK_SYSTEM_DIRS", "/usr/include|/usr/lib") monkeypatch.setenv("SPACK_MANAGED_DIRS", f"{prefix}/opt/spack") monkeypatch.setenv("SPACK_TARGET_ARGS", "") monkeypatch.delenv("SPACK_DEPENDENCIES", raising=False) cc = Executable(str(wrapper_dir / "cc")) cxx = Executable(str(wrapper_dir / "c++")) fc = Executable(str(wrapper_dir / "fc")) return {"cc": cc, "cxx": cxx, "fc": fc} @pytest.fixture def ensure_env_variables(mutable_config, mock_packages, monkeypatch, working_env): """Returns a function that takes a dictionary and updates os.environ for the test lifetime accordingly. Plugs-in mock config and repo. """ def _ensure(env_mods): for name, value in env_mods.items(): monkeypatch.setenv(name, value) return _ensure @pytest.fixture def mock_module_cmd(monkeypatch): class Logger: def __init__(self, fn=None): self.fn = fn self.calls = [] def __call__(self, *args, **kwargs): self.calls.append((args, kwargs)) if self.fn: return self.fn(*args, **kwargs) mock_module_cmd = Logger() monkeypatch.setattr(spack.build_environment, "module", mock_module_cmd) monkeypatch.setattr(spack.build_environment, "_on_cray", lambda: (True, None)) return mock_module_cmd @pytest.mark.not_on_windows("Static to Shared not supported on Win (yet)") def test_static_to_shared_library(build_environment): os.environ["SPACK_TEST_COMMAND"] = "dump-args" expected = { "linux": ( "/bin/mycc -shared" " -Wl,--disable-new-dtags" " -Wl,-soname -Wl,{2} -Wl,--whole-archive {0}" " -Wl,--no-whole-archive -o {1}" ), "darwin": ( "/bin/mycc -dynamiclib" " -Wl,--disable-new-dtags" " -install_name {1} -Wl,-force_load -Wl,{0} -o {1}" ), } static_lib = "/spack/libfoo.a" for arch in ("linux", "darwin"): for shared_lib in (None, "/spack/libbar.so"): output = _static_to_shared_library( arch, build_environment["cc"], static_lib, shared_lib, compiler_output=str ).strip() if not shared_lib: shared_lib = "{0}.{1}".format(os.path.splitext(static_lib)[0], dso_suffix) assert set(output.split()) == set( expected[arch].format(static_lib, shared_lib, os.path.basename(shared_lib)).split() ) @pytest.mark.regression("8345") @pytest.mark.usefixtures("mock_packages") @pytest.mark.not_on_windows("Module files are not supported on Windows") def test_cc_not_changed_by_modules(monkeypatch, mutable_config, working_env, compiler_factory): """Tests that external module files that are loaded cannot change the CC environment variable. """ gcc_entry = compiler_factory(spec="gcc@14.0.1 languages=c,c++") gcc_entry["modules"] = ["some_module"] mutable_config.set("packages", {"gcc": {"externals": [gcc_entry]}}) def _set_wrong_cc(x): os.environ["CC"] = "NOT_THIS_PLEASE" os.environ["ANOTHER_VAR"] = "THIS_IS_SET" monkeypatch.setattr(spack.util.module_cmd, "load_module", _set_wrong_cc) s = spack.concretize.concretize_one("cmake %gcc@14") spack.build_environment.setup_package(s.package, dirty=False) assert os.environ["CC"] != "NOT_THIS_PLEASE" assert os.environ["ANOTHER_VAR"] == "THIS_IS_SET" def test_setup_dependent_package_inherited_modules( working_env, mock_packages, install_mockery, mock_fetch ): # This will raise on regression s = spack.concretize.concretize_one("cmake-client-inheritor") PackageInstaller([s.package], fake=True).install() @pytest.mark.parametrize( "initial,modifications,expected", [ # Set and unset variables ( {"SOME_VAR_STR": "", "SOME_VAR_NUM": "0"}, {"set": {"SOME_VAR_STR": "SOME_STR", "SOME_VAR_NUM": 1}}, {"SOME_VAR_STR": "SOME_STR", "SOME_VAR_NUM": "1"}, ), ({"SOME_VAR_STR": ""}, {"unset": ["SOME_VAR_STR"]}, {"SOME_VAR_STR": None}), ( {}, # Set a variable that was not defined already {"set": {"SOME_VAR_STR": "SOME_STR"}}, {"SOME_VAR_STR": "SOME_STR"}, ), # Append and prepend to the same variable ( {"EMPTY_PATH_LIST": prep_and_join("path", "middle")}, { "prepend_path": {"EMPTY_PATH_LIST": prep_and_join("path", "first")}, "append_path": {"EMPTY_PATH_LIST": prep_and_join("path", "last")}, }, { "EMPTY_PATH_LIST": os_pathsep_join( prep_and_join("path", "first"), prep_and_join("path", "middle"), prep_and_join("path", "last"), ) }, ), # Append and prepend from empty variables ( {"EMPTY_PATH_LIST": "", "SOME_VAR_STR": ""}, { "prepend_path": {"EMPTY_PATH_LIST": prep_and_join("path", "first")}, "append_path": {"SOME_VAR_STR": prep_and_join("path", "last")}, }, { "EMPTY_PATH_LIST": prep_and_join("path", "first"), "SOME_VAR_STR": prep_and_join("path", "last"), }, ), ( {}, # Same as before but on variables that were not defined { "prepend_path": {"EMPTY_PATH_LIST": prep_and_join("path", "first")}, "append_path": {"SOME_VAR_STR": prep_and_join("path", "last")}, }, { "EMPTY_PATH_LIST": prep_and_join("path", "first"), "SOME_VAR_STR": prep_and_join("path", "last"), }, ), # Remove a path from a list ( { "EMPTY_PATH_LIST": os_pathsep_join( prep_and_join("path", "first"), prep_and_join("path", "middle"), prep_and_join("path", "last"), ) }, {"remove_path": {"EMPTY_PATH_LIST": prep_and_join("path", "middle")}}, { "EMPTY_PATH_LIST": os_pathsep_join( prep_and_join("path", "first"), prep_and_join("path", "last") ) }, ), ( {"EMPTY_PATH_LIST": prep_and_join("only", "path")}, {"remove_path": {"EMPTY_PATH_LIST": prep_and_join("only", "path")}}, {"EMPTY_PATH_LIST": ""}, ), ], ) def test_compiler_config_modifications( initial, modifications, expected, ensure_env_variables, compiler_factory, mutable_config, monkeypatch, ): # Set the environment as per prerequisites ensure_env_variables(initial) gcc_entry = compiler_factory(spec="gcc@14.0.1 languages=c,c++") gcc_entry["extra_attributes"]["environment"] = modifications mutable_config.set("packages", {"gcc": {"externals": [gcc_entry]}}) def platform_pathsep(pathlist): if Path.platform_path == Path.windows: pathlist = pathlist.replace(":", ";") return convert_to_platform_path(pathlist) pkg = spack.concretize.concretize_one("cmake %gcc@14").package # Trigger the modifications spack.build_environment.setup_package(pkg, dirty=False) # Check they were applied for name, value in expected.items(): if value is not None: value = platform_pathsep(value) assert os.environ[name] == value continue assert name not in os.environ @pytest.mark.not_on_windows("Module files are not supported on Windows") def test_load_external_modules_error(working_env, monkeypatch): """Test that load_external_modules raises an exception when a module cannot be loaded""" # Create a mock spec object with the minimum attributes needed for the test class MockSpec: def __init__(self): self.external_modules = ["non_existent_module"] def __str__(self): return "mock-external-spec" mock_spec = MockSpec() # Create a simplified SetupContext-like class that only contains what we need class MockSetupContext: def __init__(self, spec): self.external = [(spec, None)] context = MockSetupContext(mock_spec) # Mock the load_module function to raise an exception def mock_load_module(module_name): # Simulate module load failure raise spack.util.module_cmd.ModuleLoadError(module_name) monkeypatch.setattr(spack.util.module_cmd, "load_module", mock_load_module) # Test that load_external_modules raises ModuleLoadError with pytest.raises(spack.util.module_cmd.ModuleLoadError): spack.build_environment.load_external_modules(context) def test_external_config_env(mock_packages, mutable_config, working_env): cmake_config = { "externals": [ { "spec": "cmake@1.0", "prefix": "/fake/path", "extra_attributes": {"environment": {"set": {"TEST_ENV_VAR_SET": "yes it's set"}}}, } ] } spack.config.set("packages:cmake", cmake_config) cmake_client = spack.concretize.concretize_one("cmake-client") spack.build_environment.setup_package(cmake_client.package, False) assert os.environ["TEST_ENV_VAR_SET"] == "yes it's set" @pytest.mark.regression("9107") @pytest.mark.not_on_windows("Windows does not support module files") def test_spack_paths_before_module_paths( mutable_config, mock_packages, compiler_factory, monkeypatch, working_env, wrapper_dir ): gcc_entry = compiler_factory(spec="gcc@14.0.1 languages=c,c++") gcc_entry["modules"] = ["some_module"] mutable_config.set("packages", {"gcc": {"externals": [gcc_entry]}}) module_path = os.path.join("path", "to", "module") monkeypatch.setenv("SPACK_COMPILER_WRAPPER_PATH", wrapper_dir) def _set_wrong_cc(x): os.environ["PATH"] = module_path + os.pathsep + os.environ["PATH"] monkeypatch.setattr(spack.util.module_cmd, "load_module", _set_wrong_cc) s = spack.concretize.concretize_one("cmake") spack.build_environment.setup_package(s.package, dirty=False) paths = os.environ["PATH"].split(os.pathsep) assert paths.index(str(wrapper_dir)) < paths.index(module_path) def test_package_inheritance_module_setup(config, mock_packages, working_env): s = spack.concretize.concretize_one("multimodule-inheritance") pkg = s.package spack.build_environment.setup_package(pkg, False) os.environ["TEST_MODULE_VAR"] = "failed" assert pkg.use_module_variable() == "test_module_variable" assert os.environ["TEST_MODULE_VAR"] == "test_module_variable" def test_wrapper_variables( config, mock_packages, working_env, monkeypatch, installation_dir_with_headers ): """Check that build_environment supplies the needed library/include directories via the SPACK_LINK_DIRS and SPACK_INCLUDE_DIRS environment variables. """ # https://github.com/spack/spack/issues/13969 cuda_headers = HeaderList( [ "prefix/include/cuda_runtime.h", "prefix/include/cuda/atomic", "prefix/include/cuda/std/detail/libcxx/include/ctype.h", ] ) cuda_include_dirs = cuda_headers.directories assert posixpath.join("prefix", "include") in cuda_include_dirs assert ( posixpath.join("prefix", "include", "cuda", "std", "detail", "libcxx", "include") not in cuda_include_dirs ) root = spack.concretize.concretize_one("dt-diamond") for s in root.traverse(): s.set_prefix(f"/{s.name}-prefix/") dep_pkg = root["dt-diamond-left"].package dep_lib_paths = ["/test/path/to/ex1.so", "/test/path/to/subdir/ex2.so"] dep_lib_dirs = ["/test/path/to", "/test/path/to/subdir"] dep_libs = LibraryList(dep_lib_paths) dep2_pkg = root["dt-diamond-right"].package dep2_pkg.spec.set_prefix(str(installation_dir_with_headers)) setattr(dep_pkg, "libs", dep_libs) try: pkg = root.package env_mods = EnvironmentModifications() spack.build_environment.set_wrapper_variables(pkg, env_mods) env_mods.apply_modifications() def normpaths(paths): return list(os.path.normpath(p) for p in paths) link_dir_var = os.environ["SPACK_LINK_DIRS"] assert normpaths(link_dir_var.split(":")) == normpaths(dep_lib_dirs) root_libdirs = ["/dt-diamond-prefix/lib", "/dt-diamond-prefix/lib64"] rpath_dir_var = os.environ["SPACK_RPATH_DIRS"] # The 'lib' and 'lib64' subdirectories of the root package prefix # should always be rpathed and should be the first rpaths assert normpaths(rpath_dir_var.split(":")) == normpaths(root_libdirs + dep_lib_dirs) header_dir_var = os.environ["SPACK_INCLUDE_DIRS"] # The default implementation looks for header files only # in /include and subdirectories prefix = str(installation_dir_with_headers) include_dirs = normpaths(header_dir_var.split(os.pathsep)) assert os.path.join(prefix, "include") in include_dirs assert os.path.join(prefix, "include", "boost") not in include_dirs assert os.path.join(prefix, "path", "to") not in include_dirs assert os.path.join(prefix, "path", "to", "subdir") not in include_dirs finally: delattr(dep_pkg, "libs") def test_external_prefixes_last(mutable_config, mock_packages, working_env, monkeypatch): # Sanity check: under normal circumstances paths associated with # dt-diamond-left would appear first. We'll mark it as external in # the test to check if the associated paths are placed last. assert "dt-diamond-left" < "dt-diamond-right" cfg_data = syaml.load_config( """\ dt-diamond-left: externals: - spec: dt-diamond-left@1.0 prefix: /fake/path1 buildable: false """ ) spack.config.set("packages", cfg_data) top = spack.concretize.concretize_one("dt-diamond") def _trust_me_its_a_dir(path): return True monkeypatch.setattr(os.path, "isdir", _trust_me_its_a_dir) env_mods = EnvironmentModifications() spack.build_environment.set_wrapper_variables(top.package, env_mods) env_mods.apply_modifications() link_dir_var = os.environ["SPACK_LINK_DIRS"] link_dirs = link_dir_var.split(":") external_lib_paths = set( [os.path.normpath("/fake/path1/lib"), os.path.normpath("/fake/path1/lib64")] ) # The external lib paths should be the last two entries of the list and # should not appear anywhere before the last two entries assert set(os.path.normpath(x) for x in link_dirs[-2:]) == external_lib_paths assert not (set(os.path.normpath(x) for x in link_dirs[:-2]) & external_lib_paths) def test_parallel_false_is_not_propagating(default_mock_concretization): """Test that parallel=False is not propagating to dependencies""" # a foobar=bar (parallel = False) # | # b (parallel =True) s = default_mock_concretization("pkg-a foobar=bar") spack.build_environment.set_package_py_globals(s.package, context=Context.BUILD) assert s["pkg-a"].package.module.make_jobs == 1 spack.build_environment.set_package_py_globals(s["pkg-b"].package, context=Context.BUILD) assert s["pkg-b"].package.module.make_jobs == spack.config.determine_number_of_jobs( parallel=s["pkg-b"].package.parallel ) @pytest.mark.parametrize( "config_setting,expected_flag", [("runpath", "--enable-new-dtags"), ("rpath", "--disable-new-dtags")], ) @pytest.mark.skipif(sys.platform != "linux", reason="dtags make sense only on linux") def test_setting_dtags_based_on_config( config_setting, expected_flag, config, mock_packages, working_env ): # Pick a random package to be able to set compiler's variables s = spack.concretize.concretize_one("cmake") with spack.config.override("config:shared_linking", {"type": config_setting, "bind": False}): env = spack.build_environment.setup_package(s.package, dirty=False) modifications = env.group_by_name() assert "SPACK_DTAGS_TO_STRIP" in modifications assert "SPACK_DTAGS_TO_ADD" in modifications assert len(modifications["SPACK_DTAGS_TO_ADD"]) == 1 assert len(modifications["SPACK_DTAGS_TO_STRIP"]) == 1 dtags_to_add = modifications["SPACK_DTAGS_TO_ADD"][0] assert dtags_to_add.value == expected_flag def test_module_globals_available_at_setup_dependent_time( monkeypatch, mutable_config, mock_packages, working_env ): """Spack built package externaltest depends on an external package externaltool. Externaltool's setup_dependent_package needs to be able to access globals on the dependent""" def setup_dependent_package(module, dependent_spec): # Make sure set_package_py_globals was already called on # dependents # ninja is always set by the setup context and is not None dependent_module = dependent_spec.package.module assert hasattr(dependent_module, "ninja") assert dependent_module.ninja is not None dependent_spec.package.test_attr = True externaltool = spack.concretize.concretize_one("externaltest") monkeypatch.setattr( externaltool["externaltool"].package, "setup_dependent_package", setup_dependent_package ) spack.build_environment.setup_package(externaltool.package, False) assert externaltool.package.test_attr def test_build_jobs_sequential_is_sequential(): assert ( spack.config.determine_number_of_jobs( parallel=False, max_cpus=8, config=spack.config.create_from( spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 8}}), spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 8}}), ), ) == 1 ) def test_build_jobs_command_line_overrides(): assert ( spack.config.determine_number_of_jobs( parallel=True, max_cpus=1, config=spack.config.create_from( spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 10}}), spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 1}}), ), ) == 10 ) assert ( spack.config.determine_number_of_jobs( parallel=True, max_cpus=100, config=spack.config.create_from( spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 10}}), spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 100}}), ), ) == 10 ) def test_build_jobs_defaults(): assert ( spack.config.determine_number_of_jobs( parallel=True, max_cpus=10, config=spack.config.create_from( spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 1}}) ), ) == 1 ) assert ( spack.config.determine_number_of_jobs( parallel=True, max_cpus=10, config=spack.config.create_from( spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 100}}) ), ) == 10 ) class TestModuleMonkeyPatcher: def test_getting_attributes(self, default_mock_concretization): s = default_mock_concretization("libelf") module_wrapper = spack.build_environment.ModuleChangePropagator(s.package) assert module_wrapper.Libelf == s.package.module.Libelf def test_setting_attributes(self, default_mock_concretization): s = default_mock_concretization("libelf") module = s.package.module module_wrapper = spack.build_environment.ModuleChangePropagator(s.package) # Setting an attribute has an immediate effect module_wrapper.SOME_ATTRIBUTE = 1 assert module.SOME_ATTRIBUTE == 1 # We can also propagate the settings to classes in the MRO module_wrapper.propagate_changes_to_mro() for cls in s.package.__class__.__mro__: current_module = cls.module if current_module == spack.package_base: break assert current_module.SOME_ATTRIBUTE == 1 def test_effective_deptype_build_environment(default_mock_concretization): s = default_mock_concretization("dttop") # [ ] dttop@1.0 # # [b ] ^dtbuild1@1.0 # <- direct build dep # [b ] ^dtbuild2@1.0 # <- indirect build-only dep is dropped # [bl ] ^dtlink2@1.0 # <- linkable, and runtime dep of build dep # [ r ] ^dtrun2@1.0 # <- non-linkable, executable runtime dep of build dep # [bl ] ^dtlink1@1.0 # <- direct build dep # [bl ] ^dtlink3@1.0 # <- linkable, and runtime dep of build dep # [b ] ^dtbuild2@1.0 # <- indirect build-only dep is dropped # [bl ] ^dtlink4@1.0 # <- linkable, and runtime dep of build dep # [ r ] ^dtrun1@1.0 # <- run-only dep is pruned (should it be in PATH?) # [bl ] ^dtlink5@1.0 # <- children too # [ r ] ^dtrun3@1.0 # <- children too # [b ] ^dtbuild3@1.0 # <- children too expected_flags = { "dttop": UseMode.ROOT, "dtbuild1": UseMode.BUILDTIME_DIRECT, "dtlink1": UseMode.BUILDTIME_DIRECT | UseMode.BUILDTIME, "dtlink3": UseMode.BUILDTIME | UseMode.RUNTIME, "dtlink4": UseMode.BUILDTIME | UseMode.RUNTIME, "dtrun2": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE, "dtlink2": UseMode.RUNTIME, } for spec, effective_type in spack.build_environment.effective_deptypes( s, context=Context.BUILD ): assert effective_type & expected_flags.pop(spec.name) == effective_type assert not expected_flags, f"Missing {expected_flags.keys()} from effective_deptypes" def test_effective_deptype_run_environment(default_mock_concretization): s = default_mock_concretization("dttop") # [ ] dttop@1.0 # # [b ] ^dtbuild1@1.0 # <- direct build-only dep is pruned # [b ] ^dtbuild2@1.0 # <- children too # [bl ] ^dtlink2@1.0 # <- children too # [ r ] ^dtrun2@1.0 # <- children too # [bl ] ^dtlink1@1.0 # <- runtime, not executable # [bl ] ^dtlink3@1.0 # <- runtime, not executable # [b ] ^dtbuild2@1.0 # <- indirect build only dep is pruned # [bl ] ^dtlink4@1.0 # <- runtime, not executable # [ r ] ^dtrun1@1.0 # <- runtime and executable # [bl ] ^dtlink5@1.0 # <- runtime, not executable # [ r ] ^dtrun3@1.0 # <- runtime and executable # [b ] ^dtbuild3@1.0 # <- indirect build-only dep is pruned expected_flags = { "dttop": UseMode.ROOT, "dtlink1": UseMode.RUNTIME, "dtlink3": UseMode.BUILDTIME | UseMode.RUNTIME, "dtlink4": UseMode.BUILDTIME | UseMode.RUNTIME, "dtrun1": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE, "dtlink5": UseMode.RUNTIME, "dtrun3": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE, } for spec, effective_type in spack.build_environment.effective_deptypes(s, context=Context.RUN): assert effective_type & expected_flags.pop(spec.name) == effective_type assert not expected_flags, f"Missing {expected_flags.keys()} from effective_deptypes" def test_monkey_patching_works_across_virtual(default_mock_concretization): """Assert that a monkeypatched attribute is found regardless we access through the real name or the virtual name. """ s = default_mock_concretization("mpileaks ^mpich") s["mpich"].foo = "foo" assert s["mpich"].foo == "foo" assert s["mpi"].foo == "foo" def test_clear_compiler_related_runtime_variables_of_build_deps(default_mock_concretization): """Verify that Spack drops CC, CXX, FC and F77 from the dependencies related build environment variable changes if they are set in setup_run_environment. Spack manages those variables elsewhere.""" s = default_mock_concretization("build-env-compiler-var-a") ctx = spack.build_environment.SetupContext(s, context=Context.BUILD) result = {} ctx.get_env_modifications().apply_modifications(result) assert "CC" not in result assert "CXX" not in result assert "FC" not in result assert "F77" not in result assert result["ANOTHER_VAR"] == "this-should-be-present" def test_rpath_with_duplicate_link_deps(): """If we have two instances of one package in the same link sub-dag, only the newest version is rpath'ed. This is for runtime support without splicing.""" runtime_1 = spack.spec.Spec("runtime@=1.0") runtime_2 = spack.spec.Spec("runtime@=2.0") child = spack.spec.Spec("child@=1.0") root = spack.spec.Spec("root@=1.0") root.add_dependency_edge(child, depflag=dt.LINK, virtuals=()) root.add_dependency_edge(runtime_2, depflag=dt.LINK, virtuals=()) child.add_dependency_edge(runtime_1, depflag=dt.LINK, virtuals=()) rpath_deps = spack.build_environment._get_rpath_deps_from_spec(root, transitive_rpaths=True) assert child in rpath_deps assert runtime_2 in rpath_deps assert runtime_1 not in rpath_deps @pytest.mark.parametrize( "compiler_spec,target_name,expected_flags", [ # Semver versions ("gcc@4.7.2", "ivybridge", "-march=core-avx-i -mtune=core-avx-i"), ("clang@3.5", "x86_64", "-march=x86-64 -mtune=generic"), ("apple-clang@9.1.0", "x86_64", "-march=x86-64"), ("gcc@=9.2.0", "haswell", "-march=haswell -mtune=haswell"), # Check that custom string versions are accepted ("gcc@=9.2.0-foo", "icelake", "-march=icelake-client -mtune=icelake-client"), # Check that the special case for Apple's clang is treated correctly # i.e. it won't try to detect the version again ("apple-clang@=9.1.0", "x86_64", "-march=x86-64"), ], ) @pytest.mark.filterwarnings("ignore:microarchitecture specific") @pytest.mark.not_on_windows("Windows doesn't support the compiler wrapper") def test_optimization_flags(compiler_spec, target_name, expected_flags, compiler_factory): target = spack.vendor.archspec.cpu.TARGETS[target_name] compiler = spack.spec.parse_with_version_concrete(compiler_spec) opt_flags = spack.build_environment.optimization_flags(compiler, target) assert opt_flags == expected_flags @pytest.mark.skipif( str(spack.vendor.archspec.cpu.host().family) != "x86_64", reason="tests check specific x86_64 uarch flags", ) @pytest.mark.not_on_windows("Windows doesn't support the compiler wrapper") def test_optimization_flags_are_using_node_target(default_mock_concretization, monkeypatch): """Tests that we are using the target on the node to be compiled to retrieve the uarch specific flags, and not the target of the compiler. """ compiler_wrapper_pkg = default_mock_concretization("compiler-wrapper target=core2").package mpileaks = default_mock_concretization("mpileaks target=x86_64") env = EnvironmentModifications() compiler_wrapper_pkg.setup_dependent_build_environment(env, mpileaks) actions = env.group_by_name()["SPACK_TARGET_ARGS_CC"] assert len(actions) == 1 and isinstance(actions[0], spack.util.environment.SetEnv) assert actions[0].value == "-march=x86-64 -mtune=generic" @pytest.mark.regression("49827") @pytest.mark.parametrize( "gcc_config,expected_rpaths", [ ( """\ gcc: externals: - spec: gcc@14.2.0 languages:=c,c++,fortran prefix: /fake/path1 extra_attributes: compilers: c: /fake/path1 cxx: /fake/path1 fortran: /fake/path1 extra_rpaths: - /extra/rpaths1 - /extra/rpaths2 """, "/extra/rpaths1:/extra/rpaths2", ), ( """\ gcc: externals: - spec: gcc@14.2.0 languages=c,c++,fortran prefix: /fake/path1 extra_attributes: compilers: c: /fake/path1 cxx: /fake/path1 fortran: /fake/path1 """, None, ), ], ) @pytest.mark.not_on_windows("Windows doesn't use the compiler-wrapper") def test_extra_rpaths_is_set( working_env, mutable_config, mock_packages, gcc_config, expected_rpaths ): """Tests that using a compiler with an 'extra_rpaths' section will set the corresponding SPACK_COMPILER_EXTRA_RPATHS variable for the wrapper. """ cfg_data = syaml.load_config(gcc_config) spack.config.set("packages", cfg_data) mpich = spack.concretize.concretize_one("mpich %gcc@14") spack.build_environment.setup_package(mpich.package, dirty=False) if expected_rpaths is not None: assert os.environ["SPACK_COMPILER_EXTRA_RPATHS"] == expected_rpaths else: assert "SPACK_COMPILER_EXTRA_RPATHS" not in os.environ class _TestProcess: calls: Dict[str, int] = collections.defaultdict(int) terminated = False runtime = 0 def __init__(self, *, target, args, pkg, read_pipe, timeout): self.alive = None self.exitcode = 0 self._reset() self.read_pipe = read_pipe self.timeout = timeout def start(self): self.calls["start"] += 1 self.alive = True def poll(self): return True def complete(self): return None def is_alive(self): self.calls["is_alive"] += 1 return self.alive def join(self, timeout: Optional[int] = None): self.calls["join"] += 1 if timeout is not None and timeout > self.runtime: self.alive = False def terminate(self): self.calls["terminate"] += 1 self._set_terminated() self.alive = False # Do not set exit code. A non-zero exit code will trigger an error # instead of gracefully inspecting values for test @classmethod def _set_terminated(cls): cls.terminated = True @classmethod def _reset(cls): cls.calls.clear() cls.terminated = False class _TestPipe: def close(self): pass def recv(self): if _TestProcess.terminated is True: return 1 return 0 def _pipe_fn(*, duplex: bool = False) -> Tuple[_TestPipe, _TestPipe]: return _TestPipe(), _TestPipe() @pytest.fixture() def mock_build_process(monkeypatch): monkeypatch.setattr(spack.build_environment, "BuildProcess", _TestProcess) monkeypatch.setattr(multiprocessing, "Pipe", _pipe_fn) def _factory(*, runtime: int): _TestProcess.runtime = runtime return _factory @pytest.mark.parametrize( "runtime,timeout,expected_calls", [ # execution time < timeout (2, 5, {"start": 1, "join": 1, "is_alive": 1}), # execution time > timeout (5, 2, {"start": 1, "join": 1, "is_alive": 1, "terminate": 1}), ], ) def test_build_process_timeout(mock_build_process, runtime, timeout, expected_calls): """Tests that we make the correct function calls in different timeout scenarios.""" mock_build_process(runtime=runtime) process = spack.build_environment.start_build_process( pkg=None, function=None, kwargs={}, timeout=timeout ) _ = spack.build_environment.complete_build_process(process) assert _TestProcess.calls == expected_calls ================================================ FILE: lib/spack/spack/test/build_system_guess.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.cmd.create import spack.stage import spack.util.executable import spack.util.url as url_util @pytest.fixture( scope="function", params=[ ("configure", "autotools"), ("CMakeLists.txt", "cmake"), ("project.pro", "qmake"), ("pom.xml", "maven"), ("SConstruct", "scons"), ("waf", "waf"), ("argbah.rockspec", "lua"), ("setup.py", "python"), ("NAMESPACE", "r"), ("WORKSPACE", "bazel"), ("Makefile.PL", "perlmake"), ("Build.PL", "perlbuild"), ("foo.gemspec", "ruby"), ("Rakefile", "ruby"), ("setup.rb", "ruby"), ("GNUmakefile", "makefile"), ("makefile", "makefile"), ("Makefile", "makefile"), ("meson.build", "meson"), ("configure.py", "sip"), ("foobar", "generic"), ], ) def url_and_build_system(request, tmp_path: pathlib.Path): """Sets up the resources to be pulled by the stage with the appropriate file name and returns their url along with the correct build-system guess """ tar = spack.util.executable.which("tar", required=True) import spack.llnl.util.filesystem as fs with fs.working_dir(str(tmp_path)): filename, system = request.param archive_dir = tmp_path / "archive" archive_dir.mkdir() (archive_dir / filename).touch() tar("czf", "archive.tar.gz", "archive") url = url_util.path_to_file_url(str(tmp_path / "archive.tar.gz")) yield url, system def test_build_systems(url_and_build_system): url, build_system = url_and_build_system with spack.stage.Stage(url) as stage: stage.fetch() guesser = spack.cmd.create.BuildSystemAndLanguageGuesser() guesser(stage.archive_file, url) assert build_system == guesser.build_system ================================================ FILE: lib/spack/spack/test/builder.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.builder import spack.concretize import spack.paths import spack.repo from spack.llnl.util.filesystem import touch @pytest.fixture() def builder_test_repository(config): builder_test_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test") with spack.repo.use_repositories(builder_test_path) as mock_repo: yield mock_repo @pytest.mark.parametrize( "spec_str,expected_values", [ ( "callbacks@2.0", [ ("BEFORE_INSTALL_1_CALLED", "1"), ("BEFORE_INSTALL_2_CALLED", "1"), ("CALLBACKS_INSTALL_CALLED", "1"), ("AFTER_INSTALL_1_CALLED", "1"), ("TEST_VALUE", "3"), ("INSTALL_VALUE", "CALLBACKS"), ], ), # The last callback is conditional on "@1.0", check it's being executed ( "callbacks@1.0", [ ("BEFORE_INSTALL_1_CALLED", "1"), ("BEFORE_INSTALL_2_CALLED", "1"), ("CALLBACKS_INSTALL_CALLED", "1"), ("AFTER_INSTALL_1_CALLED", "1"), ("AFTER_INSTALL_2_CALLED", "1"), ("TEST_VALUE", "4"), ("INSTALL_VALUE", "CALLBACKS"), ], ), # The package below adds to "callbacks" using inheritance, test that using super() # works with builder hierarchies ( "inheritance@1.0", [ ("DERIVED_BEFORE_INSTALL_CALLED", "1"), ("BEFORE_INSTALL_1_CALLED", "1"), ("BEFORE_INSTALL_2_CALLED", "1"), ("CALLBACKS_INSTALL_CALLED", "1"), ("INHERITANCE_INSTALL_CALLED", "1"), ("AFTER_INSTALL_1_CALLED", "1"), ("AFTER_INSTALL_2_CALLED", "1"), ("TEST_VALUE", "4"), ("INSTALL_VALUE", "INHERITANCE"), ], ), # Generate custom phases using a GenericBuilder ( "custom-phases", [("CONFIGURE_CALLED", "1"), ("INSTALL_CALLED", "1"), ("LAST_PHASE", "INSTALL")], ), # Old-style package, with phase defined in base builder ("old-style-autotools@1.0", [("AFTER_AUTORECONF_1_CALLED", "1")]), ("old-style-autotools@2.0", [("AFTER_AUTORECONF_2_CALLED", "1")]), ("old-style-custom-phases", [("AFTER_CONFIGURE_CALLED", "1"), ("TEST_VALUE", "0")]), ], ) @pytest.mark.usefixtures("builder_test_repository", "config") @pytest.mark.disable_clean_stage_check def test_callbacks_and_installation_procedure(spec_str, expected_values, working_env): """Test the correct execution of callbacks and installation procedures for packages.""" s = spack.concretize.concretize_one(spec_str) builder = spack.builder.create(s.package) for phase_fn in builder: phase_fn.execute() # Check calls have produced the expected side effects for var_name, expected in expected_values: assert os.environ[var_name] == expected, os.environ @pytest.mark.usefixtures("builder_test_repository", "config") @pytest.mark.parametrize( "spec_str,method_name,expected", [ # Call a function defined on the package, which calls the same function defined # on the super(builder) ("old-style-autotools", "configure_args", ["--with-foo"]), # Call a function defined on the package, which calls the same function defined on the # super(pkg), which calls the same function defined in the super(builder) ("old-style-derived", "configure_args", ["--with-bar", "--with-foo"]), ], ) def test_old_style_compatibility_with_super(spec_str, method_name, expected): s = spack.concretize.concretize_one(spec_str) builder = spack.builder.create(s.package) value = getattr(builder, method_name)() assert value == expected @pytest.mark.not_on_windows("log_ouput cannot currently be used outside of subprocess on Windows") @pytest.mark.regression("33928") @pytest.mark.usefixtures("builder_test_repository", "config", "working_env") @pytest.mark.disable_clean_stage_check def test_build_time_tests_are_executed_from_default_builder(): s = spack.concretize.concretize_one("old-style-autotools") builder = spack.builder.create(s.package) builder.pkg.run_tests = True for phase_fn in builder: phase_fn.execute() assert os.environ.get("CHECK_CALLED") == "1", "Build time tests not executed" assert os.environ.get("INSTALLCHECK_CALLED") == "1", "Install time tests not executed" @pytest.mark.regression("34518") @pytest.mark.usefixtures("builder_test_repository", "config", "working_env") def test_monkey_patching_wrapped_pkg(): """Confirm 'run_tests' is accessible through wrappers.""" s = spack.concretize.concretize_one("old-style-autotools") builder = spack.builder.create(s.package) assert s.package.run_tests is False assert builder.pkg.run_tests is False assert builder.pkg_with_dispatcher.run_tests is False s.package.run_tests = True assert builder.pkg.run_tests is True assert builder.pkg_with_dispatcher.run_tests is True @pytest.mark.regression("34440") @pytest.mark.usefixtures("builder_test_repository", "config", "working_env") def test_monkey_patching_test_log_file(): """Confirm 'test_log_file' is accessible through wrappers.""" s = spack.concretize.concretize_one("old-style-autotools") builder = spack.builder.create(s.package) s.package.tester.test_log_file = "/some/file" assert builder.pkg.tester.test_log_file == "/some/file" assert builder.pkg_with_dispatcher.tester.test_log_file == "/some/file" # Windows context manager's __exit__ fails with ValueError ("I/O operation # on closed file"). @pytest.mark.not_on_windows("Does not run on windows") def test_install_time_test_callback(tmp_path: pathlib.Path, config, mock_packages, mock_stage): """Confirm able to run stand-alone test as a post-install callback.""" s = spack.concretize.concretize_one("py-test-callback") builder = spack.builder.create(s.package) builder.pkg.run_tests = True s.package.tester.test_log_file = str(tmp_path / "install_test.log") touch(s.package.tester.test_log_file) for phase_fn in builder: phase_fn.execute() with open(s.package.tester.test_log_file, "r", encoding="utf-8") as f: results = f.read().replace("\n", " ") assert "PyTestCallback test" in results @pytest.mark.regression("43097") @pytest.mark.usefixtures("builder_test_repository", "config") def test_mixins_with_builders(working_env): """Tests that run_after and run_before callbacks are accumulated correctly, when mixins are used with builders. """ s = spack.concretize.concretize_one("builder-and-mixins") builder = spack.builder.create(s.package) # Check that callbacks added by the mixin are in the list assert any(fn.__name__ == "before_install" for _, fn in builder._run_before_callbacks) assert any(fn.__name__ == "after_install" for _, fn in builder._run_after_callbacks) # Check that callback from the GenericBuilder are in the list too assert any(fn.__name__ == "sanity_check_prefix" for _, fn in builder._run_after_callbacks) def test_reading_api_v20_attributes(): """Tests that we can read attributes from API v2.0 builders.""" class TestBuilder(spack.builder.Builder): legacy_methods = ("configure", "install") legacy_attributes = ("foo", "bar") legacy_long_methods = ("baz", "fee") methods = spack.builder.package_methods(TestBuilder) assert methods == ("configure", "install") attributes = spack.builder.package_attributes(TestBuilder) assert attributes == ("foo", "bar") long_methods = spack.builder.package_long_methods(TestBuilder) assert long_methods == ("baz", "fee") def test_reading_api_v22_attributes(): """Tests that we can read attributes from API v2.2 builders.""" class TestBuilder(spack.builder.Builder): package_methods = ("configure", "install") package_attributes = ("foo", "bar") package_long_methods = ("baz", "fee") methods = spack.builder.package_methods(TestBuilder) assert methods == ("configure", "install") attributes = spack.builder.package_attributes(TestBuilder) assert attributes == ("foo", "bar") long_methods = spack.builder.package_long_methods(TestBuilder) assert long_methods == ("baz", "fee") @pytest.mark.regression("51917") @pytest.mark.usefixtures("builder_test_repository", "config") def test_builder_when_inheriting_just_package(working_env): """Tests that if we inherit a package from another package that has a builder defined, but we don't need to modify the builder ourselves, we'll get the builder of the base package class. """ base_spec = spack.concretize.concretize_one("callbacks") derived_spec = spack.concretize.concretize_one("inheritance-only-package") base_builder = spack.builder.create(base_spec.package) derived_builder = spack.builder.create(derived_spec.package) # The derived class doesn't redefine a builder, so we should # get the builder of the base class. assert type(base_builder) is type(derived_builder) ================================================ FILE: lib/spack/spack/test/buildrequest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.concretize import spack.deptypes as dt import spack.installer as inst import spack.repo import spack.spec def test_build_request_errors(install_mockery): with pytest.raises(ValueError, match="must be a package"): inst.BuildRequest("abc", {}) spec = spack.spec.Spec("trivial-install-test-package") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) with pytest.raises(ValueError, match="must have a concrete spec"): inst.BuildRequest(pkg_cls(spec), {}) def test_build_request_basics(install_mockery): spec = spack.concretize.concretize_one("dependent-install") assert spec.concrete # Ensure key properties match expectations request = inst.BuildRequest(spec.package, {}) assert not request.pkg.stop_before_phase assert not request.pkg.last_phase assert request.spec == spec.package.spec # Ensure key default install arguments are set assert "install_package" in request.install_args assert "install_deps" in request.install_args def test_build_request_strings(install_mockery): """Tests of BuildRequest repr and str for coverage purposes.""" # Using a package with one dependency spec = spack.concretize.concretize_one("dependent-install") assert spec.concrete # Ensure key properties match expectations request = inst.BuildRequest(spec.package, {}) # Cover __repr__ irep = request.__repr__() assert irep.startswith(request.__class__.__name__) # Cover __str__ istr = str(request) assert "package=dependent-install" in istr assert "install_args=" in istr @pytest.mark.parametrize( "root_policy,dependencies_policy,package_deptypes,dependencies_deptypes", [ ("auto", "auto", dt.BUILD | dt.LINK | dt.RUN, dt.BUILD | dt.LINK | dt.RUN), ("cache_only", "auto", dt.LINK | dt.RUN, dt.BUILD | dt.LINK | dt.RUN), ("auto", "cache_only", dt.BUILD | dt.LINK | dt.RUN, dt.LINK | dt.RUN), ("cache_only", "cache_only", dt.LINK | dt.RUN, dt.LINK | dt.RUN), ], ) def test_build_request_deptypes( install_mockery, root_policy, dependencies_policy, package_deptypes, dependencies_deptypes ): s = spack.concretize.concretize_one("dependent-install") build_request = inst.BuildRequest( s.package, {"root_policy": root_policy, "dependencies_policy": dependencies_policy} ) actual_package_deptypes = build_request.get_depflags(s.package) actual_dependency_deptypes = build_request.get_depflags(s["dependency-install"].package) assert actual_package_deptypes == package_deptypes assert actual_dependency_deptypes == dependencies_deptypes ================================================ FILE: lib/spack/spack/test/buildtask.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.concretize import spack.error import spack.installer as inst import spack.repo import spack.spec def test_build_task_errors(install_mockery): """Check expected errors when instantiating a BuildTask.""" spec = spack.spec.Spec("trivial-install-test-package") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) # The value of the request argument is expected to not be checked. for pkg in [None, "abc"]: with pytest.raises(TypeError, match="must be a package"): inst.BuildTask(pkg, None) with pytest.raises(ValueError, match="must have a concrete spec"): inst.BuildTask(pkg_cls(spec), None) # Using a concretized package now means the request argument is checked. spec = spack.concretize.concretize_one(spec) assert spec.concrete with pytest.raises(TypeError, match="is not a valid build request"): inst.BuildTask(spec.package, None) # Using a valid package and spec, the next check is the status argument. request = inst.BuildRequest(spec.package, {}) with pytest.raises(TypeError, match="is not a valid build status"): inst.BuildTask(spec.package, request, status="queued") # Now we can check that build tasks cannot be create when the status # indicates the task is/should've been removed. with pytest.raises(spack.error.InstallError, match="Cannot create a task"): inst.BuildTask(spec.package, request, status=inst.BuildStatus.REMOVED) # Also make sure to not accept an incompatible installed argument value. with pytest.raises(TypeError, match="'installed' be a 'set', not 'str'"): inst.BuildTask(spec.package, request, installed="mpileaks") def test_build_task_basics(install_mockery): spec = spack.concretize.concretize_one("dependent-install") assert spec.concrete # Ensure key properties match expectations request = inst.BuildRequest(spec.package, {}) task = inst.BuildTask(spec.package, request=request, status=inst.BuildStatus.QUEUED) assert not task.explicit assert task.priority == len(task.uninstalled_deps) assert task.key == (task.priority, task.sequence) # Ensure flagging installed works as expected assert len(task.uninstalled_deps) > 0 assert task.dependencies == task.uninstalled_deps task.flag_installed(task.dependencies) assert len(task.uninstalled_deps) == 0 assert task.priority == 0 def test_build_task_strings(install_mockery): """Tests of build_task repr and str for coverage purposes.""" # Using a package with one dependency spec = spack.concretize.concretize_one("dependent-install") assert spec.concrete # Ensure key properties match expectations request = inst.BuildRequest(spec.package, {}) task = inst.BuildTask(spec.package, request=request, status=inst.BuildStatus.QUEUED) # Cover __repr__ irep = task.__repr__() assert irep.startswith(task.__class__.__name__) assert "BuildStatus.QUEUED" in irep assert "sequence=" in irep # Cover __str__ istr = str(task) assert "status=queued" in istr # == BuildStatus.QUEUED assert "#dependencies=1" in istr assert "priority=" in istr ================================================ FILE: lib/spack/spack/test/cache_fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.config import spack.util.url as url_util from spack.fetch_strategy import CacheURLFetchStrategy, NoCacheError from spack.llnl.util.filesystem import mkdirp from spack.stage import Stage @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) def test_fetch_missing_cache(tmp_path: pathlib.Path, _fetch_method): """Ensure raise a missing cache file.""" testpath = str(tmp_path) non_existing = os.path.join(testpath, "non-existing") with spack.config.override("config:url_fetch_method", _fetch_method): url = url_util.path_to_file_url(non_existing) fetcher = CacheURLFetchStrategy(url=url) with Stage(fetcher, path=testpath): with pytest.raises(NoCacheError, match=r"No cache"): fetcher.fetch() @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) def test_fetch(tmp_path: pathlib.Path, _fetch_method): """Ensure a fetch after expanding is effectively a no-op.""" cache_dir = tmp_path / "cache" stage_dir = tmp_path / "stage" cache_dir.mkdir() stage_dir.mkdir() cache = cache_dir / "cache.tar.gz" cache.touch() url = url_util.path_to_file_url(str(cache)) with spack.config.override("config:url_fetch_method", _fetch_method): fetcher = CacheURLFetchStrategy(url=url) with Stage(fetcher, path=str(stage_dir)) as stage: source_path = stage.source_path mkdirp(source_path) fetcher.fetch() ================================================ FILE: lib/spack/spack/test/cc.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This test checks that the Spack cc compiler wrapper is parsing arguments correctly. """ import os import pytest import spack.build_environment import spack.config from spack.util.environment import SYSTEM_DIR_CASE_ENTRY, set_env from spack.util.executable import Executable, ProcessError # # Complicated compiler test command # test_args = [ "-I/test/include", "-L/test/lib", "-L/with space/lib", "-I/other/include", "arg1", "-Wl,--start-group", "arg2", "-Wl,-rpath,/first/rpath", "arg3", "-Wl,-rpath", "-Wl,/second/rpath", "-llib1", "-llib2", "arg4", "-Wl,--end-group", "-Xlinker", "-rpath", "-Xlinker", "/third/rpath", "-Xlinker", "-rpath", "-Xlinker", "/fourth/rpath", "-Wl,--rpath,/fifth/rpath", "-Wl,--rpath", "-Wl,/sixth/rpath", "-llib3", "-llib4", "arg5", "arg6", "-DGCC_ARG_WITH_PERENS=(A B C)", '"-DDOUBLE_QUOTED_ARG"', "'-DSINGLE_QUOTED_ARG'", ] # # Pieces of the test command above, as they should be parsed out. # # `_wl_rpaths` are for the compiler (with -Wl,), and `_rpaths` are raw # -rpath arguments for the linker. # test_include_paths = ["-I/test/include", "-I/other/include"] test_library_paths = ["-L/test/lib", "-L/with space/lib"] test_wl_rpaths = [ "-Wl,-rpath,/first/rpath", "-Wl,-rpath,/second/rpath", "-Wl,-rpath,/third/rpath", "-Wl,-rpath,/fourth/rpath", "-Wl,-rpath,/fifth/rpath", "-Wl,-rpath,/sixth/rpath", ] test_rpaths = [ "-rpath", "/first/rpath", "-rpath", "/second/rpath", "-rpath", "/third/rpath", "-rpath", "/fourth/rpath", "-rpath", "/fifth/rpath", "-rpath", "/sixth/rpath", ] test_args_without_paths = [ "arg1", "-Wl,--start-group", "arg2", "arg3", "-llib1", "-llib2", "arg4", "-Wl,--end-group", "-llib3", "-llib4", "arg5", "arg6", "-DGCC_ARG_WITH_PERENS=(A B C)", '"-DDOUBLE_QUOTED_ARG"', "'-DSINGLE_QUOTED_ARG'", ] #: The prefix of the package being mock installed pkg_prefix = "/spack-test-prefix" #: the "real" compiler the wrapper is expected to invoke real_cc = "/bin/mycc" # mock flags to use in the wrapper environment spack_cppflags = ["-g", "-O1", "-DVAR=VALUE"] spack_cflags = ["-Wall"] spack_cxxflags = ["-Werror"] spack_fflags = ["-w"] spack_ldflags = ["-Wl,--gc-sections", "-L", "foo"] spack_ldlibs = ["-lfoo"] lheaderpad = ["-Wl,-headerpad_max_install_names"] headerpad = ["-headerpad_max_install_names"] target_args = ["-march=znver2", "-mtune=znver2"] target_args_fc = ["-march=znver4", "-mtune=znver4"] # common compile arguments: includes, libs, -Wl linker args, other args common_compile_args = ( test_include_paths + test_library_paths + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + test_args_without_paths ) pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.fixture(scope="function") def wrapper_environment(working_env): with set_env( SPACK_CC=real_cc, SPACK_CXX=real_cc, SPACK_FC=real_cc, SPACK_PREFIX=pkg_prefix, SPACK_COMPILER_WRAPPER_PATH="test", SPACK_DEBUG_LOG_DIR=".", SPACK_DEBUG_LOG_ID="foo-hashabc", SPACK_SHORT_SPEC="foo@1.2 arch=linux-rhel6-x86_64 /hashabc", SPACK_SYSTEM_DIRS=SYSTEM_DIR_CASE_ENTRY, SPACK_MANAGED_DIRS="/path/to/spack-1/opt/spack/*|/path/to/spack-2/opt/spack/*", SPACK_CC_RPATH_ARG="-Wl,-rpath,", SPACK_CXX_RPATH_ARG="-Wl,-rpath,", SPACK_F77_RPATH_ARG="-Wl,-rpath,", SPACK_FC_RPATH_ARG="-Wl,-rpath,", SPACK_LINK_DIRS=None, SPACK_INCLUDE_DIRS=None, SPACK_RPATH_DIRS=None, SPACK_TARGET_ARGS_CC="-march=znver2 -mtune=znver2", SPACK_TARGET_ARGS_CXX="-march=znver2 -mtune=znver2", SPACK_TARGET_ARGS_FORTRAN="-march=znver4 -mtune=znver4", SPACK_CC_LINKER_ARG="-Wl,", SPACK_CXX_LINKER_ARG="-Wl,", SPACK_FC_LINKER_ARG="-Wl,", SPACK_F77_LINKER_ARG="-Wl,", SPACK_DTAGS_TO_ADD="--disable-new-dtags", SPACK_DTAGS_TO_STRIP="--enable-new-dtags", SPACK_COMPILER_FLAGS_KEEP="", SPACK_COMPILER_FLAGS_REPLACE="-Werror*|", ): yield @pytest.fixture() def wrapper_flags(): with set_env( SPACK_CPPFLAGS=" ".join(spack_cppflags), SPACK_CFLAGS=" ".join(spack_cflags), SPACK_CXXFLAGS=" ".join(spack_cxxflags), SPACK_FFLAGS=" ".join(spack_fflags), SPACK_LDFLAGS=" ".join(spack_ldflags), SPACK_LDLIBS=" ".join(spack_ldlibs), ): yield def check_args(cc, args, expected): """Check output arguments that cc produces when called with args. This assumes that cc will print debug command output with one element per line, so that we see whether arguments that should (or shouldn't) contain spaces are parsed correctly. """ cc = Executable(str(cc)) with set_env(SPACK_TEST_COMMAND="dump-args"): cc_modified_args = cc(*args, output=str).strip().split("\n") assert cc_modified_args == expected def check_args_contents(cc, args, must_contain, must_not_contain): """Check output arguments that cc produces when called with args. This assumes that cc will print debug command output with one element per line, so that we see whether arguments that should (or shouldn't) contain spaces are parsed correctly. """ cc = Executable(str(cc)) with set_env(SPACK_TEST_COMMAND="dump-args"): cc_modified_args = cc(*args, output=str).strip().split("\n") for a in must_contain: assert a in cc_modified_args for a in must_not_contain: assert a not in cc_modified_args def check_env_var(executable, var, expected): """Check environment variables updated by the passed compiler wrapper This assumes that cc will print debug output when it's environment contains SPACK_TEST_COMMAND=dump-env- """ executable = Executable(str(executable)) with set_env(SPACK_TEST_COMMAND="dump-env-" + var): output = executable(*test_args, output=str).strip() assert executable.path + ": " + var + ": " + expected == output def dump_mode(cc, args): """Make cc dump the mode it detects, and return it.""" cc = Executable(str(cc)) with set_env(SPACK_TEST_COMMAND="dump-mode"): return cc(*args, output=str).strip() def test_no_wrapper_environment(wrapper_dir): cc = Executable(str(wrapper_dir / "cc")) with pytest.raises(ProcessError): output = cc(output=str) assert "Spack compiler must be run from Spack" in output def test_modes(wrapper_environment, wrapper_dir): cc = wrapper_dir / "cc" cxx = wrapper_dir / "c++" cpp = wrapper_dir / "cpp" ld = wrapper_dir / "ld" # vcheck assert dump_mode(cc, ["-I/include", "--version"]) == "vcheck" assert dump_mode(cc, ["-I/include", "-V"]) == "vcheck" assert dump_mode(cc, ["-I/include", "-v"]) == "vcheck" assert dump_mode(cc, ["-I/include", "-dumpversion"]) == "vcheck" assert dump_mode(cc, ["-I/include", "--version", "-c"]) == "vcheck" assert dump_mode(cc, ["-I/include", "-V", "-o", "output"]) == "vcheck" # cpp assert dump_mode(cc, ["-E"]) == "cpp" assert dump_mode(cxx, ["-E"]) == "cpp" assert dump_mode(cpp, []) == "cpp" # as assert dump_mode(cc, ["-S"]) == "as" # ccld assert dump_mode(cc, []) == "ccld" assert dump_mode(cc, ["foo.c", "-o", "foo"]) == "ccld" assert dump_mode(cc, ["foo.c", "-o", "foo", "-Wl,-rpath,foo"]) == "ccld" assert dump_mode(cc, ["foo.o", "bar.o", "baz.o", "-o", "foo", "-Wl,-rpath,foo"]) == "ccld" # ld assert dump_mode(ld, []) == "ld" assert dump_mode(ld, ["foo.o", "bar.o", "baz.o", "-o", "foo", "-Wl,-rpath,foo"]) == "ld" @pytest.mark.regression("37179") def test_expected_args(wrapper_environment, wrapper_dir): cc = wrapper_dir / "cc" fc = wrapper_dir / "fc" ld = wrapper_dir / "ld" # ld_unterminated_rpath check_args( ld, ["foo.o", "bar.o", "baz.o", "-o", "foo", "-rpath"], ["ld", "--disable-new-dtags", "foo.o", "bar.o", "baz.o", "-o", "foo", "-rpath"], ) # xlinker_unterminated_rpath check_args( cc, ["foo.o", "bar.o", "baz.o", "-o", "foo", "-Xlinker", "-rpath"], [real_cc] + target_args + [ "-Wl,--disable-new-dtags", "foo.o", "bar.o", "baz.o", "-o", "foo", "-Xlinker", "-rpath", ], ) # wl_unterminated_rpath check_args( cc, ["foo.o", "bar.o", "baz.o", "-o", "foo", "-Wl,-rpath"], [real_cc] + target_args + ["-Wl,--disable-new-dtags", "foo.o", "bar.o", "baz.o", "-o", "foo", "-Wl,-rpath"], ) # Wl_parsing check_args( cc, ["-Wl,-rpath,/a,--enable-new-dtags,-rpath=/b,--rpath", "-Wl,/c"], [real_cc] + target_args + ["-Wl,--disable-new-dtags", "-Wl,-rpath,/a", "-Wl,-rpath,/b", "-Wl,-rpath,/c"], ) # Wl_parsing_with_missing_value check_args( cc, ["-Wl,-rpath=/a,-rpath=", "-Wl,--rpath="], [real_cc] + target_args + ["-Wl,--disable-new-dtags", "-Wl,-rpath,/a"], ) # Wl_parsing_NAG_is_ignored check_args( fc, ["-Wl,-Wl,,x,,y,,z"], [real_cc] + target_args_fc + ["-Wl,--disable-new-dtags", "-Wl,-Wl,,x,,y,,z"], ) # Xlinker_parsing # # -Xlinker ... -Xlinker may have compiler flags in between, like -O3 in this # example. Also check that a trailing -Xlinker (which is a compiler error) is not # dropped or given an empty argument. check_args( cc, [ "-Xlinker", "-rpath", "-O3", "-Xlinker", "/a", "-Xlinker", "--flag", "-Xlinker", "-rpath=/b", "-Xlinker", ], [real_cc] + target_args + [ "-Wl,--disable-new-dtags", "-Wl,-rpath,/a", "-Wl,-rpath,/b", "-O3", "-Xlinker", "--flag", "-Xlinker", ], ) # rpath_without_value # # cc -Wl,-rpath without a value shouldn't drop -Wl,-rpath; # same for -Xlinker check_args( cc, ["-Wl,-rpath", "-O3", "-g"], [real_cc] + target_args + ["-Wl,--disable-new-dtags", "-O3", "-g", "-Wl,-rpath"], ) check_args( cc, ["-Xlinker", "-rpath", "-O3", "-g"], [real_cc] + target_args + ["-Wl,--disable-new-dtags", "-O3", "-g", "-Xlinker", "-rpath"], ) # dep_rapth check_args(cc, test_args, [real_cc] + target_args + common_compile_args) # dep_include with set_env(SPACK_INCLUDE_DIRS="x"): check_args( cc, test_args, [real_cc] + target_args + test_include_paths + ["-Ix"] + test_library_paths + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + test_args_without_paths, ) # dep_lib # # Ensure a single dependency RPATH is added with set_env(SPACK_LINK_DIRS="x", SPACK_RPATH_DIRS="x"): check_args( cc, test_args, [real_cc] + target_args + test_include_paths + test_library_paths + ["-Lx"] + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + ["-Wl,-rpath,x"] + test_args_without_paths, ) # dep_lib_no_rpath # # Ensure a single dependency link flag is added with no dep RPATH with set_env(SPACK_LINK_DIRS="x"): check_args( cc, test_args, [real_cc] + target_args + test_include_paths + test_library_paths + ["-Lx"] + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + test_args_without_paths, ) # dep_lib_no_lib # Ensure a single dependency RPATH is added with no -L with set_env(SPACK_RPATH_DIRS="x"): check_args( cc, test_args, [real_cc] + target_args + test_include_paths + test_library_paths + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + ["-Wl,-rpath,x"] + test_args_without_paths, ) # ccld_deps # Ensure all flags are added in ccld mode with set_env( SPACK_INCLUDE_DIRS="xinc:yinc:zinc", SPACK_RPATH_DIRS="xlib:ylib:zlib", SPACK_LINK_DIRS="xlib:ylib:zlib", ): check_args( cc, test_args, [real_cc] + target_args + test_include_paths + ["-Ixinc", "-Iyinc", "-Izinc"] + test_library_paths + ["-Lxlib", "-Lylib", "-Lzlib"] + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + ["-Wl,-rpath,xlib", "-Wl,-rpath,ylib", "-Wl,-rpath,zlib"] + test_args_without_paths, ) # ccld_deps_isystem # # Ensure all flags are added in ccld mode. # When a build uses -isystem, Spack should inject it's # include paths using -isystem. Spack will insert these # after any provided -isystem includes, but before any # system directories included using -isystem with set_env( SPACK_INCLUDE_DIRS="xinc:yinc:zinc", SPACK_RPATH_DIRS="xlib:ylib:zlib", SPACK_LINK_DIRS="xlib:ylib:zlib", ): mytest_args = test_args + ["-isystem", "fooinc"] check_args( cc, mytest_args, [real_cc] + target_args + test_include_paths + ["-isystem", "fooinc", "-isystem", "xinc", "-isystem", "yinc", "-isystem", "zinc"] + test_library_paths + ["-Lxlib", "-Lylib", "-Lzlib"] + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + ["-Wl,-rpath,xlib", "-Wl,-rpath,ylib", "-Wl,-rpath,zlib"] + test_args_without_paths, ) # cc_deps # Ensure -L and RPATHs are not added in cc mode with set_env( SPACK_INCLUDE_DIRS="xinc:yinc:zinc", SPACK_RPATH_DIRS="xlib:ylib:zlib", SPACK_LINK_DIRS="xlib:ylib:zlib", ): check_args( cc, ["-c"] + test_args, [real_cc] + target_args + test_include_paths + ["-Ixinc", "-Iyinc", "-Izinc"] + test_library_paths + ["-c"] + test_args_without_paths, ) # ccld_with_system_dirs # Ensure all flags are added in ccld mode with set_env( SPACK_INCLUDE_DIRS="xinc:yinc:zinc", SPACK_RPATH_DIRS="xlib:ylib:zlib", SPACK_LINK_DIRS="xlib:ylib:zlib", ): sys_path_args = [ "-I/usr/include", "-L/usr/local/lib", "-Wl,-rpath,/usr/lib64", "-I/usr/local/include", "-L/lib64/", ] check_args( cc, sys_path_args + test_args, [real_cc] + target_args + test_include_paths + ["-Ixinc", "-Iyinc", "-Izinc"] + ["-I/usr/include", "-I/usr/local/include"] + test_library_paths + ["-Lxlib", "-Lylib", "-Lzlib"] + ["-L/usr/local/lib", "-L/lib64/"] + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + ["-Wl,-rpath,xlib", "-Wl,-rpath,ylib", "-Wl,-rpath,zlib"] + ["-Wl,-rpath,/usr/lib64"] + test_args_without_paths, ) # ccld_with_system_dirs_isystem # Ensure all flags are added in ccld mode. # Ensure that includes are in the proper # place when a build uses -isystem, and uses # system directories in the include paths with set_env( SPACK_INCLUDE_DIRS="xinc:yinc:zinc", SPACK_RPATH_DIRS="xlib:ylib:zlib", SPACK_LINK_DIRS="xlib:ylib:zlib", ): sys_path_args = [ "-isystem", "/usr/include", "-L/usr/local/lib", "-Wl,-rpath,/usr/lib64", "-isystem", "/usr/local/include", "-L/lib64/", ] check_args( cc, sys_path_args + test_args, [real_cc] + target_args + test_include_paths + ["-isystem", "xinc", "-isystem", "yinc", "-isystem", "zinc"] + ["-isystem", "/usr/include", "-isystem", "/usr/local/include"] + test_library_paths + ["-Lxlib", "-Lylib", "-Lzlib"] + ["-L/usr/local/lib", "-L/lib64/"] + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + ["-Wl,-rpath,xlib", "-Wl,-rpath,ylib", "-Wl,-rpath,zlib"] + ["-Wl,-rpath,/usr/lib64"] + test_args_without_paths, ) # ld_deps # Ensure no (extra) -I args or -Wl, are passed in ld mode with set_env( SPACK_INCLUDE_DIRS="xinc:yinc:zinc", SPACK_RPATH_DIRS="xlib:ylib:zlib", SPACK_LINK_DIRS="xlib:ylib:zlib", ): check_args( ld, test_args, ["ld"] + test_include_paths + test_library_paths + ["-Lxlib", "-Lylib", "-Lzlib"] + ["--disable-new-dtags"] + test_rpaths + ["-rpath", "xlib", "-rpath", "ylib", "-rpath", "zlib"] + test_args_without_paths, ) # ld_deps_no_rpath # Ensure SPACK_LINK_DEPS controls -L for ld with set_env(SPACK_INCLUDE_DIRS="xinc:yinc:zinc", SPACK_LINK_DIRS="xlib:ylib:zlib"): check_args( ld, test_args, ["ld"] + test_include_paths + test_library_paths + ["-Lxlib", "-Lylib", "-Lzlib"] + ["--disable-new-dtags"] + test_rpaths + test_args_without_paths, ) # ld_deps_no_link # Ensure SPACK_RPATH_DEPS controls -rpath for ld with set_env(SPACK_INCLUDE_DIRS="xinc:yinc:zinc", SPACK_RPATH_DIRS="xlib:ylib:zlib"): check_args( ld, test_args, ["ld"] + test_include_paths + test_library_paths + ["--disable-new-dtags"] + test_rpaths + ["-rpath", "xlib", "-rpath", "ylib", "-rpath", "zlib"] + test_args_without_paths, ) def test_expected_args_with_flags(wrapper_environment, wrapper_flags, wrapper_dir): cc = wrapper_dir / "cc" cxx = wrapper_dir / "c++" cpp = wrapper_dir / "cpp" fc = wrapper_dir / "fc" ld = wrapper_dir / "ld" # ld_flags check_args( ld, test_args, ["ld"] + test_include_paths + test_library_paths + ["--disable-new-dtags"] + test_rpaths + test_args_without_paths + spack_ldlibs, ) # cpp_flags check_args( cpp, test_args, ["cpp"] + test_include_paths + test_library_paths + test_args_without_paths + spack_cppflags, ) # cc_flags check_args( cc, test_args, [real_cc] + target_args + test_include_paths + ["-Lfoo"] + test_library_paths + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + test_args_without_paths + spack_cppflags + spack_cflags + ["-Wl,--gc-sections"] + spack_ldlibs, ) # cxx_flags check_args( cxx, test_args, [real_cc] + target_args + test_include_paths + ["-Lfoo"] + test_library_paths + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + test_args_without_paths + spack_cppflags + ["-Wl,--gc-sections"] + spack_ldlibs, ) # fc_flags check_args( fc, test_args, [real_cc] + target_args_fc + test_include_paths + ["-Lfoo"] + test_library_paths + ["-Wl,--disable-new-dtags"] + test_wl_rpaths + test_args_without_paths + spack_fflags + spack_cppflags + ["-Wl,--gc-sections"] + spack_ldlibs, ) # always_cflags with set_env(SPACK_ALWAYS_CFLAGS="-always1 -always2"): check_args( cc, ["-v", "--cmd-line-v-opt"], [real_cc] + ["-always1", "-always2"] + ["-v", "--cmd-line-v-opt"], ) def test_system_path_cleanup(wrapper_environment, wrapper_dir): """Ensure SPACK_COMPILER_WRAPPER_PATH is removed from PATH, even with trailing / The compiler wrapper has to ensure that it is not called nested like it would happen when gcc's collect2 looks in PATH for ld. To prevent nested calls, the compiler wrapper removes the elements of SPACK_COMPILER_WRAPPER_PATH from PATH. Autotest's generated testsuite appends a / to each element of PATH when adding AUTOTEST_PATH. Thus, ensure that PATH cleanup works even with trailing /. """ cc = wrapper_dir / "cc" system_path = "/bin:/usr/bin:/usr/local/bin" with set_env(SPACK_COMPILER_WRAPPER_PATH=str(wrapper_dir), SPACK_CC="true"): with set_env(PATH=str(wrapper_dir) + ":" + system_path): check_env_var(cc, "PATH", system_path) with set_env(PATH=str(wrapper_dir) + "/:" + system_path): check_env_var(cc, "PATH", system_path) def test_ld_deps_partial(wrapper_environment, wrapper_dir): """Make sure ld -r (partial link) is handled correctly on OS's where it doesn't accept rpaths. """ ld = wrapper_dir / "ld" with set_env(SPACK_INCLUDE_DIRS="xinc", SPACK_RPATH_DIRS="xlib", SPACK_LINK_DIRS="xlib"): # TODO: do we need to add RPATHs on other platforms like Linux? # TODO: Can't we treat them the same? os.environ["SPACK_SHORT_SPEC"] = "foo@1.2=linux-x86_64" check_args( ld, ["-r"] + test_args, ["ld"] + test_include_paths + test_library_paths + ["-Lxlib"] + ["--disable-new-dtags"] + test_rpaths + ["-rpath", "xlib"] + ["-r"] + test_args_without_paths, ) # rpaths from the underlying command will still appear # Spack will not add its own rpaths. os.environ["SPACK_SHORT_SPEC"] = "foo@1.2=darwin-x86_64" check_args( ld, ["-r"] + test_args, ["ld"] + headerpad + test_include_paths + test_library_paths + ["-Lxlib"] + ["--disable-new-dtags"] + test_rpaths + ["-r"] + test_args_without_paths, ) def test_ccache_prepend_for_cc(wrapper_environment, wrapper_dir): cc = wrapper_dir / "cc" with set_env(SPACK_CCACHE_BINARY="ccache"): os.environ["SPACK_SHORT_SPEC"] = "foo@1.2=linux-x86_64" check_args( cc, test_args, ["ccache"] + [real_cc] # ccache prepended in cc mode + target_args + common_compile_args, ) os.environ["SPACK_SHORT_SPEC"] = "foo@1.2=darwin-x86_64" check_args( cc, test_args, ["ccache"] + [real_cc] # ccache prepended in cc mode + target_args + lheaderpad + common_compile_args, ) def test_no_ccache_prepend_for_fc(wrapper_environment, wrapper_dir): fc = wrapper_dir / "fc" os.environ["SPACK_SHORT_SPEC"] = "foo@1.2=linux-x86_64" check_args( fc, test_args, # no ccache for Fortran [real_cc] + target_args_fc + common_compile_args, ) os.environ["SPACK_SHORT_SPEC"] = "foo@1.2=darwin-x86_64" check_args( fc, test_args, # no ccache for Fortran [real_cc] + target_args_fc + lheaderpad + common_compile_args, ) def test_keep_and_replace(wrapper_environment, wrapper_dir): cc = wrapper_dir / "cc" werror_specific = ["-Werror=meh"] werror = ["-Werror"] werror_all = werror_specific + werror with set_env(SPACK_COMPILER_FLAGS_KEEP="", SPACK_COMPILER_FLAGS_REPLACE="-Werror*|"): check_args_contents(cc, test_args + werror_all, ["-Wl,--end-group"], werror_all) with set_env(SPACK_COMPILER_FLAGS_KEEP="-Werror=*", SPACK_COMPILER_FLAGS_REPLACE="-Werror*|"): check_args_contents(cc, test_args + werror_all, werror_specific, werror) with set_env( SPACK_COMPILER_FLAGS_KEEP="-Werror=*", SPACK_COMPILER_FLAGS_REPLACE="-Werror*| -llib1| -Wl*|", ): check_args_contents( cc, test_args + werror_all, werror_specific, werror + ["-llib1", "-Wl,--rpath"] ) @pytest.mark.parametrize( "cfg_override,initial,expected,must_be_gone", [ # Set and unset variables ( "config:flags:keep_werror:all", ["-Werror", "-Werror=specific", "-bah"], ["-Werror", "-Werror=specific", "-bah"], [], ), ( "config:flags:keep_werror:specific", ["-Werror", "-Werror=specific", "-Werror-specific2", "-bah"], ["-Wno-error", "-Werror=specific", "-Werror-specific2", "-bah"], ["-Werror"], ), ( "config:flags:keep_werror:none", ["-Werror", "-Werror=specific", "-bah"], ["-Wno-error", "-Wno-error=specific", "-bah"], ["-Werror", "-Werror=specific"], ), # check non-standard -Werror opts like -Werror-implicit-function-declaration ( "config:flags:keep_werror:all", ["-Werror", "-Werror-implicit-function-declaration", "-bah"], ["-Werror", "-Werror-implicit-function-declaration", "-bah"], [], ), ( "config:flags:keep_werror:specific", ["-Werror", "-Werror-implicit-function-declaration", "-bah"], ["-Wno-error", "-Werror-implicit-function-declaration", "-bah"], ["-Werror"], ), ( "config:flags:keep_werror:none", ["-Werror", "-Werror-implicit-function-declaration", "-bah"], ["-Wno-error", "-bah", "-Wno-error=implicit-function-declaration"], ["-Werror", "-Werror-implicit-function-declaration"], ), ], ) @pytest.mark.usefixtures("wrapper_environment", "mutable_config") def test_flag_modification(cfg_override, initial, expected, must_be_gone, wrapper_dir): cc = wrapper_dir / "cc" spack.config.add(cfg_override) env = spack.build_environment.clean_environment() keep_werror = spack.config.get("config:flags:keep_werror") spack.build_environment._add_werror_handling(keep_werror, env) env.apply_modifications() check_args_contents(cc, test_args[:3] + initial, expected, must_be_gone) @pytest.mark.regression("9160") def test_disable_new_dtags(wrapper_environment, wrapper_flags, wrapper_dir): cc = Executable(str(wrapper_dir / "cc")) ld = Executable(str(wrapper_dir / "ld")) with set_env(SPACK_TEST_COMMAND="dump-args"): result = ld(*test_args, output=str).strip().split("\n") assert "--disable-new-dtags" in result result = cc(*test_args, output=str).strip().split("\n") assert "-Wl,--disable-new-dtags" in result @pytest.mark.regression("9160") def test_filter_enable_new_dtags(wrapper_environment, wrapper_flags, wrapper_dir): cc = Executable(str(wrapper_dir / "cc")) ld = Executable(str(wrapper_dir / "ld")) with set_env(SPACK_TEST_COMMAND="dump-args"): result = ld(*(test_args + ["--enable-new-dtags"]), output=str) result = result.strip().split("\n") assert "--enable-new-dtags" not in result result = cc(*(test_args + ["-Wl,--enable-new-dtags"]), output=str) result = result.strip().split("\n") assert "-Wl,--enable-new-dtags" not in result @pytest.mark.regression("22643") def test_linker_strips_loopopt(wrapper_environment, wrapper_flags, wrapper_dir): cc = Executable(str(wrapper_dir / "cc")) ld = Executable(str(wrapper_dir / "ld")) with set_env(SPACK_TEST_COMMAND="dump-args"): # ensure that -loopopt=0 is not present in ld mode result = ld(*(test_args + ["-loopopt=0"]), output=str) result = result.strip().split("\n") assert "-loopopt=0" not in result # ensure that -loopopt=0 is not present in ccld mode result = cc(*(test_args + ["-loopopt=0"]), output=str) result = result.strip().split("\n") assert "-loopopt=0" not in result # ensure that -loopopt=0 *is* present in cc mode # The "-c" argument is needed for cc to be detected # as compile only (cc) mode. result = cc(*(test_args + ["-loopopt=0", "-c", "x.c"]), output=str) result = result.strip().split("\n") assert "-loopopt=0" in result def test_spack_managed_dirs_are_prioritized(wrapper_environment, wrapper_dir): cc = Executable(str(wrapper_dir / "cc")) # We have two different stores with 5 packages divided over them pkg1 = "/path/to/spack-1/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-1.0-abcdef" pkg2 = "/path/to/spack-1/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-2.0-abcdef" pkg3 = "/path/to/spack-2/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-3.0-abcdef" pkg4 = "/path/to/spack-2/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-4.0-abcdef" pkg5 = "/path/to/spack-2/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-5.0-abcdef" variables = { # cppflags, ldflags from the command line, config or package.py take highest priority "SPACK_CPPFLAGS": f"-I/usr/local/include -I/external-1/include -I{pkg1}/include", "SPACK_LDFLAGS": f"-L/usr/local/lib -L/external-1/lib -L{pkg1}/lib " f"-Wl,-rpath,/usr/local/lib -Wl,-rpath,/external-1/lib -Wl,-rpath,{pkg1}/lib", # automatic -L, -Wl,-rpath, -I flags from dependencies -- on the spack side they are # already partitioned into "spack owned prefixes" and "non-spack owned prefixes" "SPACK_STORE_LINK_DIRS": f"{pkg4}/lib:{pkg5}/lib", "SPACK_STORE_RPATH_DIRS": f"{pkg4}/lib:{pkg5}/lib", "SPACK_STORE_INCLUDE_DIRS": f"{pkg4}/include:{pkg5}/include", "SPACK_LINK_DIRS": "/external-3/lib:/external-4/lib", "SPACK_RPATH_DIRS": "/external-3/lib:/external-4/lib", "SPACK_INCLUDE_DIRS": "/external-3/include:/external-4/include", } with set_env(SPACK_TEST_COMMAND="dump-args", **variables): effective_call = ( cc( # system paths "-I/usr/include", "-L/usr/lib", "-Wl,-rpath,/usr/lib", # some other externals "-I/external-2/include", "-L/external-2/lib", "-Wl,-rpath,/external-2/lib", # relative paths are considered "spack managed" since they are in the stage dir "-I..", "-L..", "-Wl,-rpath,..", # pathological but simpler for the test. # spack store paths f"-I{pkg2}/include", f"-I{pkg3}/include", f"-L{pkg2}/lib", f"-L{pkg3}/lib", f"-Wl,-rpath,{pkg2}/lib", f"-Wl,-rpath,{pkg3}/lib", "hello.c", "-o", "hello", output=str, ) .strip() .split("\n") ) dash_I = [flag[2:] for flag in effective_call if flag.startswith("-I")] dash_L = [flag[2:] for flag in effective_call if flag.startswith("-L")] dash_Wl_rpath = [flag[11:] for flag in effective_call if flag.startswith("-Wl,-rpath")] assert dash_I == [ # spack owned dirs from SPACK_*FLAGS f"{pkg1}/include", # spack owned dirs from command line & automatic flags for deps (in that order)] "..", f"{pkg2}/include", # from command line f"{pkg3}/include", # from command line f"{pkg4}/include", # from SPACK_STORE_INCLUDE_DIRS f"{pkg5}/include", # from SPACK_STORE_INCLUDE_DIRS # non-system dirs from SPACK_*FLAGS "/external-1/include", # non-system dirs from command line & automatic flags for deps (in that order) "/external-2/include", # from command line "/external-3/include", # from SPACK_INCLUDE_DIRS "/external-4/include", # from SPACK_INCLUDE_DIRS # system dirs from SPACK_*FLAGS "/usr/local/include", # system dirs from command line "/usr/include", ] assert ( dash_L == dash_Wl_rpath == [ # spack owned dirs from SPACK_*FLAGS f"{pkg1}/lib", # spack owned dirs from command line & automatic flags for deps (in that order) "..", f"{pkg2}/lib", # from command line f"{pkg3}/lib", # from command line f"{pkg4}/lib", # from SPACK_STORE_LINK_DIRS f"{pkg5}/lib", # from SPACK_STORE_LINK_DIRS # non-system dirs from SPACK_*FLAGS "/external-1/lib", # non-system dirs from command line & automatic flags for deps (in that order) "/external-2/lib", # from command line "/external-3/lib", # from SPACK_LINK_DIRS "/external-4/lib", # from SPACK_LINK_DIRS # system dirs from SPACK_*FLAGS "/usr/local/lib", # system dirs from command line "/usr/lib", ] ) ================================================ FILE: lib/spack/spack/test/ci.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import os import pathlib import subprocess from urllib.error import HTTPError import pytest import spack.ci as ci import spack.concretize import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs import spack.paths import spack.repo as repo import spack.util.git from spack.spec import Spec from spack.test.conftest import MockHTTPResponse, RepoBuilder from spack.version import Version pytestmark = [pytest.mark.usefixtures("mock_packages")] @pytest.fixture def repro_dir(tmp_path: pathlib.Path): result = tmp_path / "repro_dir" result.mkdir() with fs.working_dir(str(tmp_path)): yield result def test_filter_added_checksums_new_checksum(mock_git_package_changes): repo, filename, commits = mock_git_package_changes checksum_versions = { "3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04": Version("2.1.5"), "a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a": Version("2.1.4"), "6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200": Version("2.0.7"), "86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8": Version("2.0.0"), } with fs.working_dir(repo.packages_path): assert ci.filter_added_checksums( checksum_versions.keys(), filename, from_ref=commits[-1], to_ref=commits[-2] ) == ["3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04"] def test_filter_added_checksums_new_commit(mock_git_package_changes): repo, filename, commits = mock_git_package_changes checksum_versions = { "74253725f884e2424a0dd8ae3f69896d5377f325": Version("2.1.6"), "3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04": Version("2.1.5"), "a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a": Version("2.1.4"), "6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200": Version("2.0.7"), "86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8": Version("2.0.0"), } with fs.working_dir(repo.packages_path): assert ci.filter_added_checksums( checksum_versions, filename, from_ref=commits[-2], to_ref=commits[-3] ) == ["74253725f884e2424a0dd8ae3f69896d5377f325"] def test_pipeline_dag(config, repo_builder: RepoBuilder): r"""Test creation, pruning, and traversal of PipelineDAG using the following package dependency graph: a a /| /| c b c b |\ prune 'd' /|\ e d =====> e | g | |\ | | h | g h | \| \| f f """ repo_builder.add_package("pkg-h", dependencies=[("pkg-f", None, None)]) repo_builder.add_package("pkg-g") repo_builder.add_package("pkg-f") repo_builder.add_package("pkg-e", dependencies=[("pkg-h", None, None)]) repo_builder.add_package("pkg-d", dependencies=[("pkg-f", None, None), ("pkg-g", None, None)]) repo_builder.add_package("pkg-c") repo_builder.add_package("pkg-b", dependencies=[("pkg-d", None, None), ("pkg-e", None, None)]) repo_builder.add_package("pkg-a", dependencies=[("pkg-b", None, None), ("pkg-c", None, None)]) with repo.use_repositories(repo_builder.root): spec_a = spack.concretize.concretize_one("pkg-a") key_a = ci.common.PipelineDag.key(spec_a) key_b = ci.common.PipelineDag.key(spec_a["pkg-b"]) key_c = ci.common.PipelineDag.key(spec_a["pkg-c"]) key_d = ci.common.PipelineDag.key(spec_a["pkg-d"]) key_e = ci.common.PipelineDag.key(spec_a["pkg-e"]) key_f = ci.common.PipelineDag.key(spec_a["pkg-f"]) key_g = ci.common.PipelineDag.key(spec_a["pkg-g"]) key_h = ci.common.PipelineDag.key(spec_a["pkg-h"]) pipeline = ci.common.PipelineDag([spec_a]) expected_bottom_up_traversal = { key_a: 4, key_b: 3, key_c: 0, key_d: 1, key_e: 2, key_f: 0, key_g: 0, key_h: 1, } visited = [] for stage, node in pipeline.traverse_nodes(direction="parents"): assert expected_bottom_up_traversal[node.key] == stage visited.append(node.key) assert len(visited) == len(expected_bottom_up_traversal) assert all(k in visited for k in expected_bottom_up_traversal.keys()) expected_top_down_traversal = { key_a: 0, key_b: 1, key_c: 1, key_d: 2, key_e: 2, key_f: 4, key_g: 3, key_h: 3, } visited = [] for stage, node in pipeline.traverse_nodes(direction="children"): assert expected_top_down_traversal[node.key] == stage visited.append(node.key) assert len(visited) == len(expected_top_down_traversal) assert all(k in visited for k in expected_top_down_traversal.keys()) pipeline.prune(key_d) b_children = pipeline.nodes[key_b].children assert len(b_children) == 3 assert all([k in b_children for k in [key_e, key_f, key_g]]) # check another bottom-up traversal after pruning pkg-d expected_bottom_up_traversal = { key_a: 4, key_b: 3, key_c: 0, key_e: 2, key_f: 0, key_g: 0, key_h: 1, } visited = [] for stage, node in pipeline.traverse_nodes(direction="parents"): assert expected_bottom_up_traversal[node.key] == stage visited.append(node.key) assert len(visited) == len(expected_bottom_up_traversal) assert all(k in visited for k in expected_bottom_up_traversal.keys()) # check top-down traversal after pruning pkg-d expected_top_down_traversal = { key_a: 0, key_b: 1, key_c: 1, key_e: 2, key_f: 4, key_g: 2, key_h: 3, } visited = [] for stage, node in pipeline.traverse_nodes(direction="children"): assert expected_top_down_traversal[node.key] == stage visited.append(node.key) assert len(visited) == len(expected_top_down_traversal) assert all(k in visited for k in expected_top_down_traversal.keys()) a_deps_direct = [n.spec for n in pipeline.get_dependencies(pipeline.nodes[key_a])] assert all([s in a_deps_direct for s in [spec_a["pkg-b"], spec_a["pkg-c"]]]) @pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_import_signing_key(mock_gnupghome): signing_key_dir = spack.paths.mock_gpg_keys_path signing_key_path = os.path.join(signing_key_dir, "package-signing-key") with open(signing_key_path, encoding="utf-8") as fd: signing_key = fd.read() # Just make sure this does not raise any exceptions ci.import_signing_key(signing_key) def test_download_and_extract_artifacts(tmp_path: pathlib.Path, monkeypatch): monkeypatch.setenv("GITLAB_PRIVATE_TOKEN", "faketoken") url = "https://www.nosuchurlexists.itsfake/artifacts.zip" working_dir = tmp_path / "repro" test_artifacts_path = os.path.join( spack.paths.test_path, "data", "ci", "gitlab", "artifacts.zip" ) def _urlopen_OK(*args, **kwargs): with open(test_artifacts_path, "rb") as f: return MockHTTPResponse( "200", "OK", {"Content-Type": "application/zip"}, io.BytesIO(f.read()) ) monkeypatch.setattr(ci, "urlopen", _urlopen_OK) ci.download_and_extract_artifacts(url, str(working_dir)) found_zip = fs.find(working_dir, "artifacts.zip") assert len(found_zip) == 0 found_install = fs.find(working_dir, "install.sh") assert len(found_install) == 1 def _urlopen_500(*args, **kwargs): raise HTTPError(url, 500, "Internal Server Error", {}, None) monkeypatch.setattr(ci, "urlopen", _urlopen_500) with pytest.raises(spack.error.SpackError): ci.download_and_extract_artifacts(url, str(working_dir)) def test_ci_copy_stage_logs_to_artifacts_fail( tmp_path: pathlib.Path, default_mock_concretization, capfd ): """The copy will fail because the spec is not concrete so does not have a package.""" log_dir = tmp_path / "log_dir" concrete_spec = default_mock_concretization("printing-package") ci.copy_stage_logs_to_artifacts(concrete_spec, str(log_dir)) _, err = capfd.readouterr() assert "Unable to copy files" in err assert "No such file or directory" in err def test_ci_copy_test_logs_to_artifacts_fail(tmp_path: pathlib.Path, capfd): log_dir = tmp_path / "log_dir" ci.copy_test_logs_to_artifacts("no-such-dir", str(log_dir)) _, err = capfd.readouterr() assert "Cannot copy test logs" in err stage_dir = tmp_path / "stage_dir" stage_dir.mkdir() ci.copy_test_logs_to_artifacts(str(stage_dir), str(log_dir)) _, err = capfd.readouterr() assert "Unable to copy files" in err assert "No such file or directory" in err def test_setup_spack_repro_version( tmp_path: pathlib.Path, capfd, last_two_git_commits, monkeypatch ): c1, c2 = last_two_git_commits repro_dir = tmp_path / "repro" spack_dir = repro_dir / "spack" spack_dir.mkdir(parents=True) prefix_save = spack.paths.prefix monkeypatch.setattr(spack.paths, "prefix", "/garbage") ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() assert not ret assert "Unable to find the path" in err monkeypatch.setattr(spack.paths, "prefix", prefix_save) monkeypatch.setattr(spack.util.git, "git", lambda: None) ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) out, err = capfd.readouterr() assert not ret assert "requires git" in err class mock_git_cmd: def __init__(self, *args, **kwargs): self.returncode = 0 self.check = None def __call__(self, *args, **kwargs): if self.check: self.returncode = self.check(*args, **kwargs) else: self.returncode = 0 git_cmd = mock_git_cmd() monkeypatch.setattr(spack.util.git, "git", lambda: git_cmd) git_cmd.check = lambda *a, **k: 1 if len(a) > 2 and a[2] == c2 else 0 ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() assert not ret assert "Missing commit: {0}".format(c2) in err git_cmd.check = lambda *a, **k: 1 if len(a) > 2 and a[2] == c1 else 0 ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() assert not ret assert "Missing commit: {0}".format(c1) in err git_cmd.check = lambda *a, **k: 1 if a[0] == "clone" else 0 ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() assert not ret assert "Unable to clone" in err git_cmd.check = lambda *a, **k: 1 if a[0] == "checkout" else 0 ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() assert not ret assert "Unable to checkout" in err git_cmd.check = lambda *a, **k: 1 if "merge" in a else 0 ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1) _, err = capfd.readouterr() assert not ret assert "Unable to merge {0}".format(c1) in err def test_get_spec_filter_list(mutable_mock_env_path, mutable_mock_repo): """Tests that, given an active environment and list of touched pkgs, we get the right list of possibly-changed env specs. The test concretizes the following environment: [ ] hypre@=0.2.15+shared build_system=generic [bl ] ^openblas-with-lapack@=0.2.15 build_system=generic [ ] mpileaks@=2.3~debug~opt+shared+static build_system=generic [bl ] ^callpath@=1.0 build_system=generic [bl ] ^dyninst@=8.2 build_system=generic [bl ] ^libdwarf@=20130729 build_system=generic [bl ] ^libelf@=0.8.13 build_system=generic [b ] ^gcc@=10.2.1 build_system=generic languages='c,c++,fortran' [ l ] ^gcc-runtime@=10.2.1 build_system=generic [bl ] ^mpich@=3.0.4~debug build_system=generic and simulates a change in libdwarf. """ e1 = ev.create("test") e1.add("mpileaks") e1.add("hypre") e1.concretize() touched = {"libdwarf"} # Make sure we return the correct set of possibly affected specs, # given a dependent traversal depth and the fact that the touched # package is libdwarf. Passing traversal depth of None or something # equal to or larger than the greatest depth in the graph are # equivalent and result in traversal of all specs from the touched # package to the root. Passing negative traversal depth results in # no spec traversals. Passing any other number yields differing # numbers of possibly affected specs. full_set = { "mpileaks", "mpich", "callpath", "dyninst", "libdwarf", "libelf", "gcc", "gcc-runtime", "compiler-wrapper", } depth_2_set = { "mpich", "callpath", "dyninst", "libdwarf", "libelf", "gcc", "gcc-runtime", "compiler-wrapper", } depth_1_set = {"dyninst", "libdwarf", "libelf", "gcc", "gcc-runtime", "compiler-wrapper"} depth_0_set = {"libdwarf", "libelf", "gcc", "gcc-runtime", "compiler-wrapper"} expectations = { None: full_set, 3: full_set, 100: full_set, -1: set(), 0: depth_0_set, 1: depth_1_set, 2: depth_2_set, } for key, val in expectations.items(): affected_specs = ci.get_spec_filter_list(e1, touched, dependent_traverse_depth=key) affected_pkg_names = {s.name for s in affected_specs} assert affected_pkg_names == val @pytest.mark.regression("29947") def test_affected_specs_on_first_concretization(mutable_mock_env_path): e = ev.create("first_concretization") e.add("mpileaks~shared") e.add("mpileaks+shared") e.concretize() affected_specs = ci.get_spec_filter_list(e, {"callpath"}) mpileaks_specs = [s for s in affected_specs if s.name == "mpileaks"] assert len(mpileaks_specs) == 2, e.all_specs() @pytest.mark.not_on_windows("Reliance on bash script not supported on Windows") def test_ci_process_command(repro_dir): result = ci.process_command("help", commands=[], repro_dir=str(repro_dir)) help_sh = repro_dir / "help.sh" assert help_sh.exists() and not result @pytest.mark.not_on_windows("Reliance on bash script not supported on Windows") def test_ci_process_command_fail(repro_dir, monkeypatch): msg = "subprocess wait exception" def _fail(self, args): raise RuntimeError(msg) monkeypatch.setattr(subprocess.Popen, "__init__", _fail) with pytest.raises(RuntimeError, match=msg): ci.process_command("help", [], str(repro_dir)) def test_ci_create_buildcache(working_env, config, monkeypatch): """Test that create_buildcache returns a list of objects with the correct keys and types.""" monkeypatch.setattr(ci, "push_to_build_cache", lambda a, b, c: True) results = ci.create_buildcache( Spec(), destination_mirror_urls=["file:///fake-url-one", "file:///fake-url-two"] ) assert len(results) == 2 result1, result2 = results assert result1.success assert result1.url == "file:///fake-url-one" assert result2.success assert result2.url == "file:///fake-url-two" results = ci.create_buildcache(Spec(), destination_mirror_urls=["file:///fake-url-one"]) assert len(results) == 1 assert results[0].success assert results[0].url == "file:///fake-url-one" def test_ci_run_standalone_tests_missing_requirements( working_env, default_mock_concretization, capfd ): """This test case checks for failing prerequisite checks.""" ci.run_standalone_tests() err = capfd.readouterr()[1] assert "Job spec is required" in err args = {"job_spec": default_mock_concretization("printing-package")} ci.run_standalone_tests(**args) err = capfd.readouterr()[1] assert "Reproduction directory is required" in err @pytest.mark.not_on_windows("Reliance on bash script not supported on Windows") def test_ci_run_standalone_tests_not_installed_junit( tmp_path: pathlib.Path, repro_dir, working_env, mock_test_stage, capfd ): log_file = tmp_path / "junit.xml" ci.run_standalone_tests( log_file=str(log_file), job_spec=spack.concretize.concretize_one("printing-package"), repro_dir=str(repro_dir), fail_fast=True, ) err = capfd.readouterr()[1] assert "No installed packages" in err assert os.path.getsize(log_file) > 0 @pytest.mark.not_on_windows("Reliance on bash script not supported on Windows") def test_ci_run_standalone_tests_not_installed_cdash( tmp_path: pathlib.Path, repro_dir, working_env, mock_test_stage, capfd ): """Test run_standalone_tests with cdash and related options.""" log_file = tmp_path / "junit.xml" # Cover when CDash handler provided (with the log file as well) ci_cdash = { "url": "file://fake", "build-group": "fake-group", "project": "ci-unit-testing", "site": "fake-site", } os.environ["SPACK_CDASH_BUILD_NAME"] = "ci-test-build" os.environ["SPACK_CDASH_BUILD_STAMP"] = "ci-test-build-stamp" os.environ["CI_RUNNER_DESCRIPTION"] = "test-runner" handler = ci.CDashHandler(ci_cdash) ci.run_standalone_tests( log_file=str(log_file), job_spec=spack.concretize.concretize_one("printing-package"), repro_dir=str(repro_dir), cdash=handler, ) out = capfd.readouterr()[0] # CDash *and* log file output means log file ignored assert "xml option is ignored with CDash" in out # copy test results (though none) artifacts_dir = tmp_path / "artifacts" artifacts_dir.mkdir() handler.copy_test_results(str(tmp_path), str(artifacts_dir)) err = capfd.readouterr()[1] assert "Unable to copy files" in err assert "No such file or directory" in err def test_ci_skipped_report(tmp_path: pathlib.Path, config): """Test explicit skipping of report as well as CI's 'package' arg.""" pkg = "trivial-smoke-test" spec = spack.concretize.concretize_one(pkg) ci_cdash = { "url": "file://fake", "build-group": "fake-group", "project": "ci-unit-testing", "site": "fake-site", } os.environ["SPACK_CDASH_BUILD_NAME"] = "fake-test-build" os.environ["SPACK_CDASH_BUILD_STAMP"] = "ci-test-build-stamp" os.environ["CI_RUNNER_DESCRIPTION"] = "test-runner" handler = ci.CDashHandler(ci_cdash) reason = "Testing skip" handler.report_skipped(spec, str(tmp_path), reason=reason) reports = [name for name in tmp_path.iterdir() if str(name).endswith("Testing.xml")] assert len(reports) == 1 expected = f"Skipped {pkg} package" with open(reports[0], "r", encoding="utf-8") as f: have = [0, 0] for line in f: if expected in line: have[0] += 1 elif reason in line: have[1] += 1 assert all(count == 1 for count in have) ================================================ FILE: lib/spack/spack/test/cmd/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/test/cmd/arch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.main import SpackCommand arch = SpackCommand("arch") def test_arch(): """Sanity check ``spack arch`` to make sure it works.""" arch() arch("-f") arch("--frontend") arch("-b") arch("--backend") def test_arch_platform(): """Sanity check ``spack arch --platform`` to make sure it works.""" arch("-p") arch("--platform") arch("-f", "-p") arch("-b", "-p") def test_arch_operating_system(): """Sanity check ``spack arch --operating-system`` to make sure it works.""" arch("-o") arch("--operating-system") arch("-f", "-o") arch("-b", "-o") def test_arch_target(): """Sanity check ``spack arch --target`` to make sure it works.""" arch("-t") arch("--target") arch("-f", "-t") arch("-b", "-t") def test_display_targets(): arch("--known-targets") ================================================ FILE: lib/spack/spack/test/cmd/audit.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.audit from spack.main import SpackCommand from spack.test.conftest import MockHTTPResponse audit = SpackCommand("audit") @pytest.mark.parametrize( "pkgs,expected_returncode", [ # A single package with issues, should exit 1 (["wrong-variant-in-conflicts"], 1), # A "sane" package should exit 0 (["mpileaks"], 0), # A package with issues and a package without should exit 1 (["wrong-variant-in-conflicts", "mpileaks"], 1), (["mpileaks", "wrong-variant-in-conflicts"], 1), ], ) def test_audit_packages(pkgs, expected_returncode, mutable_config, mock_packages): """Sanity check ``spack audit packages`` to make sure it works.""" audit("packages", *pkgs, fail_on_error=False) assert audit.returncode == expected_returncode def test_audit_configs(mutable_config, mock_packages): """Sanity check ``spack audit packages`` to make sure it works.""" audit("configs", fail_on_error=False) # The mock configuration has duplicate definitions of some compilers assert audit.returncode == 1 def test_audit_packages_https(mutable_config, mock_packages, monkeypatch): """Test audit packages-https with mocked network calls.""" monkeypatch.setattr(spack.audit, "urlopen", lambda url: MockHTTPResponse(200, "OK")) # Without providing --all should fail audit("packages-https", fail_on_error=False) # The mock configuration has duplicate definitions of some compilers assert audit.returncode == 1 # This uses http and should fail audit("packages-https", "test-dependency", fail_on_error=False) assert audit.returncode == 1 # providing one or more package names with https should work audit("packages-https", "cmake", fail_on_error=True) assert audit.returncode == 0 # providing one or more package names with https should work audit("packages-https", "cmake", "conflict", fail_on_error=True) assert audit.returncode == 0 ================================================ FILE: lib/spack/spack/test/cmd/blame.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import os from pathlib import Path import pytest import spack.cmd.blame import spack.paths import spack.util.spack_json as sjson from spack.cmd.blame import ensure_full_history, git_prefix, package_repo_root from spack.llnl.util.filesystem import mkdirp, working_dir from spack.main import SpackCommand, SpackCommandError from spack.repo import RepoDescriptors from spack.util.executable import ProcessError pytestmark = pytest.mark.usefixtures("git") blame = SpackCommand("blame") def test_blame_by_modtime(mock_packages): """Sanity check the blame command to make sure it works.""" out = blame("--time", "mpich") assert "LAST_COMMIT" in out assert "AUTHOR" in out assert "EMAIL" in out def test_blame_by_percent(mock_packages): """Sanity check the blame command to make sure it works.""" out = blame("--percent", "mpich") assert "LAST_COMMIT" in out assert "AUTHOR" in out assert "EMAIL" in out def test_blame_file(): """Sanity check the blame command to make sure it works.""" with working_dir(spack.paths.prefix): out = blame(os.path.join("bin", "spack")) assert "LAST_COMMIT" in out assert "AUTHOR" in out assert "EMAIL" in out def test_blame_file_missing(): """Ensure attempt to get blame for missing file fails.""" with pytest.raises(SpackCommandError): out = blame(os.path.join("missing", "file.txt")) assert "does not exist" in out def test_blame_directory(): """Ensure attempt to get blame for path that is a directory fails.""" with pytest.raises(SpackCommandError): out = blame(".") assert "not tracked" in out def test_blame_file_outside_spack_repo(tmp_path: Path): """Ensure attempts to get blame outside a package repository are flagged.""" test_file = tmp_path / "test" test_file.write_text("This is a test") with pytest.raises(SpackCommandError): out = blame(str(test_file)) assert "not within a spack repo" in out def test_blame_spack_not_git_clone(monkeypatch): """Ensure attempt to get blame when spack not a git clone fails.""" non_git_dir = os.path.join(spack.paths.prefix, "..") monkeypatch.setattr(spack.paths, "prefix", non_git_dir) with pytest.raises(SpackCommandError): out = blame(".") assert "not in a git clone" in out def test_blame_json(mock_packages): """Ensure that we can output json as a blame.""" with working_dir(spack.paths.prefix): out = blame("--json", "mpich") # Test loading the json, and top level keys loaded = sjson.load(out) assert "authors" in out assert "totals" in out # Authors should be a list assert len(loaded["authors"]) > 0 # Each of authors and totals has these shared keys keys = ["last_commit", "lines", "percentage"] for key in keys: assert key in loaded["totals"] # But authors is a list of multiple for key in keys + ["author", "email"]: assert key in loaded["authors"][0] @pytest.mark.not_on_windows("git hangs") def test_blame_by_git(mock_packages): """Sanity check the blame command to make sure it works.""" out = blame("--git", "mpich") assert "class Mpich" in out assert ' homepage = "http://www.mpich.org"' in out def test_repo_root_local_descriptor(mock_git_version_info, monkeypatch): """Sanity check blame's package repository root using a local repo descriptor.""" # create a mock descriptor for the mock local repository MockLocalDescriptor = collections.namedtuple("MockLocalDescriptor", ["path"]) repo_path, filename, _ = mock_git_version_info git_repo_path = Path(repo_path) spack_repo_path = git_repo_path / "spack_repo" spack_repo_path.mkdir() repo_descriptor = MockLocalDescriptor(spack_repo_path) def _from_config(*args, **kwargs): return {"mock": repo_descriptor} monkeypatch.setattr(RepoDescriptors, "from_config", _from_config) # The parent of the git repository is outside the package repo root path = (git_repo_path / "..").resolve() prefix = package_repo_root((path / "..").resolve()) assert prefix is None # The base repository directory is the git root of the package repo prefix = package_repo_root(git_repo_path) assert prefix == git_repo_path # The file under the base repository directory also has the package git root prefix = package_repo_root(git_repo_path / filename) assert prefix == git_repo_path def test_repo_root_remote_descriptor(mock_git_version_info, monkeypatch): """Sanity check blame's package repository root using a remote repo descriptor.""" # create a mock descriptor for the mock local repository MockRemoteDescriptor = collections.namedtuple("MockRemoteDescriptor", ["destination"]) repo_path, filename, _ = mock_git_version_info git_repo_path = Path(repo_path) repo_descriptor = MockRemoteDescriptor(git_repo_path) def _from_config(*args, **kwargs): return {"mock": repo_descriptor} monkeypatch.setattr(RepoDescriptors, "from_config", _from_config) # The parent of the git repository is outside the package repo root path = (git_repo_path / "..").resolve() prefix = package_repo_root((path / "..").resolve()) assert prefix is None # The base repository directory is the git root of the package repo prefix = package_repo_root(git_repo_path) assert prefix == git_repo_path def test_git_prefix_bad(tmp_path: Path): """Exercise git_prefix paths with arguments that will not return success.""" assert git_prefix("no/such/file.txt") is None with pytest.raises(SystemExit): git_prefix(tmp_path) def test_ensure_full_history_shallow_works(mock_git_version_info, monkeypatch): """Ensure a git that "supports" '--unshallow' "completes" without incident.""" def _git(*args, **kwargs): if "--help" in args: return "--unshallow" else: return "" repo_path, filename, _ = mock_git_version_info shallow_dir = os.path.join(repo_path, ".git", "shallow") mkdirp(shallow_dir) # Need to patch the blame command's monkeypatch.setattr(spack.cmd.blame, "git", _git) ensure_full_history(repo_path, filename) def test_ensure_full_history_shallow_fails(mock_git_version_info, monkeypatch, capfd): """Ensure a git that supports '--unshallow' but fails generates useful error.""" error_msg = "Mock git cannot fetch." def _git(*args, **kwargs): if "--help" in args: return "--unshallow" else: raise ProcessError(error_msg) repo_path, filename, _ = mock_git_version_info shallow_dir = os.path.join(repo_path, ".git", "shallow") mkdirp(shallow_dir) # Need to patch the blame command's since 'git' already used by # mock_git_versioninfo monkeypatch.setattr(spack.cmd.blame, "git", _git) with pytest.raises(SystemExit): ensure_full_history(repo_path, filename) out = capfd.readouterr() assert error_msg in out[1] def test_ensure_full_history_shallow_old_git(mock_git_version_info, monkeypatch, capfd): """Ensure a git that doesn't support '--unshallow' fails.""" def _git(*args, **kwargs): return "" repo_path, filename, _ = mock_git_version_info shallow_dir = os.path.join(repo_path, ".git", "shallow") mkdirp(shallow_dir) # Need to patch the blame command's since 'git' already used by # mock_git_versioninfo monkeypatch.setattr(spack.cmd.blame, "git", _git) with pytest.raises(SystemExit): ensure_full_history(repo_path, filename) out = capfd.readouterr() assert "Use a newer" in out[1] ================================================ FILE: lib/spack/spack/test/cmd/bootstrap.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.bootstrap import spack.bootstrap.core import spack.cmd.mirror import spack.concretize import spack.config import spack.environment as ev import spack.main import spack.spec _bootstrap = spack.main.SpackCommand("bootstrap") @pytest.mark.parametrize("scope", [None, "site", "system", "user"]) def test_enable_and_disable(mutable_config, scope): scope_args = [] if scope: scope_args = ["--scope={0}".format(scope)] _bootstrap("enable", *scope_args) assert spack.config.get("bootstrap:enable", scope=scope) is True _bootstrap("disable", *scope_args) assert spack.config.get("bootstrap:enable", scope=scope) is False @pytest.mark.parametrize("scope", [None, "site", "system", "user"]) def test_root_get_and_set(mutable_config, tmp_path, scope): scope_args, path = [], str(tmp_path) if scope: scope_args = ["--scope={0}".format(scope)] _bootstrap("root", path, *scope_args) out = _bootstrap("root", *scope_args) assert out.strip() == path @pytest.mark.parametrize("scopes", [("site",), ("system", "user")]) def test_reset_in_file_scopes(mutable_config, scopes): # Assert files are created in the right scopes bootstrap_yaml_files = [] for s in scopes: _bootstrap("disable", "--scope={0}".format(s)) scope_path = spack.config.CONFIG.scopes[s].path bootstrap_yaml = os.path.join(scope_path, "bootstrap.yaml") assert os.path.exists(bootstrap_yaml) bootstrap_yaml_files.append(bootstrap_yaml) _bootstrap("reset", "-y") for bootstrap_yaml in bootstrap_yaml_files: assert not os.path.exists(bootstrap_yaml) def test_reset_in_environment(mutable_mock_env_path, mutable_config): env = spack.main.SpackCommand("env") env("create", "bootstrap-test") current_environment = ev.read("bootstrap-test") with current_environment: _bootstrap("disable") assert spack.config.get("bootstrap:enable") is False _bootstrap("reset", "-y") # We have no default settings in tests assert spack.config.get("bootstrap:enable") is None # Check that reset didn't delete the entire file spack_yaml = os.path.join(current_environment.path, "spack.yaml") assert os.path.exists(spack_yaml) def test_reset_in_file_scopes_overwrites_backup_files(mutable_config): # Create a bootstrap.yaml with some config _bootstrap("disable", "--scope=site") scope_path = spack.config.CONFIG.scopes["site"].path bootstrap_yaml = os.path.join(scope_path, "bootstrap.yaml") assert os.path.exists(bootstrap_yaml) # Reset the bootstrap configuration _bootstrap("reset", "-y") backup_file = bootstrap_yaml + ".bkp" assert not os.path.exists(bootstrap_yaml) assert os.path.exists(backup_file) # Iterate another time _bootstrap("disable", "--scope=site") assert os.path.exists(bootstrap_yaml) assert os.path.exists(backup_file) _bootstrap("reset", "-y") assert not os.path.exists(bootstrap_yaml) assert os.path.exists(backup_file) def test_list_sources(config): # Get the merged list and ensure we get our defaults output = _bootstrap("list") assert "github-actions" in output # Ask for a specific scope and check that the list of sources is empty output = _bootstrap("list", "--scope", "user") assert "No method available" in output @pytest.mark.parametrize("command,value", [("enable", True), ("disable", False)]) def test_enable_or_disable_sources(mutable_config, command, value): key = "bootstrap:trusted:github-actions" trusted = spack.config.get(key, default=None) assert trusted is None _bootstrap(command, "github-actions") trusted = spack.config.get(key, default=None) assert trusted is value def test_enable_or_disable_fails_with_no_method(mutable_config): with pytest.raises(RuntimeError, match="no bootstrapping method"): _bootstrap("enable", "foo") def test_enable_or_disable_fails_with_more_than_one_method(mutable_config): wrong_config = { "sources": [ {"name": "github-actions", "metadata": "$spack/share/spack/bootstrap/github-actions"}, {"name": "github-actions", "metadata": "$spack/share/spack/bootstrap/github-actions"}, ], "trusted": {}, } with spack.config.override("bootstrap", wrong_config): with pytest.raises(RuntimeError, match="more than one"): _bootstrap("enable", "github-actions") @pytest.mark.parametrize("use_existing_dir", [True, False]) def test_add_failures_for_non_existing_files( mutable_config, tmp_path: pathlib.Path, use_existing_dir ): metadata_dir = str(tmp_path) if use_existing_dir else "/foo/doesnotexist" with pytest.raises(RuntimeError, match="does not exist"): _bootstrap("add", "mock-mirror", metadata_dir) def test_add_failures_for_already_existing_name(mutable_config): with pytest.raises(RuntimeError, match="already exist"): _bootstrap("add", "github-actions", "some-place") def test_remove_failure_for_non_existing_names(mutable_config): with pytest.raises(RuntimeError, match="cannot find"): _bootstrap("remove", "mock-mirror") def test_remove_and_add_a_source(mutable_config): # Check we start with a single bootstrapping source sources = spack.bootstrap.core.bootstrapping_sources() assert len(sources) == 1 # Remove it and check the result _bootstrap("remove", "github-actions") sources = spack.bootstrap.core.bootstrapping_sources() assert not sources # Add it back and check we restored the initial state _bootstrap("add", "github-actions", "$spack/share/spack/bootstrap/github-actions-v2") sources = spack.bootstrap.core.bootstrapping_sources() assert len(sources) == 1 @pytest.mark.maybeslow @pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_bootstrap_mirror_metadata(mutable_config, linux_os, monkeypatch, tmp_path: pathlib.Path): """Test that `spack bootstrap mirror` creates a folder that can be ingested by `spack bootstrap add`. Here we don't download data, since that would be an expensive operation for a unit test. """ old_create = spack.cmd.mirror.create monkeypatch.setattr(spack.cmd.mirror, "create", lambda p, s: old_create(p, [])) monkeypatch.setattr(spack.concretize, "concretize_one", lambda p: spack.spec.Spec(p)) # Create the mirror in a temporary folder compilers = [ { "compiler": { "spec": "gcc@12.0.1", "operating_system": "{0.name}{0.version}".format(linux_os), "modules": [], "paths": { "cc": "/usr/bin", "cxx": "/usr/bin", "fc": "/usr/bin", "f77": "/usr/bin", }, } } ] with spack.config.override("compilers", compilers): _bootstrap("mirror", str(tmp_path)) # Register the mirror metadata_dir = tmp_path / "metadata" / "sources" _bootstrap("add", "--trust", "test-mirror", str(metadata_dir)) assert _bootstrap.returncode == 0 assert any(m["name"] == "test-mirror" for m in spack.bootstrap.core.bootstrapping_sources()) ================================================ FILE: lib/spack/spack/test/cmd/build_env.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pickle import sys import pytest import spack.error from spack.llnl.util.filesystem import working_dir from spack.main import SpackCommand build_env = SpackCommand("build-env") @pytest.mark.parametrize("pkg", [("pkg-c",), ("pkg-c", "--")]) @pytest.mark.usefixtures("config", "mock_packages", "working_env") def test_it_just_runs(pkg): build_env(*pkg) @pytest.mark.usefixtures("config", "mock_packages", "working_env") def test_error_when_multiple_specs_are_given(): output = build_env("libelf libdwarf", fail_on_error=False) assert "only takes one spec" in output @pytest.mark.parametrize("args", [("--", "/bin/sh", "-c", "echo test"), ("--",), ()]) @pytest.mark.usefixtures("config", "mock_packages", "working_env") def test_build_env_requires_a_spec(args): output = build_env(*args, fail_on_error=False) assert "requires a spec" in output _out_file = "env.out" @pytest.mark.parametrize("shell", ["pwsh", "bat"] if sys.platform == "win32" else ["sh"]) @pytest.mark.usefixtures("config", "mock_packages", "working_env") def test_dump(shell_as, shell, tmp_path: pathlib.Path): with working_dir(str(tmp_path)): build_env("--dump", _out_file, "pkg-c") with open(_out_file, encoding="utf-8") as f: if shell == "pwsh": assert any(line.startswith("$Env:PATH") for line in f.readlines()) elif shell == "bat": assert any(line.startswith('set "PATH=') for line in f.readlines()) else: assert any(line.startswith("PATH=") for line in f.readlines()) @pytest.mark.usefixtures("config", "mock_packages", "working_env") def test_pickle(tmp_path: pathlib.Path): with working_dir(str(tmp_path)): build_env("--pickle", _out_file, "pkg-c") environment = pickle.load(open(_out_file, "rb")) assert isinstance(environment, dict) assert "PATH" in environment def test_failure_when_uninstalled_deps(config, mock_packages): with pytest.raises( spack.error.SpackError, match="Not all dependencies of dttop are installed" ): build_env("dttop") ================================================ FILE: lib/spack/spack/test/cmd/buildcache.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import errno import json import os import pathlib import shutil import urllib.parse from datetime import datetime, timedelta from typing import Dict, List import pytest import spack.binary_distribution import spack.buildcache_migrate as migrate import spack.buildcache_prune import spack.cmd.buildcache import spack.concretize import spack.environment as ev import spack.error import spack.main import spack.mirrors.mirror import spack.spec import spack.util.url as url_util import spack.util.web as web_util from spack.installer import PackageInstaller from spack.llnl.util.filesystem import copy_tree, find, getuid from spack.llnl.util.lang import nullcontext from spack.paths import test_path from spack.url_buildcache import ( BuildcacheComponent, URLBuildcacheEntry, URLBuildcacheEntryV2, check_mirror_for_layout, get_url_buildcache_class, ) buildcache = spack.main.SpackCommand("buildcache") install = spack.main.SpackCommand("install") env = spack.main.SpackCommand("env") add = spack.main.SpackCommand("add") gpg = spack.main.SpackCommand("gpg") mirror = spack.main.SpackCommand("mirror") uninstall = spack.main.SpackCommand("uninstall") pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.fixture() def mock_get_specs(database, monkeypatch): specs = database.query_local() monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", lambda: specs) @pytest.fixture() def mock_get_specs_multiarch(database, monkeypatch): specs = [spec.copy() for spec in database.query_local()] # make one spec that is NOT the test architecture for spec in specs: if spec.name == "mpileaks": spec.architecture = spack.spec.ArchSpec("linux-rhel7-x86_64") break monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", lambda: specs) @pytest.mark.db @pytest.mark.regression("13757") def test_buildcache_list_duplicates(mock_get_specs): output = buildcache("list", "mpileaks", "@2.3") assert output.count("mpileaks") == 3 @pytest.mark.db @pytest.mark.regression("17827") def test_buildcache_list_allarch(database, mock_get_specs_multiarch): output = buildcache("list", "--allarch") assert output.count("mpileaks") == 3 output = buildcache("list") assert output.count("mpileaks") == 2 def tests_buildcache_create_env( install_mockery, mock_fetch, tmp_path: pathlib.Path, mutable_mock_env_path ): """ "Ensure that buildcache create creates output files from env""" pkg = "trivial-install-test-package" env("create", "--without-view", "test") with ev.read("test"): add(pkg) install() buildcache("push", "--unsigned", str(tmp_path)) spec = spack.concretize.concretize_one(pkg) mirror_url = tmp_path.as_uri() cache_class = get_url_buildcache_class( layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ) cache_entry = cache_class(mirror_url, spec, allow_unsigned=True) assert cache_entry.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL]) cache_entry.destroy() def test_buildcache_create_fails_on_noargs(tmp_path: pathlib.Path): """Ensure that buildcache create fails when given no args or environment.""" with pytest.raises(spack.main.SpackCommandError): buildcache("push", "--unsigned", str(tmp_path)) @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_buildcache_create_fail_on_perm_denied( install_mockery, mock_fetch, tmp_path: pathlib.Path ): """Ensure that buildcache create fails on permission denied error.""" install("trivial-install-test-package") tmp_path.chmod(0) with pytest.raises(OSError) as error: buildcache("push", "--unsigned", str(tmp_path), "trivial-install-test-package") assert error.value.errno == errno.EACCES tmp_path.chmod(0o700) def test_update_key_index( tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, mock_packages, mock_fetch, mock_stage, mock_gnupghome, ): """Test the update-index command with the --keys option""" working_dir = tmp_path / "working_dir" working_dir.mkdir() mirror_dir = working_dir / "mirror" mirror_url = mirror_dir.as_uri() mirror("add", "test-mirror", mirror_url) gpg("create", "Test Signing Key", "nobody@nowhere.com") s = spack.concretize.concretize_one("libdwarf") # Install a package install("--fake", s.name) # Put installed package in the buildcache, which, because we're signing # it, should result in the public key getting pushed to the buildcache # as well. buildcache("push", str(mirror_dir), s.name) # Now make sure that when we pass the "--keys" argument to update-index # it causes the index to get update. buildcache("update-index", "--keys", str(mirror_dir)) key_dir_list = os.listdir( os.path.join(str(mirror_dir), spack.binary_distribution.buildcache_relative_keys_path()) ) uninstall("-y", s.name) mirror("rm", "test-mirror") assert "keys.manifest.json" in key_dir_list def test_buildcache_autopush(tmp_path: pathlib.Path, install_mockery, mock_fetch): """Test buildcache with autopush""" mirror_dir = tmp_path / "mirror" mirror_autopush_dir = tmp_path / "mirror_autopush" mirror("add", "--unsigned", "mirror", mirror_dir.as_uri()) mirror("add", "--autopush", "--unsigned", "mirror-autopush", mirror_autopush_dir.as_uri()) s = spack.concretize.concretize_one("libdwarf") # Install and generate build cache index PackageInstaller([s.package], fake=True, explicit=True).install() manifest_file = URLBuildcacheEntry.get_manifest_filename(s) specs_dirs = os.path.join( *URLBuildcacheEntry.get_relative_path_components(BuildcacheComponent.SPEC), s.name ) assert not (mirror_dir / specs_dirs / manifest_file).exists() assert (mirror_autopush_dir / specs_dirs / manifest_file).exists() def test_buildcache_sync( mutable_mock_env_path, install_mockery, mock_packages, mock_fetch, mock_stage, tmp_path: pathlib.Path, ): """ Make sure buildcache sync works in an environment-aware manner, ignoring any specs that may be in the mirror but not in the environment. """ working_dir = tmp_path / "working_dir" working_dir.mkdir() src_mirror_dir = working_dir / "src_mirror" src_mirror_url = src_mirror_dir.as_uri() dest_mirror_dir = working_dir / "dest_mirror" dest_mirror_url = dest_mirror_dir.as_uri() in_env_pkg = "trivial-install-test-package" out_env_pkg = "libdwarf" def verify_mirror_contents(): dest_list = os.listdir( os.path.join( str(dest_mirror_dir), spack.binary_distribution.buildcache_relative_specs_path() ) ) found_pkg = False for p in dest_list: assert out_env_pkg not in p if in_env_pkg in p: found_pkg = True assert found_pkg, f"Expected to find {in_env_pkg} in {dest_mirror_dir}" # Install a package and put it in the buildcache s = spack.concretize.concretize_one(out_env_pkg) install("--fake", s.name) buildcache("push", "-u", "-f", src_mirror_url, s.name) env("create", "--without-view", "test") with ev.read("test"): add(in_env_pkg) install() buildcache("push", "-u", "-f", src_mirror_url, in_env_pkg) # Now run the spack buildcache sync command with all the various options # for specifying mirrors # Use urls to specify mirrors buildcache("sync", src_mirror_url, dest_mirror_url) verify_mirror_contents() shutil.rmtree(str(dest_mirror_dir)) # Use local directory paths to specify fs locations buildcache("sync", str(src_mirror_dir), str(dest_mirror_dir)) verify_mirror_contents() shutil.rmtree(str(dest_mirror_dir)) # Use mirror names to specify mirrors mirror("add", "src", src_mirror_url) mirror("add", "dest", dest_mirror_url) mirror("add", "ignored", "file:///dummy/io") buildcache("sync", "src", "dest") verify_mirror_contents() shutil.rmtree(str(dest_mirror_dir)) cache_class = get_url_buildcache_class( layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ) def manifest_insert(manifest, spec, dest_url): manifest[spec.dag_hash()] = { "src": cache_class.get_manifest_url(spec, src_mirror_url), "dest": cache_class.get_manifest_url(spec, dest_url), } manifest_file = str(tmp_path / "manifest_dest.json") with open(manifest_file, "w", encoding="utf-8") as fd: test_env = ev.active_environment() assert test_env is not None manifest: Dict[str, Dict[str, str]] = {} for spec in test_env.specs_by_hash.values(): manifest_insert(manifest, spec, dest_mirror_url) json.dump(manifest, fd) buildcache("sync", "--manifest-glob", manifest_file) verify_mirror_contents() shutil.rmtree(str(dest_mirror_dir)) manifest_file = str(tmp_path / "manifest_bad_dest.json") with open(manifest_file, "w", encoding="utf-8") as fd: manifest_2: Dict[str, Dict[str, str]] = {} for spec in test_env.specs_by_hash.values(): manifest_insert(manifest_2, spec, url_util.join(dest_mirror_url, "invalid_path")) json.dump(manifest_2, fd) # Trigger the warning output = buildcache("sync", "--manifest-glob", manifest_file, "dest", "ignored") assert "Ignoring unused argument: ignored" in output verify_mirror_contents() shutil.rmtree(str(dest_mirror_dir)) def test_buildcache_create_install( mutable_mock_env_path, install_mockery, mock_packages, mock_fetch, mock_stage, tmp_path: pathlib.Path, ): """ "Ensure that buildcache create creates output files""" pkg = "trivial-install-test-package" install(pkg) buildcache("push", "--unsigned", str(tmp_path), pkg) mirror_url = tmp_path.as_uri() spec = spack.concretize.concretize_one(pkg) cache_class = get_url_buildcache_class( layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION ) cache_entry = cache_class(mirror_url, spec, allow_unsigned=True) manifest_path = os.path.join( str(tmp_path), *cache_class.get_relative_path_components(BuildcacheComponent.SPEC), spec.name, cache_class.get_manifest_filename(spec), ) assert os.path.exists(manifest_path) cache_entry.read_manifest() spec_blob_record = cache_entry.get_blob_record(BuildcacheComponent.SPEC) tarball_blob_record = cache_entry.get_blob_record(BuildcacheComponent.TARBALL) spec_blob_path = os.path.join( str(tmp_path), *cache_class.get_blob_path_components(spec_blob_record) ) assert os.path.exists(spec_blob_path) tarball_blob_path = os.path.join( str(tmp_path), *cache_class.get_blob_path_components(tarball_blob_record) ) assert os.path.exists(tarball_blob_path) cache_entry.destroy() def _mock_uploader(tmp_path: pathlib.Path): class DontUpload(spack.binary_distribution.Uploader): def __init__(self): super().__init__( spack.mirrors.mirror.Mirror.from_local_path(str(tmp_path)), False, False ) self.pushed = [] def push(self, specs: List[spack.spec.Spec]): self.pushed.extend(s.name for s in specs) return [], [] return DontUpload() @pytest.mark.parametrize( "things_to_install,expected", [ ( "", [ "dttop", "dtbuild1", "dtbuild2", "dtlink2", "dtrun2", "dtlink1", "dtlink3", "dtlink4", "dtrun1", "dtlink5", "dtrun3", "dtbuild3", ], ), ( "dependencies", [ "dtbuild1", "dtbuild2", "dtlink2", "dtrun2", "dtlink1", "dtlink3", "dtlink4", "dtrun1", "dtlink5", "dtrun3", "dtbuild3", ], ), ("package", ["dttop"]), ], ) def test_correct_specs_are_pushed( things_to_install, expected, tmp_path: pathlib.Path, monkeypatch, default_mock_concretization, temporary_store, ): spec = default_mock_concretization("dttop") PackageInstaller([spec.package], explicit=True, fake=True).install() slash_hash = f"/{spec.dag_hash()}" uploader = _mock_uploader(tmp_path) monkeypatch.setattr( spack.binary_distribution, "make_uploader", lambda *args, **kwargs: uploader ) buildcache_create_args = ["create", "--unsigned"] if things_to_install != "": buildcache_create_args.extend(["--only", things_to_install]) buildcache_create_args.extend([str(tmp_path), slash_hash]) buildcache(*buildcache_create_args) # Order is not guaranteed, so we can't just compare lists assert set(uploader.pushed) == set(expected) # Ensure no duplicates assert len(set(uploader.pushed)) == len(uploader.pushed) @pytest.mark.parametrize("signed", [True, False]) def test_push_and_install_with_mirror_marked_unsigned_does_not_require_extra_flags( tmp_path: pathlib.Path, mutable_database, mock_gnupghome, signed ): """Tests whether marking a mirror as unsigned makes it possible to push and install to/from it without requiring extra flags on the command line (and no signing keys configured).""" # Create a named mirror with signed set to True or False add_flag = "--signed" if signed else "--unsigned" mirror("add", add_flag, "my-mirror", str(tmp_path)) spec = mutable_database.query_local("libelf", installed=True)[0] # Push if signed: # Need to pass "--unsigned" to override the mirror's default args = ["push", "--update-index", "--unsigned", "my-mirror", f"/{spec.dag_hash()}"] else: # No need to pass "--unsigned" if the mirror is unsigned args = ["push", "--update-index", "my-mirror", f"/{spec.dag_hash()}"] buildcache(*args) spec.package.do_uninstall(force=True) PackageInstaller( [spec.package], explicit=True, root_policy="cache_only", dependencies_policy="cache_only", unsigned=True if signed else None, ).install() def test_skip_no_redistribute(mock_packages, config): specs = list(spack.concretize.concretize_one("no-redistribute-dependent").traverse()) filtered = spack.cmd.buildcache._skip_no_redistribute_for_public(specs) assert not any(s.name == "no-redistribute" for s in filtered) assert any(s.name == "no-redistribute-dependent" for s in filtered) def test_best_effort_vs_fail_fast_when_dep_not_installed(tmp_path: pathlib.Path, mutable_database): """When --fail-fast is passed, the push command should fail if it immediately finds an uninstalled dependency. Otherwise, failure to push one dependency shouldn't prevent the others from being pushed.""" mirror("add", "--unsigned", "my-mirror", str(tmp_path)) # Uninstall mpich so that its dependent mpileaks can't be pushed for s in mutable_database.query_local("mpich"): s.package.do_uninstall(force=True) with pytest.raises(spack.cmd.buildcache.PackagesAreNotInstalledError, match="mpich"): buildcache("push", "--update-index", "--fail-fast", "my-mirror", "mpileaks^mpich") # nothing should be pushed due to --fail-fast. assert not os.listdir(tmp_path) assert not spack.binary_distribution.update_cache_and_get_specs() with pytest.raises(spack.cmd.buildcache.PackageNotInstalledError): buildcache("push", "--update-index", "my-mirror", "mpileaks^mpich") specs = spack.binary_distribution.update_cache_and_get_specs() # everything but mpich should be pushed mpileaks = mutable_database.query_local("mpileaks^mpich")[0] assert set(specs) == {s for s in mpileaks.traverse() if s.name != "mpich"} def test_push_without_build_deps( tmp_path: pathlib.Path, temporary_store, mock_packages, mutable_config ): """Spack should not error when build deps are uninstalled and --without-build-dependenies is passed.""" mirror("add", "--unsigned", "my-mirror", str(tmp_path)) s = spack.concretize.concretize_one("dtrun3") PackageInstaller([s.package], explicit=True, fake=True).install() s["dtbuild3"].package.do_uninstall() # fails when build deps are required with pytest.raises(spack.error.SpackError, match="package not installed"): buildcache( "push", "--update-index", "--with-build-dependencies", "my-mirror", f"/{s.dag_hash()}" ) # succeeds when build deps are not required buildcache( "push", "--update-index", "--without-build-dependencies", "my-mirror", f"/{s.dag_hash()}" ) assert spack.binary_distribution.update_cache_and_get_specs() == [s] @pytest.fixture(scope="function") def v2_buildcache_layout(tmp_path: pathlib.Path): def _layout(signedness: str = "signed"): source_path = str(pathlib.Path(test_path) / "data" / "mirrors" / "v2_layout" / signedness) test_mirror_path = tmp_path / "mirror" copy_tree(source_path, test_mirror_path) return test_mirror_path return _layout def test_check_mirror_for_layout(v2_buildcache_layout, mutable_config, capfd): """Check printed warning in the presence of v2 layout binary mirrors""" test_mirror_path = v2_buildcache_layout("unsigned") check_mirror_for_layout(spack.mirrors.mirror.Mirror.from_local_path(str(test_mirror_path))) err = str(capfd.readouterr()[1]) assert all([word in err for word in ["Warning", "missing", "layout"]]) def test_url_buildcache_entry_v2_exists( v2_buildcache_layout, mock_packages, mutable_config, do_not_check_runtimes_on_reuse ): """Test existence check for v2 buildcache entries""" test_mirror_path = v2_buildcache_layout("unsigned") mirror_url = pathlib.Path(test_mirror_path).as_uri() mirror("add", "v2mirror", mirror_url) output = buildcache("list", "-a", "-l") assert "Fetching an index from a v2 binary mirror layout" in output assert "deprecated" in output v2_cache_class = URLBuildcacheEntryV2 # If you don't give it a spec, it returns False build_cache = v2_cache_class(mirror_url) assert not build_cache.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL]) spec = spack.concretize.concretize_one("libdwarf") # In v2 we have to ask for both, because we need to have the spec to have the tarball build_cache = v2_cache_class(mirror_url, spec, allow_unsigned=True) assert not build_cache.exists([BuildcacheComponent.TARBALL]) assert not build_cache.exists([BuildcacheComponent.SPEC]) # But if we do ask for both, they should be there in this case assert build_cache.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL]) spec_path = build_cache._get_spec_url(spec, mirror_url, ext=".spec.json")[7:] tarball_path = build_cache._get_tarball_url(spec, mirror_url)[7:] os.remove(tarball_path) build_cache = v2_cache_class(mirror_url, spec, allow_unsigned=True) assert not build_cache.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL]) os.remove(spec_path) build_cache = v2_cache_class(mirror_url, spec, allow_unsigned=True) assert not build_cache.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL]) @pytest.mark.parametrize("signing", ["unsigned", "signed"]) def test_install_v2_layout( signing, v2_buildcache_layout, mock_packages, mutable_config, mutable_mock_env_path, install_mockery, mock_gnupghome, do_not_check_runtimes_on_reuse, ): """Ensure we can still install from signed and unsigned v2 buildcache""" test_mirror_path = v2_buildcache_layout(signing) mirror("add", "my-mirror", str(test_mirror_path)) # Trust original signing key (no-op if this is the unsigned pass) buildcache("keys", "--install", "--trust") output = install("--fake", "--no-check-signature", "libdwarf") assert "Extracting libelf" in output assert "libelf: Successfully installed" in output assert "Extracting libdwarf" in output assert "libdwarf: Successfully installed" in output assert "Installing a spec from a v2 binary mirror layout" in output assert "Fetching an index from a v2 binary mirror layout" in output assert "deprecated" in output def test_basic_migrate_unsigned(v2_buildcache_layout, mutable_config): """Make sure first unsigned migration results in usable buildcache, leaving the previous layout in place. Also test that a subsequent one doesn't need to migrate anything, and that using --delete-existing removes the previous layout""" test_mirror_path = v2_buildcache_layout("unsigned") mirror("add", "my-mirror", str(test_mirror_path)) output = buildcache("migrate", "--unsigned", "my-mirror") # The output indicates both specs were migrated assert output.count("Successfully migrated") == 6 build_cache_path = str(test_mirror_path / "build_cache") # Without "--delete-existing" and "--yes-to-all", migration leaves the # previous layout in place assert os.path.exists(build_cache_path) assert os.path.isdir(build_cache_path) # Now list the specs available under the new layout output = buildcache("list", "--allarch") assert "libdwarf" in output and "libelf" in output output = buildcache("migrate", "--unsigned", "--delete-existing", "--yes-to-all", "my-mirror") # A second migration of the same mirror indicates neither spec # needs to be migrated assert output.count("No need to migrate") == 6 # When we provide "--delete-existing" and "--yes-to-all", migration # removes the old layout assert not os.path.exists(build_cache_path) def test_basic_migrate_signed(v2_buildcache_layout, mock_gnupghome, mutable_config): """Test a signed migration requires a signing key, requires the public key originally used to sign the pkgs, fails and prints reasonable messages if those requirements are unmet, and eventually succeeds when they are met.""" test_mirror_path = v2_buildcache_layout("signed") mirror("add", "my-mirror", str(test_mirror_path)) with pytest.raises(migrate.MigrationException) as error: buildcache("migrate", "my-mirror") # Without a signing key spack fails and explains why assert error.value.message == "Signed migration requires exactly one secret key in keychain" # Create a signing key and trust the key used to sign the pkgs originally gpg("create", "New Test Signing Key", "noone@nowhere.org") output = buildcache("migrate", "my-mirror") # Without trusting the original signing key, spack fails with an explanation assert "Failed to verify signature of libelf" in output assert "Failed to verify signature of libdwarf" in output assert "did you mean to perform an unsigned migration" in output # Trust original signing key (since it's in the original layout location, # this is where the monkeypatched attribute is used) output = buildcache("keys", "--install", "--trust") output = buildcache("migrate", "my-mirror") # Once we have the proper keys, migration should succeed assert "Successfully migrated libelf" in output assert "Successfully migrated libelf" in output # Now list the specs available under the new layout output = buildcache("list", "--allarch") assert "libdwarf" in output and "libelf" in output def test_unsigned_migrate_of_signed_mirror(v2_buildcache_layout, mutable_config): """Test spack can do an unsigned migration of a signed buildcache by ignoring signatures and skipping re-signing.""" test_mirror_path = v2_buildcache_layout("signed") mirror("add", "my-mirror", str(test_mirror_path)) output = buildcache("migrate", "--unsigned", "--delete-existing", "--yes-to-all", "my-mirror") # Now list the specs available under the new layout output = buildcache("list", "--allarch") assert "libdwarf" in output and "libelf" in output # We should find two spec manifest files, one for each spec file_list = find(test_mirror_path, "*.spec.manifest.json") assert len(file_list) == 6 assert any(["libdwarf" in file for file in file_list]) assert any(["libelf" in file for file in file_list]) # The two spec manifest files should be unsigned for file_path in file_list: with open(file_path, "r", encoding="utf-8") as fd: assert json.load(fd) def test_migrate_requires_index(v2_buildcache_layout, mutable_config): """Test spack fails with a reasonable error message when mirror does not have an index""" test_mirror_path = v2_buildcache_layout("unsigned") v2_index_path = test_mirror_path / "build_cache" / "index.json" v2_index_hash_path = test_mirror_path / "build_cache" / "index.json.hash" os.remove(str(v2_index_path)) os.remove(str(v2_index_hash_path)) mirror("add", "my-mirror", str(test_mirror_path)) with pytest.raises(migrate.MigrationException) as error: buildcache("migrate", "--unsigned", "my-mirror") # If the buildcache has no index, spack fails and explains why assert error.value.message == "Buildcache migration requires a buildcache index" @pytest.mark.parametrize("dry_run", [False, True]) def test_buildcache_prune_no_orphans(tmp_path, mutable_database, mock_gnupghome, dry_run): mirror("add", "--unsigned", "my-mirror", str(tmp_path)) spec = mutable_database.query_local("libelf", installed=True)[0] buildcache("push", "--update-index", "my-mirror", f"/{spec.dag_hash()}") cmd_args = ["prune", "my-mirror"] if dry_run: cmd_args.append("--dry-run") output = buildcache(*cmd_args) assert "0 orphaned objects from mirror:" in output @pytest.mark.parametrize("dry_run", [False, True]) def test_buildcache_prune_orphaned_blobs(tmp_path, mutable_database, mock_gnupghome, dry_run): # Create a mirror and push a package to it mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) spec = mutable_database.query_local("libelf", installed=True)[0] buildcache("push", "--update-index", "my-mirror", f"/{spec.dag_hash()}") cache_entry = URLBuildcacheEntry( mirror_url=f"file://{mirror_directory}", spec=spec, allow_unsigned=True ) blob_urls = [ URLBuildcacheEntry.get_blob_url(mirror_url=f"file://{mirror_directory}", record=blob) for blob in cache_entry.read_manifest().data ] # Remove the manifest from the cache, orphaning the blobs manifest_url = URLBuildcacheEntry.get_manifest_url( spec, mirror_url=f"file://{mirror_directory}" ) web_util.remove_url(manifest_url) # Ensure the blobs are still there before pruning assert all(web_util.url_exists(blob_url) for blob_url in blob_urls) cmd_args = ["prune", "my-mirror"] if dry_run: cmd_args.append("--dry-run") output = buildcache(*cmd_args) # Ensure the blobs are gone after pruning (or not if dry_run is True) assert all(web_util.url_exists(blob_url) == dry_run for blob_url in blob_urls) assert "Found 2 blob(s) with no manifest" in output cache_entry.destroy() @pytest.mark.parametrize("dry_run", [False, True]) def test_buildcache_prune_orphaned_manifest(tmp_path, mutable_database, mock_gnupghome, dry_run): # Create a mirror and push a package to it mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) spec = mutable_database.query_local("libelf", installed=True)[0] buildcache("push", "--update-index", "my-mirror", f"/{spec.dag_hash()}") # Create a cache entry and read the manifest, which should succeed # as we haven't pruned anything yet cache_entry = URLBuildcacheEntry( mirror_url=f"file://{mirror_directory}", spec=spec, allow_unsigned=True ) manifest = cache_entry.read_manifest() manifest_url = f"file://{cache_entry.get_manifest_url(spec=spec, mirror_url=mirror_directory)}" # Remove the blobs from the cache, orphaning the manifest for blob_file in manifest.data: blob_url = cache_entry.get_blob_url(mirror_url=mirror_directory, record=blob_file) web_util.remove_url(url=f"file://{blob_url}") cmd_args = ["prune", "my-mirror"] if dry_run: cmd_args.append("--dry-run") output = buildcache(*cmd_args) # Ensure the manifest is gone after pruning (or not if dry_run is True) assert web_util.url_exists(manifest_url) == dry_run assert "Found 1 manifest(s) that are missing blobs" in output cache_entry.destroy() @pytest.mark.parametrize("dry_run", [False, True]) def test_buildcache_prune_direct_with_keeplist( tmp_path: pathlib.Path, mutable_database, mock_gnupghome, dry_run ): """Test direct pruning functionality with a keeplist file""" mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Install and push multiple packages specs = mutable_database.query_local("libelf", installed=True) spec1 = specs[0] cache_entry = URLBuildcacheEntry( mirror_url=f"file://{mirror_directory}", spec=spec1, allow_unsigned=True ) manifest_url = cache_entry.get_manifest_url(spec1, f"file://{mirror_directory}") # Push the first spec (package only, no dependencies) buildcache("push", "--only", "package", "--update-index", "my-mirror", f"/{spec1.dag_hash()}") # Create a keeplist file that includes only spec1 keeplist_file = tmp_path / "keeplist.txt" keeplist_file.write_text(f"{spec1.dag_hash()}\n") # Run direct pruning cmd_args = ["prune", "my-mirror", "--keeplist", str(keeplist_file)] if dry_run: cmd_args.append("--dry-run") output = buildcache(*cmd_args) # Since all packages are in the keeplist, nothing should be pruned assert web_util.url_exists(manifest_url) assert "No specs to prune - all specs are in the keeplist" in output @pytest.mark.parametrize("dry_run", [False, True]) def test_buildcache_prune_direct_removes_unlisted( tmp_path: pathlib.Path, mutable_database, mock_gnupghome, dry_run ): """Test that direct pruning removes specs not in the keeplist""" mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Install and push a package (package only, no dependencies) specs = mutable_database.query_local("libelf", installed=True) spec1 = specs[0] buildcache("push", "--only", "package", "--update-index", "my-mirror", f"/{spec1.dag_hash()}") # Create a keeplist file that excludes the pushed package keeplist_file = tmp_path / "keeplist.txt" keeplist_file.write_text("0" * 32) cache_entry = URLBuildcacheEntry( mirror_url=f"file://{mirror_directory}", spec=spec1, allow_unsigned=True ) manifest_url = cache_entry.get_manifest_url(spec1, f"file://{mirror_directory}") assert web_util.url_exists(manifest_url) # Run direct pruning cmd_args = ["prune", "my-mirror", "--keeplist", str(keeplist_file)] if dry_run: cmd_args.append("--dry-run") buildcache(*cmd_args) assert web_util.url_exists(manifest_url) == dry_run def test_buildcache_prune_direct_empty_keeplist_fails( tmp_path: pathlib.Path, mutable_database, mock_gnupghome ): """Test that direct pruning fails with an empty keeplist file""" mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Create empty keeplist file keeplist_file = tmp_path / "empty_keeplist.txt" keeplist_file.write_text("") # Should fail with empty keeplist with pytest.raises(spack.buildcache_prune.BuildcachePruningException): buildcache("prune", "my-mirror", "--keeplist", str(keeplist_file)) @pytest.mark.parametrize("dry_run", [False, True]) def test_buildcache_prune_with_invalid_keep_hash( tmp_path: pathlib.Path, mutable_database, mock_gnupghome, dry_run: bool ): mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Create a keeplist file that includes an invalid hash keeplist_file = tmp_path / "keeplist.txt" keeplist_file.write_text("this_is_an_invalid_hash") cmd_args = ["prune", "my-mirror", "--keeplist", str(keeplist_file)] if dry_run: cmd_args.append("--dry-run") with pytest.raises(spack.buildcache_prune.BuildcachePruningException): buildcache(*cmd_args) def test_buildcache_prune_new_specs_race_condition( tmp_path: pathlib.Path, mutable_database, mock_gnupghome, monkeypatch: pytest.MonkeyPatch ): """Test that specs uploaded after pruning begins are protected""" mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) spec = mutable_database.query_local("libelf", installed=True)[0] buildcache("push", "--only", "package", "--update-index", "my-mirror", f"/{spec.dag_hash()}") cache_entry = URLBuildcacheEntry( mirror_url=f"file://{mirror_directory}", spec=spec, allow_unsigned=True ) manifest_url = cache_entry.get_manifest_url(spec, f"file://{mirror_directory}") def mock_stat_url(url: str): """ Mock the stat_url function for testing. For the specific spec created in this test, fake its mtime so that it appears to have been created after the pruning started. """ if url == manifest_url: return 1, datetime.now().timestamp() + timedelta(minutes=10).total_seconds() parsed_url = urllib.parse.urlparse(url) stat_result = pathlib.Path(parsed_url.path).stat() return stat_result.st_size, stat_result.st_mtime monkeypatch.setattr(web_util, "stat_url", mock_stat_url) keeplist_file = tmp_path / "keeplist.txt" keeplist_file.write_text("0" * 32) # Run end-to-end buildcache prune - this should not delete `libelf`, despite it # not being in the keeplist, because its mtime is after the pruning started assert web_util.url_exists(manifest_url) buildcache("prune", "my-mirror", "--keeplist", str(keeplist_file)) assert web_util.url_exists(manifest_url) def create_env_from_concrete_spec(spec: spack.spec.Spec): """Build cache index view source is current active environment""" # Create a unique environment for this spec only env_name = f"specenv-{spec.dag_hash()}" if not ev.exists(env_name): env("create", "--without-view", env_name) e = ev.environment_from_name_or_dir(env_name) with e: add(f"{spec.name}/{spec.dag_hash()}") # This should handle updating the environment to mark all packages as installed install() return e def args_for_active_env(spec: spack.spec.Spec): """Build cache index view source is an active environment""" env = create_env_from_concrete_spec(spec) return [env, []] def args_for_env_by_path(spec: spack.spec.Spec): """Build cache index view source is an environment path""" env = create_env_from_concrete_spec(spec) return [nullcontext(), [env.path]] def args_for_env_by_name(spec: spack.spec.Spec): """Build cache index view source is a managed environment name""" env = create_env_from_concrete_spec(spec) return [nullcontext(), [env.name]] def read_specs_in_index(mirror_directory, view): """Read specs hashes from a build cache index (ie. a database file) This assumes the layout of the database holds the spec hashes under: database -> installs -> hashes... """ mirror_metadata = spack.binary_distribution.MirrorMetadata( f"file://{mirror_directory}", spack.mirrors.mirror.SUPPORTED_LAYOUT_VERSIONS[0], view ) fetcher = spack.binary_distribution.DefaultIndexFetcher(mirror_metadata, None) result = fetcher.conditional_fetch() db_dict = json.loads(result.data) return set([h for h in db_dict["database"]["installs"]]) def test_buildcache_create_view_failure(tmp_path, mutable_config, mutable_mock_env_path): mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # No spec sources should raise an exception with pytest.raises(spack.main.SpackCommandError): command_args = ["update-index", "--name", "test_view", "my-mirror"] buildcache(*command_args) # spec sources should raise an exception expect = "no such environment 'DEADBEEF'" with pytest.raises(spack.error.SpackError, match=expect): command_args = ["update-index", "--name", "test_view", "my-mirror", "DEADBEEF"] buildcache(*command_args) def test_buildcache_create_view_empty( tmp_path, mutable_config, mutable_database, mutable_mock_env_path ): mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Push a spec to the cache mpileaks_specs = mutable_database.query_local("mpileaks") buildcache("push", "my-mirror", mpileaks_specs[0].format("{/hash}")) # Make sure the view doesn't exist yet with pytest.raises(spack.binary_distribution.FetchIndexError): hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Write a minimal lockfile (this is not validated/read by an environment) empty_manifest = tmp_path / "emptylock" / "spack.yaml" empty_manifest.parent.mkdir(exist_ok=False) empty_manifest.write_text("spack: {}", encoding="utf-8") empty_lockfile = tmp_path / "emptylock" / "spack.lock" empty_lockfile.write_text( '{"_meta": {"lockfile-version": 1}, "roots": [], "concrete_specs": {}}', encoding="utf-8" ) # Create a view with no specs command_args = [ "update-index", "--force", "--name", "test_view", "my-mirror", str(empty_lockfile.parent), ] out = buildcache(*command_args) assert "No specs found for view, creating an empty index" in out hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Assert there are no hashes in the view assert not hashes_in_view @pytest.mark.parametrize( "source_args", (args_for_active_env, args_for_env_by_path, args_for_env_by_name) ) def test_buildcache_create_view( tmp_path, mutable_config, mutable_database, mutable_mock_env_path, source_args ): mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Push all of the mpileaks packages to the cache mpileaks_specs = mutable_database.query_local("mpileaks") for s in mpileaks_specs: buildcache("push", "my-mirror", s.format("{/hash}")) # Grab all of the hashes for each mpileaks spec mpileaks_0_hashes = set([s.dag_hash() for s in mpileaks_specs[0].traverse()]) # Make sure the view doesn't exist yet with pytest.raises(spack.binary_distribution.FetchIndexError): hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Create a view using a parametrized source that contains one of the mpileaks context, extra_args = source_args(mpileaks_specs[0]) with context: command_args = ["update-index", "--name", "test_view", "my-mirror"] + extra_args buildcache(*command_args) hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes assert hashes_in_view == mpileaks_0_hashes # Test fail to overwrite without force expect = "Index already exists. To overwrite or update pass --force or --append respectively" with pytest.raises(spack.error.SpackError, match=expect): command_args = [ "update-index", "--name", "test_view", "my-mirror", mpileaks_specs[2].format("{/hash}"), ] buildcache(*command_args) hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes assert hashes_in_view == mpileaks_0_hashes @pytest.mark.parametrize( "source_args", (args_for_active_env, args_for_env_by_path, args_for_env_by_name) ) def test_buildcache_create_view_append( tmp_path, mutable_config, mutable_database, mutable_mock_env_path, source_args ): mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Push all of the mpileaks packages to the cache mpileaks_specs = mutable_database.query_local("mpileaks") for s in mpileaks_specs: buildcache("push", "my-mirror", s.format("{/hash}")) # Grab all of the hashes for each mpileaks spec mpileaks_0_hashes = set([s.dag_hash() for s in mpileaks_specs[0].traverse()]) mpileaks_1_hashes = set([s.dag_hash() for s in mpileaks_specs[1].traverse()]) # Make sure the view doesn't exist yet with pytest.raises(spack.binary_distribution.FetchIndexError): hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Test append to empty index view context, extra_args = source_args(mpileaks_specs[0]) with context: command_args = [ "update-index", "-y", "--append", "--name", "test_view", "my-mirror", ] + extra_args buildcache(*command_args) hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes assert hashes_in_view == mpileaks_0_hashes # Test append to existing index view context, extra_args = source_args(mpileaks_specs[1]) with context: command_args = [ "update-index", "-y", "--append", "--name", "test_view", "my-mirror", ] + extra_args buildcache(*command_args) hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Assert all of the hashes for mpileaks_0_hashes and mpileaks_1_hashes exist in the view, # and no other hashes assert hashes_in_view == (mpileaks_0_hashes | mpileaks_1_hashes) @pytest.mark.parametrize( "source_args", (args_for_active_env, args_for_env_by_path, args_for_env_by_name) ) def test_buildcache_create_view_overwrite( tmp_path, mutable_config, mutable_database, mutable_mock_env_path, source_args ): mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Push all of the mpileaks packages to the cache mpileaks_specs = mutable_database.query_local("mpileaks") for s in mpileaks_specs: buildcache("push", "my-mirror", s.format("{/hash}")) # Grab all of the hashes for each mpileaks spec mpileaks_0_hashes = set([s.dag_hash() for s in mpileaks_specs[0].traverse()]) mpileaks_1_hashes = set([s.dag_hash() for s in mpileaks_specs[1].traverse()]) # Make sure the view doesn't exist yet with pytest.raises(spack.binary_distribution.FetchIndexError): hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Create a view using a parametrized source that contains one of the mpileaks context, extra_args = source_args(mpileaks_specs[0]) with context: command_args = ["update-index", "--name", "test_view", "my-mirror"] + extra_args buildcache(*command_args) hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes assert hashes_in_view == mpileaks_0_hashes # Override a view with force context, extra_args = source_args(mpileaks_specs[1]) with context: command_args = ["update-index", "--force", "--name", "test_view", "my-mirror"] + extra_args buildcache(*command_args) hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Assert all of the hashes for mpileaks_1_hashes exist in the view, and no other hashes assert hashes_in_view == mpileaks_1_hashes def test_buildcache_create_view_non_active_env( tmp_path, mutable_config, mutable_database, mutable_mock_env_path ): mirror_directory = str(tmp_path) mirror("add", "--unsigned", "my-mirror", mirror_directory) # Push all of the mpileaks packages to the cache mpileaks_specs = mutable_database.query_local("mpileaks") for s in mpileaks_specs: buildcache("push", "my-mirror", s.format("{/hash}")) # Grab all of the hashes for each mpileaks spec mpileaks_0_hashes = set([s.dag_hash() for s in mpileaks_specs[0].traverse()]) # Make sure the view doesn't exist yet with pytest.raises(spack.binary_distribution.FetchIndexError): hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Create a view from an environment name that is different from the current active environment _, extra_args = args_for_env_by_name( mpileaks_specs[0] ) # Get args for env by name using mpileaks[0] context, _ = args_for_active_env( mpileaks_specs[1] ) # Get the context for an active env using mpileaks[1] with context: command_args = ["update-index", "--name", "test_view", "my-mirror"] + extra_args buildcache(*command_args) hashes_in_view = read_specs_in_index(mirror_directory, "test_view") # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes assert hashes_in_view == mpileaks_0_hashes @pytest.mark.parametrize("view", (None, "test_view")) @pytest.mark.disable_clean_stage_check def test_buildcache_check_index_full( tmp_path, mutable_config, mutable_database, mutable_mock_env_path, view ): view_args = ["--name", view] if view is not None else [] mirror_directory = str(tmp_path) mirror("add", "--unsigned", *view_args, "my-mirror", mirror_directory) # Push all of the mpileaks packages to the cache mpileaks_specs = mutable_database.query_local("mpileaks") for s in mpileaks_specs: buildcache("push", "my-mirror", s.format("{/hash}")) # Update index using and active environment containing mpileaks[0] context, extra_args = args_for_active_env(mpileaks_specs[0]) with context: buildcache("update-index", "my-mirror", *extra_args) out = buildcache("check-index", "--verify", "exists", "manifests", "blobs", "--", "my-mirror") # Everything thing be good here assert "Index exists in mirror: my-mirror" assert "Missing specs: 0" assert "Missing blobs: 0" if view: assert "Unindexed specs: n/a" in out else: assert "Unindexed specs: 0" in out # Remove the index blob # This creates index not for both view and non-view indices # For non-view indices this is also a missing blob index_name = "index.manifest.json" if view: index_name = os.path.join(view, index_name) blob_path = tmp_path / "blobs" / "sha256" with open(tmp_path / "v3" / "manifests" / "index" / index_name, "r", encoding="utf-8") as fd: manifest = json.load(fd) print(manifest) digest = manifest["data"][0]["checksum"] blob_path = blob_path / digest[:2] / digest # Delete the index manifest os.remove(blob_path) out = buildcache("check-index", "--verify", "exists", "manifests", "blobs", "--", "my-mirror") # Everything thing be good here assert "Index does not exist in mirror: my-mirror" assert "Missing specs: 0" if view: assert "Unindexed specs: n/a" in out assert "Missing blobs: 0" else: assert "The index blob is missing" in out assert "Unindexed specs: 15" in out assert "Missing blobs: 1" def test_buildcache_push_with_group( tmp_path: pathlib.Path, monkeypatch, install_mockery, mock_fetch, mutable_mock_env_path ): """Tests that --group pushes only specs from the requested group.""" env_dir = tmp_path / "myenv" env_dir.mkdir() (env_dir / "spack.yaml").write_text( """\ spack: specs: - libelf - group: extra specs: - libdwarf view: false """ ) mirror_dir = tmp_path / "mirror" mirror_dir.mkdir() uploader = _mock_uploader(mirror_dir) monkeypatch.setattr( spack.binary_distribution, "make_uploader", lambda *args, **kwargs: uploader ) with ev.Environment(env_dir) as e: e.concretize() e.write() for _, root in e.concretized_specs(): PackageInstaller([root.package], explicit=True, fake=True).install() buildcache("push", "--unsigned", "--only", "package", "--group", "extra", str(mirror_dir)) assert uploader.pushed == ["libdwarf"] def test_buildcache_push_with_multiple_groups( tmp_path: pathlib.Path, monkeypatch, install_mockery, mock_fetch, mutable_mock_env_path ): """Tests that --group can be repeated to push specs from multiple groups.""" env_dir = tmp_path / "myenv" env_dir.mkdir() (env_dir / "spack.yaml").write_text( """\ spack: specs: - libelf - group: extra specs: - libdwarf view: false """ ) mirror_dir = tmp_path / "mirror" mirror_dir.mkdir() uploader = _mock_uploader(mirror_dir) monkeypatch.setattr( spack.binary_distribution, "make_uploader", lambda *args, **kwargs: uploader ) with ev.Environment(env_dir) as e: e.concretize() e.write() for _, root in e.concretized_specs(): PackageInstaller([root.package], explicit=True, fake=True).install() buildcache( "push", "--unsigned", "--only", "package", "--group", "default", "--group", "extra", str(mirror_dir), ) assert set(uploader.pushed) == {"libelf", "libdwarf"} assert len(uploader.pushed) == len(set(uploader.pushed)) def test_buildcache_push_group_nonexistent_errors(tmp_path: pathlib.Path, mutable_mock_env_path): """Tests that --group with a nonexistent group name raises an error.""" env_dir = tmp_path / "myenv" env_dir.mkdir() (env_dir / "spack.yaml").write_text( """\ spack: specs: - libelf view: false """ ) mirror_dir = tmp_path / "mirror" mirror_dir.mkdir() with ev.Environment(env_dir): with pytest.raises(spack.main.SpackCommandError): buildcache( "push", "--unsigned", "--group", "nonexistent", str(mirror_dir), fail_on_error=True ) def test_buildcache_push_group_and_specs_mutually_exclusive( tmp_path: pathlib.Path, mutable_mock_env_path ): """Tests that --group and explicit specs on the command line are mutually exclusive.""" env_dir = tmp_path / "myenv" env_dir.mkdir() (env_dir / "spack.yaml").write_text( """\ spack: specs: - libelf view: false """ ) mirror_dir = tmp_path / "mirror" mirror_dir.mkdir() with ev.Environment(env_dir): with pytest.raises(spack.main.SpackCommandError): buildcache( "push", "--unsigned", "--group", "default", str(mirror_dir), "libelf", fail_on_error=True, ) def test_buildcache_push_group_requires_active_env(tmp_path: pathlib.Path): """Tests that ck--group without an active environment produces an error.""" mirror_dir = tmp_path / "mirror" mirror_dir.mkdir() with pytest.raises(spack.main.SpackCommandError): buildcache("push", "--unsigned", "--group", "default", str(mirror_dir), fail_on_error=True) ================================================ FILE: lib/spack/spack/test/cmd/cd.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.main import SpackCommand cd = SpackCommand("cd") def test_cd(): """Sanity check the cd command to make sure it works.""" out = cd() assert "To set up shell support" in out ================================================ FILE: lib/spack/spack/test/cmd/checksum.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import pathlib import pytest import spack.cmd.checksum import spack.concretize import spack.error import spack.package_base import spack.repo import spack.stage import spack.util.web from spack.main import SpackCommand from spack.package_base import ManualDownloadRequiredError from spack.stage import interactive_version_filter from spack.version import Version spack_checksum = SpackCommand("checksum") @pytest.fixture def no_add(monkeypatch): def add_versions_to_pkg(pkg, version_lines, open_in_editor): raise AssertionError("Should not be called") monkeypatch.setattr(spack.cmd.checksum, "add_versions_to_pkg", add_versions_to_pkg) @pytest.fixture def can_fetch_versions(monkeypatch, no_add): """Fake successful version detection.""" def fetch_remote_versions(pkg, concurrency): return {Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz"} def get_checksums_for_versions(url_by_version, package_name, **kwargs): return { v: "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" for v in url_by_version } def url_exists(url, curl=None): return True monkeypatch.setattr( spack.package_base.PackageBase, "fetch_remote_versions", fetch_remote_versions ) monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions) monkeypatch.setattr(spack.util.web, "url_exists", url_exists) @pytest.fixture def cannot_fetch_versions(monkeypatch, no_add): """Fake unsuccessful version detection.""" def fetch_remote_versions(pkg, concurrency): return {} def get_checksums_for_versions(url_by_version, package_name, **kwargs): return {} def url_exists(url, curl=None): return False monkeypatch.setattr( spack.package_base.PackageBase, "fetch_remote_versions", fetch_remote_versions ) monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions) monkeypatch.setattr(spack.util.web, "url_exists", url_exists) @pytest.mark.parametrize( "arguments,expected", [ (["--batch", "patch"], (True, False, False, False, False)), (["--latest", "patch"], (False, True, False, False, False)), (["--preferred", "patch"], (False, False, True, False, False)), (["--add-to-package", "patch"], (False, False, False, True, False)), (["--verify", "patch"], (False, False, False, False, True)), ], ) def test_checksum_args(arguments, expected): parser = argparse.ArgumentParser() spack.cmd.checksum.setup_parser(parser) args = parser.parse_args(arguments) check = args.batch, args.latest, args.preferred, args.add_to_package, args.verify assert check == expected @pytest.mark.parametrize( "arguments,expected", [ (["--batch", "preferred-test"], "version of preferred-test"), (["--latest", "preferred-test"], "Found 1 version"), (["--preferred", "preferred-test"], "Found 1 version"), (["--verify", "preferred-test"], "Verified 1 of 1"), (["--verify", "zlib", "1.2.13"], "1.2.13 [-] No previous checksum"), ], ) def test_checksum(arguments, expected, mock_packages, can_fetch_versions): output = spack_checksum(*arguments) assert expected in output # --verify doesn't print versions strings like other flags if "--verify" not in arguments: assert "version(" in output def input_from_commands(*commands): """Create a function that returns the next command from a list of inputs for interactive spack checksum. If None is encountered, this is equivalent to EOF / ^D.""" commands = iter(commands) def _input(prompt): cmd = next(commands) if cmd is None: raise EOFError assert isinstance(cmd, str) return cmd return _input def test_checksum_interactive_filter(): # Filter effectively by 1:1.0, then checksum. input = input_from_commands("f", "@1:", "f", "@:1.0", "c") assert interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, input=input, ) == { Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", } def test_checksum_interactive_return_from_filter_prompt(): # Enter and then exit filter subcommand. input = input_from_commands("f", None, "c") assert interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, input=input, ) == { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", } def test_checksum_interactive_quit_returns_none(): # Quit after filtering something out (y to confirm quit) input = input_from_commands("f", "@1:", "q", "y") assert ( interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, input=input, ) is None ) def test_checksum_interactive_reset_resets(): # Filter 1:, then reset, then filter :0, should just given 0.9 (it was filtered out # before reset) input = input_from_commands("f", "@1:", "r", "f", ":0", "c") assert interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, input=input, ) == {Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz"} def test_checksum_interactive_ask_each(): # Ask each should run on the filtered list. First select 1.x, then select only the second # entry, which is 1.0.1. input = input_from_commands("f", "@1:", "a", "n", "y", "n") assert interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, input=input, ) == {Version("1.0.1"): "https://www.example.com/pkg-1.0.1.tar.gz"} def test_checksum_interactive_quit_from_ask_each(): # Enter ask each mode, select the second item, then quit from submenu, then checksum, which # should still include the last item at which ask each stopped. input = input_from_commands("a", "n", "y", None, "c") assert interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, input=input, ) == { Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", } def test_checksum_interactive_nothing_left(): """If nothing is left after interactive filtering, return an empty dict.""" input = input_from_commands("f", "@2", "c") assert ( interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, input=input, ) == {} ) def test_checksum_interactive_new_only(): # The 1.0 version is known already, and should be dropped on `n`. input = input_from_commands("n", "c") assert interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, known_versions=[Version("1.0")], input=input, ) == { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", } def test_checksum_interactive_top_n(): """Test integers select top n versions""" input = input_from_commands("2", "c") assert interactive_version_filter( { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", Version("0.9"): "https://www.example.com/pkg-0.9.tar.gz", }, input=input, ) == { Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz", Version("1.0"): "https://www.example.com/pkg-1.0.tar.gz", } def test_checksum_interactive_unrecognized_command(): """Unrecognized commands should be ignored""" input = input_from_commands("-1", "0", "hello", "c") v = {Version("1.1"): "https://www.example.com/pkg-1.1.tar.gz"} assert interactive_version_filter(v.copy(), input=input) == v def test_checksum_versions(mock_packages, can_fetch_versions, monkeypatch): pkg_cls = spack.repo.PATH.get_pkg_class("zlib") versions = [str(v) for v in pkg_cls.versions] output = spack_checksum("zlib", *versions) assert "Found 3 versions" in output assert "version(" in output def test_checksum_missing_version(mock_packages, cannot_fetch_versions): output = spack_checksum("preferred-test", "99.99.99", fail_on_error=False) assert "Could not find any remote versions" in output output = spack_checksum("--add-to-package", "preferred-test", "99.99.99", fail_on_error=False) assert "Could not find any remote versions" in output def test_checksum_deprecated_version(mock_packages, can_fetch_versions): output = spack_checksum("deprecated-versions", "1.1.0", fail_on_error=False) assert "Version 1.1.0 is deprecated" in output output = spack_checksum( "--add-to-package", "deprecated-versions", "1.1.0", fail_on_error=False ) assert "Version 1.1.0 is deprecated" in output def test_checksum_url(mock_packages, config): pkg_cls = spack.repo.PATH.get_pkg_class("zlib") with pytest.raises(spack.error.SpecSyntaxError): spack_checksum(f"{pkg_cls.url}") def test_checksum_verification_fails(default_mock_concretization, capfd, can_fetch_versions): spec = spack.concretize.concretize_one("zlib") pkg = spec.package versions = list(pkg.versions.keys()) version_hashes = {versions[0]: "abadhash", Version("0.1"): "123456789"} with pytest.raises(SystemExit): spack.cmd.checksum.print_checksum_status(pkg, version_hashes) out = str(capfd.readouterr()) assert out.count("Correct") == 0 assert "No previous checksum" in out assert "Invalid checksum" in out def test_checksum_manual_download_fails(mock_packages, monkeypatch): """Confirm that checksumming a manually downloadable package fails.""" name = "zlib" pkg_cls = spack.repo.PATH.get_pkg_class(name) versions = [str(v) for v in pkg_cls.versions] monkeypatch.setattr(spack.package_base.PackageBase, "manual_download", True) # First check that the exception is raised with the default download # instructions. with pytest.raises(ManualDownloadRequiredError, match=f"required for {name}"): spack_checksum(name, *versions) # Now check that the exception is raised with custom download instructions. error = "Cannot calculate the checksum for a manually downloaded package." monkeypatch.setattr(spack.package_base.PackageBase, "download_instr", error) with pytest.raises(ManualDownloadRequiredError, match=error): spack_checksum(name, *versions) def test_upate_package_contents(tmp_path: pathlib.Path): """Test that the package.py file is updated with the new versions.""" pkg_path = tmp_path / "package.py" pkg_path.write_text( """\ from spack.package import * class Zlib(Package): homepage = "http://zlib.net" url = "http://zlib.net/fossils/zlib-1.2.11.tar.gz" version("1.2.11", sha256="c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1") version("1.2.8", sha256="36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d") version("1.2.3", sha256="1795c7d067a43174113fdf03447532f373e1c6c57c08d61d9e4e9be5e244b05e") variant("pic", default=True, description="test") def install(self, spec, prefix): make("install") """ ) version_lines = """\ version("1.2.13", sha256="abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890") version("1.2.5", sha256="abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890") version("1.2.3", sha256="1795c7d067a43174113fdf03447532f373e1c6c57c08d61d9e4e9be5e244b05e") """ # ruff: disable[E501] # two new versions are added assert spack.cmd.checksum.add_versions_to_pkg(str(pkg_path), version_lines) == 2 assert ( pkg_path.read_text() == """\ from spack.package import * class Zlib(Package): homepage = "http://zlib.net" url = "http://zlib.net/fossils/zlib-1.2.11.tar.gz" version("1.2.13", sha256="abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890") # FIXME version("1.2.11", sha256="c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1") version("1.2.8", sha256="36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d") version("1.2.5", sha256="abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890") # FIXME version("1.2.3", sha256="1795c7d067a43174113fdf03447532f373e1c6c57c08d61d9e4e9be5e244b05e") variant("pic", default=True, description="test") def install(self, spec, prefix): make("install") """ ) # ruff: enable[E501] ================================================ FILE: lib/spack/spack/test/cmd/ci.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import json import os import pathlib import shutil from typing import NamedTuple import pytest import spack.vendor.jsonschema import spack.binary_distribution import spack.ci as ci import spack.cmd import spack.cmd.ci import spack.concretize import spack.environment as ev import spack.hash_types as ht import spack.main import spack.paths import spack.repo import spack.spec import spack.stage import spack.util.spack_yaml as syaml import spack.util.web import spack.version from spack.ci import gitlab as gitlab_generator from spack.ci.common import PipelineDag, PipelineOptions, SpackCIConfig from spack.ci.generator_registry import generator from spack.cmd.ci import FAILED_CREATE_BUILDCACHE_CODE from spack.error import SpackError from spack.llnl.util.filesystem import mkdirp, working_dir from spack.schema.database_index import schema as db_idx_schema from spack.test.conftest import MockHTTPResponse, RepoBuilder config_cmd = spack.main.SpackCommand("config") ci_cmd = spack.main.SpackCommand("ci") env_cmd = spack.main.SpackCommand("env") mirror_cmd = spack.main.SpackCommand("mirror") gpg_cmd = spack.main.SpackCommand("gpg") install_cmd = spack.main.SpackCommand("install") uninstall_cmd = spack.main.SpackCommand("uninstall") buildcache_cmd = spack.main.SpackCommand("buildcache") pytestmark = [ pytest.mark.usefixtures("mock_packages"), pytest.mark.not_on_windows("does not run on windows"), pytest.mark.maybeslow, ] @pytest.fixture() def ci_base_environment(working_env, tmp_path: pathlib.Path): os.environ["CI_PROJECT_DIR"] = str(tmp_path) os.environ["CI_PIPELINE_ID"] = "7192" os.environ["CI_JOB_NAME"] = "mock" @pytest.fixture(scope="function") def mock_git_repo(git, tmp_path: pathlib.Path): """Create a mock git repo with two commits, the last one creating a .gitlab-ci.yml""" repo_path = str(tmp_path / "mockspackrepo") mkdirp(repo_path) with working_dir(repo_path): git("init") git("config", "--local", "user.email", "testing@spack.io") git("config", "--local", "user.name", "Spack Testing") # This path is used to satisfy git root detection and detection of environment changed path_to_env = os.path.sep.join(("no", "such", "env", "path", "spack.yaml")) os.makedirs(os.path.dirname(path_to_env)) with open(path_to_env, "w", encoding="utf-8") as f: f.write( """ spack: specs: - a """ ) git("add", path_to_env) with open("README.md", "w", encoding="utf-8") as f: f.write("# Introduction") # initial commit with README git("add", "README.md") git("-c", "commit.gpgsign=false", "commit", "-m", "initial commit") with open(".gitlab-ci.yml", "w", encoding="utf-8") as f: f.write( """ testjob: script: - echo "success" """ ) # second commit, adding a .gitlab-ci.yml git("add", ".gitlab-ci.yml") git("-c", "commit.gpgsign=false", "commit", "-m", "add a .gitlab-ci.yml") yield repo_path @pytest.fixture() def ci_generate_test( tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, ci_base_environment ): """Returns a function that creates a new test environment, and runs 'spack generate' on it, given the content of the spack.yaml file. Additional positional arguments will be added to the 'spack generate' call. """ def _func(spack_yaml_content, *args, fail_on_error=True): spack_yaml = tmp_path / "spack.yaml" spack_yaml.write_text(spack_yaml_content) ev.create("test", init_file=spack_yaml, with_view=False) outputfile = tmp_path / ".gitlab-ci.yml" with ev.read("test"): output = ci_cmd( "generate", "--output-file", str(outputfile), *args, fail_on_error=fail_on_error ) return spack_yaml, outputfile, output return _func def test_ci_generate_with_env(ci_generate_test, tmp_path: pathlib.Path, mock_binary_index): """Make sure we can get a .gitlab-ci.yml from an environment file which has the gitlab-ci, cdash, and mirrors sections. """ mirror_url = tmp_path / "ci-mirror" spack_yaml, outputfile, _ = ci_generate_test( f"""\ spack: definitions: - old-gcc-pkgs: - archive-files - callpath # specify ^openblas-with-lapack to ensure that builtin.mock repo flake8 # package (which can also provide lapack) is not chosen, as it violates # a package-level check which requires exactly one fetch strategy (this # is apparently not an issue for other tests that use it). - hypre@0.2.15 ^openblas-with-lapack specs: - matrix: - [$old-gcc-pkgs] mirrors: buildcache-destination: {mirror_url} ci: pipeline-gen: - submapping: - match: - arch=test-debian6-core2 build-job: tags: - donotcare image: donotcare - match: - arch=test-debian6-m1 build-job: tags: - donotcare image: donotcare - cleanup-job: image: donotcare tags: [donotcare] - reindex-job: script:: [hello, world] custom_attribute: custom! cdash: build-group: Not important url: https://my.fake.cdash project: Not used site: Nothing """, "--artifacts-root", str(tmp_path / "my_artifacts_root"), ) yaml_contents = syaml.load(outputfile.read_text()) assert "workflow" in yaml_contents assert "rules" in yaml_contents["workflow"] assert yaml_contents["workflow"]["rules"] == [{"when": "always"}] assert "stages" in yaml_contents assert len(yaml_contents["stages"]) == 6 assert yaml_contents["stages"][0] == "stage-0" assert yaml_contents["stages"][5] == "stage-rebuild-index" assert "rebuild-index" in yaml_contents rebuild_job = yaml_contents["rebuild-index"] assert ( rebuild_job["script"][0] == f"spack -v buildcache update-index --keys {mirror_url.as_uri()}" ) assert rebuild_job["custom_attribute"] == "custom!" assert "variables" in yaml_contents assert "SPACK_ARTIFACTS_ROOT" in yaml_contents["variables"] assert yaml_contents["variables"]["SPACK_ARTIFACTS_ROOT"] == "my_artifacts_root" def test_ci_generate_with_env_missing_section( ci_generate_test, tmp_path: pathlib.Path, mock_binary_index ): """Make sure we get a reasonable message if we omit gitlab-ci section""" env_yaml = f"""\ spack: specs: - archive-files mirrors: buildcache-destination: {tmp_path / "ci-mirror"} """ expect = "Environment does not have a `ci` configuration" with pytest.raises(ci.SpackCIError, match=expect): ci_generate_test(env_yaml) def test_ci_generate_with_cdash_token( ci_generate_test, tmp_path: pathlib.Path, mock_binary_index, monkeypatch ): """Make sure we it doesn't break if we configure cdash""" monkeypatch.setenv("SPACK_CDASH_AUTH_TOKEN", "notreallyatokenbutshouldnotmatter") spack_yaml_content = f"""\ spack: specs: - archive-files mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare cdash: build-group: Not important url: {(tmp_path / "cdash").as_uri()} project: Not used site: Nothing """ def _urlopen(*args, **kwargs): return MockHTTPResponse.with_json(200, "OK", headers={}, body={}) monkeypatch.setattr(ci.common, "_urlopen", _urlopen) spack_yaml, original_file, output = ci_generate_test(spack_yaml_content) yaml_contents = syaml.load(original_file.read_text()) # That fake token should have resulted in being unable to # register build group with cdash, but the workload should # still have been generated. assert "Failed to create or retrieve buildgroup" in output expected_keys = ["rebuild-index", "stages", "variables", "workflow"] assert all([key in yaml_contents.keys() for key in expected_keys]) def test_ci_generate_with_custom_settings( ci_generate_test, tmp_path: pathlib.Path, mock_binary_index, monkeypatch ): """Test use of user-provided scripts and attributes""" monkeypatch.setattr(spack, "get_version", lambda: "0.15.3") monkeypatch.setattr(spack, "get_spack_commit", lambda: "big ol commit sha") spack_yaml, outputfile, _ = ci_generate_test( f"""\ spack: specs: - archive-files mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare variables: ONE: plain-string-value TWO: ${{INTERP_ON_BUILD}} before_script: - mkdir /some/path - pushd /some/path - git clone ${{SPACK_REPO}} - cd spack - git checkout ${{SPACK_REF}} - popd script: - spack -d ci rebuild after_script: - rm -rf /some/path/spack custom_attribute: custom! artifacts: paths: - some/custom/artifact """ ) yaml_contents = syaml.load(outputfile.read_text()) assert yaml_contents["variables"]["SPACK_VERSION"] == "0.15.3" assert yaml_contents["variables"]["SPACK_CHECKOUT_VERSION"] == "big ol commit sha" assert any("archive-files" in key for key in yaml_contents) for ci_key, ci_obj in yaml_contents.items(): if "archive-files" not in ci_key: continue # Ensure we have variables, possibly interpolated assert ci_obj["variables"]["ONE"] == "plain-string-value" assert ci_obj["variables"]["TWO"] == "${INTERP_ON_BUILD}" # Ensure we have scripts verbatim assert ci_obj["before_script"] == [ "mkdir /some/path", "pushd /some/path", "git clone ${SPACK_REPO}", "cd spack", "git checkout ${SPACK_REF}", "popd", ] assert ci_obj["script"][1].startswith("cd ") ci_obj["script"][1] = "cd ENV" assert ci_obj["script"] == [ "spack -d ci rebuild", "cd ENV", "spack env activate --without-view .", "spack spec /$SPACK_JOB_SPEC_DAG_HASH", "spack ci rebuild", ] assert ci_obj["after_script"] == ["rm -rf /some/path/spack"] # Ensure we have the custom attributes assert "some/custom/artifact" in ci_obj["artifacts"]["paths"] assert ci_obj["custom_attribute"] == "custom!" def test_ci_generate_pkg_with_deps(ci_generate_test, tmp_path: pathlib.Path, ci_base_environment): """Test pipeline generation for a package w/ dependencies""" spack_yaml, outputfile, _ = ci_generate_test( f"""\ spack: specs: - dependent-install mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - submapping: - match: - dependent-install build-job: tags: - donotcare - match: - dependency-install build-job: tags: - donotcare """ ) yaml_contents = syaml.load(outputfile.read_text()) found = [] for ci_key, ci_obj in yaml_contents.items(): if "dependency-install" in ci_key: assert "stage" in ci_obj assert ci_obj["stage"] == "stage-0" found.append("dependency-install") if "dependent-install" in ci_key: assert "stage" in ci_obj assert ci_obj["stage"] == "stage-1" found.append("dependent-install") assert "dependent-install" in found assert "dependency-install" in found def test_ci_generate_for_pr_pipeline(ci_generate_test, tmp_path: pathlib.Path, monkeypatch): """Test generation of a PR pipeline with disabled rebuild-index""" monkeypatch.setenv("SPACK_PIPELINE_TYPE", "spack_pull_request") spack_yaml, outputfile, _ = ci_generate_test( f"""\ spack: specs: - dependent-install mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - submapping: - match: - dependent-install build-job: tags: - donotcare - match: - dependency-install build-job: tags: - donotcare - cleanup-job: image: donotcare tags: [donotcare] rebuild-index: False """ ) yaml_contents = syaml.load(outputfile.read_text()) assert "rebuild-index" not in yaml_contents assert "variables" in yaml_contents assert "SPACK_PIPELINE_TYPE" in yaml_contents["variables"] assert ( ci.common.PipelineType[yaml_contents["variables"]["SPACK_PIPELINE_TYPE"]] == ci.common.PipelineType.PULL_REQUEST ) def test_ci_generate_with_external_pkg(ci_generate_test, tmp_path: pathlib.Path, monkeypatch): """Make sure we do not generate jobs for external pkgs""" spack_yaml, outputfile, _ = ci_generate_test( f"""\ spack: specs: - archive-files - externaltest mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - submapping: - match: - archive-files - externaltest build-job: tags: - donotcare image: donotcare """ ) yaml_contents = syaml.load(outputfile.read_text()) # Check that the "externaltool" package was not erroneously staged assert all("externaltool" not in key for key in yaml_contents) def test_ci_rebuild_missing_config(tmp_path: pathlib.Path, working_env, mutable_mock_env_path): spack_yaml = tmp_path / "spack.yaml" spack_yaml.write_text( """ spack: specs: - archive-files """ ) env_cmd("create", "test", str(spack_yaml)) env_cmd("activate", "--without-view", "--sh", "test") out = ci_cmd("rebuild", fail_on_error=False) assert "env containing ci" in out env_cmd("deactivate") def _signing_key(): signing_key_path = pathlib.Path(spack.paths.mock_gpg_keys_path) / "package-signing-key" return signing_key_path.read_text() class RebuildEnv(NamedTuple): broken_spec_file: pathlib.Path ci_job_url: str ci_pipeline_url: str env_dir: pathlib.Path log_dir: pathlib.Path mirror_dir: pathlib.Path mirror_url: str repro_dir: pathlib.Path root_spec_dag_hash: str test_dir: pathlib.Path working_dir: pathlib.Path def create_rebuild_env( tmp_path: pathlib.Path, pkg_name: str, broken_tests: bool = False ) -> RebuildEnv: scratch = tmp_path / "working_dir" log_dir = scratch / "logs" repro_dir = scratch / "repro" test_dir = scratch / "test" env_dir = scratch / "concrete_env" mirror_dir = scratch / "mirror" broken_specs_path = scratch / "naughty-list" mirror_url = mirror_dir.as_uri() ci_job_url = "https://some.domain/group/project/-/jobs/42" ci_pipeline_url = "https://some.domain/group/project/-/pipelines/7" env_dir.mkdir(parents=True) with open(env_dir / "spack.yaml", "w", encoding="utf-8") as f: f.write( f""" spack: definitions: - packages: [{pkg_name}] specs: - $packages mirrors: buildcache-destination: {mirror_dir} ci: broken-specs-url: {broken_specs_path.as_uri()} broken-tests-packages: {json.dumps([pkg_name] if broken_tests else [])} pipeline-gen: - submapping: - match: - {pkg_name} build-job: tags: - donotcare image: donotcare cdash: build-group: Not important url: https://my.fake.cdash project: Not used site: Nothing """ ) with ev.Environment(env_dir) as env: env.concretize() env.write() shutil.copy(env_dir / "spack.yaml", tmp_path / "spack.yaml") root_spec_dag_hash = env.concrete_roots()[0].dag_hash() return RebuildEnv( broken_spec_file=broken_specs_path / root_spec_dag_hash, ci_job_url=ci_job_url, ci_pipeline_url=ci_pipeline_url, env_dir=env_dir, log_dir=log_dir, mirror_dir=mirror_dir, mirror_url=mirror_url, repro_dir=repro_dir, root_spec_dag_hash=root_spec_dag_hash, test_dir=test_dir, working_dir=scratch, ) def activate_rebuild_env(tmp_path: pathlib.Path, pkg_name: str, rebuild_env: RebuildEnv): env_cmd("activate", "--without-view", "--sh", "-d", ".") # Create environment variables as gitlab would do it os.environ.update( { "SPACK_ARTIFACTS_ROOT": str(rebuild_env.working_dir), "SPACK_JOB_LOG_DIR": str(rebuild_env.log_dir), "SPACK_JOB_REPRO_DIR": str(rebuild_env.repro_dir), "SPACK_JOB_TEST_DIR": str(rebuild_env.test_dir), "SPACK_LOCAL_MIRROR_DIR": str(rebuild_env.mirror_dir), "SPACK_CONCRETE_ENV_DIR": str(rebuild_env.env_dir), "CI_PIPELINE_ID": "7192", "SPACK_SIGNING_KEY": _signing_key(), "SPACK_JOB_SPEC_DAG_HASH": rebuild_env.root_spec_dag_hash, "SPACK_JOB_SPEC_PKG_NAME": pkg_name, "SPACK_COMPILER_ACTION": "NONE", "SPACK_CDASH_BUILD_NAME": pkg_name, "SPACK_REMOTE_MIRROR_URL": rebuild_env.mirror_url, "SPACK_PIPELINE_TYPE": "spack_protected_branch", "CI_JOB_URL": rebuild_env.ci_job_url, "CI_PIPELINE_URL": rebuild_env.ci_pipeline_url, "CI_PROJECT_DIR": str(tmp_path / "ci-project"), } ) @pytest.mark.parametrize("broken_tests", [True, False]) def test_ci_rebuild_mock_success( tmp_path: pathlib.Path, working_env, mutable_mock_env_path, install_mockery, mock_gnupghome, mock_fetch, mock_binary_index, monkeypatch, broken_tests, ): pkg_name = "archive-files" rebuild_env = create_rebuild_env(tmp_path, pkg_name, broken_tests) monkeypatch.setattr(spack.cmd.ci, "SPACK_COMMAND", "echo") with working_dir(rebuild_env.env_dir): activate_rebuild_env(tmp_path, pkg_name, rebuild_env) out = ci_cmd("rebuild", "--tests", fail_on_error=False) # We didn"t really run the build so build output file(s) are missing assert "Unable to copy files" in out assert "No such file or directory" in out if broken_tests: # We generate a skipped tests report in this case assert "Unable to run stand-alone tests" in out else: # No installation means no package to test and no test log to copy assert "Cannot copy test logs" in out def test_ci_rebuild_mock_failure_to_push( tmp_path: pathlib.Path, working_env, mutable_mock_env_path, install_mockery, mock_gnupghome, mock_fetch, mock_binary_index, ci_base_environment, monkeypatch, ): pkg_name = "trivial-install-test-package" rebuild_env = create_rebuild_env(tmp_path, pkg_name) # Mock the install script succuess def mock_success(*args, **kwargs): return 0 monkeypatch.setattr(ci, "process_command", mock_success) # Mock failure to push to the build cache def mock_push_or_raise(*args, **kwargs): raise spack.binary_distribution.PushToBuildCacheError( "Encountered problem pushing binary : " ) monkeypatch.setattr(spack.binary_distribution.Uploader, "push_or_raise", mock_push_or_raise) with working_dir(rebuild_env.env_dir): activate_rebuild_env(tmp_path, pkg_name, rebuild_env) with pytest.raises(spack.main.SpackCommandError) as e: ci_cmd("rebuild") assert e.value.code == FAILED_CREATE_BUILDCACHE_CODE def test_ci_require_signing( tmp_path: pathlib.Path, working_env, mutable_mock_env_path, mock_gnupghome, ci_base_environment, monkeypatch, ): spack_yaml = tmp_path / "spack.yaml" spack_yaml.write_text( f""" spack: specs: - archive-files mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare """ ) env_cmd("activate", "--without-view", "--sh", "-d", str(spack_yaml.parent)) # Run without the variable to make sure we don't accidentally require signing output = ci_cmd("rebuild", fail_on_error=False) assert "spack must have exactly one signing key" not in output # Now run with the variable to make sure it works monkeypatch.setenv("SPACK_REQUIRE_SIGNING", "True") output = ci_cmd("rebuild", fail_on_error=False) assert "spack must have exactly one signing key" in output env_cmd("deactivate") def test_ci_nothing_to_rebuild( tmp_path: pathlib.Path, working_env, mutable_mock_env_path, install_mockery, monkeypatch, mock_fetch, ci_base_environment, mock_binary_index, ): scratch = tmp_path / "working_dir" mirror_dir = scratch / "mirror" mirror_url = mirror_dir.as_uri() with open(tmp_path / "spack.yaml", "w", encoding="utf-8") as f: f.write( f""" spack: definitions: - packages: [archive-files] specs: - $packages mirrors: buildcache-destination: {mirror_url} ci: pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare """ ) install_cmd("archive-files") buildcache_cmd("push", "-f", "-u", "--update-index", mirror_url, "archive-files") with working_dir(tmp_path): env_cmd("create", "test", "./spack.yaml") with ev.read("test") as env: env.concretize() # Create environment variables as gitlab would do it os.environ.update( { "SPACK_ARTIFACTS_ROOT": str(scratch), "SPACK_JOB_LOG_DIR": "log_dir", "SPACK_JOB_REPRO_DIR": "repro_dir", "SPACK_JOB_TEST_DIR": "test_dir", "SPACK_CONCRETE_ENV_DIR": str(tmp_path), "SPACK_JOB_SPEC_DAG_HASH": env.concrete_roots()[0].dag_hash(), "SPACK_JOB_SPEC_PKG_NAME": "archive-files", "SPACK_COMPILER_ACTION": "NONE", } ) ci_out = ci_cmd("rebuild") assert "No need to rebuild archive-files" in ci_out env_cmd("deactivate") @pytest.mark.disable_clean_stage_check def test_push_to_build_cache( tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, mock_fetch, mock_gnupghome, ci_base_environment, mock_binary_index, ): scratch = tmp_path / "working_dir" mirror_dir = scratch / "mirror" mirror_url = mirror_dir.as_uri() ci.import_signing_key(_signing_key()) with working_dir(tmp_path): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( f"""\ spack: definitions: - packages: [patchelf] specs: - $packages mirrors: buildcache-destination: {mirror_url} ci: pipeline-gen: - submapping: - match: - patchelf build-job: tags: - donotcare image: donotcare - cleanup-job: tags: - nonbuildtag image: basicimage - any-job: tags: - nonbuildtag image: basicimage custom_attribute: custom! """ ) env_cmd("create", "test", "./spack.yaml") with ev.read("test") as current_env: current_env.concretize() install_cmd("--keep-stage") concrete_spec = list(current_env.roots())[0] spec_json = concrete_spec.to_json(hash=ht.dag_hash) json_path = str(tmp_path / "spec.json") with open(json_path, "w", encoding="utf-8") as ypfd: ypfd.write(spec_json) for s in concrete_spec.traverse(): ci.push_to_build_cache(s, mirror_url, True) # Now test the --prune-dag (default) option of spack ci generate mirror_cmd("add", "test-ci", mirror_url) outputfile_pruned = str(tmp_path / "pruned_pipeline.yml") ci_cmd("generate", "--output-file", outputfile_pruned) with open(outputfile_pruned, encoding="utf-8") as f: contents = f.read() yaml_contents = syaml.load(contents) # Make sure there are no other spec jobs or rebuild-index assert set(yaml_contents.keys()) == {"no-specs-to-rebuild", "workflow"} the_elt = yaml_contents["no-specs-to-rebuild"] assert "tags" in the_elt assert "nonbuildtag" in the_elt["tags"] assert "image" in the_elt assert the_elt["image"] == "basicimage" assert the_elt["custom_attribute"] == "custom!" assert "rules" in yaml_contents["workflow"] assert yaml_contents["workflow"]["rules"] == [{"when": "always"}] outputfile_not_pruned = str(tmp_path / "unpruned_pipeline.yml") ci_cmd("generate", "--no-prune-dag", "--output-file", outputfile_not_pruned) # Test the --no-prune-dag option of spack ci generate with open(outputfile_not_pruned, encoding="utf-8") as f: contents = f.read() yaml_contents = syaml.load(contents) found_spec_job = False for ci_key in yaml_contents.keys(): if "patchelf" in ci_key: the_elt = yaml_contents[ci_key] assert "variables" in the_elt job_vars = the_elt["variables"] assert "SPACK_SPEC_NEEDS_REBUILD" in job_vars assert job_vars["SPACK_SPEC_NEEDS_REBUILD"] == "False" assert the_elt["custom_attribute"] == "custom!" found_spec_job = True assert found_spec_job mirror_cmd("rm", "test-ci") # Test generating buildcache index while we have bin mirror buildcache_cmd("update-index", mirror_url) # Validate resulting buildcache (database) index layout_version = spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION mirror_metadata = spack.binary_distribution.MirrorMetadata(mirror_url, layout_version) index_fetcher = spack.binary_distribution.DefaultIndexFetcher(mirror_metadata, None) result = index_fetcher.conditional_fetch() spack.vendor.jsonschema.validate(json.loads(result.data), db_idx_schema) # Now that index is regenerated, validate "buildcache list" output assert "patchelf" in buildcache_cmd("list") logs_dir = scratch / "logs_dir" logs_dir.mkdir() ci.copy_stage_logs_to_artifacts(concrete_spec, str(logs_dir)) assert "spack-build-out.txt.gz" in os.listdir(logs_dir) def test_push_to_build_cache_exceptions(monkeypatch, tmp_path: pathlib.Path, capfd): def push_or_raise(*args, **kwargs): raise spack.binary_distribution.PushToBuildCacheError("Error: Access Denied") monkeypatch.setattr(spack.binary_distribution.Uploader, "push_or_raise", push_or_raise) # Input doesn't matter, as we are faking exceptional output url = tmp_path.as_uri() ci.push_to_build_cache(spack.spec.Spec(), url, False) assert f"Problem writing to {url}: Error: Access Denied" in capfd.readouterr().err @pytest.mark.parametrize("match_behavior", ["first", "merge"]) @pytest.mark.parametrize("git_version", ["big ol commit sha", None]) def test_ci_generate_override_runner_attrs( ci_generate_test, tmp_path: pathlib.Path, monkeypatch, match_behavior, git_version ): """Test that we get the behavior we want with respect to the provision of runner attributes like tags, variables, and scripts, both when we inherit them from the top level, as well as when we override one or more at the runner level""" monkeypatch.setattr(spack, "spack_version", "0.20.0.test0") monkeypatch.setattr(spack, "get_version", lambda: "0.20.0.test0 (blah)") monkeypatch.setattr(spack, "get_spack_commit", lambda: git_version) spack_yaml, outputfile, _ = ci_generate_test( f"""\ spack: specs: - dependent-install - pkg-a mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - match_behavior: {match_behavior} submapping: - match: - dependent-install build-job: tags: - specific-one variables: THREE: specificvarthree - match: - dependency-install - match: - pkg-a build-job: tags: - specific-a-2 - match: - pkg-a build-job-remove: tags: - toplevel2 build-job: tags: - specific-a variables: ONE: specificvarone TWO: specificvartwo before_script:: - - custom pre step one script:: - - custom main step after_script:: - custom post step one - build-job: tags: - toplevel - toplevel2 variables: ONE: toplevelvarone TWO: toplevelvartwo before_script: - - pre step one - pre step two script:: - - main step after_script: - - post step one - cleanup-job: image: donotcare tags: [donotcare] """ ) yaml_contents = syaml.load(outputfile.read_text()) assert "variables" in yaml_contents global_vars = yaml_contents["variables"] assert "SPACK_VERSION" in global_vars assert global_vars["SPACK_VERSION"] == "0.20.0.test0 (blah)" assert "SPACK_CHECKOUT_VERSION" in global_vars assert global_vars["SPACK_CHECKOUT_VERSION"] == git_version or "v0.20.0.test0" for ci_key in yaml_contents.keys(): if ci_key.startswith("pkg-a"): # Make sure pkg-a's attributes override variables, and all the # scripts. Also, make sure the 'toplevel' tag doesn't # appear twice, but that a's specific extra tag does appear the_elt = yaml_contents[ci_key] assert the_elt["variables"]["ONE"] == "specificvarone" assert the_elt["variables"]["TWO"] == "specificvartwo" assert "THREE" not in the_elt["variables"] assert len(the_elt["tags"]) == (2 if match_behavior == "first" else 3) assert "specific-a" in the_elt["tags"] if match_behavior == "merge": assert "specific-a-2" in the_elt["tags"] assert "toplevel" in the_elt["tags"] assert "toplevel2" not in the_elt["tags"] assert len(the_elt["before_script"]) == 1 assert the_elt["before_script"][0] == "custom pre step one" assert len(the_elt["script"]) == 1 assert the_elt["script"][0] == "custom main step" assert len(the_elt["after_script"]) == 1 assert the_elt["after_script"][0] == "custom post step one" if "dependency-install" in ci_key: # Since the dependency-install match omits any # runner-attributes, make sure it inherited all the # top-level attributes. the_elt = yaml_contents[ci_key] assert the_elt["variables"]["ONE"] == "toplevelvarone" assert the_elt["variables"]["TWO"] == "toplevelvartwo" assert "THREE" not in the_elt["variables"] assert len(the_elt["tags"]) == 2 assert "toplevel" in the_elt["tags"] assert "toplevel2" in the_elt["tags"] assert len(the_elt["before_script"]) == 2 assert the_elt["before_script"][0] == "pre step one" assert the_elt["before_script"][1] == "pre step two" assert len(the_elt["script"]) == 1 assert the_elt["script"][0] == "main step" assert len(the_elt["after_script"]) == 1 assert the_elt["after_script"][0] == "post step one" if "dependent-install" in ci_key: # The dependent-install match specifies that we keep the two # top level variables, but add a third specific one. It # also adds a custom tag which should be combined with # the top-level tag. the_elt = yaml_contents[ci_key] assert the_elt["variables"]["ONE"] == "toplevelvarone" assert the_elt["variables"]["TWO"] == "toplevelvartwo" assert the_elt["variables"]["THREE"] == "specificvarthree" assert len(the_elt["tags"]) == 3 assert "specific-one" in the_elt["tags"] assert "toplevel" in the_elt["tags"] assert "toplevel2" in the_elt["tags"] assert len(the_elt["before_script"]) == 2 assert the_elt["before_script"][0] == "pre step one" assert the_elt["before_script"][1] == "pre step two" assert len(the_elt["script"]) == 1 assert the_elt["script"][0] == "main step" assert len(the_elt["after_script"]) == 1 assert the_elt["after_script"][0] == "post step one" def test_ci_rebuild_index( tmp_path: pathlib.Path, working_env, mutable_mock_env_path, install_mockery, mock_fetch, mock_binary_index, ): scratch = tmp_path / "working_dir" mirror_dir = scratch / "mirror" mirror_url = mirror_dir.as_uri() with open(tmp_path / "spack.yaml", "w", encoding="utf-8") as f: f.write( f""" spack: specs: - callpath mirrors: buildcache-destination: {mirror_url} ci: pipeline-gen: - submapping: - match: - patchelf build-job: tags: - donotcare image: donotcare """ ) with working_dir(tmp_path): env_cmd("create", "test", "./spack.yaml") with ev.read("test"): concrete_spec = spack.concretize.concretize_one("callpath") with open(tmp_path / "spec.json", "w", encoding="utf-8") as f: f.write(concrete_spec.to_json(hash=ht.dag_hash)) install_cmd("--fake", str(tmp_path / "spec.json")) buildcache_cmd("push", "-u", "-f", mirror_url, "callpath") ci_cmd("rebuild-index") output = buildcache_cmd("list", "-L", "--allarch") assert concrete_spec.dag_hash() + " callpath" in output def test_ci_get_stack_changed(mock_git_repo, monkeypatch): """Test that we can detect the change to .gitlab-ci.yml in a mock spack git repo.""" monkeypatch.setattr(spack.paths, "prefix", mock_git_repo) fake_env_path = os.path.join( spack.paths.prefix, os.path.sep.join(("no", "such", "env", "path")) ) assert ci.stack_changed(fake_env_path) is True def test_ci_generate_prune_untouched( ci_generate_test, monkeypatch, tmp_path: pathlib.Path, repo_builder: RepoBuilder ): """Test pipeline generation with pruning works to eliminate specs that were not affected by a change""" monkeypatch.setenv("SPACK_PRUNE_UNTOUCHED", "TRUE") # enables pruning of untouched specs def fake_compute_affected(repo, rev1=None, rev2=None): if "mock" in os.path.basename(repo.root): return ["libdwarf"] else: return ["pkg-c"] def fake_stack_changed(env_path): return False def fake_change_revisions(env_path): return "HEAD^", "HEAD" repo_builder.add_package("pkg-a", dependencies=[("pkg-b", None, None)]) repo_builder.add_package("pkg-b", dependencies=[("pkg-c", None, None)]) repo_builder.add_package("pkg-c") repo_builder.add_package("pkg-d") monkeypatch.setattr(ci, "compute_affected_packages", fake_compute_affected) monkeypatch.setattr(ci, "stack_changed", fake_stack_changed) monkeypatch.setattr(ci, "get_change_revisions", fake_change_revisions) with spack.repo.use_repositories(repo_builder.root, override=False): spack_yaml, outputfile, _ = ci_generate_test( f"""\ spack: specs: - archive-files - callpath - pkg-a - pkg-d mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - build-job: tags: - donotcare image: donotcare """ ) # Dependency graph rooted at callpath # callpath -> dyninst -> libelf # -> libdwarf -> libelf # -> mpich env_hashes = {} with ev.read("test") as active_env: active_env.concretize() for s in active_env.all_specs(): env_hashes[s.name] = s.dag_hash() yaml_contents = syaml.load(outputfile.read_text()) generated_hashes = [] for ci_key in yaml_contents.keys(): if "variables" in yaml_contents[ci_key]: generated_hashes.append(yaml_contents[ci_key]["variables"]["SPACK_JOB_SPEC_DAG_HASH"]) assert env_hashes["archive-files"] not in generated_hashes assert env_hashes["pkg-d"] not in generated_hashes for spec_name in [ "callpath", "dyninst", "mpich", "libdwarf", "libelf", "pkg-a", "pkg-b", "pkg-c", ]: assert env_hashes[spec_name] in generated_hashes def test_ci_subcommands_without_mirror( tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, ci_base_environment, mock_binary_index, ): """Make sure we catch if there is not a mirror and report an error""" with open(tmp_path / "spack.yaml", "w", encoding="utf-8") as f: f.write( """\ spack: specs: - archive-files ci: pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare """ ) with working_dir(tmp_path): env_cmd("create", "test", "./spack.yaml") with ev.read("test"): # Check the 'generate' subcommand expect = "spack ci generate requires a mirror named 'buildcache-destination'" with pytest.raises(ci.SpackCIError, match=expect): ci_cmd("generate", "--output-file", str(tmp_path / ".gitlab-ci.yml")) # Also check the 'rebuild-index' subcommand output = ci_cmd("rebuild-index", fail_on_error=False) assert "spack ci rebuild-index requires an env containing a mirror" in output def test_ci_generate_read_broken_specs_url( tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, ci_base_environment, ): """Verify that `broken-specs-url` works as intended""" spec_a = spack.concretize.concretize_one("pkg-a") a_dag_hash = spec_a.dag_hash() spec_flattendeps = spack.concretize.concretize_one("dependent-install") flattendeps_dag_hash = spec_flattendeps.dag_hash() broken_specs_url = tmp_path.as_uri() # Mark 'a' as broken (but not 'dependent-install') broken_spec_a_url = "{0}/{1}".format(broken_specs_url, a_dag_hash) job_stack = "job_stack" a_job_url = "a_job_url" ci.write_broken_spec( broken_spec_a_url, spec_a.name, job_stack, a_job_url, "pipeline_url", spec_a.to_dict() ) # Test that `spack ci generate` notices this broken spec and fails. with open(tmp_path / "spack.yaml", "w", encoding="utf-8") as f: f.write( f"""\ spack: specs: - dependent-install - pkg-a mirrors: buildcache-destination: {(tmp_path / "ci-mirror").as_uri()} ci: broken-specs-url: "{broken_specs_url}" pipeline-gen: - submapping: - match: - pkg-a - dependent-install - pkg-b - dependency-install build-job: tags: - donotcare image: donotcare """ ) with working_dir(tmp_path): env_cmd("create", "test", "./spack.yaml") with ev.read("test"): # Check output of the 'generate' subcommand output = ci_cmd("generate", fail_on_error=False) assert "known to be broken" in output expected = ( f"{spec_a.name}/{a_dag_hash[:7]} (in stack {job_stack}) was " f"reported broken here: {a_job_url}" ) assert expected in output not_expected = f"dependent-install/{flattendeps_dag_hash[:7]} (in stack" assert not_expected not in output def test_ci_generate_external_signing_job( install_mockery, ci_generate_test, tmp_path: pathlib.Path, monkeypatch ): """Verify that in external signing mode: 1) each rebuild jobs includes the location where the binary hash information is written and 2) we properly generate a final signing job in the pipeline.""" monkeypatch.setenv("SPACK_PIPELINE_TYPE", "spack_protected_branch") _, outputfile, _ = ci_generate_test( f"""\ spack: specs: - archive-files mirrors: buildcache-destination: {(tmp_path / "ci-mirror").as_uri()} ci: pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare - signing-job: tags: - nonbuildtag - secretrunner image: name: customdockerimage entrypoint: [] variables: IMPORTANT_INFO: avalue script:: - echo hello custom_attribute: custom! """ ) yaml_contents = syaml.load(outputfile.read_text()) assert "sign-pkgs" in yaml_contents signing_job = yaml_contents["sign-pkgs"] assert "tags" in signing_job signing_job_tags = signing_job["tags"] for expected_tag in ["notary", "protected", "aws"]: assert expected_tag in signing_job_tags assert signing_job["custom_attribute"] == "custom!" def test_ci_reproduce( tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, monkeypatch, last_two_git_commits, ci_base_environment, mock_binary_index, ): repro_dir = tmp_path / "repro_dir" image_name = "org/image:tag" with open(tmp_path / "spack.yaml", "w", encoding="utf-8") as f: f.write( f""" spack: definitions: - packages: [archive-files] specs: - $packages mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: {image_name} """ ) with working_dir(tmp_path), ev.Environment(".") as env: env.concretize() env.write() def fake_download_and_extract_artifacts(url, work_dir, merge_commit_test=True): with working_dir(tmp_path), ev.Environment(".") as env: if not os.path.exists(repro_dir): repro_dir.mkdir() job_spec = env.concrete_roots()[0] with open(repro_dir / "archivefiles.json", "w", encoding="utf-8") as f: f.write(job_spec.to_json(hash=ht.dag_hash)) artifacts_root = repro_dir / "jobs_scratch_dir" pipeline_path = artifacts_root / "pipeline.yml" ci_cmd( "generate", "--output-file", str(pipeline_path), "--artifacts-root", str(artifacts_root), ) job_name = gitlab_generator.get_job_name(job_spec) with open(repro_dir / "repro.json", "w", encoding="utf-8") as f: f.write( json.dumps( { "job_name": job_name, "job_spec_json": "archivefiles.json", "ci_project_dir": str(repro_dir), } ) ) with open(repro_dir / "install.sh", "w", encoding="utf-8") as f: f.write("#!/bin/sh\n\n#fake install\nspack install blah\n") with open(repro_dir / "spack_info.txt", "w", encoding="utf-8") as f: if merge_commit_test: f.write( f"\nMerge {last_two_git_commits[1]} into {last_two_git_commits[0]}\n\n" ) else: f.write(f"\ncommit {last_two_git_commits[1]}\n\n") return "jobs_scratch_dir" monkeypatch.setattr(ci, "download_and_extract_artifacts", fake_download_and_extract_artifacts) rep_out = ci_cmd( "reproduce-build", "https://example.com/api/v1/projects/1/jobs/2/artifacts", "--working-dir", str(repro_dir), ) # Make sure the script was generated assert (repro_dir / "start.sh").exists() # Make sure we tell the user where it is when not in interactive mode assert f"$ {repro_dir}/start.sh" in rep_out # Ensure the correct commits are used assert f"checkout_commit: {last_two_git_commits[0]}" in rep_out assert f"merge_commit: {last_two_git_commits[1]}" in rep_out # Test re-running in dirty working dir with pytest.raises(SpackError, match=f"{repro_dir}"): rep_out = ci_cmd( "reproduce-build", "https://example.com/api/v1/projects/1/jobs/2/artifacts", "--working-dir", str(repro_dir), ) # Cleanup between tests shutil.rmtree(repro_dir) # Test --use-local-head rep_out = ci_cmd( "reproduce-build", "https://example.com/api/v1/projects/1/jobs/2/artifacts", "--use-local-head", "--working-dir", str(repro_dir), ) # Make sure we are checkout out the HEAD commit without a merge commit assert "checkout_commit: HEAD" in rep_out assert "merge_commit: None" in rep_out # Test the case where the spack_info.txt is not a merge commit monkeypatch.setattr( ci, "download_and_extract_artifacts", lambda url, wd: fake_download_and_extract_artifacts(url, wd, False), ) # Cleanup between tests shutil.rmtree(repro_dir) rep_out = ci_cmd( "reproduce-build", "https://example.com/api/v1/projects/1/jobs/2/artifacts", "--working-dir", str(repro_dir), ) # Make sure the script was generated assert (repro_dir / "start.sh").exists() # Make sure we tell the user where it is when not in interactive mode assert f"$ {repro_dir}/start.sh" in rep_out # Ensure the correct commit is used (different than HEAD) assert f"checkout_commit: {last_two_git_commits[1]}" in rep_out assert "merge_commit: None" in rep_out @pytest.mark.parametrize( "url_in,url_out", [ ( "https://example.com/api/v4/projects/1/jobs/2/artifacts", "https://example.com/api/v4/projects/1/jobs/2/artifacts", ), ( "https://example.com/spack/spack/-/jobs/123456/artifacts/download", "https://example.com/spack/spack/-/jobs/123456/artifacts/download", ), ( "https://example.com/spack/spack/-/jobs/123456", "https://example.com/spack/spack/-/jobs/123456/artifacts/download", ), ( "https://example.com/spack/spack/-/jobs/////123456////?x=y#z", "https://example.com/spack/spack/-/jobs/123456/artifacts/download", ), ], ) def test_reproduce_build_url_validation(url_in, url_out): assert spack.cmd.ci._gitlab_artifacts_url(url_in) == url_out def test_reproduce_build_url_validation_fails(): """Wrong URLs should cause an exception""" with pytest.raises(spack.main.SpackCommandError): ci_cmd("reproduce-build", "example.com/spack/spack/-/jobs/123456/artifacts/download") with pytest.raises(spack.main.SpackCommandError): ci_cmd("reproduce-build", "https://example.com/spack/spack/-/issues") with pytest.raises(spack.main.SpackCommandError): ci_cmd("reproduce-build", "https://example.com/spack/spack/-") @pytest.mark.parametrize( "subcmd", [(""), ("generate"), ("rebuild-index"), ("rebuild"), ("reproduce-build")] ) def test_ci_help(subcmd): """Make sure `spack ci` --help describes the (sub)command help.""" out = spack.main.SpackCommand("ci")(subcmd, "--help", fail_on_error=False) usage = " ci {0}{1}[".format(subcmd, " " if subcmd else "") assert usage in out def test_docstring_utils(): def example_function(): """\ this is the first line this is not the first line """ pass assert spack.cmd.doc_first_line(example_function) == "this is the first line" assert spack.cmd.doc_dedented(example_function) == ( "this is the first line\n\nthis is not the first line\n" ) def test_gitlab_config_scopes(install_mockery, ci_generate_test, tmp_path: pathlib.Path): """Test pipeline generation with included configs""" # Create an included config scope configs_path = tmp_path / "gitlab" / "configs" configs_path.mkdir(parents=True, exist_ok=True) with open(configs_path / "ci.yaml", "w", encoding="utf-8") as fd: fd.write( """ ci: pipeline-gen: - reindex-job: variables: CI_JOB_SIZE: small KUBERNETES_CPU_REQUEST: 10 KUBERNETES_MEMORY_REQUEST: 100 tags: ["spack", "service"] """ ) rel_configs_path = configs_path.relative_to(tmp_path) manifest, outputfile, _ = ci_generate_test( f"""\ spack: config: install_tree: root: {tmp_path / "opt"} include: - {rel_configs_path} - path: {rel_configs_path} - {configs_path} - when: 'False' path: https://dummy.io view: false specs: - dependent-install mirrors: buildcache-destination: {tmp_path / "ci-mirror"} ci: pipeline-gen: - build-job: image: "ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01" tags: ["some_tag"] """ ) yaml_contents = syaml.load(outputfile.read_text()) assert "rebuild-index" in yaml_contents rebuild_job = yaml_contents["rebuild-index"] assert "tags" in rebuild_job assert "variables" in rebuild_job rebuild_tags = rebuild_job["tags"] rebuild_vars = rebuild_job["variables"] assert all([t in rebuild_tags for t in ["spack", "service"]]) expected_vars = ["CI_JOB_SIZE", "KUBERNETES_CPU_REQUEST", "KUBERNETES_MEMORY_REQUEST"] assert all([v in rebuild_vars for v in expected_vars]) # Read the concrete environment and ensure the relative path was updated conc_env_path = tmp_path / "jobs_scratch_dir" / "concrete_environment" conc_env_manifest = conc_env_path / "spack.yaml" env_manifest = syaml.load(conc_env_manifest.read_text()) assert "include" in env_manifest["spack"] # Ensure relative path include correctly updated # Ensure the relocated concrete env includes point to the same location rel_conc_path = env_manifest["spack"]["include"][0] abs_conc_path = (conc_env_path / rel_conc_path).absolute().resolve() assert str(abs_conc_path) == os.path.join(ev.as_env_dir("test"), "gitlab", "configs") # Ensure relative path include with "path" correctly updated # Ensure the relocated concrete env includes point to the same location rel_conc_path = env_manifest["spack"]["include"][1]["path"] abs_conc_path = (conc_env_path / rel_conc_path).absolute().resolve() assert str(abs_conc_path) == os.path.join(ev.as_env_dir("test"), "gitlab", "configs") # Ensure absolute path is unchanged # Ensure the relocated concrete env includes point to the same location abs_config_path = env_manifest["spack"]["include"][2] assert str(abs_config_path) == str(configs_path) # Ensure URL path is unchanged url_config_path = env_manifest["spack"]["include"][3]["path"] assert str(url_config_path) == "https://dummy.io" def test_ci_generate_mirror_config( tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, monkeypatch, ci_base_environment, mock_binary_index, ): """Make sure the correct mirror gets used as the buildcache destination""" fst, snd = (tmp_path / "first").as_uri(), (tmp_path / "second").as_uri() with open(tmp_path / "spack.yaml", "w", encoding="utf-8") as f: f.write( f"""\ spack: specs: - archive-files mirrors: some-mirror: {fst} buildcache-destination: {snd} ci: pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare """ ) with ev.Environment(tmp_path): ci_cmd("generate", "--output-file", str(tmp_path / ".gitlab-ci.yml")) with open(tmp_path / ".gitlab-ci.yml", encoding="utf-8") as f: pipeline_doc = syaml.load(f) assert fst not in pipeline_doc["rebuild-index"]["script"][0] assert snd in pipeline_doc["rebuild-index"]["script"][0] def dynamic_mapping_setup(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: specs: - pkg-a mirrors: buildcache-destination: https://my.fake.mirror ci: pipeline-gen: - dynamic-mapping: endpoint: https://fake.spack.io/mapper require: ["variables"] ignore: ["ignored_field"] allow: ["variables", "retry"] """ ) spec_a = spack.concretize.concretize_one("pkg-a") return gitlab_generator.get_job_name(spec_a) def test_ci_dynamic_mapping_empty( tmp_path: pathlib.Path, working_env, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, ci_base_environment, ): # The test will always return an empty dictionary def _urlopen(*args, **kwargs): return MockHTTPResponse.with_json(200, "OK", headers={}, body={}) monkeypatch.setattr(ci.common, "_urlopen", _urlopen) _ = dynamic_mapping_setup(tmp_path) with working_dir(str(tmp_path)): env_cmd("create", "test", "./spack.yaml") outputfile = str(tmp_path / ".gitlab-ci.yml") with ev.read("test"): output = ci_cmd("generate", "--output-file", outputfile) assert "Response missing required keys: ['variables']" in output def test_ci_dynamic_mapping_full( tmp_path: pathlib.Path, working_env, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, ci_base_environment, ): def _urlopen(*args, **kwargs): return MockHTTPResponse.with_json( 200, "OK", headers={}, body={"variables": {"MY_VAR": "hello"}, "ignored_field": 0, "unallowed_field": 0}, ) monkeypatch.setattr(ci.common, "_urlopen", _urlopen) label = dynamic_mapping_setup(tmp_path) with working_dir(str(tmp_path)): env_cmd("create", "test", "./spack.yaml") outputfile = str(tmp_path / ".gitlab-ci.yml") with ev.read("test"): ci_cmd("generate", "--output-file", outputfile) with open(outputfile, encoding="utf-8") as of: pipeline_doc = syaml.load(of.read()) assert label in pipeline_doc job = pipeline_doc[label] assert job.get("variables", {}).get("MY_VAR") == "hello" assert "ignored_field" not in job assert "unallowed_field" not in job def test_ci_generate_unknown_generator( ci_generate_test, tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment, ): """Ensure unrecognized ci targets are detected and the user sees an intelligible and actionable message""" src_mirror_url = tmp_path / "ci-src-mirror" bin_mirror_url = tmp_path / "ci-bin-mirror" spack_yaml_contents = f""" spack: specs: - archive-files mirrors: some-mirror: {src_mirror_url} buildcache-destination: {bin_mirror_url} ci: target: unknown pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare """ expect = "Spack CI module cannot generate a pipeline for format unknown" with pytest.raises(ci.SpackCIError, match=expect): ci_generate_test(spack_yaml_contents) def test_ci_generate_copy_only( ci_generate_test, tmp_path: pathlib.Path, monkeypatch, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment, ): """Ensure the correct jobs are generated for a copy-only pipeline, and verify that pipeline manifest is produced containing the right number of entries.""" src_mirror_url = tmp_path / "ci-src-mirror" bin_mirror_url = tmp_path / "ci-bin-mirror" copy_mirror_url = tmp_path / "ci-copy-mirror" monkeypatch.setenv("SPACK_PIPELINE_TYPE", "spack_copy_only") monkeypatch.setenv("SPACK_COPY_BUILDCACHE", copy_mirror_url) spack_yaml_contents = f""" spack: specs: - archive-files mirrors: buildcache-source: fetch: {src_mirror_url} push: {src_mirror_url} source: False binary: True buildcache-destination: fetch: {bin_mirror_url} push: {bin_mirror_url} source: False binary: True ci: target: gitlab pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare """ _, output_file, _ = ci_generate_test(spack_yaml_contents) with open(output_file, encoding="utf-8") as of: pipeline_doc = syaml.load(of.read()) expected_keys = ["copy", "rebuild-index", "stages", "variables", "workflow"] assert all([k in pipeline_doc for k in expected_keys]) # Make sure there are only two jobs and two stages stages = pipeline_doc["stages"] copy_stage = "copy" rebuild_index_stage = "stage-rebuild-index" assert len(stages) == 2 assert stages[0] == copy_stage assert stages[1] == rebuild_index_stage rebuild_index_job = pipeline_doc["rebuild-index"] assert rebuild_index_job["stage"] == rebuild_index_stage copy_job = pipeline_doc["copy"] assert copy_job["stage"] == copy_stage # Make sure a pipeline manifest was generated output_directory = os.path.dirname(output_file) assert "SPACK_ARTIFACTS_ROOT" in pipeline_doc["variables"] artifacts_root = pipeline_doc["variables"]["SPACK_ARTIFACTS_ROOT"] pipeline_manifest_path = os.path.join( output_directory, artifacts_root, "specs_to_copy", "copy_rebuilt_specs.json" ) assert os.path.exists(pipeline_manifest_path) assert os.path.isfile(pipeline_manifest_path) with open(pipeline_manifest_path, encoding="utf-8") as fd: manifest_data = json.load(fd) with ev.read("test") as active_env: active_env.concretize() for s in active_env.all_specs(): assert s.dag_hash() in manifest_data @generator("unittestgenerator") def generate_unittest_pipeline( pipeline: PipelineDag, spack_ci: SpackCIConfig, options: PipelineOptions ): """Define a custom pipeline generator for the target 'unittestgenerator'.""" output_file = options.output_file assert output_file is not None with open(output_file, "w", encoding="utf-8") as fd: fd.write("unittestpipeline\n") for _, node in pipeline.traverse_nodes(direction="children"): release_spec = node.spec fd.write(f" {release_spec.name}\n") def test_ci_generate_alternate_target( ci_generate_test, tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment, ): """Ensure the above pipeline generator was correctly registered and is used to generate a pipeline for the stack/config defined here.""" bin_mirror_url = tmp_path / "ci-bin-mirror" spack_yaml_contents = f""" spack: specs: - archive-files - externaltest mirrors: buildcache-destination: {bin_mirror_url} ci: target: unittestgenerator pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare """ _, output_file, _ = ci_generate_test(spack_yaml_contents, "--no-prune-externals") with open(output_file, encoding="utf-8") as of: pipeline_doc = of.read() assert pipeline_doc.startswith("unittestpipeline") assert "externaltest" in pipeline_doc def test_ci_generate_forward_variables( ci_generate_test, tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment, ): """Ensure the above pipeline generator was correctly registered and is used to generate a pipeline for the stack/config defined here.""" bin_mirror_url = tmp_path / "ci-bin-mirror" spack_yaml_contents = f""" spack: specs: - archive-files - externaltest mirrors: buildcache-destination: {bin_mirror_url} ci: target: gitlab pipeline-gen: - submapping: - match: - archive-files build-job: tags: - donotcare image: donotcare """ noforward_vars = ["NO_FORWARD_VAR"] forward_vars = ["TEST_VAR", "ANOTHER_TEST_VAR"] for v in forward_vars + noforward_vars: os.environ[v] = f"{v}_BEEF" fwd_arg = " --forward-variable " _, output_file, _ = ci_generate_test( spack_yaml_contents, fwd_arg.strip(), *fwd_arg.join(forward_vars).split() ) with open(output_file, encoding="utf-8") as fd: pipeline_yaml = syaml.load(fd.read()) for v in forward_vars: assert v in pipeline_yaml["variables"] assert pipeline_yaml["variables"][v] == f"{v}_BEEF" for v in noforward_vars: assert v not in pipeline_yaml["variables"] @pytest.fixture def fetch_versions_match(monkeypatch): """Fake successful checksums returned from downloaded tarballs.""" def get_checksums_for_versions(url_by_version, package_name, **kwargs): pkg_cls = spack.repo.PATH.get_pkg_class(package_name) return {v: pkg_cls.versions[v]["sha256"] for v in url_by_version} monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions) monkeypatch.setattr(spack.util.web, "url_exists", lambda url: True) @pytest.fixture def fetch_versions_invalid(monkeypatch): """Fake successful checksums returned from downloaded tarballs.""" def get_checksums_for_versions(url_by_version, package_name, **kwargs): return { v: "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" for v in url_by_version } monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions) monkeypatch.setattr(spack.util.web, "url_exists", lambda url: True) @pytest.mark.parametrize("versions", [["2.1.4"], ["2.1.4", "2.1.5"]]) def test_ci_validate_standard_versions_valid(capfd, mock_packages, fetch_versions_match, versions): spec = spack.spec.Spec("diff-test") pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) version_list = [spack.version.Version(v) for v in versions] assert spack.cmd.ci.validate_standard_versions(pkg, version_list) out, err = capfd.readouterr() for version in versions: assert f"Validated diff-test@{version}" in out @pytest.mark.parametrize("versions", [["2.1.4"], ["2.1.4", "2.1.5"]]) def test_ci_validate_standard_versions_invalid( capfd, mock_packages, fetch_versions_invalid, versions ): spec = spack.spec.Spec("diff-test") pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) version_list = [spack.version.Version(v) for v in versions] assert spack.cmd.ci.validate_standard_versions(pkg, version_list) is False out, err = capfd.readouterr() for version in versions: assert f"Invalid checksum found diff-test@{version}" in err @pytest.mark.parametrize("versions", [[("1.0", -2)], [("1.1", -4), ("2.0", -6)]]) def test_ci_validate_git_versions_valid( capfd, monkeypatch, mock_packages, mock_git_version_info, versions ): spec = spack.spec.Spec("diff-test") pkg_class = spack.repo.PATH.get_pkg_class(spec.name) pkg = pkg_class(spec) version_list = [spack.version.Version(v) for v, _ in versions] repo_path, filename, commits = mock_git_version_info version_commit_dict = { spack.version.Version(v): {"tag": f"v{v}", "commit": commits[c]} for v, c in versions } monkeypatch.setattr(pkg_class, "git", repo_path) monkeypatch.setattr(pkg_class, "versions", version_commit_dict) assert spack.cmd.ci.validate_git_versions(pkg, version_list) out, err = capfd.readouterr() for version in version_list: assert f"Validated diff-test@{version}" in out @pytest.mark.parametrize("versions", [[("1.0", -3)], [("1.1", -5), ("2.0", -5)]]) def test_ci_validate_git_versions_bad_tag( capfd, monkeypatch, mock_packages, mock_git_version_info, versions ): spec = spack.spec.Spec("diff-test") pkg_class = spack.repo.PATH.get_pkg_class(spec.name) pkg = pkg_class(spec) version_list = [spack.version.Version(v) for v, _ in versions] repo_path, filename, commits = mock_git_version_info version_commit_dict = { spack.version.Version(v): {"tag": f"v{v}", "commit": commits[c]} for v, c in versions } monkeypatch.setattr(pkg_class, "git", repo_path) monkeypatch.setattr(pkg_class, "versions", version_commit_dict) assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False out, err = capfd.readouterr() for version in version_list: assert f"Mismatched tag <-> commit found for diff-test@{version}" in err @pytest.mark.parametrize("versions", [[("1.0", -2)], [("1.1", -4), ("2.0", -6), ("3.0", -6)]]) def test_ci_validate_git_versions_invalid( capfd, monkeypatch, mock_packages, mock_git_version_info, versions ): spec = spack.spec.Spec("diff-test") pkg_class = spack.repo.PATH.get_pkg_class(spec.name) pkg = pkg_class(spec) version_list = [spack.version.Version(v) for v, _ in versions] repo_path, filename, commits = mock_git_version_info version_commit_dict = { spack.version.Version(v): { "tag": f"v{v}", "commit": "abcdefabcdefabcdefabcdefabcdefabcdefabc", } for v, c in versions } monkeypatch.setattr(pkg_class, "git", repo_path) monkeypatch.setattr(pkg_class, "versions", version_commit_dict) assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False out, err = capfd.readouterr() for version in version_list: assert f"Invalid commit for diff-test@{version}" in err def mock_packages_path(path): def packages_path(): return path return packages_path @pytest.fixture def verify_standard_versions_valid(monkeypatch): def validate_standard_versions(pkg, versions): for version in versions: print(f"Validated {pkg.name}@{version}") return True monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions) @pytest.fixture def verify_git_versions_valid(monkeypatch): def validate_git_versions(pkg, versions): for version in versions: print(f"Validated {pkg.name}@{version}") return True monkeypatch.setattr(spack.cmd.ci, "validate_git_versions", validate_git_versions) @pytest.fixture def verify_standard_versions_invalid(monkeypatch): def validate_standard_versions(pkg, versions): for version in versions: print(f"Invalid checksum found {pkg.name}@{version}") return False monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions) @pytest.fixture def verify_standard_versions_invalid_duplicates(monkeypatch): def validate_standard_versions(pkg, versions): for version in versions: if str(version) == "2.1.7": print(f"Validated {pkg.name}@{version}") else: print(f"Invalid checksum found {pkg.name}@{version}") return False monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions) @pytest.fixture def verify_git_versions_invalid(monkeypatch): def validate_git_versions(pkg, versions): for version in versions: print(f"Invalid commit for {pkg.name}@{version}") return False monkeypatch.setattr(spack.cmd.ci, "validate_git_versions", validate_git_versions) def test_ci_verify_versions_valid( monkeypatch, mock_packages, mock_git_package_changes, verify_standard_versions_valid, verify_git_versions_valid, ): repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(repo): monkeypatch.setattr(spack.repo, "builtin_repo", lambda: repo) out = ci_cmd("verify-versions", commits[-1], commits[-3]) assert "Validated diff-test@2.1.5" in out assert "Validated diff-test@2.1.6" in out def test_ci_verify_versions_invalid( monkeypatch, mock_packages, mock_git_package_changes, verify_standard_versions_invalid, verify_git_versions_invalid, ): repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(repo): monkeypatch.setattr(spack.repo, "builtin_repo", lambda: repo) out = ci_cmd("verify-versions", commits[-1], commits[-3], fail_on_error=False) assert "Invalid checksum found diff-test@2.1.5" in out assert "Invalid commit for diff-test@2.1.6" in out def test_ci_verify_versions_standard_duplicates( monkeypatch, mock_packages, mock_git_package_changes, verify_standard_versions_invalid_duplicates, ): repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(repo): monkeypatch.setattr(spack.repo, "builtin_repo", lambda: repo) out = ci_cmd("verify-versions", commits[-3], commits[-4], fail_on_error=False) print(f"'{out}'") assert "Validated diff-test@2.1.7" in out assert "Invalid checksum found diff-test@2.1.8" in out def test_ci_verify_versions_manual_package(monkeypatch, mock_packages, mock_git_package_changes): repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(repo): monkeypatch.setattr(spack.repo, "builtin_repo", lambda: repo) pkg_class = spack.repo.PATH.get_pkg_class("diff-test") monkeypatch.setattr(pkg_class, "manual_download", True) out = ci_cmd("verify-versions", commits[-1], commits[-2]) assert "Skipping manual download package: diff-test" in out ================================================ FILE: lib/spack/spack/test/cmd/clean.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.caches import spack.cmd.clean import spack.llnl.util.filesystem as fs import spack.main import spack.package_base import spack.stage import spack.store clean = spack.main.SpackCommand("clean") @pytest.fixture() def mock_calls_for_clean(monkeypatch): counts = {} class Counter: def __init__(self, name): self.name = name counts[name] = 0 def __call__(self, *args, **kwargs): counts[self.name] += 1 monkeypatch.setattr(spack.package_base.PackageBase, "do_clean", Counter("package")) monkeypatch.setattr(spack.stage, "purge", Counter("stages")) monkeypatch.setattr(spack.caches.FETCH_CACHE, "destroy", Counter("downloads"), raising=False) monkeypatch.setattr(spack.caches.MISC_CACHE, "destroy", Counter("caches")) monkeypatch.setattr(spack.store.STORE.failure_tracker, "clear_all", Counter("failures")) monkeypatch.setattr(spack.cmd.clean, "remove_python_cache", Counter("python_cache")) monkeypatch.setattr(spack.cmd.clean, "remove_python_cache", Counter("python_cache")) monkeypatch.setattr(fs, "remove_directory_contents", Counter("bootstrap")) yield counts all_effects = ["stages", "downloads", "caches", "failures", "python_cache", "bootstrap"] @pytest.mark.usefixtures("mock_packages") @pytest.mark.parametrize( "command_line,effects", [ ("mpileaks", ["package"]), ("-s", ["stages"]), ("-sd", ["stages", "downloads"]), ("-m", ["caches"]), ("-f", ["failures"]), ("-p", ["python_cache"]), ("-a", all_effects), ("", []), ], ) def test_function_calls(command_line, effects, mock_calls_for_clean, mutable_config): mutable_config.set("bootstrap", {"root": "fake"}) # Call the command with the supplied command line clean(command_line) # Assert that we called the expected functions the correct # number of times for name in ["package"] + all_effects: assert mock_calls_for_clean[name] == (1 if name in effects else 0) def test_remove_python_cache(tmp_path: pathlib.Path, monkeypatch): cache_files = ["file1.pyo", "file2.pyc"] source_file = "file1.py" def _setup_files(directory): # Create a python cache and source file. cache_dir = fs.join_path(directory, "__pycache__") fs.mkdirp(cache_dir) fs.touch(fs.join_path(directory, source_file)) fs.touch(fs.join_path(directory, cache_files[0])) for filename in cache_files: # Ensure byte code files in python cache directory fs.touch(fs.join_path(cache_dir, filename)) def _check_files(directory): # Ensure the python cache created by _setup_files is removed # and the source file is not. assert os.path.exists(fs.join_path(directory, source_file)) assert not os.path.exists(fs.join_path(directory, cache_files[0])) assert not os.path.exists(fs.join_path(directory, "__pycache__")) source_dir = fs.join_path(str(tmp_path), "lib", "spack", "spack") var_dir = fs.join_path(str(tmp_path), "var", "spack", "stuff") for d in [source_dir, var_dir]: _setup_files(d) # Patching the path variables from-import'd by spack.cmd.clean is needed # to ensure the paths used by the command for this test reflect the # temporary directory locations and not those from spack.paths when # the clean command's module was imported. monkeypatch.setattr(spack.cmd.clean, "lib_path", source_dir) monkeypatch.setattr(spack.cmd.clean, "var_path", var_dir) spack.cmd.clean.remove_python_cache() for d in [source_dir, var_dir]: _check_files(d) ================================================ FILE: lib/spack/spack/test/cmd/commands.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp import os import pathlib import shutil import sys import textwrap import pytest import spack.cmd import spack.cmd.commands import spack.config import spack.main import spack.paths from spack.cmd.commands import _dest_to_fish_complete, _positional_to_subroutine from spack.util.executable import Executable def commands(*args: str) -> str: """Run `spack commands args...` and return output as a string. It's a separate process so that we run through the main Spack command logic and avoid caching issues.""" python = Executable(sys.executable) return python(spack.paths.spack_script, "commands", *args, output=str) def test_names(): """Test default output of spack commands.""" out1 = commands().strip().splitlines() assert out1 == spack.cmd.all_commands() assert "rm" not in out1 out2 = commands("--aliases").strip().splitlines() assert out1 != out2 assert "rm" in out2 out3 = commands("--format=names").strip().splitlines() assert out1 == out3 def test_subcommands(): """Test subcommand traversal.""" out1 = commands("--format=subcommands") assert "spack mirror create" in out1 assert "spack buildcache list" in out1 assert "spack repo add" in out1 assert "spack pkg diff" in out1 assert "spack url parse" in out1 assert "spack view symlink" in out1 assert "spack rm" not in out1 assert "spack compiler add" not in out1 out2 = commands("--aliases", "--format=subcommands") assert "spack mirror create" in out2 assert "spack buildcache list" in out2 assert "spack repo add" in out2 assert "spack pkg diff" in out2 assert "spack url parse" in out2 assert "spack view symlink" in out2 assert "spack rm" in out2 assert "spack compiler add" in out2 def test_alias_overrides_builtin(mutable_config: spack.config.Configuration, capfd): """Test that spack commands cannot be overridden by aliases.""" mutable_config.set("config:aliases", {"install": "find"}) cmd, args = spack.main.resolve_alias("install", ["install", "-v"]) assert cmd == "install" and args == ["install", "-v"] out = capfd.readouterr().err assert "Alias 'install' (mapping to 'find') attempts to override built-in command" in out def test_alias_with_space(mutable_config: spack.config.Configuration, capfd): """Test that spack aliases with spaces are rejected.""" mutable_config.set("config:aliases", {"foo bar": "find"}) cmd, args = spack.main.resolve_alias("install", ["install", "-v"]) assert cmd == "install" and args == ["install", "-v"] out = capfd.readouterr().err assert "Alias 'foo bar' (mapping to 'find') contains a space, which is not supported" in out def test_alias_resolves_properly(mutable_config: spack.config.Configuration): """Test that spack aliases resolve properly.""" mutable_config.set("config:aliases", {"my_find": "find"}) cmd, args = spack.main.resolve_alias("my_find", ["my_find", "-v"]) assert cmd == "find" and args == ["find", "-v"] def test_rst(): """Do some simple sanity checks of the rst writer.""" out1 = commands("--format=rst") assert "spack mirror create" in out1 assert "spack buildcache list" in out1 assert "spack repo add" in out1 assert "spack pkg diff" in out1 assert "spack url parse" in out1 assert "spack view symlink" in out1 assert "spack rm" not in out1 assert "spack compiler add" not in out1 out2 = commands("--aliases", "--format=rst") assert "spack mirror create" in out2 assert "spack buildcache list" in out2 assert "spack repo add" in out2 assert "spack pkg diff" in out2 assert "spack url parse" in out2 assert "spack view symlink" in out2 assert "spack rm" in out2 assert "spack compiler add" in out2 def test_rst_with_input_files(tmp_path: pathlib.Path): filename = tmp_path / "file.rst" with filename.open("w") as f: f.write( """ .. _cmd-spack-fetch: cmd-spack-list: .. _cmd-spack-stage: _cmd-spack-install: .. _cmd-spack-patch: """ ) out = commands("--format=rst", str(filename)) for name in ["fetch", "stage", "patch"]: assert (":ref:`More documentation `" % name) in out for name in ["list", "install"]: assert (":ref:`More documentation `" % name) not in out def test_rst_with_header(tmp_path: pathlib.Path): local_commands = spack.main.SpackCommand("commands") fake_header = "this is a header!\n\n" filename = tmp_path / "header.txt" with filename.open("w") as f: f.write(fake_header) out = local_commands("--format=rst", "--header", str(filename)) assert out.startswith(fake_header) with pytest.raises(spack.main.SpackCommandError): local_commands("--format=rst", "--header", "asdfjhkf") def test_rst_update(tmp_path: pathlib.Path): update_file = tmp_path / "output" commands("--update", str(update_file)) assert update_file.exists() def test_update_with_header(tmp_path: pathlib.Path): update_file = tmp_path / "output" commands("--update", str(update_file)) assert update_file.exists() fake_header = "this is a header!\n\n" filename = tmp_path / "header.txt" with filename.open("w") as f: f.write(fake_header) commands("--update", str(update_file), "--header", str(filename)) def test_bash_completion(): """Test the bash completion writer.""" out1 = commands("--format=bash") # Make sure header not included assert "_bash_completion_spack() {" not in out1 assert "_all_packages() {" not in out1 # Make sure subcommands appear assert "_spack_remove() {" in out1 assert "_spack_compiler_find() {" in out1 # Make sure aliases don't appear assert "_spack_rm() {" not in out1 assert "_spack_compiler_add() {" not in out1 # Make sure options appear assert "-h --help" in out1 # Make sure subcommands are called for function in _positional_to_subroutine.values(): assert function in out1 out2 = commands("--aliases", "--format=bash") # Make sure aliases appear assert "_spack_rm() {" in out2 assert "_spack_compiler_add() {" in out2 def test_bash_completion_choices(): """Test that bash completion includes choices for positional arguments.""" out = commands("--format=bash") # `spack env view` has a positional `action` with choices assert 'SPACK_COMPREPLY="disable enable regenerate"' in out def test_fish_completion(): """Test the fish completion writer.""" out1 = commands("--format=fish") # Make sure header not included assert "function __fish_spack_argparse" not in out1 assert "complete -c spack --erase" not in out1 # Make sure subcommands appear assert "__fish_spack_using_command remove" in out1 assert "__fish_spack_using_command compiler find" in out1 # Make sure aliases don't appear assert "__fish_spack_using_command rm" not in out1 assert "__fish_spack_using_command compiler add" not in out1 # Make sure options appear assert "-s h -l help" in out1 # Make sure subcommands are called for complete_cmd in _dest_to_fish_complete.values(): assert complete_cmd in out1 out2 = commands("--aliases", "--format=fish") # Make sure aliases appear assert "__fish_spack_using_command rm" in out2 assert "__fish_spack_using_command compiler add" in out2 @pytest.mark.parametrize("shell", ["bash", "fish"]) def test_update_completion_arg(shell, tmp_path: pathlib.Path, monkeypatch): """Test the update completion flag.""" (tmp_path / shell).mkdir() mock_infile = tmp_path / shell / f"spack-completion.{shell}" mock_outfile = tmp_path / f"spack-completion.{shell}" mock_args = { shell: { "aliases": True, "format": shell, "header": str(mock_infile), "update": str(mock_outfile), } } # make a mock completion file missing the --update-completion argument real_args = spack.cmd.commands.update_completion_args shutil.copy(real_args[shell]["header"], mock_args[shell]["header"]) with open(real_args[shell]["update"], encoding="utf-8") as old: old_file = old.read() with open(mock_args[shell]["update"], "w", encoding="utf-8") as mock: mock.write(old_file.replace("update-completion", "")) monkeypatch.setattr(spack.cmd.commands, "update_completion_args", mock_args) local_commands = spack.main.SpackCommand("commands") # ensure things fail if --update-completion isn't specified alone with pytest.raises(spack.main.SpackCommandError): local_commands("--update-completion", "-a") # ensure arg is restored assert "update-completion" not in mock_outfile.read_text() local_commands("--update-completion") assert "update-completion" in mock_outfile.read_text() # Note: this test is never expected to be supported on Windows @pytest.mark.not_on_windows("Shell completion script generator fails on windows") @pytest.mark.parametrize("shell", ["bash", "fish"]) def test_updated_completion_scripts(shell, tmp_path: pathlib.Path): """Make sure our shell tab completion scripts remain up-to-date.""" width = 72 lines = textwrap.wrap( "It looks like Spack's command-line interface has been modified. " "If differences are more than your global 'include:' scopes, please " "update Spack's shell tab completion scripts by running:", width, ) lines.append("\n spack commands --update-completion\n") lines.extend( textwrap.wrap( "and adding the changed files (minus your global 'include:' scopes) " "to your pull request.", width, ) ) msg = "\n".join(lines) header = os.path.join(spack.paths.share_path, shell, f"spack-completion.{shell}") script = f"spack-completion.{shell}" old_script = os.path.join(spack.paths.share_path, script) new_script = str(tmp_path / script) commands("--aliases", "--format", shell, "--header", header, "--update", new_script) if not filecmp.cmp(old_script, new_script): # If there is a diff, something is wrong: in that case output what the diff is. import difflib with open(old_script, "r", encoding="utf-8") as f1, open( new_script, "r", encoding="utf-8" ) as f2: l1 = f1.readlines() l2 = f2.readlines() diff = difflib.unified_diff(l1, l2, fromfile=old_script, tofile=new_script) msg += "\nDiff failure:\n\n" + "".join(diff) raise AssertionError(msg) ================================================ FILE: lib/spack/spack/test/cmd/common/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/test/cmd/common/arguments.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import pytest import spack.cmd import spack.cmd.common.arguments as arguments import spack.config import spack.environment as ev import spack.main @pytest.fixture() def job_parser(): # --jobs needs to write to a command_line config scope, so this is the only # scope we create. p = argparse.ArgumentParser() arguments.add_common_arguments(p, ["jobs"]) scopes = [spack.config.InternalConfigScope("command_line", {"config": {}})] with spack.config.use_configuration(*scopes): yield p def test_setting_jobs_flag(job_parser): namespace = job_parser.parse_args(["-j", "24"]) assert namespace.jobs == 24 assert spack.config.get("config:build_jobs", scope="command_line") == 24 def test_omitted_job_flag(job_parser): namespace = job_parser.parse_args([]) assert namespace.jobs is None assert spack.config.get("config:build_jobs") is None def test_negative_integers_not_allowed_for_parallel_jobs(job_parser): with pytest.raises(ValueError) as exc_info: job_parser.parse_args(["-j", "-2"]) assert "expected a positive integer" in str(exc_info.value) @pytest.mark.parametrize( "specs,cflags,propagation,negated_variants", [ (['coreutils cflags="-O3 -g"'], ["-O3", "-g"], [False, False], []), (['coreutils cflags=="-O3 -g"'], ["-O3", "-g"], [True, True], []), (["coreutils", "cflags=-O3 -g"], ["-O3", "-g"], [False, False], []), (["coreutils", "cflags==-O3 -g"], ["-O3", "-g"], [True, True], []), (["coreutils", "cflags=-O3", "-g"], ["-O3"], [False], ["g"]), ], ) @pytest.mark.regression("12951") def test_parse_spec_flags_with_spaces(specs, cflags, propagation, negated_variants): spec_list = spack.cmd.parse_specs(specs) assert len(spec_list) == 1 s = spec_list.pop() compiler_flags = [flag for flag in s.compiler_flags["cflags"]] flag_propagation = [flag.propagate for flag in s.compiler_flags["cflags"]] assert compiler_flags == cflags assert flag_propagation == propagation assert list(s.variants.keys()) == negated_variants for v in negated_variants: assert "~{0}".format(v) in s def test_match_spec_env(mock_packages, mutable_mock_env_path): """ Concretize a spec with non-default options in an environment. Make sure that when we ask for a matching spec when the environment is active that we get the instance concretized in the environment. """ # Initial sanity check: we are planning on choosing a non-default # value, so make sure that is in fact not the default. check_defaults = spack.cmd.parse_specs(["pkg-a"], concretize=True)[0] assert not check_defaults.satisfies("foobar=baz") e = ev.create("test") e.add("pkg-a foobar=baz") e.concretize() with e: env_spec = spack.cmd.matching_spec_from_env(spack.cmd.parse_specs(["pkg-a"])[0]) assert env_spec.satisfies("foobar=baz") assert env_spec.concrete def test_multiple_env_match_raises_error(mock_packages, mutable_mock_env_path): e = ev.create("test") e.add("pkg-a foobar=baz") e.add("pkg-a foobar=fee") e.concretize() with e: with pytest.raises(ev.SpackEnvironmentError) as exc_info: spack.cmd.matching_spec_from_env(spack.cmd.parse_specs(["pkg-a"])[0]) assert "matches multiple specs" in exc_info.value.message def test_root_and_dep_match_returns_root(mock_packages, mutable_mock_env_path): e = ev.create("test") e.add("pkg-b@0.9") e.add("pkg-a foobar=bar") # Depends on b, should choose b@1.0 e.concretize() with e: # This query matches the root b and b as a dependency of a. In that # case the root instance should be preferred. env_spec1 = spack.cmd.matching_spec_from_env(spack.cmd.parse_specs(["pkg-b"])[0]) assert env_spec1.satisfies("@0.9") env_spec2 = spack.cmd.matching_spec_from_env(spack.cmd.parse_specs(["pkg-b@1.0"])[0]) assert env_spec2 @pytest.mark.parametrize( "arg,conf", [ ("--reuse", True), ("--fresh", False), ("--reuse-deps", "dependencies"), ("--fresh-roots", "dependencies"), ], ) def test_concretizer_arguments(mutable_config, mock_packages, arg, conf): """Ensure that ConfigSetAction is doing the right thing.""" spec = spack.main.SpackCommand("spec") assert spack.config.get("concretizer:reuse", None, scope="command_line") is None spec(arg, "zlib") assert spack.config.get("concretizer:reuse", None) == conf assert spack.config.get("concretizer:reuse", None, scope="command_line") == conf def test_use_buildcache_type(): assert arguments.use_buildcache("only") == ("only", "only") assert arguments.use_buildcache("never") == ("never", "never") assert arguments.use_buildcache("auto") == ("auto", "auto") assert arguments.use_buildcache("package:never,dependencies:only") == ("never", "only") assert arguments.use_buildcache("only,package:never") == ("never", "only") assert arguments.use_buildcache("package:only,package:never") == ("never", "auto") assert arguments.use_buildcache("auto , package: only") == ("only", "auto") with pytest.raises(argparse.ArgumentTypeError): assert arguments.use_buildcache("pkg:only,deps:never") with pytest.raises(argparse.ArgumentTypeError): assert arguments.use_buildcache("sometimes") def test_missing_config_scopes_are_valid_scope_arguments(mock_missing_dir_include_scopes): """Test that if an included scope does not have a directory or file, we can still specify it as a scope as an argument""" a = argparse.ArgumentParser() a.add_argument( "--scope", action=arguments.ConfigScope, default=lambda: spack.config.default_modify_scope(), help="configuration scope to modify", ) namespace = a.parse_args(["--scope", "sub_base"]) assert namespace.scope == "sub_base" def test_missing_config_scopes_not_valid_read_scope(mock_missing_dir_include_scopes): """Ensures that if a missing include scope is the subject of a read operation, we fail at the argparse level""" a = argparse.ArgumentParser() a.add_argument( "--scope", action=arguments.ConfigScope, type=arguments.config_scope_readable_validator, default=lambda: spack.config.default_modify_scope(), help="configuration scope to modify", ) with pytest.raises(SystemExit): a.parse_args(["--scope", "sub_base"]) ================================================ FILE: lib/spack/spack/test/cmd/common/spec_strings.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import spack.cmd.common.spec_strings def test_spec_strings(tmp_path: pathlib.Path): (tmp_path / "example.py").write_text( """\ def func(x): print("dont fix %s me" % x, 3) return x.satisfies("+foo %gcc +bar") and x.satisfies("%gcc +baz") """ ) (tmp_path / "example.json").write_text( """\ { "spec": [ "+foo %gcc +bar~nope ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another %gcc ", "%gcc +baz" ], "%gcc x=y": 2 } """ ) (tmp_path / "example.yaml").write_text( """\ spec: - "+foo %gcc +bar" - "%gcc +baz" - "this is fine %clang" "%gcc x=y": 2 """ ) issues = set() def collect_issues(path: str, line: int, col: int, old: str, new: str): issues.add((path, line, col, old, new)) # check for issues with custom handler spack.cmd.common.spec_strings._check_spec_strings( [ str(tmp_path / "nonexistent.py"), str(tmp_path / "example.py"), str(tmp_path / "example.json"), str(tmp_path / "example.yaml"), ], handler=collect_issues, ) assert issues == { ( str(tmp_path / "example.json"), 3, 9, "+foo %gcc +bar~nope ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another %gcc ", "+foo +bar~nope %gcc ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another %gcc ", ), (str(tmp_path / "example.json"), 4, 9, "%gcc +baz", "+baz %gcc"), (str(tmp_path / "example.json"), 6, 5, "%gcc x=y", "x=y %gcc"), (str(tmp_path / "example.py"), 3, 23, "+foo %gcc +bar", "+foo +bar %gcc"), (str(tmp_path / "example.py"), 3, 57, "%gcc +baz", "+baz %gcc"), (str(tmp_path / "example.yaml"), 2, 5, "+foo %gcc +bar", "+foo +bar %gcc"), (str(tmp_path / "example.yaml"), 3, 5, "%gcc +baz", "+baz %gcc"), (str(tmp_path / "example.yaml"), 5, 1, "%gcc x=y", "x=y %gcc"), } # fix the issues in the files spack.cmd.common.spec_strings._check_spec_strings( [ str(tmp_path / "nonexistent.py"), str(tmp_path / "example.py"), str(tmp_path / "example.json"), str(tmp_path / "example.yaml"), ], handler=spack.cmd.common.spec_strings._spec_str_fix_handler, ) assert ( (tmp_path / "example.json").read_text() == """\ { "spec": [ "+foo +bar~nope %gcc ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another %gcc ", "+baz %gcc" ], "x=y %gcc": 2 } """ ) assert ( (tmp_path / "example.py").read_text() == """\ def func(x): print("dont fix %s me" % x, 3) return x.satisfies("+foo +bar %gcc") and x.satisfies("+baz %gcc") """ ) assert ( (tmp_path / "example.yaml").read_text() == """\ spec: - "+foo +bar %gcc" - "+baz %gcc" - "this is fine %clang" "x=y %gcc": 2 """ ) ================================================ FILE: lib/spack/spack/test/cmd/compiler.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import shutil import pytest import spack.cmd.compiler import spack.compilers.config import spack.config import spack.main import spack.util.pattern import spack.version compiler = spack.main.SpackCommand("compiler") pytestmark = [pytest.mark.usefixtures("mock_packages")] @pytest.fixture def compilers_dir(mock_executable): """Create a directory with some mock compiler scripts in it. Scripts are: - clang - clang++ - gcc - g++ - gfortran-8 """ clang_path = mock_executable( "clang", output=""" if [ "$1" = "--version" ]; then echo "clang version 11.0.0 (clang-1100.0.33.16)" echo "Target: x86_64-apple-darwin18.7.0" echo "Thread model: posix" echo "InstalledDir: /dummy" else echo "clang: error: no input files" exit 1 fi """, ) shutil.copy(clang_path, clang_path.parent / "clang++") gcc_script = """ if [ "$1" = "-dumpversion" ]; then echo "8" elif [ "$1" = "-dumpfullversion" ]; then echo "8.4.0" elif [ "$1" = "--version" ]; then echo "{0} (GCC) 8.4.0 20120313 (Red Hat 8.4.0-1)" echo "Copyright (C) 2010 Free Software Foundation, Inc." else echo "{1}: fatal error: no input files" echo "compilation terminated." exit 1 fi """ mock_executable("gcc-8", output=gcc_script.format("gcc", "gcc-8")) mock_executable("g++-8", output=gcc_script.format("g++", "g++-8")) mock_executable("gfortran-8", output=gcc_script.format("GNU Fortran", "gfortran-8")) return clang_path.parent @pytest.mark.not_on_windows("Cannot execute bash script on Windows") @pytest.mark.regression("11678,13138") def test_compiler_find_without_paths(no_packages_yaml, working_env, mock_executable): """Tests that 'spack compiler find' looks into PATH by default, if no specific path is given. """ gcc_path = mock_executable("gcc", output='echo "0.0.0"') os.environ["PATH"] = str(gcc_path.parent) output = compiler("find", "--scope=site") assert "gcc" in output @pytest.mark.regression("37996") def test_compiler_remove(mutable_config): """Tests that we can remove a compiler from configuration.""" assert any( compiler.satisfies("gcc@=9.4.0") for compiler in spack.compilers.config.all_compilers() ) args = spack.util.pattern.Bunch(all=True, compiler_spec="gcc@9.4.0", add_paths=[], scope=None) spack.cmd.compiler.compiler_remove(args) assert not any( compiler.satisfies("gcc@=9.4.0") for compiler in spack.compilers.config.all_compilers() ) @pytest.mark.regression("37996") def test_removing_compilers_from_multiple_scopes(mutable_config): # Duplicate "site" scope into "user" scope site_config = spack.config.get("packages", scope="site") spack.config.set("packages", site_config, scope="user") assert any( compiler.satisfies("gcc@=9.4.0") for compiler in spack.compilers.config.all_compilers() ) args = spack.util.pattern.Bunch(all=True, compiler_spec="gcc@9.4.0", add_paths=[], scope=None) spack.cmd.compiler.compiler_remove(args) assert not any( compiler.satisfies("gcc@=9.4.0") for compiler in spack.compilers.config.all_compilers() ) @pytest.mark.not_on_windows("Cannot execute bash script on Windows") def test_compiler_add(mutable_config, mock_executable): """Tests that we can add a compiler to configuration.""" expected_version = "4.5.3" gcc_path = mock_executable( "gcc", output=f"""\ for arg in "$@"; do if [ "$arg" = -dumpversion ]; then echo '{expected_version}' fi done """, ) bin_dir = gcc_path.parent root_dir = bin_dir.parent compilers_before_find = set(spack.compilers.config.all_compilers()) args = spack.util.pattern.Bunch( all=None, compiler_spec=None, add_paths=[str(root_dir)], scope=None, mixed_toolchain=False, jobs=1, ) spack.cmd.compiler.compiler_find(args) compilers_after_find = set(spack.compilers.config.all_compilers()) compilers_added_by_find = compilers_after_find - compilers_before_find assert len(compilers_added_by_find) == 1 new_compiler = compilers_added_by_find.pop() assert new_compiler.version == spack.version.Version(expected_version) @pytest.mark.not_on_windows("Cannot execute bash script on Windows") @pytest.mark.regression("17590") def test_compiler_find_prefer_no_suffix(no_packages_yaml, working_env, compilers_dir): """Ensure that we'll pick 'clang' over 'clang-gpu' when there is a choice.""" clang_path = compilers_dir / "clang" shutil.copy(clang_path, clang_path.parent / "clang-gpu") shutil.copy(clang_path, clang_path.parent / "clang++-gpu") os.environ["PATH"] = str(compilers_dir) output = compiler("find", "--scope=site") assert "llvm@11.0.0" in output assert "gcc@8.4.0" in output compilers = spack.compilers.config.all_compilers_from(no_packages_yaml, scope="site") clang = [x for x in compilers if x.satisfies("llvm@11")] assert len(clang) == 1 assert clang[0].extra_attributes["compilers"]["c"] == str(compilers_dir / "clang") assert clang[0].extra_attributes["compilers"]["cxx"] == str(compilers_dir / "clang++") @pytest.mark.not_on_windows("Cannot execute bash script on Windows") def test_compiler_find_path_order(no_packages_yaml, working_env, compilers_dir): """When the same compiler version is found in two PATH directories, only the first entry in PATH is kept and a warning is emitted for the duplicate. """ new_dir = compilers_dir / "first_in_path" new_dir.mkdir() for name in ("gcc-8", "g++-8", "gfortran-8"): shutil.copy(compilers_dir / name, new_dir / name) # Set PATH to have the new folder searched first os.environ["PATH"] = f"{str(new_dir)}:{str(compilers_dir)}" with pytest.warns(UserWarning, match="gcc@"): compiler("find", "--scope=site") compilers = spack.compilers.config.all_compilers(scope="site") gcc = [x for x in compilers if x.satisfies("gcc@8.4")] # Duplicate is dropped. Only the first entry in PATH is kept assert len(gcc) == 1 assert gcc[0].extra_attributes["compilers"] == { "c": str(new_dir / "gcc-8"), "cxx": str(new_dir / "g++-8"), "fortran": str(new_dir / "gfortran-8"), } def test_compiler_list_empty(no_packages_yaml, compilers_dir, monkeypatch): """Spack should not automatically search for compilers when listing them and none are available. And when stdout is not a tty like in tests, there should be no output and no error exit code. """ monkeypatch.setenv("PATH", str(compilers_dir), prepend=":") out = compiler("list") assert not out assert compiler.returncode == 0 @pytest.mark.parametrize( "external,expected", [ ( { "spec": "gcc@=7.7.7 languages=c,cxx,fortran os=foobar target=x86_64", "prefix": "/path/to/fake", "modules": ["gcc/7.7.7", "foobar"], "extra_attributes": { "compilers": { "c": "/path/to/fake/gcc", "cxx": "/path/to/fake/g++", "fortran": "/path/to/fake/gfortran", }, "flags": {"fflags": "-ffree-form"}, }, }, """gcc@7.7.7 languages=c,cxx,fortran os=foobar target=x86_64: paths: cc = /path/to/fake/gcc cxx = /path/to/fake/g++ \t\tf77 = /path/to/fake/gfortran \t\tfc = /path/to/fake/gfortran \tflags: \t\tfflags = ['-ffree-form'] \tmodules = ['gcc/7.7.7', 'foobar'] \toperating system = foobar """, ) ], ) def test_compilers_shows_packages_yaml( external, expected, no_packages_yaml, working_env, compilers_dir ): """Spack should see a single compiler defined from packages.yaml""" external["prefix"] = external["prefix"].format(prefix=os.path.dirname(compilers_dir)) gcc_entry = {"externals": [external]} packages = spack.config.get("packages") packages["gcc"] = gcc_entry spack.config.set("packages", packages) out = compiler("list", fail_on_error=True) assert out.count("gcc@7.7.7") == 1 ================================================ FILE: lib/spack/spack/test/cmd/concretize.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.environment as ev from spack import spack_version from spack.main import SpackCommand pytestmark = pytest.mark.usefixtures("mutable_config", "mutable_mock_repo") env = SpackCommand("env") add = SpackCommand("add") concretize = SpackCommand("concretize") unification_strategies = [False, True, "when_possible"] @pytest.mark.parametrize("unify", unification_strategies) def test_concretize_all_test_dependencies(unify, mutable_config, mutable_mock_env_path): """Check all test dependencies are concretized.""" env("create", "test") with ev.read("test") as e: mutable_config.set("concretizer:unify", unify) add("depb") concretize("--test", "all") assert e.matching_spec("test-dependency") @pytest.mark.parametrize("unify", unification_strategies) def test_concretize_root_test_dependencies_not_recursive( unify, mutable_config, mutable_mock_env_path ): """Check that test dependencies are not concretized recursively.""" env("create", "test") with ev.read("test") as e: mutable_config.set("concretizer:unify", unify) add("depb") concretize("--test", "root") assert e.matching_spec("test-dependency") is None @pytest.mark.parametrize("unify", unification_strategies) def test_concretize_root_test_dependencies_are_concretized( unify, mutable_config, mutable_mock_env_path ): """Check that root test dependencies are concretized.""" env("create", "test") with ev.read("test") as e: mutable_config.set("concretizer:unify", unify) add("pkg-a") add("pkg-b") concretize("--test", "root") assert e.matching_spec("test-dependency") data = e._to_lockfile_dict() assert data["spack"]["version"] == spack_version ================================================ FILE: lib/spack/spack/test/cmd/config.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import functools import json import os import pathlib import re import pytest import spack.concretize import spack.config import spack.database import spack.environment as ev import spack.llnl.util.filesystem as fs import spack.main import spack.schema.config import spack.store import spack.util.spack_yaml as syaml config = spack.main.SpackCommand("config") env = spack.main.SpackCommand("env") pytestmark = pytest.mark.usefixtures("mock_packages") def _create_config(scope=None, data={}, section="packages"): scope = scope or spack.config.default_modify_scope() cfg_file = spack.config.CONFIG.get_config_filename(scope, section) with open(cfg_file, "w", encoding="utf-8") as f: syaml.dump(data, stream=f) return cfg_file @pytest.fixture() def config_yaml_v015(mutable_config): """Create a packages.yaml in the old format""" old_data = { "config": {"install_tree": "/fake/path", "install_path_scheme": "{name}-{version}"} } return functools.partial(_create_config, data=old_data, section="config") scope_path_re = r"\(([^\)]+)\)" @pytest.mark.parametrize( "path,types", [ (False, []), (True, []), (False, ["path"]), (False, ["env"]), (False, ["internal", "include"]), ], ) def test_config_scopes(path, types, mutable_mock_env_path): ev.create("test") scopes_cmd = ["scopes"] if path: scopes_cmd.append("-p") if types: scopes_cmd.extend(["-t", *types]) output = config(*scopes_cmd).split() if not types or any(i in ("all", "internal") for i in types): assert "command_line" in output assert "_builtin" in output if types: if not any(i in ("all", "path", "include") for i in types): assert "site" not in output if not any(i in ("all", "env", "include", "path") for i in types): assert not output or all(":" not in x for x in output) if not any(i in ("all", "env", "path") for i in types): assert not output or all(not x.startswith("env:") for x in output) if not any(i in ("all", "internal") for i in types): assert "command_line" not in output assert "_builtin" not in output if path: paths = (x[1] for x in (re.fullmatch(scope_path_re, s) for s in output) if x) assert all(os.sep in x for x in paths) @pytest.mark.parametrize("type", ["path", "include", "internal", "env"]) def test_config_scopes_include(type): """Ensure that `spack config scopes -vt TYPE outputs only scopes of that type.""" scopes_cmd = ["scopes", "-vt", type] output = config(*scopes_cmd).strip() lines = output.split("\n") assert not output or all([type in line for line in lines[1:]]) def test_config_scopes_section(mutable_config): scopes_cmd = ["scopes", "-v", "packages"] output = config(*scopes_cmd).strip() lines = output.split("\n") lines_by_scope_name = {line.split()[0]: line for line in lines} assert "absent" in lines_by_scope_name["command_line"] assert "absent" in lines_by_scope_name["_builtin"] assert "active" in lines_by_scope_name["site"] def test_include_overrides(mutable_config): output = config("scopes").strip() lines = output.split("\n") assert "user" in lines assert "system" in lines assert "site" in lines assert "_builtin" in lines mutable_config.push_scope(spack.config.InternalConfigScope("override", {"include:": []})) # overridden scopes are not shown without `-v` output = config("scopes").strip() lines = output.split("\n") assert "user" not in lines assert "system" not in lines assert "site" not in lines # scopes with ConfigScopePriority.DEFAULTS remain assert "_builtin" in lines # overridden scopes are shown with `-v` and marked 'override' output = config("scopes", "-v").strip() lines = output.split("\n") assert "override" in next(line for line in lines if line.startswith("user")) assert "override" in next(line for line in lines if line.startswith("system")) assert "override" in next(line for line in lines if line.startswith("site")) def test_blame_override(mutable_config): # includes are present when section is specified output = config("blame", "include").strip() include_path = re.escape(os.path.join(mutable_config.scopes["site"].path, "include.yaml")) assert re.search(rf"{include_path}:\d+\s+\- path: base", output) # includes are also present when section is NOT specified output = config("blame").strip() assert re.search(rf"{include_path}:\d+\s+\- path: base", output) mutable_config.push_scope(spack.config.InternalConfigScope("override", {"include:": []})) # site includes are not present when overridden output = config("blame", "include").strip() assert not re.search(rf"{include_path}:\d+\s+\- path: base", output) assert "include: []" in output output = config("blame").strip() assert not re.search(rf"{include_path}:\d+\s+\- path: base", output) assert "include: []" in output def test_config_scopes_path(mutable_config): scopes_cmd = ["scopes", "-p"] output = config(*scopes_cmd).strip() lines = output.split("\n") lines_by_scope_name = {line.split()[0]: line for line in lines} assert f"{os.sep}user{os.sep}" in lines_by_scope_name["user"] assert f"{os.sep}system{os.sep}" in lines_by_scope_name["system"] assert f"{os.sep}site{os.sep}" in lines_by_scope_name["site"] def test_get_config_scope(mock_low_high_config): assert config("get", "compilers").strip() == "compilers: {}" def test_get_config_roundtrip(mutable_config): """Test that ``spack config get [--json]
`` roundtrips correctly.""" json_roundtrip = json.loads(config("get", "--json", "config")) yaml_roundtrip = syaml.load(config("get", "config")) assert json_roundtrip["config"] == yaml_roundtrip["config"] == mutable_config.get("config") def test_get_all_config_roundtrip(mutable_config): """Test that ``spack config get [--json]`` roundtrips correctly.""" json_roundtrip = json.loads(config("get", "--json")) yaml_roundtrip = syaml.load(config("get")) assert json_roundtrip == yaml_roundtrip for section in spack.config.SECTION_SCHEMAS: assert json_roundtrip["spack"][section] == mutable_config.get(section) def test_get_config_scope_merged(mock_low_high_config): low_path = mock_low_high_config.scopes["low"].path high_path = mock_low_high_config.scopes["high"].path fs.mkdirp(low_path) fs.mkdirp(high_path) with open(os.path.join(low_path, "repos.yaml"), "w", encoding="utf-8") as f: f.write( """\ repos: repo3: repo3 """ ) with open(os.path.join(high_path, "repos.yaml"), "w", encoding="utf-8") as f: f.write( """\ repos: repo1: repo1 repo2: repo2 """ ) assert ( config("get", "repos").strip() == """repos: repo1: repo1 repo2: repo2 repo3: repo3""" ) def test_config_edit(mutable_config, working_env): """Ensure `spack config edit` edits the right paths.""" dms = spack.config.default_modify_scope("compilers") dms_path = spack.config.CONFIG.scopes[dms].path user_path = spack.config.CONFIG.scopes["user"].path comp_path = os.path.join(dms_path, "compilers.yaml") repos_path = os.path.join(user_path, "repos.yaml") assert config("edit", "--print-file", "compilers").strip() == comp_path assert config("edit", "--print-file", "repos").strip() == repos_path def test_config_get_gets_spack_yaml(mutable_mock_env_path): with ev.create("test") as env: assert "mpileaks" not in config("get") env.add("mpileaks") env.write() assert "mpileaks" in config("get") def test_config_edit_edits_spack_yaml(mutable_mock_env_path): env = ev.create("test") with env: assert config("edit", "--print-file").strip() == env.manifest_path def test_config_add_with_scope_adds_to_scope(mutable_config, mutable_mock_env_path): """Test adding to non-env config scope with an active environment""" env = ev.create("test") with env: config("--scope=user", "add", "config:install_tree:root:/usr") assert spack.config.get("config:install_tree:root", scope="user") == "/usr" def test_config_edit_fails_correctly_with_no_env(mutable_mock_env_path): output = config("edit", "--print-file", fail_on_error=False) assert "requires a section argument or an active environment" in output def test_config_list(): output = config("list") assert "compilers" in output assert "packages" in output def test_config_add(mutable_empty_config): config("add", "config:dirty:true") output = config("get", "config") assert ( output == """config: dirty: true """ ) def test_config_add_list(mutable_empty_config): config("add", "config:template_dirs:test1") config("add", "config:template_dirs:[test2]") config("add", "config:template_dirs:test3") output = config("get", "config") assert ( output == """config: template_dirs: - test3 - test2 - test1 """ ) def test_config_add_override(mutable_empty_config): config("--scope", "site", "add", "config:template_dirs:test1") config("add", "config:template_dirs:[test2]") output = config("get", "config") assert ( output == """config: template_dirs: - test2 - test1 """ ) config("add", "config::template_dirs:[test2]") output = config("get", "config") assert ( output == """config: template_dirs: - test2 """ ) def test_config_add_override_leaf(mutable_empty_config): config("--scope", "site", "add", "config:template_dirs:test1") config("add", "config:template_dirs:[test2]") output = config("get", "config") assert ( output == """config: template_dirs: - test2 - test1 """ ) config("add", "config:template_dirs::[test2]") output = config("get", "config") assert ( output == """config: 'template_dirs:': - test2 """ ) def test_config_add_update_dict(mutable_empty_config): config("add", "packages:hdf5:version:[1.0.0]") output = config("get", "packages") expected = "packages:\n hdf5:\n version: [1.0.0]\n" assert output == expected def test_config_with_c_argument(mutable_empty_config): # I don't know how to add a spack argument to a Spack Command, so we test this way config_file = "config:install_tree:root:/path/to/config.yaml" parser = spack.main.make_argument_parser() args = parser.parse_args(["-c", config_file]) assert config_file in args.config_vars # Add the path to the config config("add", args.config_vars[0]) output = config("get", "config") assert "config:\n install_tree:\n root: /path/to/config.yaml" in output def test_config_add_ordered_dict(mutable_empty_config): config("add", "mirrors:first:/path/to/first") config("add", "mirrors:second:/path/to/second") output = config("get", "mirrors") assert ( output == """mirrors: first: /path/to/first second: /path/to/second """ ) def test_config_add_interpret_oneof(mutable_empty_config): # Regression test for a bug that would raise a validation error config("add", "packages:all:target:[x86_64]") config("add", "packages:all:variants:~shared") def test_config_add_invalid_fails(mutable_empty_config): config("add", "packages:all:variants:+debug") with pytest.raises((spack.config.ConfigFormatError, AttributeError)): config("add", "packages:all:True") def test_config_add_from_file(mutable_empty_config, tmp_path: pathlib.Path): contents = """spack: config: dirty: true """ file = str(tmp_path / "spack.yaml") with open(file, "w", encoding="utf-8") as f: f.write(contents) config("add", "-f", file) output = config("get", "config") assert ( output == """config: dirty: true """ ) def test_config_add_from_file_multiple(mutable_empty_config, tmp_path: pathlib.Path): contents = """spack: config: dirty: true template_dirs: [test1] """ file = str(tmp_path / "spack.yaml") with open(file, "w", encoding="utf-8") as f: f.write(contents) config("add", "-f", file) output = config("get", "config") assert ( output == """config: dirty: true template_dirs: [test1] """ ) def test_config_add_override_from_file(mutable_empty_config, tmp_path: pathlib.Path): config("--scope", "site", "add", "config:template_dirs:test1") contents = """spack: config:: template_dirs: [test2] """ file = str(tmp_path / "spack.yaml") with open(file, "w", encoding="utf-8") as f: f.write(contents) config("add", "-f", file) output = config("get", "config") assert ( output == """config: template_dirs: [test2] """ ) def test_config_add_override_leaf_from_file(mutable_empty_config, tmp_path: pathlib.Path): config("--scope", "site", "add", "config:template_dirs:test1") contents = """spack: config: template_dirs:: [test2] """ file = str(tmp_path / "spack.yaml") with open(file, "w", encoding="utf-8") as f: f.write(contents) config("add", "-f", file) output = config("get", "config") assert ( output == """config: 'template_dirs:': [test2] """ ) def test_config_add_update_dict_from_file(mutable_empty_config, tmp_path: pathlib.Path): config("add", "packages:all:require:['%gcc']") # contents to add to file contents = """spack: packages: all: target: [x86_64] """ # create temp file and add it to config file = str(tmp_path / "spack.yaml") with open(file, "w", encoding="utf-8") as f: f.write(contents) config("add", "-f", file) # get results output = config("get", "packages") # added config comes before prior config expected = """packages: all: target: [x86_64] require: ['%gcc'] """ assert expected == output def test_config_add_invalid_file_fails(tmp_path: pathlib.Path): # contents to add to file # invalid because version requires a list contents = """spack: packages: hdf5: version: 1.0.0 """ # create temp file and add it to config file = str(tmp_path / "spack.yaml") with open(file, "w", encoding="utf-8") as f: f.write(contents) with pytest.raises((spack.config.ConfigFormatError)): config("add", "-f", file) def test_config_remove_value(mutable_empty_config): config("add", "config:dirty:true") config("remove", "config:dirty:true") output = config("get", "config") assert ( output == """config: {} """ ) def test_config_remove_alias_rm(mutable_empty_config): config("add", "config:dirty:true") config("rm", "config:dirty:true") output = config("get", "config") assert ( output == """config: {} """ ) def test_config_remove_dict(mutable_empty_config): config("add", "config:dirty:true") config("rm", "config:dirty") output = config("get", "config") assert ( output == """config: {} """ ) def test_remove_from_list(mutable_empty_config): config("add", "config:template_dirs:test1") config("add", "config:template_dirs:[test2]") config("add", "config:template_dirs:test3") config("remove", "config:template_dirs:test2") output = config("get", "config") assert ( output == """config: template_dirs: - test3 - test1 """ ) def test_remove_list(mutable_empty_config): config("add", "config:template_dirs:test1") config("add", "config:template_dirs:[test2]") config("add", "config:template_dirs:test3") config("remove", "config:template_dirs:[test2]") output = config("get", "config") assert ( output == """config: template_dirs: - test3 - test1 """ ) def test_config_add_to_env(mutable_empty_config, mutable_mock_env_path): env("create", "test") with ev.read("test"): config("add", "config:dirty:true") output = config("get") expected = """ config: dirty: true """ assert expected in output def test_config_add_to_env_preserve_comments( mutable_empty_config, mutable_mock_env_path, tmp_path: pathlib.Path ): filepath = str(tmp_path / "spack.yaml") manifest = """# comment spack: # comment # comment specs: # comment - foo # comment # comment view: true # comment packages: # comment # comment all: # comment # comment compiler: [gcc] # comment """ with open(filepath, "w", encoding="utf-8") as f: f.write(manifest) env = ev.Environment(str(tmp_path)) with env: config("add", "config:dirty:true") output = config("get") assert "# comment" in output assert "dirty: true" in output def test_config_remove_from_env(mutable_empty_config, mutable_mock_env_path): env("create", "test") with ev.read("test"): config("add", "config:dirty:true") output = config("get") assert "dirty: true" in output with ev.read("test"): config("rm", "config:dirty") output = config("get") assert "dirty: true" not in output def test_config_update_not_needed(mutable_config): data_before = spack.config.get("repos") config("update", "-y", "repos") data_after = spack.config.get("repos") assert data_before == data_after def test_config_update_shared_linking(mutable_config): # Old syntax: config:shared_linking:rpath/runpath # New syntax: config:shared_linking:{type:rpath/runpath,bind:True/False} with spack.config.override("config:shared_linking", "runpath"): assert spack.config.get("config:shared_linking:type") == "runpath" assert not spack.config.get("config:shared_linking:bind") def test_config_prefer_upstream( tmp_path_factory: pytest.TempPathFactory, install_mockery, mock_fetch, mutable_config, gen_mock_layout, monkeypatch, ): """Check that when a dependency package is recorded as installed in an upstream database that it is not reinstalled. """ mock_db_root = str(tmp_path_factory.mktemp("mock_db_root")) prepared_db = spack.database.Database(mock_db_root, layout=gen_mock_layout("a")) for spec in ["hdf5 +mpi", "hdf5 ~mpi", "boost+debug~icu+graph", "dependency-install", "patch"]: dep = spack.concretize.concretize_one(spec) prepared_db.add(dep) downstream_db_root = str(tmp_path_factory.mktemp("mock_downstream_db_root")) db_for_test = spack.database.Database(downstream_db_root, upstream_dbs=[prepared_db]) monkeypatch.setattr(spack.store.STORE, "db", db_for_test) output = config("prefer-upstream") scope = spack.config.default_modify_scope("packages") cfg_file = spack.config.CONFIG.get_config_filename(scope, "packages") packages = syaml.load(open(cfg_file, encoding="utf-8"))["packages"] # Make sure only the non-default variants are set. assert packages["boost"] == {"variants": "+debug +graph", "version": ["1.63.0"]} assert packages["dependency-install"] == {"version": ["2.0"]} # Ensure that neither variant gets listed for hdf5, since they conflict assert packages["hdf5"] == {"version": ["2.3"]} # Make sure a message about the conflicting hdf5's was given. assert "- hdf5" in output def test_environment_config_update(tmp_path: pathlib.Path, mutable_config, monkeypatch): with open(tmp_path / "spack.yaml", "w", encoding="utf-8") as f: f.write( """\ spack: config: ccache: true """ ) def update_config(data): data["config"]["ccache"] = False return True monkeypatch.setattr(spack.schema.config, "update", update_config) with ev.Environment(str(tmp_path)): config("update", "-y", "config") with ev.Environment(str(tmp_path)) as e: assert not e.manifest.yaml_content["spack"]["config"]["ccache"] _GROUP_OVERRIDE_SPACK_YAML = """\ spack: specs: - group: mygroup specs: - zlib override: packages: zlib: version: ['1.2.13'] """ @pytest.mark.parametrize("cmd_str", ["get", "blame"]) def test_config_with_group_shows_override_packages(cmd_str, tmp_path, mutable_config): """Tests that packages should show that group's override packages config, when the option is given. """ (tmp_path / "spack.yaml").write_text(_GROUP_OVERRIDE_SPACK_YAML) with ev.Environment(str(tmp_path)): output = config(cmd_str, "packages") assert "1.2.13" not in output if cmd_str == "blame": assert "env:groups:mygroup" not in output output = config(cmd_str, "--group=mygroup", "packages") assert "1.2.13" in output if cmd_str == "blame": assert "env:groups:mygroup" in output @pytest.mark.parametrize("cmd_str", ["get", "blame"]) def test_config_with_group_requires_active_environment(cmd_str, mutable_config): """Tests that using groups outside an environment should give a clear error.""" output = config(cmd_str, "--group=mygroup", "packages", fail_on_error=False) assert config.returncode != 0 assert "--group requires an active environment" in output @pytest.mark.parametrize("cmd_str", ["get", "blame"]) def test_config_with_unknown_group_gives_clear_error(cmd_str, tmp_path, mutable_config): """Tests that using a non-existing group gives a clear error.""" (tmp_path / "spack.yaml").write_text("spack:\n specs:\n - zlib\n") with ev.Environment(str(tmp_path)): output = config(cmd_str, "--group=nonexistent", "packages", fail_on_error=False) assert config.returncode != 0 assert "'nonexistent' not found in" in output ================================================ FILE: lib/spack/spack/test/cmd/create.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import tarfile import pytest import spack.cmd.create import spack.url from spack.main import SpackCommand from spack.url import UndetectableNameError from spack.util.executable import which create = SpackCommand("create") @pytest.mark.parametrize( "args,name,expected", [ # Basic package cases (["/test-package"], "test-package", [r"TestPackage(Package)", r"def install(self"]), ( ["-n", "test-named-package", "file://example.tar.gz"], "test-named-package", [r"TestNamedPackage(Package)", r"def install(self"], ), (["file://example.tar.gz"], "example", [r"Example(Package)", r"def install(self"]), ( ["-n", "test-license"], "test-license", [r'license("UNKNOWN", checked_by="github_user1")'], ), # Template-specific cases ( ["-t", "autoreconf", "/test-autoreconf"], "test-autoreconf", [ r"TestAutoreconf(AutotoolsPackage)", r'depends_on("autoconf', r"def autoreconf(self", r"def configure_args(self", ], ), ( ["-t", "autotools", "/test-autotools"], "test-autotools", [r"TestAutotools(AutotoolsPackage)", r"def configure_args(self"], ), ( ["-t", "bazel", "/test-bazel"], "test-bazel", [r"TestBazel(Package)", r'depends_on("bazel', r"bazel()"], ), (["-t", "bundle", "/test-bundle"], "test-bundle", [r"TestBundle(BundlePackage)"]), ( ["-t", "cmake", "/test-cmake"], "test-cmake", [r"TestCmake(CMakePackage)", r"def cmake_args(self"], ), ( ["-t", "intel", "/test-intel"], "test-intel", [r"TestIntel(IntelOneApiPackage)", r"setup_environment"], ), ( ["-t", "makefile", "/test-makefile"], "test-makefile", [r"TestMakefile(MakefilePackage)", r"def edit(self", r"makefile"], ), ( ["-t", "meson", "/test-meson"], "test-meson", [r"TestMeson(MesonPackage)", r"def meson_args(self"], ), ( ["-t", "octave", "/test-octave"], "octave-test-octave", [r"OctaveTestOctave(OctavePackage)", r'extends("octave', r'depends_on("octave'], ), ( ["-t", "perlbuild", "/test-perlbuild"], "perl-test-perlbuild", [ r"PerlTestPerlbuild(PerlPackage)", r'depends_on("perl-module-build', r"def configure_args(self", ], ), ( ["-t", "perlmake", "/test-perlmake"], "perl-test-perlmake", [r"PerlTestPerlmake(PerlPackage)", r'depends_on("perl-', r"def configure_args(self"], ), ( ["-t", "python", "/test-python"], "py-test-python", [r"PyTestPython(PythonPackage)", r'depends_on("py-', r"def config_settings(self"], ), ( ["-t", "qmake", "/test-qmake"], "test-qmake", [r"TestQmake(QMakePackage)", r"def qmake_args(self"], ), ( ["-t", "r", "/test-r"], "r-test-r", [r"RTestR(RPackage)", r'depends_on("r-', r"def configure_args(self"], ), ( ["-t", "scons", "/test-scons"], "test-scons", [r"TestScons(SConsPackage)", r"def build_args(self"], ), ( ["-t", "sip", "/test-sip"], "py-test-sip", [r"PyTestSip(SIPPackage)", r"def configure_args(self"], ), (["-t", "waf", "/test-waf"], "test-waf", [r"TestWaf(WafPackage)", r"configure_args()"]), ], ) def test_create_template(mock_test_repo, args, name, expected): """Test template creation.""" repo, repodir = mock_test_repo create("--skip-editor", *args) filename = repo.filename_for_package_name(name) assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as package_file: content = package_file.read() for entry in expected: assert entry in content black = which("black", required=False) if not black: pytest.skip("checking blackness of `spack create` output requires black") black("--check", "--diff", filename) @pytest.mark.parametrize( "name,expected", [(" ", "name must be provided"), ("bad#name", "name can only contain")] ) def test_create_template_bad_name(mock_test_repo, name, expected): """Test template creation with bad name options.""" output = create("--skip-editor", "-n", name, fail_on_error=False) assert expected in output assert create.returncode != 0 def test_build_system_guesser_no_stage(): """Test build system guesser when stage not provided.""" guesser = spack.cmd.create.BuildSystemAndLanguageGuesser() # Ensure get the expected build system with pytest.raises(AttributeError, match="'NoneType' object has no attribute"): guesser(None, "/the/url/does/not/matter") def test_build_system_guesser_octave(tmp_path: pathlib.Path): """ Test build system guesser for the special case, where the same base URL identifies the build system rather than guessing the build system from files contained in the archive. """ url, expected = "downloads.sourceforge.net/octave/", "octave" guesser = spack.cmd.create.BuildSystemAndLanguageGuesser() # Ensure get the expected build system guesser(str(tmp_path / "archive.tar.gz"), url) assert guesser.build_system == expected # Also ensure get the correct template bs = spack.cmd.create.get_build_system(None, url, guesser) assert bs == expected @pytest.mark.parametrize( "url,expected", [("testname", "testname"), ("file://example.com/archive.tar.gz", "archive")] ) def test_get_name_urls(url, expected): """Test get_name with different URLs.""" name = spack.cmd.create.get_name(None, url) assert name == expected def test_get_name_error(monkeypatch, capfd): """Test get_name UndetectableNameError exception path.""" def _parse_name_offset(path, v): raise UndetectableNameError(path) monkeypatch.setattr(spack.url, "parse_name_offset", _parse_name_offset) url = "downloads.sourceforge.net/noapp/" with pytest.raises(SystemExit): spack.cmd.create.get_name(None, url) captured = capfd.readouterr() assert "Couldn't guess a name" in str(captured) def test_no_url(): """Test creation of package without a URL.""" create("--skip-editor", "-n", "create-new-package") @pytest.mark.parametrize( "source_files,languages", [ (["fst.c", "snd.C"], ["c", "cxx"]), (["fst.c", "snd.cxx"], ["c", "cxx"]), (["fst.F", "snd.cc"], ["cxx", "fortran"]), (["fst.f", "snd.c"], ["c", "fortran"]), (["fst.jl", "snd.py"], []), ], ) def test_language_and_build_system_detection(tmp_path: pathlib.Path, source_files, languages): """Test that languages are detected from tarball, and the build system is guessed from the most top-level build system file.""" def add(tar: tarfile.TarFile, name: str, type): tarinfo = tarfile.TarInfo(name) tarinfo.type = type tar.addfile(tarinfo) tarball = str(tmp_path / "example.tar.gz") with tarfile.open(tarball, "w:gz") as tar: add(tar, "./third-party/", tarfile.DIRTYPE) add(tar, "./third-party/example/", tarfile.DIRTYPE) add(tar, "./third-party/example/CMakeLists.txt", tarfile.REGTYPE) # false positive add(tar, "./configure", tarfile.REGTYPE) # actual build system add(tar, "./src/", tarfile.DIRTYPE) for file in source_files: add(tar, f"src/{file}", tarfile.REGTYPE) guesser = spack.cmd.create.BuildSystemAndLanguageGuesser() guesser(str(tarball), "https://example.com") assert guesser.build_system == "autotools" assert guesser.languages == languages ================================================ FILE: lib/spack/spack/test/cmd/debug.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform import spack import spack.cmd.debug import spack.platforms import spack.repo import spack.spec from spack.main import SpackCommand from spack.test.conftest import _return_none debug = SpackCommand("debug") def test_report(): out = debug("report") host_platform = spack.platforms.host() host_os = host_platform.default_operating_system() host_target = host_platform.default_target() architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target))) assert spack.spack_version in out assert spack.get_spack_commit() in out assert platform.python_version() in out assert str(architecture) in out def test_get_builtin_repo_info_local_repo(mock_git_version_info, monkeypatch): """Confirm local git repo descriptor returns expected path.""" path = mock_git_version_info[0] def _from_config(*args, **kwargs): return {"builtin": spack.repo.LocalRepoDescriptor("builtin", path)} monkeypatch.setattr(spack.repo.RepoDescriptors, "from_config", _from_config) assert path in spack.cmd.debug._get_builtin_repo_info() def test_get_builtin_repo_info_unsupported_type(mock_git_version_info, monkeypatch): """Confirm None is return if the 'builtin' repo descriptor's type is unsupported.""" def _from_config(*args, **kwargs): path = mock_git_version_info[0] return {"builtin": path} monkeypatch.setattr(spack.repo.RepoDescriptors, "from_config", _from_config) assert spack.cmd.debug._get_builtin_repo_info() is None def test_get_builtin_repo_info_no_builtin(monkeypatch): """Confirm None is return if there is no 'builtin' repo descriptor.""" def _from_config(*args, **kwargs): return {"local": "/assumes/no/descriptor/needed"} monkeypatch.setattr(spack.repo.RepoDescriptors, "from_config", _from_config) assert spack.cmd.debug._get_builtin_repo_info() is None def test_get_builtin_repo_info_bad_destination(mock_git_version_info, monkeypatch): """Confirm git failure of a repository returns None.""" def _from_config(*args, **kwargs): path = mock_git_version_info[0] return {"builtin": spack.repo.LocalRepoDescriptor("builtin", f"{path}/missing")} monkeypatch.setattr(spack.repo.RepoDescriptors, "from_config", _from_config) assert spack.cmd.debug._get_builtin_repo_info() is None def test_get_spack_repo_info_no_commit(monkeypatch): """Confirm the version is returned if there is no spack commit.""" monkeypatch.setattr(spack, "get_spack_commit", _return_none) assert spack.cmd.debug._get_spack_repo_info() == spack.spack_version ================================================ FILE: lib/spack/spack/test/cmd/deconcretize.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.environment as ev from spack.main import SpackCommand, SpackCommandError deconcretize = SpackCommand("deconcretize") @pytest.fixture(scope="function") def test_env(mutable_mock_env_path, mock_packages): ev.create("test") with ev.read("test") as e: e.add("pkg-a@2.0 foobar=bar ^pkg-b@1.0") e.add("pkg-a@1.0 foobar=bar ^pkg-b@0.9") e.concretize() e.write() def test_deconcretize_dep(test_env): with ev.read("test") as e: deconcretize("-y", "pkg-b@1.0") specs = [s for s, _ in e.concretized_specs()] assert len(specs) == 1 assert specs[0].satisfies("pkg-a@1.0") def test_deconcretize_all_dep(test_env): with ev.read("test") as e: with pytest.raises(SpackCommandError): deconcretize("-y", "pkg-b") deconcretize("-y", "--all", "pkg-b") specs = [s for s, _ in e.concretized_specs()] assert len(specs) == 0 def test_deconcretize_root(test_env): with ev.read("test") as e: output = deconcretize("-y", "--root", "pkg-b@1.0") assert "No matching specs to deconcretize" in output assert len(e.concretized_roots) == 2 deconcretize("-y", "--root", "pkg-a@2.0") specs = [s for s, _ in e.concretized_specs()] assert len(specs) == 1 assert specs[0].satisfies("pkg-a@1.0") def test_deconcretize_all_root(test_env): with ev.read("test") as e: with pytest.raises(SpackCommandError): deconcretize("-y", "--root", "pkg-a") output = deconcretize("-y", "--root", "--all", "pkg-b") assert "No matching specs to deconcretize" in output assert len(e.concretized_roots) == 2 deconcretize("-y", "--root", "--all", "pkg-a") specs = [s for s, _ in e.concretized_specs()] assert len(specs) == 0 def test_deconcretize_all(test_env): with ev.read("test") as e: with pytest.raises(SpackCommandError): deconcretize() deconcretize("-y", "--all") specs = [s for s, _ in e.concretized_specs()] assert len(specs) == 0 ================================================ FILE: lib/spack/spack/test/cmd/dependencies.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import pytest import spack.store from spack.llnl.util.tty.color import color_when from spack.main import SpackCommand dependencies = SpackCommand("dependencies") MPIS = [ "intel-parallel-studio", "low-priority-provider", "mpich", "mpich2", "multi-provider-mpi", "zmpi", ] COMPILERS = ["gcc", "llvm", "compiler-with-deps"] MPI_DEPS = ["fake"] COMPILER_DEPS = ["binutils-for-test", "zlib"] @pytest.mark.parametrize( "cli_args,expected", [ (["mpileaks"], set(["callpath"] + MPIS + COMPILERS)), ( ["--transitive", "mpileaks"], set( ["callpath", "dyninst", "libdwarf", "libelf"] + MPIS + MPI_DEPS + COMPILERS + COMPILER_DEPS ), ), (["--transitive", "--deptype=link,run", "dtbuild1"], {"dtlink2", "dtrun2"}), (["--transitive", "--deptype=build", "dtbuild1"], {"dtbuild2", "dtlink2"}), (["--transitive", "--deptype=link", "dtbuild1"], {"dtlink2"}), ], ) def test_direct_dependencies(cli_args, expected, mock_packages): out = dependencies(*cli_args) result = set(re.split(r"\s+", out.strip())) assert expected == result @pytest.mark.db def test_direct_installed_dependencies(mock_packages, database): with color_when(False): out = dependencies("--installed", "mpileaks^mpich") root = spack.store.STORE.db.query_one("mpileaks ^mpich") lines = [line for line in out.strip().split("\n") if line and not line.startswith("--")] hashes = {re.split(r"\s+", line)[0] for line in lines} expected = {s.dag_hash(7) for s in root.dependencies()} assert expected == hashes @pytest.mark.db def test_transitive_installed_dependencies(mock_packages, database): with color_when(False): out = dependencies("--installed", "--transitive", "mpileaks^zmpi") root = spack.store.STORE.db.query_one("mpileaks ^zmpi") lines = [line for line in out.strip().split("\n") if line and not line.startswith("--")] hashes = {re.split(r"\s+", line)[0] for line in lines} expected = {s.dag_hash(7) for s in root.traverse(root=False)} assert expected == hashes ================================================ FILE: lib/spack/spack/test/cmd/dependents.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import pytest import spack.store from spack.llnl.util.tty.color import color_when from spack.main import SpackCommand dependents = SpackCommand("dependents") def test_immediate_dependents(mock_packages): out = dependents("libelf") actual = set(re.split(r"\s+", out.strip())) assert actual == set( [ "dyninst", "libdwarf", "patch-a-dependency", "patch-several-dependencies", "quantum-espresso", "conditionally-patch-dependency", ] ) def test_transitive_dependents(mock_packages): out = dependents("--transitive", "libelf") actual = set(re.split(r"\s+", out.strip())) assert actual == { "callpath", "dyninst", "libdwarf", "mixing-parent", "mpileaks", "multivalue-variant", "singlevalue-variant-dependent", "trilinos", "patch-a-dependency", "patch-several-dependencies", "quantum-espresso", "conditionally-patch-dependency", } @pytest.mark.db def test_immediate_installed_dependents(mock_packages, database): with color_when(False): out = dependents("--installed", "libelf") lines = [li for li in out.strip().split("\n") if not li.startswith("--")] hashes = set([re.split(r"\s+", li)[0] for li in lines if li]) expected = set( [spack.store.STORE.db.query_one(s).dag_hash(7) for s in ["dyninst", "libdwarf"]] ) libelf = spack.store.STORE.db.query_one("libelf") expected = set([d.dag_hash(7) for d in libelf.dependents()]) assert expected == hashes @pytest.mark.db def test_transitive_installed_dependents(mock_packages, database): with color_when(False): out = dependents("--installed", "--transitive", "fake") lines = [li for li in out.strip().split("\n") if li and not li.startswith("--")] hashes = set([re.split(r"\s+", li)[0] for li in lines]) expected = set( [ spack.store.STORE.db.query_one(s).dag_hash(7) for s in ["zmpi", "callpath^zmpi", "mpileaks^zmpi"] ] ) assert expected == hashes ================================================ FILE: lib/spack/spack/test/cmd/deprecate.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.concretize import spack.spec import spack.store from spack.enums import InstallRecordStatus from spack.main import SpackCommand install = SpackCommand("install") uninstall = SpackCommand("uninstall") deprecate = SpackCommand("deprecate") find = SpackCommand("find") # Unit tests should not be affected by the user's managed environments pytestmark = pytest.mark.usefixtures("mutable_mock_env_path") def test_deprecate(mock_packages, mock_archive, mock_fetch, install_mockery): install("--fake", "libelf@0.8.13") install("--fake", "libelf@0.8.10") all_installed = spack.store.STORE.db.query("libelf") assert len(all_installed) == 2 deprecate("-y", "libelf@0.8.10", "libelf@0.8.13") non_deprecated = spack.store.STORE.db.query("libelf") all_available = spack.store.STORE.db.query("libelf", installed=InstallRecordStatus.ANY) assert all_available == all_installed assert non_deprecated == spack.store.STORE.db.query("libelf@0.8.13") def test_deprecate_fails_no_such_package(mock_packages, mock_archive, mock_fetch, install_mockery): """Tests that deprecating a spec that is not installed fails. Tests that deprecating without the ``-i`` option in favor of a spec that is not installed fails.""" output = deprecate("-y", "libelf@0.8.10", "libelf@0.8.13", fail_on_error=False) assert "Spec 'libelf@0.8.10' matches no installed packages" in output install("--fake", "libelf@0.8.10") output = deprecate("-y", "libelf@0.8.10", "libelf@0.8.13", fail_on_error=False) assert "Spec 'libelf@0.8.13' matches no installed packages" in output def test_deprecate_install(mock_packages, mock_archive, mock_fetch, install_mockery, monkeypatch): """Tests that the -i option allows us to deprecate in favor of a spec that is not yet installed. """ install("--fake", "libelf@0.8.10") to_deprecate = spack.store.STORE.db.query("libelf") assert len(to_deprecate) == 1 deprecate("-y", "-i", "libelf@0.8.10", "libelf@0.8.13") non_deprecated = spack.store.STORE.db.query("libelf") deprecated = spack.store.STORE.db.query("libelf", installed=InstallRecordStatus.DEPRECATED) assert deprecated == to_deprecate assert len(non_deprecated) == 1 assert non_deprecated[0].satisfies("libelf@0.8.13") def test_deprecate_deps(mock_packages, mock_archive, mock_fetch, install_mockery): """Test that the deprecate command deprecates all dependencies properly.""" install("--fake", "libdwarf@20130729 ^libelf@0.8.13") install("--fake", "libdwarf@20130207 ^libelf@0.8.10") new_spec = spack.concretize.concretize_one("libdwarf@20130729^libelf@0.8.13") old_spec = spack.concretize.concretize_one("libdwarf@20130207^libelf@0.8.10") all_installed = spack.store.STORE.db.query() deprecate("-y", "-d", "libdwarf@20130207", "libdwarf@20130729") non_deprecated = spack.store.STORE.db.query() all_available = spack.store.STORE.db.query(installed=InstallRecordStatus.ANY) deprecated = spack.store.STORE.db.query(installed=InstallRecordStatus.DEPRECATED) assert all_available == all_installed assert sorted(all_available) == sorted(deprecated + non_deprecated) assert sorted(non_deprecated) == sorted(new_spec.traverse()) assert sorted(deprecated) == sorted([old_spec, old_spec["libelf"]]) def test_uninstall_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery): """Tests that we can still uninstall deprecated packages.""" install("--fake", "libelf@0.8.13") install("--fake", "libelf@0.8.10") deprecate("-y", "libelf@0.8.10", "libelf@0.8.13") non_deprecated = spack.store.STORE.db.query() uninstall("-y", "libelf@0.8.10") assert spack.store.STORE.db.query() == spack.store.STORE.db.query( installed=InstallRecordStatus.ANY ) assert spack.store.STORE.db.query() == non_deprecated def test_deprecate_already_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery): """Tests that we can re-deprecate a spec to change its deprecator.""" install("--fake", "libelf@0.8.13") install("--fake", "libelf@0.8.12") install("--fake", "libelf@0.8.10") deprecated_spec = spack.concretize.concretize_one("libelf@0.8.10") deprecate("-y", "libelf@0.8.10", "libelf@0.8.12") deprecator = spack.store.STORE.db.deprecator(deprecated_spec) assert deprecator == spack.concretize.concretize_one("libelf@0.8.12") deprecate("-y", "libelf@0.8.10", "libelf@0.8.13") non_deprecated = spack.store.STORE.db.query("libelf") all_available = spack.store.STORE.db.query("libelf", installed=InstallRecordStatus.ANY) assert len(non_deprecated) == 2 assert len(all_available) == 3 deprecator = spack.store.STORE.db.deprecator(deprecated_spec) assert deprecator == spack.concretize.concretize_one("libelf@0.8.13") def test_deprecate_deprecator(mock_packages, mock_archive, mock_fetch, install_mockery): """Tests that when a deprecator spec is deprecated, its deprecatee specs are updated to point to the new deprecator.""" install("--fake", "libelf@0.8.13") install("--fake", "libelf@0.8.12") install("--fake", "libelf@0.8.10") first_deprecated_spec = spack.concretize.concretize_one("libelf@0.8.10") second_deprecated_spec = spack.concretize.concretize_one("libelf@0.8.12") final_deprecator = spack.concretize.concretize_one("libelf@0.8.13") deprecate("-y", "libelf@0.8.10", "libelf@0.8.12") deprecator = spack.store.STORE.db.deprecator(first_deprecated_spec) assert deprecator == second_deprecated_spec deprecate("-y", "libelf@0.8.12", "libelf@0.8.13") non_deprecated = spack.store.STORE.db.query("libelf") all_available = spack.store.STORE.db.query("libelf", installed=InstallRecordStatus.ANY) assert len(non_deprecated) == 1 assert len(all_available) == 3 first_deprecator = spack.store.STORE.db.deprecator(first_deprecated_spec) assert first_deprecator == final_deprecator second_deprecator = spack.store.STORE.db.deprecator(second_deprecated_spec) assert second_deprecator == final_deprecator def test_concretize_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery): """Tests that the concretizer throws an error if we concretize to a deprecated spec""" install("--fake", "libelf@0.8.13") install("--fake", "libelf@0.8.10") deprecate("-y", "libelf@0.8.10", "libelf@0.8.13") spec = spack.spec.Spec("libelf@0.8.10") with pytest.raises(spack.spec.SpecDeprecatedError): spack.concretize.concretize_one(spec) @pytest.mark.usefixtures("mock_packages", "mock_archive", "mock_fetch", "install_mockery") @pytest.mark.regression("46915") def test_deprecate_spec_with_external_dependency( mutable_config, temporary_store, tmp_path: pathlib.Path ): """Tests that we can deprecate a spec that has an external dependency""" packages_yaml = { "libelf": { "buildable": False, "externals": [{"spec": "libelf@0.8.13", "prefix": str(tmp_path / "libelf")}], } } mutable_config.set("packages", packages_yaml) install("--fake", "dyninst ^libdwarf@=20111030") install("--fake", "libdwarf@=20130729") # Ensure we are using the external libelf db = temporary_store.db libelf = db.query_one("libelf") assert libelf.external deprecated_spec = db.query_one("libdwarf@=20111030") new_libdwarf = db.query_one("libdwarf@=20130729") deprecate("-y", "libdwarf@=20111030", "libdwarf@=20130729") assert db.deprecator(deprecated_spec) == new_libdwarf ================================================ FILE: lib/spack/spack/test/cmd/dev_build.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.concretize import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs import spack.main import spack.repo import spack.spec import spack.store from spack.main import SpackCommand dev_build = SpackCommand("dev-build") install = SpackCommand("install") env = SpackCommand("env") pytestmark = [pytest.mark.disable_clean_stage_check] def test_dev_build_basics(tmp_path: pathlib.Path, install_mockery): spec = spack.concretize.concretize_one( spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmp_path}") ) assert "dev_path" in spec.variants with fs.working_dir(str(tmp_path)): with open(spec.package.filename, "w", encoding="utf-8") as f: # type: ignore f.write(spec.package.original_string) # type: ignore dev_build("dev-build-test-install@0.0.0") assert spec.package.filename in os.listdir(spec.prefix) with open(os.path.join(spec.prefix, spec.package.filename), "r", encoding="utf-8") as f: assert f.read() == spec.package.replacement_string assert os.path.exists(str(tmp_path)) def test_dev_build_before(tmp_path: pathlib.Path, install_mockery, installer_variant): spec = spack.concretize.concretize_one( spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmp_path}") ) with fs.working_dir(str(tmp_path)): with open(spec.package.filename, "w", encoding="utf-8") as f: # type: ignore f.write(spec.package.original_string) # type: ignore dev_build("-b", "edit", "dev-build-test-install@0.0.0") assert spec.package.filename in os.listdir(os.getcwd()) # type: ignore with open(spec.package.filename, "r", encoding="utf-8") as f: # type: ignore assert f.read() == spec.package.original_string # type: ignore assert not os.path.exists(spec.prefix) @pytest.mark.parametrize("last_phase", ["edit", "install"]) def test_dev_build_until(tmp_path: pathlib.Path, install_mockery, last_phase, installer_variant): spec = spack.concretize.concretize_one( spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmp_path}") ) with fs.working_dir(str(tmp_path)): with open(spec.package.filename, "w", encoding="utf-8") as f: # type: ignore f.write(spec.package.original_string) # type: ignore dev_build("--until", last_phase, "dev-build-test-install@0.0.0") assert spec.package.filename in os.listdir(os.getcwd()) # type: ignore with open(spec.package.filename, "r", encoding="utf-8") as f: # type: ignore assert f.read() == spec.package.replacement_string # type: ignore assert not os.path.exists(spec.prefix) assert not spack.store.STORE.db.query(spec, installed=True) def test_dev_build_before_until(tmp_path: pathlib.Path, install_mockery, installer_variant): spec = spack.concretize.concretize_one( spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={tmp_path}") ) with fs.working_dir(str(tmp_path)): with open(spec.package.filename, "w", encoding="utf-8") as f: f.write(spec.package.original_string) with pytest.raises(spack.main.SpackCommandError): dev_build("-u", "edit", "-b", "edit", "dev-build-test-install@0.0.0") bad_phase = "phase_that_does_not_exist" not_allowed = "is not a valid phase" not_installed = "was not installed" out = dev_build("-u", bad_phase, "dev-build-test-install@0.0.0", fail_on_error=False) assert bad_phase in out assert not_allowed in out if installer_variant == "old": assert not_installed in out out = dev_build("-b", bad_phase, "dev-build-test-install@0.0.0", fail_on_error=False) assert bad_phase in out assert not_allowed in out if installer_variant == "old": assert not_installed in out def _print_spack_short_spec(*args): print(f"SPACK_SHORT_SPEC={os.environ['SPACK_SHORT_SPEC']}") def test_dev_build_drop_in( tmp_path: pathlib.Path, mock_packages, monkeypatch, install_mockery, working_env ): monkeypatch.setattr(os, "execvp", _print_spack_short_spec) with fs.working_dir(str(tmp_path)): output = dev_build("-b", "edit", "--drop-in", "sh", "dev-build-test-install@0.0.0") assert "SPACK_SHORT_SPEC=dev-build-test-install@0.0.0" in output def test_dev_build_fails_already_installed(tmp_path: pathlib.Path, install_mockery): spec = spack.concretize.concretize_one( spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % str(tmp_path)) ) with fs.working_dir(str(tmp_path)): with open(spec.package.filename, "w", encoding="utf-8") as f: f.write(spec.package.original_string) dev_build("dev-build-test-install@0.0.0") output = dev_build("dev-build-test-install@0.0.0", fail_on_error=False) assert "Already installed in %s" % spec.prefix in output def test_dev_build_fails_no_spec(): output = dev_build(fail_on_error=False) assert "requires a package spec argument" in output def test_dev_build_fails_multiple_specs(mock_packages): output = dev_build("libelf", "libdwarf", fail_on_error=False) assert "only takes one spec" in output def test_dev_build_fails_nonexistent_package_name(mock_packages): output = "" try: dev_build("no_such_package") assert False, "no exception was raised!" except spack.repo.UnknownPackageError as e: output = e.message assert "Package 'no_such_package' not found" in output def test_dev_build_fails_no_version(mock_packages): output = dev_build("dev-build-test-install", fail_on_error=False) assert "dev-build spec must have a single, concrete version" in output def test_dev_build_can_parse_path_with_at_symbol(tmp_path: pathlib.Path, install_mockery): special_char_dir = tmp_path / "tmp@place" special_char_dir.mkdir() spec = spack.concretize.concretize_one( f'dev-build-test-install@0.0.0 dev_path="{special_char_dir}"' ) with fs.working_dir(str(special_char_dir)): with open(spec.package.filename, "w", encoding="utf-8") as f: f.write(spec.package.original_string) dev_build("dev-build-test-install@0.0.0") assert spec.package.filename in os.listdir(spec.prefix) def test_dev_build_env( tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, installer_variant ): """Test Spack does dev builds for packages in develop section of env.""" # setup dev-build-test-install package for dev build build_dir = tmp_path / "build" build_dir.mkdir() spec = spack.concretize.concretize_one( spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % str(build_dir)) ) with fs.working_dir(str(build_dir)): with open(spec.package.filename, "w", encoding="utf-8") as f: f.write(spec.package.original_string) # setup environment envdir = tmp_path / "env" envdir.mkdir() with fs.working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( f"""\ spack: specs: - dev-build-test-install@0.0.0 develop: dev-build-test-install: spec: dev-build-test-install@0.0.0 path: {os.path.relpath(str(build_dir), start=str(envdir))} """ ) env("create", "test", "./spack.yaml") with ev.read("test"): install() assert spec.package.filename in os.listdir(spec.prefix) with open(os.path.join(spec.prefix, spec.package.filename), "r", encoding="utf-8") as f: assert f.read() == spec.package.replacement_string def test_dev_build_env_with_vars( tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, monkeypatch, installer_variant ): """Test Spack does dev builds for packages in develop section of env (path with variables).""" # setup dev-build-test-install package for dev build build_dir = tmp_path / "build" build_dir.mkdir() spec = spack.concretize.concretize_one( spack.spec.Spec(f"dev-build-test-install@0.0.0 dev_path={str(build_dir)}") ) # store the build path in an environment variable that will be used in the environment monkeypatch.setenv("CUSTOM_BUILD_PATH", str(build_dir)) with fs.working_dir(str(build_dir)), open(spec.package.filename, "w", encoding="utf-8") as f: f.write(spec.package.original_string) # setup environment envdir = tmp_path / "env" envdir.mkdir() with fs.working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( """\ spack: specs: - dev-build-test-install@0.0.0 develop: dev-build-test-install: spec: dev-build-test-install@0.0.0 path: $CUSTOM_BUILD_PATH """ ) env("create", "test", "./spack.yaml") with ev.read("test"): install() assert spec.package.filename in os.listdir(spec.prefix) with open(os.path.join(spec.prefix, spec.package.filename), "r", encoding="utf-8") as f: assert f.read() == spec.package.replacement_string def test_dev_build_env_version_mismatch( tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, installer_variant ): """Test Spack constraints concretization by develop specs.""" # setup dev-build-test-install package for dev build build_dir = tmp_path / "build" build_dir.mkdir() spec = spack.concretize.concretize_one( spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % str(tmp_path)) ) with fs.working_dir(str(build_dir)): with open(spec.package.filename, "w", encoding="utf-8") as f: f.write(spec.package.original_string) # setup environment envdir = tmp_path / "env" envdir.mkdir() with fs.working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( f"""\ spack: specs: - dev-build-test-install@0.0.0 develop: dev-build-test-install: spec: dev-build-test-install@1.1.1 path: {build_dir} """ ) env("create", "test", "./spack.yaml") with ev.read("test"): with pytest.raises((RuntimeError, spack.error.UnsatisfiableSpecError)): install() def test_dev_build_multiple( tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, mock_fetch ): """Test spack install with multiple developer builds Test that only the root needs to be specified in the environment Test that versions known only from the dev specs are included in the solve, even if they come from a non-root """ # setup dev-build-test-install package for dev build # Wait to concretize inside the environment to set dev_path on the specs; # without the environment, the user would need to set dev_path for both the # root and dependency if they wanted a dev build for both. leaf_dir = tmp_path / "leaf" leaf_dir.mkdir() leaf_spec = spack.spec.Spec("dev-build-test-install@=1.0.0") # non-existing version leaf_pkg_cls = spack.repo.PATH.get_pkg_class(leaf_spec.name) with fs.working_dir(str(leaf_dir)): with open(leaf_pkg_cls.filename, "w", encoding="utf-8") as f: # type: ignore f.write(leaf_pkg_cls.original_string) # type: ignore # setup dev-build-test-dependent package for dev build # don't concretize outside environment -- dev info will be wrong root_dir = tmp_path / "root" root_dir.mkdir() root_spec = spack.spec.Spec("dev-build-test-dependent@0.0.0") root_pkg_cls = spack.repo.PATH.get_pkg_class(root_spec.name) with fs.working_dir(str(root_dir)): with open(root_pkg_cls.filename, "w", encoding="utf-8") as f: # type: ignore f.write(root_pkg_cls.original_string) # type: ignore # setup environment envdir = tmp_path / "env" envdir.mkdir() with fs.working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( f"""\ spack: specs: - dev-build-test-dependent@0.0.0 develop: dev-build-test-install: path: {leaf_dir} spec: dev-build-test-install@=1.0.0 dev-build-test-dependent: spec: dev-build-test-dependent@0.0.0 path: {root_dir} """ ) env("create", "test", "./spack.yaml") with ev.read("test"): # Do concretization inside environment for dev info # These specs are the source of truth to compare against the installs leaf_spec = spack.concretize.concretize_one(leaf_spec) root_spec = spack.concretize.concretize_one(root_spec) # Do install install() for spec in (leaf_spec, root_spec): filename = spec.package.filename # type: ignore assert filename in os.listdir(spec.prefix) with open(os.path.join(spec.prefix, filename), "r", encoding="utf-8") as f: assert f.read() == spec.package.replacement_string def test_dev_build_env_dependency( tmp_path: pathlib.Path, install_mockery, mock_fetch, mutable_mock_env_path ): """ Test non-root specs in an environment are properly marked for dev builds. """ # setup dev-build-test-install package for dev build build_dir = tmp_path / "build" build_dir.mkdir() spec = spack.spec.Spec("dependent-of-dev-build@0.0.0") dep_spec = spack.spec.Spec("dev-build-test-install") with fs.working_dir(str(build_dir)): dep_pkg_cls = spack.repo.PATH.get_pkg_class(dep_spec.name) with open(dep_pkg_cls.filename, "w", encoding="utf-8") as f: # type: ignore f.write(dep_pkg_cls.original_string) # type: ignore # setup environment envdir = tmp_path / "env" envdir.mkdir() with fs.working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( f"""\ spack: specs: - dependent-of-dev-build@0.0.0 develop: dev-build-test-install: spec: dev-build-test-install@0.0.0 path: {os.path.relpath(str(build_dir), start=str(envdir))} """ ) env("create", "test", "./spack.yaml") with ev.read("test"): # concretize in the environment to get the dev build info # equivalent to setting dev_build and dev_path variants # on all specs above spec = spack.concretize.concretize_one(spec) dep_spec = spack.concretize.concretize_one(dep_spec) install() # Ensure that both specs installed properly assert dep_spec.package.filename in os.listdir(dep_spec.prefix) assert os.path.exists(spec.prefix) # Ensure variants set properly; ensure build_dir is absolute and normalized for dep in (dep_spec, spec["dev-build-test-install"]): assert dep.satisfies("dev_path=%s" % str(build_dir)) assert spec.satisfies("^dev_path=*") @pytest.mark.parametrize("test_spec", ["dev-build-test-install", "dependent-of-dev-build"]) def test_dev_build_rebuild_on_source_changes( test_spec, tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, mock_fetch ): """Test dev builds rebuild on changes to source code. ``test_spec = dev-build-test-install`` tests rebuild for changes to package ``test_spec = dependent-of-dev-build`` tests rebuild for changes to dep """ # setup dev-build-test-install package for dev build build_dir = tmp_path / "build" build_dir.mkdir() spec = spack.concretize.concretize_one( spack.spec.Spec("dev-build-test-install@0.0.0 dev_path=%s" % str(build_dir)) ) def reset_string(): with fs.working_dir(str(build_dir)): with open(spec.package.filename, "w", encoding="utf-8") as f: # type: ignore f.write(spec.package.original_string) # type: ignore reset_string() # setup environment envdir = tmp_path / "env" envdir.mkdir() with fs.working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( f"""\ spack: specs: - {test_spec}@0.0.0 develop: dev-build-test-install: spec: dev-build-test-install@0.0.0 path: {build_dir} """ ) env("create", "test", "./spack.yaml") with ev.read("test"): install() reset_string() # so the package will accept rebuilds fs.touch(os.path.join(str(build_dir), "test")) output = install() assert f"Installing {test_spec}" in output ================================================ FILE: lib/spack/spack/test/cmd/develop.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import shutil import pytest import spack.concretize import spack.config import spack.environment as ev import spack.llnl.util.filesystem as fs import spack.package_base import spack.spec import spack.stage import spack.util.git import spack.util.path from spack.error import SpackError from spack.fetch_strategy import URLFetchStrategy from spack.main import SpackCommand add = SpackCommand("add") develop = SpackCommand("develop") env = SpackCommand("env") @pytest.mark.usefixtures("mutable_mock_env_path", "mock_packages", "mock_fetch", "mutable_config") class TestDevelop: def check_develop(self, env, spec, path=None, build_dir=None): path = path or spec.name # check in memory representation assert spec.name in env.dev_specs dev_specs_entry = env.dev_specs[spec.name] assert dev_specs_entry["path"] == path assert dev_specs_entry["spec"] == str(spec) # check yaml representation dev_config = spack.config.get("develop", {}) assert spec.name in dev_config yaml_entry = dev_config[spec.name] assert yaml_entry["spec"] == str(spec) if path == spec.name: # default paths aren't written out assert "path" not in yaml_entry else: assert yaml_entry["path"] == path if build_dir is not None: scope = env.scope_name assert build_dir == spack.config.get( "packages:{}:package_attributes:build_directory".format(spec.name), scope ) def test_develop_no_path_no_clone(self): env("create", "test") with ev.read("test") as e: # develop checks that the path exists fs.mkdirp(os.path.join(e.path, "mpich")) develop("--no-clone", "mpich@1.0") self.check_develop(e, spack.spec.Spec("mpich@=1.0")) def test_develop_no_clone(self, tmp_path: pathlib.Path): env("create", "test") with ev.read("test") as e: develop("--no-clone", "-p", str(tmp_path), "mpich@1.0") self.check_develop(e, spack.spec.Spec("mpich@=1.0"), str(tmp_path)) def test_develop_no_version(self, tmp_path: pathlib.Path): env("create", "test") with ev.read("test") as e: develop("--no-clone", "-p", str(tmp_path), "mpich") self.check_develop(e, spack.spec.Spec("mpich@=main"), str(tmp_path)) def test_develop(self): env("create", "test") with ev.read("test") as e: develop("mpich@1.0") self.check_develop(e, spack.spec.Spec("mpich@=1.0")) def test_develop_no_args(self): env("create", "test") with ev.read("test") as e: # develop and remove it develop("mpich@1.0") shutil.rmtree(os.path.join(e.path, "mpich")) # test develop with no args develop() self.check_develop(e, spack.spec.Spec("mpich@=1.0")) def test_develop_build_directory(self): env("create", "test") with ev.read("test") as e: develop("-b", "test_build_dir", "mpich@1.0") self.check_develop(e, spack.spec.Spec("mpich@=1.0"), None, "test_build_dir") def test_develop_twice(self): env("create", "test") with ev.read("test") as e: develop("mpich@1.0") self.check_develop(e, spack.spec.Spec("mpich@=1.0")) develop("mpich@1.0") # disk representation isn't updated unless we write # second develop command doesn't change it, so we don't write # but we check disk representation e.write() self.check_develop(e, spack.spec.Spec("mpich@=1.0")) assert len(e.dev_specs) == 1 def test_develop_update_path(self, tmp_path: pathlib.Path): env("create", "test") with ev.read("test") as e: develop("mpich@1.0") develop("-p", str(tmp_path), "mpich@1.0") self.check_develop(e, spack.spec.Spec("mpich@=1.0"), str(tmp_path)) assert len(e.dev_specs) == 1 def test_develop_update_spec(self): env("create", "test") with ev.read("test") as e: develop("mpich@1.0") develop("mpich@2.0") self.check_develop(e, spack.spec.Spec("mpich@=2.0")) assert len(e.dev_specs) == 1 def test_develop_applies_changes(self, monkeypatch): env("create", "test") with ev.read("test") as e: e.add("mpich@1.0") e.concretize() e.write() monkeypatch.setattr(spack.stage.Stage, "steal_source", lambda x, y: None) develop("mpich@1.0") # Check modifications actually worked spec = next(e.roots()) assert spec.satisfies("dev_path=*") def test_develop_applies_changes_parents(self, monkeypatch): env("create", "test") with ev.read("test") as e: e.add("hdf5^mpich@1.0") e.concretize() e.write() orig_hash = next(e.roots()).dag_hash() monkeypatch.setattr(spack.stage.Stage, "steal_source", lambda x, y: None) develop("mpich@1.0") # Check modifications actually worked new_hdf5 = next(e.roots()) assert new_hdf5.dag_hash() != orig_hash assert new_hdf5["mpi"].satisfies("dev_path=*") def test_develop_applies_changes_spec_conflict(self, monkeypatch): env("create", "test") with ev.read("test") as e: e.add("mpich@1.0") e.concretize() e.write() monkeypatch.setattr(spack.stage.Stage, "steal_source", lambda x, y: None) with pytest.raises(ev.SpackEnvironmentDevelopError, match="conflicts with concrete"): develop("mpich@1.1") def test_develop_applies_changes_path(self, monkeypatch): env("create", "test") with ev.read("test") as e: e.add("mpich@1.0") e.concretize() e.write() # canonicalize paths relative to env testpath1 = spack.util.path.canonicalize_path("test/path1", e.path) testpath2 = spack.util.path.canonicalize_path("test/path2", e.path) monkeypatch.setattr(spack.stage.Stage, "steal_source", lambda x, y: None) # Testing that second call to develop successfully changes both config and specs for path in (testpath1, testpath2): develop("--path", path, "mpich@1.0") # Check modifications actually worked spec = next(e.roots()) assert spec.satisfies(f"dev_path={path}") assert spack.config.get("develop:mpich:path") == path def test_develop_no_modify(self, monkeypatch): env("create", "test") with ev.read("test") as e: e.add("mpich@1.0") e.concretize() e.write() monkeypatch.setattr(spack.stage.Stage, "steal_source", lambda x, y: None) develop("--no-modify-concrete-specs", "mpich@1.0") # Check modifications were not applied spec = next(e.roots()) assert not spec.satisfies("dev_path=*") def test_develop_canonicalize_path(self, monkeypatch): env("create", "test") with ev.read("test") as e: e.add("mpich@1.0") e.concretize() e.write() path = "../$user" abspath = spack.util.path.canonicalize_path(path, e.path) def check_path(stage, dest): assert dest == abspath monkeypatch.setattr(spack.stage.Stage, "steal_source", check_path) develop("-p", path, "mpich@1.0") self.check_develop(e, spack.spec.Spec("mpich@=1.0"), path) # Check modifications actually worked spec = next(e.roots()) assert spec.satisfies("dev_path=%s" % abspath) def test_develop_canonicalize_path_no_args(self, monkeypatch): env("create", "test") with ev.read("test") as e: e.add("mpich@1.0") e.concretize() e.write() path = "$user" abspath = spack.util.path.canonicalize_path(path, e.path) def check_path(stage, dest): assert dest == abspath monkeypatch.setattr(spack.stage.Stage, "steal_source", check_path) # Defensive check to ensure canonicalization failures don't pollute FS assert abspath.startswith(e.path) # Create path to allow develop to modify env fs.mkdirp(abspath) develop("--no-clone", "-p", path, "mpich@1.0") self.check_develop(e, spack.spec.Spec("mpich@=1.0"), path) # Remove path to ensure develop with no args runs staging code os.rmdir(abspath) develop() self.check_develop(e, spack.spec.Spec("mpich@=1.0"), path) # Check modifications actually worked spec = next(e.roots()) assert spec.satisfies("dev_path=%s" % abspath) def _git_commit_list(git_repo_dir): git = spack.util.git.git() with fs.working_dir(git_repo_dir): output = git("log", "--pretty=format:%h", "-n", "20", output=str) return output.strip().split() def test_develop_full_git_repo( mutable_mock_env_path, mock_git_version_info, install_mockery, mock_packages, monkeypatch, mutable_config, request, ): repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False ) spec = spack.concretize.concretize_one("git-test-commit@1.2") try: spec.package.do_stage() commits = _git_commit_list(spec.package.stage[0].source_path) # Outside of "spack develop" Spack will only pull exactly the commit it # needs, with no additional history assert len(commits) == 1 finally: spec.package.do_clean() # Now use "spack develop": look at the resulting dev_path and make # sure the git repo pulled includes the full branch history (or rather, # more than just one commit). env("create", "test") with ev.read("test") as e: add("git-test-commit@1.2") e.concretize() e.write() develop("git-test-commit@1.2") e.write() spec = e.all_specs()[0] develop_dir = spec.variants["dev_path"].value commits = _git_commit_list(develop_dir) assert len(commits) > 1 def test_recursive(mutable_mock_env_path, install_mockery, mock_fetch): env("create", "test") with ev.read("test") as e: add("indirect-mpich@1.0") e.concretize() e.write() specs = e.all_specs() assert len(specs) > 1 develop("--recursive", "mpich") expected_dev_specs = ["mpich", "direct-mpich", "indirect-mpich"] for spec in expected_dev_specs: assert spec in e.dev_specs spec = next(e.roots()) for dep in spec.traverse(): assert dep.satisfies("dev_path=*") == (dep.name in expected_dev_specs) def test_develop_fails_with_multiple_concrete_versions( mutable_mock_env_path, install_mockery, mock_fetch, mutable_config ): env("create", "test") with ev.read("test") as e: add("indirect-mpich@1.0") add("indirect-mpich@0.9") mutable_config.set("concretizer:unify", False) e.concretize() with pytest.raises(SpackError) as develop_error: develop("indirect-mpich", fail_on_error=True) error_str = "has multiple concrete instances in the graph" assert error_str in str(develop_error.value) def test_concretize_dev_path_with_at_symbol_in_env( mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages ): spec_like = "develop-test@develop" develop_dir = tmp_path / "build@location" develop_dir.mkdir() env("create", "test_at_sym") with ev.read("test_at_sym") as e: add(spec_like) e.concretize() e.write() develop(f"--path={develop_dir}", spec_like) result = e.concrete_roots() assert len(result) == 1 cspec = result[0] assert cspec.satisfies(spec_like), cspec assert cspec.is_develop, cspec assert str(develop_dir) in cspec.variants["dev_path"], cspec def _failing_fn(*args, **kwargs): # This stands in for a function that should never be called as # part of a test. assert False @pytest.mark.parametrize("_devpath_should_exist", [True, False]) @pytest.mark.disable_clean_stage_check def test_develop_with_devpath_staging( monkeypatch, mutable_mock_env_path, mock_packages, tmp_path: pathlib.Path, mock_archive, install_mockery, mock_fetch, mock_resource_fetch, mock_stage, _devpath_should_exist, ): # If the specified develop path exists, a resource should not be # downloaded at all at install time. Otherwise, it should be. env("create", "test") develop_dir = tmp_path / "build@location" if _devpath_should_exist: develop_dir.mkdir() monkeypatch.setattr(URLFetchStrategy, "fetch", _failing_fn) spec_like = "simple-resource@1.0" with ev.read("test") as e: e.add(spec_like) e.concretize() e.write() develop(f"--path={develop_dir}", spec_like) e.install_all() expected_resource_path = develop_dir / "resource.tgz" if _devpath_should_exist: # If we made it here, we didn't try to download anything. pass else: assert os.path.exists(expected_resource_path) ================================================ FILE: lib/spack/spack/test/cmd/diff.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pytest import spack.cmd.diff import spack.concretize import spack.main import spack.paths import spack.repo import spack.util.spack_json as sjson import spack.version install_cmd = spack.main.SpackCommand("install") diff_cmd = spack.main.SpackCommand("diff") find_cmd = spack.main.SpackCommand("find") # Note that the hash of p1 will differ depending on the variant chosen # we probably always want to omit that from diffs # p1____ # | \ # p2 v1 # | ____/ | # p3 p4 # i1 and i2 provide v1 (and both have the same dependencies) # All packages have an associated variant @pytest.fixture def test_repo(config): builder_test_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "diff") with spack.repo.use_repositories(builder_test_path) as mock_repo: yield mock_repo def test_diff_ignore(test_repo): specA = spack.concretize.concretize_one("p1+usev1") specB = spack.concretize.concretize_one("p1~usev1") c1 = spack.cmd.diff.compare_specs(specA, specB, to_string=False) def match(function, name, args): limit = len(args) return function.name == name and list(args[:limit]) == list(function.args[:limit]) def find(function_list, name, args): return any(match(f, name, args) for f in function_list) assert find(c1["a_not_b"], "node_os", ["p4"]) c2 = spack.cmd.diff.compare_specs(specA, specB, ignore_packages=["v1"], to_string=False) assert not find(c2["a_not_b"], "node_os", ["p4"]) assert find(c2["intersect"], "node_os", ["p3"]) # Check ignoring changes on multiple packages specA = spack.concretize.concretize_one("p1+usev1 ^p3+p3var") specA = spack.concretize.concretize_one("p1~usev1 ^p3~p3var") c3 = spack.cmd.diff.compare_specs(specA, specB, to_string=False) assert find(c3["a_not_b"], "variant_value", ["p3", "p3var"]) c4 = spack.cmd.diff.compare_specs(specA, specB, ignore_packages=["v1", "p3"], to_string=False) assert not find(c4["a_not_b"], "node_os", ["p4"]) assert not find(c4["a_not_b"], "variant_value", ["p3"]) def test_diff_cmd(install_mockery, mock_fetch, mock_archive, mock_packages): """Test that we can install two packages and diff them""" specA = spack.concretize.concretize_one("mpileaks") specB = spack.concretize.concretize_one("mpileaks+debug") # Specs should be the same as themselves c = spack.cmd.diff.compare_specs(specA, specA, to_string=True) assert len(c["a_not_b"]) == 0 assert len(c["b_not_a"]) == 0 # Calculate the comparison (c) c = spack.cmd.diff.compare_specs(specA, specB, to_string=True) # these particular diffs should have the same length b/c there aren't # any node differences -- just value differences. assert len(c["a_not_b"]) == len(c["b_not_a"]) # ensure that variant diffs are in here the result assert ["variant_value", "mpileaks debug False"] in c["a_not_b"] assert ["variant_value", "mpileaks debug True"] in c["b_not_a"] # ensure that hash diffs are in here the result assert ["hash", "mpileaks %s" % specA.dag_hash()] in c["a_not_b"] assert ["hash", "mpileaks %s" % specB.dag_hash()] in c["b_not_a"] def test_diff_runtimes(install_mockery, mock_fetch, mock_archive, mock_packages): """Test that we can install two packages and diff them""" specA = spack.concretize.concretize_one("mpileaks") specB = specA.copy() specB["gcc-runtime"].versions = spack.version.VersionList([spack.version.Version("0.0.0")]) # Specs should be the same as themselves c = spack.cmd.diff.compare_specs(specA, specB, to_string=True) assert ["version", "gcc-runtime 0.0.0"] in c["b_not_a"] def test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages): """Test with and without the --first option""" install_cmd("--fake", "mpileaks") # Only one version of mpileaks will work diff_cmd("mpileaks", "mpileaks") # 2 specs are required for a diff with pytest.raises(spack.main.SpackCommandError): diff_cmd("mpileaks") with pytest.raises(spack.main.SpackCommandError): diff_cmd("mpileaks", "mpileaks", "mpileaks") # Ensure they are the same assert "No differences" in diff_cmd("mpileaks", "mpileaks") output = diff_cmd("--json", "mpileaks", "mpileaks") result = sjson.load(output) assert not result["a_not_b"] assert not result["b_not_a"] assert "mpileaks" in result["a_name"] assert "mpileaks" in result["b_name"] # spot check attributes in the intersection to ensure they describe the spec assert "intersect" in result assert all( ["node", dep] in result["intersect"] for dep in ("mpileaks", "callpath", "dyninst", "libelf", "libdwarf", "mpich") ) assert all( len([diff for diff in result["intersect"] if diff[0] == attr]) == 9 for attr in ( "version", "node_target", "node_platform", "node_os", "node", "package_hash", "hash", ) ) # After we install another version, it should ask us to disambiguate install_cmd("mpileaks+debug") # There are two versions of mpileaks with pytest.raises(spack.main.SpackCommandError): diff_cmd("mpileaks", "mpileaks+debug") # But if we tell it to use the first, it won't try to disambiguate assert "variant" in diff_cmd("--first", "mpileaks", "mpileaks+debug") # This matches them exactly debug_hash = find_cmd("--format", "{hash}", "mpileaks+debug").strip() no_debug_hashes = find_cmd("--format", "{hash}", "mpileaks~debug") no_debug_hash = no_debug_hashes.split()[0] output = diff_cmd( "--json", "mpileaks/{0}".format(debug_hash), "mpileaks/{0}".format(no_debug_hash) ) result = sjson.load(output) assert ["hash", "mpileaks %s" % debug_hash] in result["a_not_b"] assert ["variant_value", "mpileaks debug True"] in result["a_not_b"] assert ["hash", "mpileaks %s" % no_debug_hash] in result["b_not_a"] assert ["variant_value", "mpileaks debug False"] in result["b_not_a"] ================================================ FILE: lib/spack/spack/test/cmd/edit.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import spack.paths import spack.repo import spack.util.editor from spack.main import SpackCommand edit = SpackCommand("edit") def test_edit_packages(monkeypatch, mock_packages: spack.repo.RepoPath): """Test spack edit pkg-a pkg-b""" path_a = mock_packages.filename_for_package_name("pkg-a") path_b = mock_packages.filename_for_package_name("pkg-b") called = False def editor(*args: str, **kwargs): nonlocal called called = True assert args[0] == path_a assert args[1] == path_b monkeypatch.setattr(spack.util.editor, "editor", editor) edit("pkg-a", "pkg-b") assert called def test_edit_files(monkeypatch, mock_packages): """Test spack edit --build-system autotools cmake""" called = False def editor(*args: str, **kwargs): nonlocal called called = True from spack_repo.builtin_mock.build_systems import autotools, cmake # type: ignore assert os.path.samefile(args[0], autotools.__file__) assert os.path.samefile(args[1], cmake.__file__) monkeypatch.setattr(spack.util.editor, "editor", editor) edit("--build-system", "autotools", "cmake") assert called def test_edit_non_default_build_system(monkeypatch, mock_packages, mutable_config): called = False def editor(*args: str, **kwargs): nonlocal called called = True from spack_repo.builtin_mock.build_systems import autotools, cmake # type: ignore assert os.path.samefile(args[0], autotools.__file__) assert os.path.samefile(args[1], cmake.__file__) monkeypatch.setattr(spack.util.editor, "editor", editor) # set up an additional repo extra_repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "requirements_test" with spack.repo.use_repositories(str(extra_repo_dir), override=False): edit("--build-system", "builtin_mock.autotools", "builtin_mock.cmake") assert called ================================================ FILE: lib/spack/spack/test/cmd/env.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import contextlib import filecmp import glob import io import os import pathlib import shutil import sys from argparse import Namespace from typing import Any, Dict, Optional import pytest import spack.cmd.env import spack.concretize import spack.config import spack.environment as ev import spack.environment.depfile as depfile import spack.error import spack.llnl.util.filesystem as fs import spack.llnl.util.link_tree import spack.llnl.util.tty as tty import spack.main import spack.modules import spack.modules.tcl import spack.package_base import spack.paths import spack.repo import spack.schema.env import spack.solver.asp import spack.stage import spack.store import spack.util.environment import spack.util.spack_json as sjson import spack.util.spack_yaml from spack.cmd.env import _env_create from spack.installer import PackageInstaller from spack.llnl.util.filesystem import readlink from spack.llnl.util.lang import dedupe from spack.main import SpackCommand, SpackCommandError from spack.spec import Spec from spack.stage import stage_prefix from spack.test.conftest import RepoBuilder from spack.traverse import traverse_nodes from spack.util.executable import Executable from spack.util.path import substitute_path_variables from spack.version import Version # TODO-27021 # everything here uses the mock_env_path pytestmark = [ pytest.mark.usefixtures("mutable_config", "mutable_mock_env_path", "mutable_mock_repo"), pytest.mark.maybeslow, pytest.mark.not_on_windows("Envs unsupported on Windows"), ] env = SpackCommand("env") install = SpackCommand("install") add = SpackCommand("add") change = SpackCommand("change") config = SpackCommand("config") remove = SpackCommand("remove") concretize = SpackCommand("concretize") stage = SpackCommand("stage") uninstall = SpackCommand("uninstall") find = SpackCommand("find") develop = SpackCommand("develop") module = SpackCommand("module") sep = os.sep def setup_combined_multiple_env(): env("create", "test1") test1 = ev.read("test1") with test1: add("mpich@1.0") test1.concretize() test1.write() env("create", "test2") test2 = ev.read("test2") with test2: add("libelf") test2.concretize() test2.write() env("create", "--include-concrete", "test1", "--include-concrete", "test2", "combined_env") combined = ev.read("combined_env") return test1, test2, combined @pytest.fixture() def environment_from_manifest(tmp_path: pathlib.Path): """Returns a new environment named 'test' from the content of a manifest file.""" def _create(content): spack_yaml = tmp_path / ev.manifest_name spack_yaml.write_text(content) return _env_create("test", init_file=str(spack_yaml)) return _create def check_mpileaks_and_deps_in_view(viewdir: pathlib.Path): """Check that the expected install directories exist.""" assert (viewdir / ".spack" / "mpileaks").exists() assert (viewdir / ".spack" / "libdwarf").exists() def check_viewdir_removal(viewdir: pathlib.Path): """Check that the uninstall/removal worked.""" assert not (viewdir / ".spack").exists() or list((viewdir / ".spack").iterdir()) == [ viewdir / "projections.yaml" ] def test_env_track_nonexistent_path_fails(): with pytest.raises(spack.main.SpackCommandError): env("track", "path/does/not/exist") assert "doesn't contain an environment" in env.output def test_env_track_existing_env_fails(): env("create", "track_test") with pytest.raises(spack.main.SpackCommandError): env("track", "--name", "track_test", ev.environment_dir_from_name("track_test")) assert "environment named track_test already exists" in env.output def test_env_track_valid(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): # create an independent environment env("create", "-d", ".") # test tracking an environment in known store env("track", "--name", "test1", ".") # test removing environment to ensure independent isn't deleted env("rm", "-y", "test1") assert os.path.isfile("spack.yaml") def test_env_untrack_valid(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): # create an independent environment env("create", "-d", ".") # test tracking an environment in known store env("track", "--name", "test_untrack", ".") env("untrack", "--yes-to-all", "test_untrack") # check that environment was successfully untracked out = env("ls") assert "test_untrack" not in out def test_env_untrack_invalid_name(): # test untracking an environment that doesn't exist env_name = "invalid_environment_untrack" out = env("untrack", env_name) assert f"Environment '{env_name}' does not exist" in out def test_env_untrack_when_active(tmp_path: pathlib.Path): env_name = "test_untrack_active" with fs.working_dir(str(tmp_path)): # create an independent environment env("create", "-d", ".") # test tracking an environment in known store env("track", "--name", env_name, ".") active_env = ev.read(env_name) with active_env: output = env("untrack", "--yes-to-all", env_name, fail_on_error=False) assert env.error is not None # check that environment could not be untracked while active assert f"'{env_name}' can't be untracked while activated" in output env("untrack", "-f", env_name) out = env("ls") assert env_name not in out def test_env_untrack_managed(): env_name = "test_untrack_managed" # create an managed environment env("create", env_name) with pytest.raises(spack.main.SpackCommandError): env("untrack", env_name) # check that environment could not be untracked while active assert f"'{env_name}' is not a tracked env" in env.output @pytest.fixture() def installed_environment( tmp_path: pathlib.Path, mock_fetch, mock_packages, mock_archive, install_mockery ): spack_yaml = tmp_path / "spack.yaml" @contextlib.contextmanager def _installed_environment(content): spack_yaml.write_text(content) with fs.working_dir(tmp_path): env("create", "test", "./spack.yaml") with ev.read("test") as current_environment: current_environment.concretize() current_environment.install_all(fake=True) current_environment.write(regenerate=True) with ev.read("test") as current_environment: yield current_environment return _installed_environment @pytest.fixture() def template_combinatorial_env(tmp_path: pathlib.Path): """Returns a template base environment for tests. Since the environment configuration is extended using str.format, we need double '{' escaping for the projections. """ view_dir = tmp_path / "view" return f"""\ spack: definitions: - packages: [mpileaks, callpath] - targets: ['target=x86_64', 'target=core2'] specs: - matrix: - [$packages] - [$targets] view: combinatorial: root: {view_dir} {{view_config}} projections: 'all': '{{{{architecture.target}}}}/{{{{name}}}}-{{{{version}}}}' """ def test_add(): e = ev.create("test") e.add("mpileaks") assert Spec("mpileaks") in e.user_specs def test_change_match_spec(): env("create", "test") e = ev.read("test") with e: add("mpileaks@2.1") add("mpileaks@2.2") change("--match-spec", "mpileaks@2.2", "mpileaks@2.3") assert not any(x.intersects("mpileaks@2.2") for x in e.user_specs) assert any(x.intersects("mpileaks@2.3") for x in e.user_specs) def test_change_multiple_matches(): env("create", "test") e = ev.read("test") with e: add("mpileaks@2.1") add("mpileaks@2.2") add("libelf@0.8.12%clang") change("--match-spec", "mpileaks", "-a", "mpileaks%gcc") assert all(x.intersects("%gcc") for x in e.user_specs if x.name == "mpileaks") assert any(x.intersects("%clang") for x in e.user_specs if x.name == "libelf") def test_env_add_virtual(): env("create", "test") e = ev.read("test") e.add("mpi") e.concretize() assert len(e.concretized_roots) == 1 spec = e.specs_by_hash[e.concretized_roots[0].hash] assert spec.intersects("mpi") def test_env_add_nonexistent_fails(): env("create", "test") e = ev.read("test") with pytest.raises(ev.SpackEnvironmentError, match=r"no such package"): e.add("thispackagedoesnotexist") def test_env_list(mutable_mock_env_path): env("create", "foo") env("create", "bar") env("create", "baz") out = env("list") assert "foo" in out assert "bar" in out assert "baz" in out # make sure `spack env list` skips invalid things in var/spack/env (mutable_mock_env_path / ".DS_Store").touch() out = env("list") assert "foo" in out assert "bar" in out assert "baz" in out assert ".DS_Store" not in out def test_env_remove(): env("create", "foo") env("create", "bar") out = env("list") assert "foo" in out assert "bar" in out foo = ev.read("foo") with foo: with pytest.raises(SpackCommandError): env("remove", "-y", "foo") assert "foo" in env("list") env("remove", "-y", "foo") out = env("list") assert "foo" not in out assert "bar" in out env("remove", "-y", "bar") out = env("list") assert "foo" not in out assert "bar" not in out def test_env_rename_managed(): # Need real environment with pytest.raises(spack.main.SpackCommandError): env("rename", "foo", "bar") assert "The specified name does not correspond to a managed spack environment" in env.output env("create", "foo") out = env("list") assert "foo" in out out = env("rename", "foo", "bar") assert "Successfully renamed environment foo to bar" in out out = env("list") assert "foo" not in out assert "bar" in out bar = ev.read("bar") with bar: # Cannot rename active environment with pytest.raises(spack.main.SpackCommandError): env("rename", "bar", "baz") assert "Cannot rename active environment" in env.output env("create", "qux") # Cannot rename to an active environment (even with force flag) with pytest.raises(spack.main.SpackCommandError): env("rename", "-f", "qux", "bar") assert "bar is an active environment" in env.output # Can rename inactive environment when another's active out = env("rename", "qux", "quux") assert "Successfully renamed environment qux to quux" in out out = env("list") assert "bar" in out assert "baz" not in out env("create", "baz") # Cannot rename to existing environment without --force with pytest.raises(spack.main.SpackCommandError): env("rename", "bar", "baz") errmsg = ( "The new name corresponds to an existing environment;" " specify the --force flag to overwrite it." ) assert errmsg in env.output env("rename", "-f", "bar", "baz") out = env("list") assert "bar" not in out assert "baz" in out def test_env_rename_independent(tmp_path: pathlib.Path): # Need real environment with pytest.raises(spack.main.SpackCommandError): env("rename", "-d", "./non-existing", "./also-non-existing") assert "The specified path does not correspond to a valid spack environment" in env.output anon_foo = str(tmp_path / "foo") env("create", "-d", anon_foo) anon_bar = str(tmp_path / "bar") out = env("rename", "-d", anon_foo, anon_bar) assert f"Successfully renamed environment {anon_foo} to {anon_bar}" in out assert not ev.is_env_dir(anon_foo) assert ev.is_env_dir(anon_bar) # Cannot rename active environment anon_baz = str(tmp_path / "baz") env("activate", "--sh", "-d", anon_bar) with pytest.raises(spack.main.SpackCommandError): env("rename", "-d", anon_bar, anon_baz) assert "Cannot rename active environment" in env.output env("deactivate", "--sh") assert ev.is_env_dir(anon_bar) assert not ev.is_env_dir(anon_baz) # Cannot rename to existing environment without --force env("create", "-d", anon_baz) with pytest.raises(spack.main.SpackCommandError): env("rename", "-d", anon_bar, anon_baz) errmsg = ( "The new path corresponds to an existing environment;" " specify the --force flag to overwrite it." ) assert errmsg in env.output assert ev.is_env_dir(anon_bar) assert ev.is_env_dir(anon_baz) env("rename", "-f", "-d", anon_bar, anon_baz) assert not ev.is_env_dir(anon_bar) assert ev.is_env_dir(anon_baz) # Cannot rename to existing (non-environment) path without --force qux = tmp_path / "qux" qux.mkdir() anon_qux = str(qux) assert not ev.is_env_dir(anon_qux) with pytest.raises(spack.main.SpackCommandError): env("rename", "-d", anon_baz, anon_qux) errmsg = "The new path already exists; specify the --force flag to overwrite it." assert errmsg in env.output env("rename", "-f", "-d", anon_baz, anon_qux) assert not ev.is_env_dir(anon_baz) assert ev.is_env_dir(anon_qux) def test_concretize(): e = ev.create("test") e.add("mpileaks") e.concretize() assert len(e.concretized_roots) == 1 assert e.concretized_roots[0].root == Spec("mpileaks") def test_env_specs_partition(install_mockery, mock_fetch): e = ev.create("test") e.add("cmake-client") e.concretize() # Single not installed root spec. roots_already_installed, roots_to_install = e._partition_roots_by_install_status() assert len(roots_already_installed) == 0 assert len(roots_to_install) == 1 assert roots_to_install[0].name == "cmake-client" # Single installed root. e.install_all(fake=True) roots_already_installed, roots_to_install = e._partition_roots_by_install_status() assert len(roots_already_installed) == 1 assert roots_already_installed[0].name == "cmake-client" assert len(roots_to_install) == 0 # One installed root, one not installed root. e.add("mpileaks") e.concretize() roots_already_installed, roots_to_install = e._partition_roots_by_install_status() assert len(roots_already_installed) == 1 assert len(roots_to_install) == 1 assert roots_already_installed[0].name == "cmake-client" assert roots_to_install[0].name == "mpileaks" def test_env_install_all(install_mockery, mock_fetch): e = ev.create("test") e.add("cmake-client") e.concretize() e.install_all(fake=True) spec = next(x for x in e.all_specs_generator() if x.name == "cmake-client") assert spec.installed def test_env_install_single_spec(install_mockery, mock_fetch, installer_variant): env("create", "test") install = SpackCommand("install") e = ev.read("test") with e: install("--fake", "--add", "cmake-client") e = ev.read("test") assert len(e.concretized_roots) == 1 item = e.concretized_roots[0] assert list(e.user_specs) == [Spec("cmake-client")] assert item.root == Spec("cmake-client") assert e.specs_by_hash[item.hash].name == "cmake-client" @pytest.mark.parametrize("unify", [True, False, "when_possible"]) @pytest.mark.parametrize("reuse", [True, False]) def test_env_install_include_concrete_env( unify, reuse, install_mockery, mock_fetch, mutable_config ): test1, test2, combined = setup_combined_multiple_env() if unify is False: combined.manifest.set_default_view(False) with combined: mutable_config.set("concretizer:unify", unify) mutable_config.set("concretizer:reuse", reuse) combined.add("mpileaks") combined.concretize() combined.write() install("--fake") test1_user_spec_hashes = [x.hash for x in test1.concretized_roots] test2_user_spec_hashes = [x.hash for x in test2.concretized_roots] for spec in combined.all_specs(): assert spec.installed assert test1_user_spec_hashes == [ x.hash for x in combined.included_concretized_roots[test1.path] ] assert test2_user_spec_hashes == [ x.hash for x in combined.included_concretized_roots[test2.path] ] mpileaks_hash = combined.concretized_roots[0].hash mpileaks = combined.specs_by_hash[mpileaks_hash] if unify is False and reuse is False: # check that unification is not by accident assert mpileaks["mpi"].dag_hash() not in test1_user_spec_hashes else: assert mpileaks["mpi"].dag_hash() in test1_user_spec_hashes assert mpileaks["libelf"].dag_hash() in test2_user_spec_hashes def test_env_roots_marked_explicit(install_mockery, mock_fetch, installer_variant): install = SpackCommand("install") install("--fake", "dependent-install") # Check one explicit, one implicit install dependent = spack.store.STORE.db.query(explicit=True) dependency = spack.store.STORE.db.query(explicit=False) assert len(dependent) == 1 assert len(dependency) == 1 env("create", "test") with ev.read("test") as e: # make implicit install a root of the env e.add(dependency[0].name) e.concretize() e.install_all() explicit = spack.store.STORE.db.query(explicit=True) assert len(explicit) == 2 def test_env_modifications_error_on_activate(install_mockery, mock_fetch, monkeypatch, capfd): env("create", "test") install = SpackCommand("install") e = ev.read("test") with e: install("--fake", "--add", "cmake-client") def setup_error(pkg, env): raise RuntimeError("cmake-client had issues!") pkg = spack.repo.PATH.get_pkg_class("cmake-client") monkeypatch.setattr(pkg, "setup_run_environment", setup_error) ev.shell.activate(e) _, err = capfd.readouterr() assert "cmake-client had issues!" in err assert "Warning: could not load runtime environment" in err def test_activate_adds_transitive_run_deps_to_path(install_mockery, mock_fetch, monkeypatch): env("create", "test") install = SpackCommand("install") e = ev.read("test") with e: install("--add", "--fake", "depends-on-run-env") env_variables = {} ev.shell.activate(e).apply_modifications(env_variables) assert env_variables["DEPENDENCY_ENV_VAR"] == "1" def test_env_definition_symlink(install_mockery, mock_fetch, tmp_path: pathlib.Path): filepath = str(tmp_path / "spack.yaml") filepath_mid = str(tmp_path / "spack_mid.yaml") env("create", "test") e = ev.read("test") e.add("mpileaks") os.rename(e.manifest_path, filepath) os.symlink(filepath, filepath_mid) os.symlink(filepath_mid, e.manifest_path) e.concretize() e.write() assert os.path.islink(e.manifest_path) assert os.path.islink(filepath_mid) def test_env_install_two_specs_same_dep( install_mockery, mock_fetch, tmp_path: pathlib.Path, monkeypatch ): """Test installation of two packages that share a dependency with no connection and the second specifying the dependency as a 'build' dependency. """ path = tmp_path / "spack.yaml" with fs.working_dir(str(tmp_path)): with open(str(path), "w", encoding="utf-8") as f: f.write( """\ spack: specs: - pkg-a - depb """ ) env("create", "test", "spack.yaml") with ev.read("test"): out = install("--fake") # Ensure both packages reach install phase processing and are installed out = str(out) assert "depb: Successfully installed" in out assert "pkg-a: Successfully installed" in out depb = spack.store.STORE.db.query_one("depb", installed=True) assert depb, "Expected depb to be installed" a = spack.store.STORE.db.query_one("pkg-a", installed=True) assert a, "Expected pkg-a to be installed" def test_remove_after_concretize(): e = ev.create("test") e.add("mpileaks") e.concretize() e.add("python") e.concretize() e.remove("mpileaks") assert Spec("mpileaks") not in e.user_specs assert any(s.name == "mpileaks" for s in e.all_specs_generator()) e.add("mpileaks") assert any(s.name == "mpileaks" for s in e.user_specs) e.remove("mpileaks", force=True) assert Spec("mpileaks") not in e.user_specs assert not any(s.name == "mpileaks" for s in e.all_specs_generator()) def test_remove_before_concretize(mutable_config): """Tests the effect of concretization after adding and removing specs""" with ev.create("test") as e: mutable_config.set("concretizer:unify", True) e.add("mpileaks") e.concretize() assert len(e.concretized_roots) == 1 assert e.concrete_roots()[0].satisfies("mpileaks") e.remove("mpileaks") e.concretize() assert not e.concretized_roots def test_remove_command(): env("create", "test") assert "test" in env("list") with ev.read("test"): add("mpileaks") with ev.read("test"): assert "mpileaks" in find() assert "mpileaks@" not in find() assert "mpileaks@" not in find("--show-concretized") with ev.read("test"): remove("mpileaks") with ev.read("test"): assert "mpileaks" not in find() assert "mpileaks@" not in find() assert "mpileaks@" not in find("--show-concretized") with ev.read("test"): add("mpileaks") with ev.read("test"): assert "mpileaks" in find() assert "mpileaks@" not in find() assert "mpileaks@" not in find("--show-concretized") with ev.read("test"): concretize() with ev.read("test"): assert "mpileaks" in find() assert "mpileaks@" not in find() assert "mpileaks@" in find("--show-concretized") with ev.read("test"): remove("mpileaks") with ev.read("test"): assert "mpileaks" not in find() # removed but still in last concretized specs assert "mpileaks@" in find("--show-concretized") with ev.read("test"): concretize() with ev.read("test"): assert "mpileaks" not in find() assert "mpileaks@" not in find() # now the lockfile is regenerated and it's gone. assert "mpileaks@" not in find("--show-concretized") def test_remove_command_all(): # Need separate ev.read calls for each command to ensure we test round-trip to disk env("create", "test") test_pkgs = ("mpileaks", "zlib") with ev.read("test"): for name in test_pkgs: add(name) with ev.read("test"): for name in test_pkgs: assert name in find() assert f"{name}@" not in find() with ev.read("test"): remove("-a") with ev.read("test"): for name in test_pkgs: assert name not in find() def test_bad_remove_included_env(): env("create", "test") test = ev.read("test") with test: add("mpileaks") test.concretize() test.write() env("create", "--include-concrete", "test", "combined_env") with pytest.raises(SpackCommandError): env("remove", "test") def test_force_remove_included_env(): env("create", "test") test = ev.read("test") with test: add("mpileaks") test.concretize() test.write() env("create", "--include-concrete", "test", "combined_env") rm_output = env("remove", "-f", "-y", "test") list_output = env("list") assert "'test' is used by environment 'combined_env'" in rm_output assert "test" not in list_output def test_environment_status(tmp_path: pathlib.Path, monkeypatch): with fs.working_dir(str(tmp_path)): assert "No active environment" in env("status") with ev.create("test"): assert "In environment test" in env("status") with ev.create_in_dir("local_dir"): assert os.path.join(os.getcwd(), "local_dir") in env("status") e = ev.create_in_dir("myproject") e.write() with fs.working_dir(str(tmp_path / "myproject")): with e: assert "in current directory" in env("status") def test_env_status_broken_view( mutable_mock_env_path, mock_archive, mock_fetch, mock_custom_repository, install_mockery, tmp_path: pathlib.Path, ): with ev.create_in_dir(tmp_path): install("--add", "--fake", "trivial-install-test-package") # switch to a new repo that doesn't include the installed package # test that Spack detects the missing package and warns the user with spack.repo.use_repositories(mock_custom_repository): with ev.Environment(tmp_path): output = env("status") assert "includes out of date packages or repos" in output # Test that the warning goes away when it's fixed with ev.Environment(tmp_path): output = env("status") assert "includes out of date packages or repos" not in output def test_env_activate_broken_view( mutable_mock_env_path, mock_archive, mock_fetch, mock_custom_repository, install_mockery ): with ev.create("test"): install("--add", "--fake", "trivial-install-test-package") # switch to a new repo that doesn't include the installed package # test that Spack detects the missing package and fails gracefully with spack.repo.use_repositories(mock_custom_repository): wrong_repo = env("activate", "--sh", "test") assert "Warning: could not load runtime environment" in wrong_repo assert "Unknown namespace: builtin_mock" in wrong_repo # test replacing repo fixes it normal_repo = env("activate", "--sh", "test") assert "Warning: could not load runtime environment" not in normal_repo assert "Unknown namespace: builtin_mock" not in normal_repo def test_to_lockfile_dict(): e = ev.create("test") e.add("mpileaks") e.concretize() context_dict = e._to_lockfile_dict() e_copy = ev.create("test_copy") e_copy._read_lockfile_dict(context_dict) assert e.specs_by_hash == e_copy.specs_by_hash def test_env_repo(): e = ev.create("test") e.add("mpileaks") e.write() with ev.read("test"): concretize() pkg_cls = e.repo.get_pkg_class("mpileaks") assert pkg_cls.name == "mpileaks" assert pkg_cls.namespace == "builtin_mock" def test_user_removed_spec(environment_from_manifest): """Ensure a user can remove from any position in the spack.yaml file.""" before = environment_from_manifest( """\ spack: specs: - mpileaks - hypre - libelf """ ) before.concretize() before.write() # user modifies yaml externally to spack and removes hypre with open(before.manifest_path, "w", encoding="utf-8") as f: f.write( """\ spack: specs: - mpileaks - libelf """ ) after = ev.read("test") after.concretize() after.write() read = ev.read("test") assert not any(x.name == "hypre" for x in read.all_specs_generator()) def test_lockfile_spliced_specs(environment_from_manifest, install_mockery): """Test that an environment can round-trip a spliced spec.""" # Create a local install for zmpi to splice in # Default concretization is not using zmpi zmpi = spack.concretize.concretize_one("zmpi") PackageInstaller([zmpi.package], fake=True).install() e1 = environment_from_manifest( f""" spack: specs: - mpileaks concretizer: splice: explicit: - target: mpi replacement: zmpi/{zmpi.dag_hash()} """ ) with e1: e1.concretize() e1.write() # By reading into a second environment, we force a round trip to json e2 = _env_create("test2", init_file=e1.lock_path) # The one spec is mpileaks for _, spec in e2.concretized_specs(): assert spec.spliced assert spec["mpi"].satisfies(f"zmpi@{zmpi.version}") assert spec["mpi"].build_spec.satisfies(zmpi) def test_init_from_lockfile(environment_from_manifest): """Test that an environment can be instantiated from a lockfile.""" e1 = environment_from_manifest( """ spack: specs: - mpileaks - hypre - libelf """ ) e1.concretize() e1.write() e2 = _env_create("test2", init_file=e1.lock_path) for s1, s2 in zip(e1.user_specs, e2.user_specs): assert s1 == s2 for r1, r2 in zip(e1.concretized_roots, e2.concretized_roots): assert r1 == r2 assert e1.specs_by_hash == e2.specs_by_hash def test_init_from_yaml(environment_from_manifest): """Test that an environment can be instantiated from a lockfile.""" e1 = environment_from_manifest( """ spack: specs: - mpileaks - hypre - libelf """ ) e1.concretize() e1.write() e2 = _env_create("test2", init_file=e1.manifest_path) for s1, s2 in zip(e1.user_specs, e2.user_specs): assert s1 == s2 assert not e2.concretized_roots assert not e2.specs_by_hash @pytest.mark.parametrize("use_name", (True, False)) def test_init_from_env(use_name, environment_from_manifest): """Test that an environment can be instantiated from an environment dir""" e1 = environment_from_manifest( """ spack: specs: - mpileaks - hypre - libelf """ ) with e1: # Test that relative paths in the env are not rewritten # Test that relative paths outside the env are dev_config = { "libelf": {"spec": "libelf", "path": "./libelf"}, "mpileaks": {"spec": "mpileaks", "path": "../mpileaks"}, } spack.config.set("develop", dev_config) fs.touch(os.path.join(e1.path, "libelf")) e1.concretize() e1.write() e2 = _env_create("test2", init_file="test" if use_name else e1.path) for s1, s2 in zip(e1.user_specs, e2.user_specs): assert s1 == s2 assert e2.concretized_roots == e1.concretized_roots assert e2.specs_by_hash == e1.specs_by_hash assert os.path.exists(os.path.join(e2.path, "libelf")) with e2: assert e2.dev_specs["libelf"]["path"] == "./libelf" assert e2.dev_specs["mpileaks"]["path"] == os.path.join( os.path.dirname(e1.path), "mpileaks" ) def test_init_from_env_no_spackfile(tmp_path): with pytest.raises(ev.SpackEnvironmentError, match="not a valid environment"): _env_create("test", init_file=str(tmp_path)) def test_init_from_yaml_relative_includes(tmp_path: pathlib.Path): files = [ "relative_copied/packages.yaml", "./relative_copied/compilers.yaml", "repos.yaml", "./config.yaml", ] manifest = f""" spack: specs: [] include: {files} """ e1_path = tmp_path / "e1" e1_manifest = e1_path / "spack.yaml" e1_path.mkdir(parents=True, exist_ok=True) with open(e1_manifest, "w", encoding="utf-8") as f: f.write(manifest) for f in files: (e1_path / f).parent.mkdir(parents=True, exist_ok=True) (e1_path / f).touch() e2 = _env_create("test2", init_file=str(e1_manifest)) for f in files: assert os.path.exists(os.path.join(e2.path, f)) # TODO: Should we be supporting relative path rewrites when creating new env from existing? # TODO: If so, then this should confirm that the absolute include paths in the new env exist. def test_init_from_yaml_relative_includes_outside_env(tmp_path: pathlib.Path): """Ensure relative includes to files outside the environment fail.""" files = ["../outside_env/repos.yaml"] manifest = f""" spack: specs: [] include: - path: {files[0]} """ # subdir to ensure parent of environment dir is not shared e1_path = tmp_path / "e1_subdir" / "e1" e1_manifest = e1_path / "spack.yaml" e1_path.mkdir(parents=True, exist_ok=True) with open(e1_manifest, "w", encoding="utf-8") as f: f.write(manifest) for f in files: file_path = e1_path / f file_path.parent.mkdir(parents=True, exist_ok=True) file_path.touch() with pytest.raises(ValueError, match="does not exist"): _ = _env_create("test2", init_file=str(e1_manifest)) def test_env_view_external_prefix(tmp_path: pathlib.Path, mutable_database, mock_packages): fake_prefix = tmp_path / "a-prefix" fake_bin = fake_prefix / "bin" fake_bin.mkdir(parents=True, exist_ok=False) manifest_dir = tmp_path / "environment" manifest_dir.mkdir(parents=True, exist_ok=False) manifest_file = manifest_dir / ev.manifest_name manifest_file.write_text( """\ spack: specs: - pkg-a view: true """ ) external_config = io.StringIO( """\ packages: pkg-a: externals: - spec: pkg-a@2.0 prefix: {a_prefix} buildable: false """.format(a_prefix=str(fake_prefix)) ) external_config_dict = spack.util.spack_yaml.load_config(external_config) test_scope = spack.config.InternalConfigScope("env-external-test", data=external_config_dict) with spack.config.override(test_scope): e = ev.create("test", manifest_file) e.concretize() # Note: normally installing specs in a test environment requires doing # a fake install, but not for external specs since no actions are # taken to install them. The installation commands also include # post-installation functions like DB-registration, so are important # to do (otherwise the package is not considered installed). e.install_all() e.write() env_mod = spack.util.environment.EnvironmentModifications() e.add_view_to_env(env_mod, "default") env_variables: Dict[str, str] = {} env_mod.apply_modifications(env_variables) assert str(fake_bin) in env_variables["PATH"] def test_init_with_file_and_remove(tmp_path: pathlib.Path, monkeypatch): """Ensure a user can remove from any position in the spack.yaml file.""" path = tmp_path / "spack.yaml" with fs.working_dir(str(tmp_path)): with open(str(path), "w", encoding="utf-8") as f: f.write( """\ spack: specs: - mpileaks """ ) env("create", "test", "spack.yaml") out = env("list") assert "test" in out with ev.read("test"): assert "mpileaks" in find() env("remove", "-y", "test") out = env("list") assert "test" not in out def test_env_with_config(environment_from_manifest): e = environment_from_manifest( """ spack: specs: - mpileaks packages: mpileaks: version: ["2.2"] """ ) with e: e.concretize() mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) mpileaks = e.specs_by_hash[mpileaks_hash] assert mpileaks.satisfies("mpileaks@2.2") def test_with_config_bad_include_create(environment_from_manifest): """Confirm missing required include raises expected exception.""" err = "does not exist" with pytest.raises(ValueError, match=err): environment_from_manifest( """ spack: include: - /no/such/directory """ ) def test_with_config_bad_include_activate(environment_from_manifest, tmp_path: pathlib.Path): env_root = tmp_path / "env-root" env_root.mkdir() include1 = env_root / "include1.yaml" include1.touch() spack_yaml = env_root / ev.manifest_name spack_yaml.write_text( """ spack: include: - ./include1.yaml """ ) with ev.Environment(env_root) as e: e.concretize() # We've created an environment with included config file (which does # exist). Now we remove it and check that we get a sensible error. os.remove(include1) with pytest.raises(ValueError, match="does not exist"): ev.activate(ev.Environment(env_root)) assert ev.active_environment() is None def test_env_with_include_config_files_same_basename( tmp_path: pathlib.Path, environment_from_manifest ): file1 = tmp_path / "path" / "to" / "included-config.yaml" file1.parent.mkdir(parents=True, exist_ok=True) with open(file1, "w", encoding="utf-8") as f: f.write( """\ packages: libelf: version: ["0.8.10"] """ ) file2 = tmp_path / "second" / "path" / "included-config.yaml" file2.parent.mkdir(parents=True, exist_ok=True) with open(file2, "w", encoding="utf-8") as f: f.write( """\ packages: mpileaks: version: ["2.2"] """ ) e = environment_from_manifest( f""" spack: include: - {file1} - {file2} specs: - libelf - mpileaks """ ) with e: e.concretize() mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) mpileaks = e.specs_by_hash[mpileaks_hash] assert mpileaks.satisfies("mpileaks@2.2") libelf_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("libelf")) libelf = e.specs_by_hash[libelf_hash] assert libelf.satisfies("libelf@0.8.10") @pytest.fixture(scope="function") def packages_file(tmp_path: pathlib.Path): """Return the path to the packages configuration file.""" raw_yaml = """ packages: mpileaks: version: ["2.2"] """ config_dir = tmp_path / "testconfig" config_dir.mkdir() filename = config_dir / "packages.yaml" filename.write_text(raw_yaml) yield filename def mpileaks_env_config(include_path): """Return the contents of an environment that includes the provided path and lists mpileaks as the sole spec.""" return """\ spack: include: - {0} specs: - mpileaks """.format(include_path) def test_env_with_included_config_file(mutable_mock_env_path, packages_file): """Test inclusion of a relative packages configuration file added to an existing environment. """ env_root = mutable_mock_env_path env_root.mkdir(parents=True, exist_ok=True) include_filename = "included-config.yaml" included_path = env_root / include_filename shutil.move(str(packages_file), included_path) spack_yaml = env_root / ev.manifest_name spack_yaml.write_text( f"""\ spack: include: - {os.path.join(".", include_filename)} specs: - mpileaks """ ) e = ev.Environment(env_root) with e: e.concretize() mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) mpileaks = e.specs_by_hash[mpileaks_hash] assert mpileaks.satisfies("mpileaks@2.2") def test_config_change_existing( mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mutable_config ): """Test ``config change`` with config in the ``spack.yaml`` as well as an included file scope. """ env_path = tmp_path / "test_config" env_path.mkdir(parents=True, exist_ok=True) included_file = "included-packages.yaml" included_path = env_path / included_file with open(included_path, "w", encoding="utf-8") as f: f.write( """\ packages: mpich: require: - spec: "@3.0.2" libelf: require: "@0.8.10" bowtie: require: - one_of: ["@1.3.0", "@1.2.0"] """ ) spack_yaml = env_path / ev.manifest_name spack_yaml.write_text( f"""\ spack: packages: mpich: require: - spec: "+debug" include: - {os.path.join(".", included_file)} specs: [] """ ) mutable_config.set("config:misc_cache", str(tmp_path / "cache")) e = ev.Environment(env_path) with e: # List of requirements, flip a variant config("change", "packages:mpich:require:~debug") test_spec = spack.concretize.concretize_one("mpich") assert test_spec.satisfies("@3.0.2~debug") # List of requirements, change the version (in a different scope) config("change", "packages:mpich:require:@3.0.3") test_spec = spack.concretize.concretize_one("mpich") assert test_spec.satisfies("@3.0.3") # "require:" as a single string, also try specifying # a spec string that requires enclosing in quotes as # part of the config path config("change", 'packages:libelf:require:"@0.8.12:"') spack.concretize.concretize_one("libelf@0.8.12") # No need for assert, if there wasn't a failure, we # changed the requirement successfully. # Use change to add a requirement for a package that # has no requirements defined config("change", "packages:fftw:require:+mpi") test_spec = spack.concretize.concretize_one("fftw") assert test_spec.satisfies("+mpi") config("change", "packages:fftw:require:~mpi") test_spec = spack.concretize.concretize_one("fftw") assert test_spec.satisfies("~mpi") config("change", "packages:fftw:require:@1.0") test_spec = spack.concretize.concretize_one("fftw") assert test_spec.satisfies("@1.0~mpi") # Use "--match-spec" to change one spec in a "one_of" # list config("change", "packages:bowtie:require:@1.2.2", "--match-spec", "@1.2.0") # confirm that we can concretize to either value spack.concretize.concretize_one("bowtie@1.3.0") spack.concretize.concretize_one("bowtie@1.2.2") # confirm that we cannot concretize to the old value with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one("bowtie@1.2.0") def test_config_change_new( mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mutable_config ): spack_yaml = tmp_path / ev.manifest_name spack_yaml.write_text( """\ spack: specs: [] """ ) with ev.Environment(tmp_path): config("change", "packages:mpich:require:~debug") with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one("mpich+debug") spack.concretize.concretize_one("mpich~debug") # Now check that we raise an error if we need to add a require: constraint # when preexisting config manually specified it as a singular spec spack_yaml.write_text( """\ spack: specs: [] packages: mpich: require: "@3.0.3" """ ) with ev.Environment(tmp_path): assert spack.concretize.concretize_one("mpich").satisfies("@3.0.3") with pytest.raises(spack.error.ConfigError, match="not a list"): config("change", "packages:mpich:require:~debug") def test_env_with_included_config_file_url( tmp_path: pathlib.Path, mutable_empty_config, packages_file ): """Test configuration inclusion of a file whose path is a URL before the environment is concretized.""" spack_yaml = tmp_path / "spack.yaml" with spack_yaml.open("w") as f: f.write("spack:\n include:\n - {0}\n".format(packages_file.as_uri())) env = ev.Environment(str(tmp_path)) ev.activate(env) cfg = spack.config.get("packages") assert cfg["mpileaks"]["version"] == ["2.2"] def test_env_with_included_config_scope(mutable_mock_env_path, packages_file): """Test inclusion of a package file from the environment's configuration stage directory. This test is intended to represent a case where a remote file has already been staged.""" env_root = mutable_mock_env_path config_scope_path = env_root / "config" # Copy the packages.yaml file to the environment configuration # directory, so it is picked up during concretization. (Using # copy instead of rename in case the fixture scope changes.) config_scope_path.mkdir(parents=True, exist_ok=True) include_filename = packages_file.name included_path = config_scope_path / include_filename fs.copy(str(packages_file), included_path) # Configure the environment to include file(s) from the environment's # remote configuration stage directory. spack_yaml = env_root / ev.manifest_name spack_yaml.write_text(mpileaks_env_config(config_scope_path)) # Ensure the concretized environment reflects contents of the # packages.yaml file. e = ev.Environment(env_root) with e: e.concretize() mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) mpileaks = e.specs_by_hash[mpileaks_hash] assert mpileaks.satisfies("mpileaks@2.2") def test_env_with_included_config_var_path(tmp_path: pathlib.Path, packages_file): """Test inclusion of a package configuration file with path variables "staged" in the environment's configuration stage directory.""" included_file = str(packages_file) env_path = tmp_path config_var_path = os.path.join("$tempdir", "included-packages.yaml") spack_yaml = env_path / ev.manifest_name spack_yaml.write_text(mpileaks_env_config(config_var_path)) config_real_path = substitute_path_variables(config_var_path) shutil.move(included_file, config_real_path) assert os.path.exists(config_real_path) e = ev.Environment(env_path) with e: e.concretize() mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) mpileaks = e.specs_by_hash[mpileaks_hash] assert mpileaks.satisfies("mpileaks@2.2") def test_env_with_included_config_precedence(tmp_path: pathlib.Path): """Test included scope and manifest precedence when including a package configuration file.""" included_file = "included-packages.yaml" included_path = tmp_path / included_file with open(included_path, "w", encoding="utf-8") as f: f.write( """\ packages: mpileaks: version: ["2.2"] libelf: version: ["0.8.10"] """ ) spack_yaml = tmp_path / ev.manifest_name spack_yaml.write_text( f"""\ spack: packages: libelf: version: ["0.8.12"] include: - {os.path.join(".", included_file)} specs: - mpileaks """ ) e = ev.Environment(tmp_path) with e: e.concretize() mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) mpileaks = e.specs_by_hash[mpileaks_hash] # ensure included scope took effect assert mpileaks.satisfies("mpileaks@2.2") # ensure env file takes precedence assert mpileaks["libelf"].satisfies("libelf@0.8.12") def test_env_with_included_configs_precedence(tmp_path: pathlib.Path): """Test precedence of multiple included configuration files.""" file1 = "high-config.yaml" file2 = "low-config.yaml" spack_yaml = tmp_path / ev.manifest_name spack_yaml.write_text( f"""\ spack: include: - {os.path.join(".", file1)} # this one should take precedence - {os.path.join(".", file2)} specs: - mpileaks """ ) with open(tmp_path / file1, "w", encoding="utf-8") as f: f.write( """\ packages: libelf: version: ["0.8.10"] # this should override libelf version below """ ) with open(tmp_path / file2, "w", encoding="utf-8") as f: f.write( """\ packages: mpileaks: version: ["2.2"] libelf: version: ["0.8.12"] """ ) e = ev.Environment(tmp_path) with e: e.concretize() mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec("mpileaks")) mpileaks = e.specs_by_hash[mpileaks_hash] # ensure the included package spec took precedence over manifest spec assert mpileaks.satisfies("mpileaks@2.2") # ensure the first included package spec took precedence over one from second assert mpileaks["libelf"].satisfies("libelf@0.8.10") @pytest.mark.regression("39248") def test_bad_env_yaml_format_remove(mutable_mock_env_path): badenv = "badenv" env("create", badenv) filename = mutable_mock_env_path / "spack.yaml" with open(filename, "w", encoding="utf-8") as f: f.write( """\ - mpileaks """ ) assert badenv in env("list") env("remove", "-y", badenv) assert badenv not in env("list") @pytest.mark.regression("39248") @pytest.mark.parametrize( "error,message,contents", [ ( spack.config.ConfigFormatError, "not of type", """\ spack: specs: mpi@2.0 """, ), ( ev.SpackEnvironmentConfigError, "duplicate key", """\ spack: packages: all: providers: mpi: [mvapich2] mpi: [mpich] """, ), ( spack.config.ConfigFormatError, "'specks' was unexpected", """\ spack: specks: - libdwarf """, ), ], ) def test_bad_env_yaml_create_fails( tmp_path: pathlib.Path, mutable_mock_env_path, error, message, contents ): """Ensure creation with invalid yaml does NOT create or leave the environment.""" filename = tmp_path / ev.manifest_name filename.write_text(contents) env_name = "bad_env" with pytest.raises(error, match=message): env("create", env_name, str(filename)) assert env_name not in env("list") manifest = mutable_mock_env_path / env_name / ev.manifest_name assert not os.path.exists(str(manifest)) @pytest.mark.regression("39248") @pytest.mark.parametrize("answer", ["-y", ""]) def test_multi_env_remove(mutable_mock_env_path, monkeypatch, answer): """Test removal (or not) of a valid and invalid environment""" remove_environment = answer == "-y" monkeypatch.setattr(tty, "get_yes_or_no", lambda prompt, default: remove_environment) environments = ["goodenv", "badenv"] for e in environments: env("create", e) # Ensure the bad environment contains invalid yaml filename = mutable_mock_env_path / environments[1] / ev.manifest_name filename.write_text( """\ - libdwarf """ ) assert all(e in env("list") for e in environments) args = [answer] if answer else [] args.extend(environments) output = env("remove", *args, fail_on_error=False) if remove_environment is True: # Successfully removed (and reported removal) of *both* environments assert not all(e in env("list") for e in environments) assert output.count("Successfully removed") == len(environments) else: # Not removing any of the environments assert all(e in env("list") for e in environments) def test_env_loads(install_mockery, mock_fetch, mock_modules_root): env("create", "test") with ev.read("test"): add("mpileaks") concretize() install("--fake") module("tcl", "refresh", "-y") with ev.read("test"): env("loads") e = ev.read("test") loads_file = os.path.join(e.path, "loads") assert os.path.exists(loads_file) with open(loads_file, encoding="utf-8") as f: contents = f.read() assert "module load mpileaks" in contents @pytest.mark.disable_clean_stage_check def test_stage(mock_stage, mock_fetch, install_mockery): env("create", "test") with ev.read("test"): add("mpileaks") add("zmpi") concretize() stage() root = str(mock_stage) def check_stage(spec): spec = spack.concretize.concretize_one(spec) for dep in spec.traverse(): stage_name = f"{stage_prefix}{dep.name}-{dep.version}-{dep.dag_hash()}" if dep.external: assert not os.path.exists(os.path.join(root, stage_name)) else: assert os.path.isdir(os.path.join(root, stage_name)) check_stage("mpileaks") check_stage("zmpi") def test_env_commands_die_with_no_env_arg(): # these fail in argparse when given no arg with pytest.raises(SpackCommandError): env("create") with pytest.raises(SpackCommandError): env("remove") # these have an optional env arg and raise errors via tty.die with pytest.raises(SpackCommandError): env("loads") # This should NOT raise an error with no environment # it just tells the user there isn't an environment env("status") def test_env_blocks_uninstall(mock_stage, mock_fetch, install_mockery): env("create", "test") with ev.read("test"): add("mpileaks") install("--fake") out = uninstall("-y", "mpileaks", fail_on_error=False) assert uninstall.returncode == 1 assert "The following environments still reference these specs" in out def test_roots_display_with_variants(): env("create", "test") with ev.read("test"): add("boost+shared") with ev.read("test"): out = find() assert "boost+shared" in out def test_uninstall_keeps_in_env(mock_stage, mock_fetch, install_mockery): # 'spack uninstall' without --remove should not change the environment # spack.yaml file, just uninstall specs env("create", "test") with ev.read("test"): add("mpileaks") add("libelf") install("--fake") test = ev.read("test") # Save this spec to check later if it is still in the env (mpileaks_hash,) = list(x for x, y in test.specs_by_hash.items() if y.name == "mpileaks") user_specs_before = test.user_specs user_spec_hashes_before = {x.hash for x in test.concretized_roots} with ev.read("test"): uninstall("-ya") test = ev.read("test") assert {x.hash for x in test.concretized_roots} == user_spec_hashes_before assert test.user_specs.specs == user_specs_before.specs assert mpileaks_hash in test.specs_by_hash assert not test.specs_by_hash[mpileaks_hash].installed def test_uninstall_removes_from_env(mock_stage, mock_fetch, install_mockery): # 'spack uninstall --remove' should update the environment env("create", "test") with ev.read("test"): add("mpileaks") add("libelf") install("--fake") with ev.read("test"): uninstall("-y", "-a", "--remove") test = ev.read("test") assert not test.specs_by_hash assert not test.concretized_roots assert not test.user_specs def test_indirect_build_dep(repo_builder: RepoBuilder): """Simple case of X->Y->Z where Y is a build/link dep and Z is a build-only dep. Make sure this concrete DAG is preserved when writing the environment out and reading it back. """ repo_builder.add_package("z") repo_builder.add_package("y", dependencies=[("z", "build", None)]) repo_builder.add_package("x", dependencies=[("y", None, None)]) with spack.repo.use_repositories(repo_builder.root): x_spec = Spec("x") x_concretized = spack.concretize.concretize_one(x_spec) _env_create("test", with_view=False) e = ev.read("test") e.add(x_spec) e.concretize() e.write() e_read = ev.read("test") assert len(e_read.concretized_roots) == 1 x_env_hash = e_read.concretized_roots[0].hash x_env_spec = e_read.specs_by_hash[x_env_hash] assert x_env_spec == x_concretized def test_store_different_build_deps(repo_builder: RepoBuilder): r"""Ensure that an environment can store two instances of a build-only dependency:: x y /| (l) | (b) (b) | y z2 \| (b) z1 """ repo_builder.add_package("z") repo_builder.add_package("y", dependencies=[("z", "build", None)]) repo_builder.add_package("x", dependencies=[("y", None, None), ("z", "build", None)]) with spack.repo.use_repositories(repo_builder.root): y_spec = Spec("y ^z@3") y_concretized = spack.concretize.concretize_one(y_spec) x_spec = Spec("x ^z@2") x_concretized = spack.concretize.concretize_one(x_spec) # Even though x chose a different 'z', the y it chooses should be identical # *aside* from the dependency on 'z'. The dag_hash() will show the difference # in build dependencies. assert x_concretized["y"].eq_node(y_concretized) assert x_concretized["y"].dag_hash() != y_concretized.dag_hash() _env_create("test", with_view=False) e = ev.read("test") e.add(y_spec) e.add(x_spec) e.concretize() e.write() e_read = ev.read("test") y_env_hash, x_env_hash = [x.hash for x in e_read.concretized_roots] y_read = e_read.specs_by_hash[y_env_hash] x_read = e_read.specs_by_hash[x_env_hash] # make sure the DAG hashes and build deps are preserved after # a round trip to/from the lockfile assert x_read["z"] != y_read["z"] assert x_read["z"].dag_hash() != y_read["z"].dag_hash() assert x_read["y"].eq_node(y_read) assert x_read["y"].dag_hash() != y_read.dag_hash() def test_env_updates_view_install(tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery): view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") with ev.read("test"): add("mpileaks") install("--fake") check_mpileaks_and_deps_in_view(view_dir) def test_env_view_fails( tmp_path: pathlib.Path, mock_packages, mock_stage, mock_fetch, install_mockery ): # We currently ignore file-file conflicts for the prefix merge, # so in principle there will be no errors in this test. But # the .spack metadata dir is handled separately and is more strict. # It also throws on file-file conflicts. That's what we're checking here # by adding the same package twice to a view. view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") with ev.read("test"): add("libelf") add("libelf cflags=-g") with pytest.raises( ev.SpackEnvironmentViewError, match="two specs project to the same prefix" ): install("--fake") def test_env_view_fails_dir_file( tmp_path: pathlib.Path, mock_packages, mock_stage, mock_fetch, install_mockery ): # This environment view fails to be created because a file # and a dir are in the same path. Test that it mentions the problematic path. view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") with ev.read("test"): add("view-file") add("view-dir") with pytest.raises( spack.llnl.util.link_tree.MergeConflictSummary, match=os.path.join("bin", "x") ): install() def test_env_view_succeeds_symlinked_dir_file( tmp_path: pathlib.Path, mock_packages, mock_stage, mock_fetch, install_mockery ): # A symlinked dir and an ordinary dir merge happily view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") with ev.read("test"): add("view-symlinked-dir") add("view-dir") install() x_dir = os.path.join(str(view_dir), "bin", "x") assert os.path.exists(os.path.join(x_dir, "file_in_dir")) assert os.path.exists(os.path.join(x_dir, "file_in_symlinked_dir")) def test_env_without_view_install(tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery): # Test enabling a view after installing specs env("create", "--without-view", "test") test_env = ev.read("test") with pytest.raises(ev.SpackEnvironmentError): test_env.default_view view_dir = tmp_path / "view" with ev.read("test"): add("mpileaks") install("--fake") env("view", "enable", str(view_dir)) # After enabling the view, the specs should be linked into the environment # view dir check_mpileaks_and_deps_in_view(view_dir) @pytest.mark.parametrize("env_name", [True, False]) def test_env_include_concrete_env_yaml(env_name): env("create", "test") test = ev.read("test") with test: add("mpileaks") test.concretize() test.write() environ = "test" if env_name else test.path env("create", "--include-concrete", environ, "combined_env") combined = ev.read("combined_env") combined_yaml = combined.manifest["spack"] assert ev.lockfile_include_key in combined_yaml assert test.path in combined_yaml[ev.lockfile_include_key] @pytest.mark.regression("45766") @pytest.mark.parametrize("format", ["v1", "v2", "v3"]) def test_env_include_concrete_old_env(format): lockfile = os.path.join(spack.paths.test_path, "data", "legacy_env", f"{format}.lock") # create an env from old .lock file -- this does not update the format env("create", "old-env", lockfile) env("create", "--include-concrete", "old-env", "test") assert ev.read("old-env").all_specs() == ev.read("test").all_specs() def test_env_bad_include_concrete_env(): with pytest.raises(ev.SpackEnvironmentError): env("create", "--include-concrete", "nonexistent_env", "combined_env") def test_env_not_concrete_include_concrete_env(): env("create", "test") test = ev.read("test") with test: add("mpileaks") with pytest.raises(ev.SpackEnvironmentError): env("create", "--include-concrete", "test", "combined_env") def test_env_multiple_include_concrete_envs(): test1, test2, combined = setup_combined_multiple_env() combined_yaml = combined.manifest["spack"] assert test1.path in combined_yaml[ev.lockfile_include_key][0] assert test2.path in combined_yaml[ev.lockfile_include_key][1] # No local specs in the combined env assert not combined_yaml["specs"] def test_env_include_concrete_envs_lockfile(): test1, test2, combined = setup_combined_multiple_env() combined_yaml = combined.manifest["spack"] assert ev.lockfile_include_key in combined_yaml assert test1.path in combined_yaml[ev.lockfile_include_key] with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert set( entry["hash"] for entry in lockfile_as_dict[ev.lockfile_include_key][test1.path]["roots"] ) == set(test1.specs_by_hash) assert set( entry["hash"] for entry in lockfile_as_dict[ev.lockfile_include_key][test2.path]["roots"] ) == set(test2.specs_by_hash) def test_env_include_concrete_add_env(): test1, test2, combined = setup_combined_multiple_env() # create new env & concretize env("create", "new") new_env = ev.read("new") with new_env: add("mpileaks") new_env.concretize() new_env.write() # add new env to combined combined.included_concrete_env_root_dirs.append(new_env.path) # assert thing haven't changed yet with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert new_env.path not in lockfile_as_dict[ev.lockfile_include_key].keys() # concretize combined env with new env combined.concretize() combined.write() # assert changes with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert new_env.path in lockfile_as_dict[ev.lockfile_include_key].keys() def test_env_include_concrete_remove_env(): test1, test2, combined = setup_combined_multiple_env() # remove test2 from combined combined.included_concrete_env_root_dirs = [test1.path] # assert test2 is still in combined's lockfile with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert test2.path in lockfile_as_dict[ev.lockfile_include_key].keys() # reconcretize combined combined.concretize() combined.write() # assert test2 is not in combined's lockfile with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert test2.path not in lockfile_as_dict[ev.lockfile_include_key].keys() def configure_reuse(reuse_mode, combined_env) -> Optional[ev.Environment]: override_env = None _config: Dict[Any, Any] = {} if reuse_mode == "true": _config = {"concretizer": {"reuse": True}} elif reuse_mode == "from_environment": _config = {"concretizer": {"reuse": {"from": [{"type": "environment"}]}}} elif reuse_mode == "from_environment_test1": _config = {"concretizer": {"reuse": {"from": [{"type": "environment", "path": "test1"}]}}} elif reuse_mode == "from_environment_external_test": # Create a new environment called external_test that enables the "debug" # The default is "~debug" env("create", "external_test") override_env = ev.read("external_test") with override_env: add("mpich@1.0 +debug") override_env.concretize() override_env.write() # Reuse from the environment that is not included. # Specify the requirement for the debug variant. By default this would concretize to use # mpich@3.0 but with include concrete the mpich@1.0 +debug version from the # "external_test" environment will be used. _config = { "concretizer": {"reuse": {"from": [{"type": "environment", "path": "external_test"}]}}, "packages": {"mpich": {"require": ["+debug"]}}, } elif reuse_mode == "from_environment_raise": _config = { "concretizer": {"reuse": {"from": [{"type": "environment", "path": "not-a-real-env"}]}} } # Disable unification in these tests to avoid confusing reuse due to unification using an # include concrete spec vs reuse due to the reuse configuration _config["concretizer"].update({"unify": False}) combined_env.manifest.configuration.update(_config) combined_env.manifest.changed = True combined_env.write() return override_env @pytest.mark.parametrize( "reuse_mode", [ "true", "from_environment", "from_environment_test1", "from_environment_external_test", "from_environment_raise", ], ) def test_env_include_concrete_reuse(reuse_mode): # The default mpi version is 3.x provided by mpich in the mock repo. # This test verifies that concretizing with an included concrete # environment with "concretizer:reuse:true" the included # concrete spec overrides the default with mpi@1.0. test1, _, combined = setup_combined_multiple_env() # Set the reuse mode for the environment override_env = configure_reuse(reuse_mode, combined) if override_env: # If there is an override environment (ie. testing reuse with # an external environment) update it here. test1 = override_env # Capture the test1 specs included by combined test1_specs_by_hash = test1.specs_by_hash try: # Add mpileaks to the combined environment with combined: add("mpileaks") combined.concretize() comb_specs_by_hash = combined.specs_by_hash # create reference env with mpileaks that does not use reuse # This should concretize to the default version of mpich (3.0) env("create", "new") ref_env = ev.read("new") with ref_env: add("mpileaks") ref_env.concretize() ref_specs_by_hash = ref_env.specs_by_hash # Ensure that the mpich used by the mpileaks is the mpich from the reused test environment comb_mpileaks_spec = [s for s in comb_specs_by_hash.values() if s.name == "mpileaks"] test1_mpich_spec = [s for s in test1_specs_by_hash.values() if s.name == "mpich"] assert len(comb_mpileaks_spec) == 1 assert len(test1_mpich_spec) == 1 assert comb_mpileaks_spec[0]["mpich"].dag_hash() == test1_mpich_spec[0].dag_hash() # None of the references specs (using mpich@3) reuse specs from test1. # This tests that the reuse is not happening coincidently assert not any([s in test1_specs_by_hash for s in ref_specs_by_hash]) # Make sure the raise tests raises assert "raise" not in reuse_mode except ev.SpackEnvironmentError: assert "raise" in reuse_mode @pytest.mark.parametrize("unify", [True, False, "when_possible"]) def test_env_include_concrete_env_reconcretized(mutable_config, unify): """Double check to make sure that concrete_specs for the local specs is empty after reconcretizing. """ _, _, combined = setup_combined_multiple_env() with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert not lockfile_as_dict["roots"] assert not lockfile_as_dict["concrete_specs"] with combined: mutable_config.set("concretizer:unify", unify) combined.concretize() combined.write() with open(combined.lock_path, encoding="utf-8") as f: lockfile_as_dict = combined._read_lockfile(f) assert not lockfile_as_dict["roots"] assert not lockfile_as_dict["concrete_specs"] def test_concretize_include_concrete_env(): """Tests that if we update an included environment, and later we re-concretize the environment that includes it, we use the latest version of the concrete specs. """ test1, _, combined = setup_combined_multiple_env() # Update test1 environment with test1: add("mpileaks") test1.concretize() test1.write() # Check the test1 environment includes mpileaks, while the combined environment does not assert Spec("mpileaks") in {x.root for x in test1.concretized_roots} assert Spec("mpileaks") not in { x.root for x in combined.included_concretized_roots[test1.path] } # If we update the combined environment, it will include mpileaks too combined.concretize() combined.write() assert Spec("mpileaks") in {x.root for x in combined.included_concretized_roots[test1.path]} def test_concretize_nested_include_concrete_envs(): env("create", "test1") test1 = ev.read("test1") with test1: add("zlib") test1.concretize() test1.write() env("create", "--include-concrete", "test1", "test2") test2 = ev.read("test2") with test2: add("libelf") test2.concretize() test2.write() env("create", "--include-concrete", "test2", "test3") test3 = ev.read("test3") with open(test3.lock_path, encoding="utf-8") as f: lockfile_as_dict = test3._read_lockfile(f) assert test2.path in lockfile_as_dict[ev.lockfile_include_key] assert ( test1.path in lockfile_as_dict[ev.lockfile_include_key][test2.path][ev.lockfile_include_key] ) assert Spec("zlib") in {x.root for x in test3.included_concretized_roots[test1.path]} def test_concretize_nested_included_concrete(): """Confirm that nested included environments use specs concretized at environment creation time and change with reconcretization.""" env("create", "test1") test1 = ev.read("test1") with test1: add("zlib") test1.concretize() test1.write() # test2 should include test1 with zlib env("create", "--include-concrete", "test1", "test2") test2 = ev.read("test2") with test2: add("libelf") test2.concretize() test2.write() assert Spec("zlib") in {x.root for x in test2.included_concretized_roots[test1.path]} # Modify/re-concretize test1 to replace zlib with mpileaks with test1: remove("zlib") add("mpileaks") test1.concretize() test1.write() # test3 should include the latest concretization of test1 env("create", "--include-concrete", "test1", "test3") test3 = ev.read("test3") with test3: add("callpath") test3.concretize() test3.write() included_roots = test3.included_concretized_roots[test1.path] assert len(included_roots) == 1 assert Spec("mpileaks") in {x.root for x in included_roots} # The last concretization of test4's included environments should have test2 # with the original concretized test1 spec and test3 with the re-concretized # test1 spec. env("create", "--include-concrete", "test2", "--include-concrete", "test3", "test4") test4 = ev.read("test4") def included_included_spec(path1, path2): included_path1 = test4.included_concrete_spec_data[path1] included_path2 = included_path1[ev.lockfile_include_key][path2] return included_path2["roots"][0]["spec"] included_test2_test1 = included_included_spec(test2.path, test1.path) assert "zlib" in included_test2_test1 included_test3_test1 = included_included_spec(test3.path, test1.path) assert "mpileaks" in included_test3_test1 # test4's concretized specs should reflect the original concretization. concrete_specs = [s for s, _ in test4.concretized_specs()] expected = [Spec(s) for s in ["libelf", "zlib", "mpileaks", "callpath"]] assert all(s in concrete_specs for s in expected) # Re-concretize test2 to reflect the new concretization of included test1 # to remove zlib and write it out so it can be picked up by test4. # Re-concretize test4 to reflect the re-concretization of included test2 # and ensure that its included specs are up-to-date test2.concretize() test2.write() test4.concretize() concrete_specs = [s for s, _ in test4.concretized_specs()] assert Spec("zlib") not in concrete_specs # Expecting mpileaks to appear only once expected = [Spec(s) for s in ["libelf", "mpileaks", "callpath"]] assert len(concrete_specs) == 3 and all(s in concrete_specs for s in expected) def test_env_config_view_default( environment_from_manifest, mock_stage, mock_fetch, install_mockery ): # This config doesn't mention whether a view is enabled environment_from_manifest( """ spack: specs: - mpileaks """ ) with ev.read("test"): install("--fake") e = ev.read("test") # Check that metadata folder for this spec exists assert os.path.isdir(os.path.join(e.default_view.view()._root, ".spack", "mpileaks")) def test_env_updates_view_install_package( tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery ): view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") with ev.read("test"): install("--fake", "--add", "mpileaks") assert os.path.exists(str(view_dir / ".spack/mpileaks")) def test_env_updates_view_add_concretize( tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery ): view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") install("--fake", "mpileaks") with ev.read("test"): add("mpileaks") concretize() check_mpileaks_and_deps_in_view(view_dir) def test_env_updates_view_uninstall( tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery ): view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") with ev.read("test"): install("--fake", "--add", "mpileaks") check_mpileaks_and_deps_in_view(view_dir) with ev.read("test"): uninstall("-ay") check_viewdir_removal(view_dir) def test_env_updates_view_uninstall_referenced_elsewhere( tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery ): view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") install("--fake", "mpileaks") with ev.read("test"): add("mpileaks") concretize() check_mpileaks_and_deps_in_view(view_dir) with ev.read("test"): uninstall("-ay") check_viewdir_removal(view_dir) def test_env_updates_view_remove_concretize( tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery ): view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") install("--fake", "mpileaks") with ev.read("test"): add("mpileaks") concretize() check_mpileaks_and_deps_in_view(view_dir) with ev.read("test"): remove("mpileaks") concretize() check_viewdir_removal(view_dir) def test_env_updates_view_force_remove( tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery ): view_dir = tmp_path / "view" env("create", "--with-view=%s" % view_dir, "test") with ev.read("test"): install("--add", "--fake", "mpileaks") check_mpileaks_and_deps_in_view(view_dir) with ev.read("test"): remove("-f", "mpileaks") check_viewdir_removal(view_dir) def test_env_activate_view_fails(mock_stage, mock_fetch, install_mockery): """Sanity check on env activate to make sure it requires shell support""" out = env("activate", "test") assert "To set up shell support" in out def test_stack_yaml_definitions(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [mpileaks, callpath] specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("mpileaks") in test.user_specs assert Spec("callpath") in test.user_specs def test_stack_yaml_definitions_as_constraints(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [mpileaks, callpath] - mpis: [mpich, openmpi] specs: - matrix: - [$packages] - [$^mpis] """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("mpileaks^mpich") in test.user_specs assert Spec("callpath^mpich") in test.user_specs assert Spec("mpileaks^openmpi") in test.user_specs assert Spec("callpath^openmpi") in test.user_specs def test_stack_yaml_definitions_as_constraints_on_matrix(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [mpileaks, callpath] - mpis: - matrix: - [mpich] - ['@3.0.4', '@3.0.3'] specs: - matrix: - [$packages] - [$^mpis] """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("mpileaks^mpich@3.0.4") in test.user_specs assert Spec("callpath^mpich@3.0.4") in test.user_specs assert Spec("mpileaks^mpich@3.0.3") in test.user_specs assert Spec("callpath^mpich@3.0.3") in test.user_specs @pytest.mark.regression("12095") def test_stack_yaml_definitions_write_reference(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [mpileaks, callpath] - indirect: [$packages] specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") with ev.read("test"): concretize() test = ev.read("test") assert Spec("mpileaks") in test.user_specs assert Spec("callpath") in test.user_specs def test_stack_yaml_add_to_list(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [mpileaks, callpath] specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") with ev.read("test"): add("-l", "packages", "libelf") test = ev.read("test") assert Spec("libelf") in test.user_specs assert Spec("mpileaks") in test.user_specs assert Spec("callpath") in test.user_specs def test_stack_yaml_remove_from_list(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [mpileaks, callpath] specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") with ev.read("test"): remove("-l", "packages", "mpileaks") test = ev.read("test") assert Spec("mpileaks") not in test.user_specs assert Spec("callpath") in test.user_specs def test_stack_yaml_remove_from_list_force(tmp_path: pathlib.Path): spack_yaml = tmp_path / ev.manifest_name spack_yaml.write_text( """\ spack: definitions: - packages: [mpileaks, callpath] specs: - matrix: - [$packages] - [^mpich, ^zmpi] """ ) env("create", "test", str(spack_yaml)) with ev.read("test"): concretize() remove("-f", "-l", "packages", "mpileaks") find_output = find("-c") assert "mpileaks" not in find_output test = ev.read("test") assert len(test.user_specs) == 2 assert Spec("callpath ^zmpi") in test.user_specs assert Spec("callpath ^mpich") in test.user_specs def test_stack_yaml_remove_from_matrix_no_effect(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: - matrix: - [mpileaks, callpath] - [target=default_target] specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") with ev.read("test") as e: before = e.user_specs.specs remove("-l", "packages", "mpileaks") after = e.user_specs.specs assert before == after def test_stack_yaml_force_remove_from_matrix(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: - matrix: - [mpileaks, callpath] - [target=default_target] specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") with ev.read("test") as e: e.concretize() before_user = e.user_specs.specs concretized_roots_before = e.concretized_roots remove("-f", "-l", "packages", "mpileaks") after_user = e.user_specs.specs concretized_roots_after = e.concretized_roots assert before_user == after_user mpileaks_spec = Spec("mpileaks target=default_target") assert mpileaks_spec in {x.root for x in concretized_roots_before} assert mpileaks_spec not in {x.root for x in concretized_roots_after} def test_stack_definition_extension(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [libelf, mpileaks] - packages: [callpath] specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("libelf") in test.user_specs assert Spec("mpileaks") in test.user_specs assert Spec("callpath") in test.user_specs def test_stack_definition_conditional_false(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [libelf, mpileaks] - packages: [callpath] when: 'False' specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("libelf") in test.user_specs assert Spec("mpileaks") in test.user_specs assert Spec("callpath") not in test.user_specs def test_stack_definition_conditional_true(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [libelf, mpileaks] - packages: [callpath] when: 'True' specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("libelf") in test.user_specs assert Spec("mpileaks") in test.user_specs assert Spec("callpath") in test.user_specs def test_stack_definition_conditional_with_variable(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [libelf, mpileaks] - packages: [callpath] when: platform == 'test' specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("libelf") in test.user_specs assert Spec("mpileaks") in test.user_specs assert Spec("callpath") in test.user_specs def test_stack_definition_conditional_with_satisfaction(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [libelf, mpileaks] when: arch.satisfies('platform=foo') # will be "test" when testing - packages: [callpath] when: arch.satisfies('platform=test') specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("libelf") not in test.user_specs assert Spec("mpileaks") not in test.user_specs assert Spec("callpath") in test.user_specs def test_stack_definition_complex_conditional(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [libelf, mpileaks] - packages: [callpath] when: re.search(r'foo', hostname) and env['test'] == 'THISSHOULDBEFALSE' specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") test = ev.read("test") assert Spec("libelf") in test.user_specs assert Spec("mpileaks") in test.user_specs assert Spec("callpath") not in test.user_specs def test_stack_definition_conditional_invalid_variable(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [libelf, mpileaks] - packages: [callpath] when: bad_variable == 'test' specs: - $packages """ ) with fs.working_dir(str(tmp_path)): with pytest.raises(NameError): env("create", "test", "./spack.yaml") def test_stack_definition_conditional_add_write(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: definitions: - packages: [libelf, mpileaks] - packages: [callpath] when: platform == 'test' specs: - $packages """ ) with fs.working_dir(str(tmp_path)): env("create", "test", "./spack.yaml") with ev.read("test"): add("-l", "packages", "zmpi") test = ev.read("test") packages_lists = list( filter(lambda x: "packages" in x, test.manifest["spack"]["definitions"]) ) assert len(packages_lists) == 2 assert "callpath" not in packages_lists[0]["packages"] assert "callpath" in packages_lists[1]["packages"] assert "zmpi" in packages_lists[0]["packages"] assert "zmpi" not in packages_lists[1]["packages"] def test_stack_combinatorial_view( installed_environment, template_combinatorial_env, tmp_path: pathlib.Path ): """Tests creating a default view for a combinatorial stack.""" view_dir = tmp_path / "view" with installed_environment(template_combinatorial_env.format(view_config="")) as test: for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" assert current_dir.exists() and current_dir.is_dir() def test_stack_view_select( installed_environment, template_combinatorial_env, tmp_path: pathlib.Path ): view_dir = tmp_path / "view" content = template_combinatorial_env.format(view_config="select: ['target=x86_64']\n") with installed_environment(content) as test: for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" assert current_dir.exists() is spec.satisfies("target=x86_64") def test_stack_view_exclude( installed_environment, template_combinatorial_env, tmp_path: pathlib.Path ): view_dir = tmp_path / "view" content = template_combinatorial_env.format(view_config="exclude: [callpath]\n") with installed_environment(content) as test: for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" assert current_dir.exists() is not spec.satisfies("callpath") def test_stack_view_select_and_exclude( installed_environment, template_combinatorial_env, tmp_path: pathlib.Path ): view_dir = tmp_path / "view" content = template_combinatorial_env.format( view_config="""select: ['target=x86_64'] exclude: [callpath] """ ) with installed_environment(content) as test: for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" assert current_dir.exists() is ( spec.satisfies("target=x86_64") and not spec.satisfies("callpath") ) def test_view_link_roots( installed_environment, template_combinatorial_env, tmp_path: pathlib.Path ): view_dir = tmp_path / "view" content = template_combinatorial_env.format( view_config="""select: ['target=x86_64'] exclude: [callpath] link: 'roots' """ ) with installed_environment(content) as test: for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" expected_exists = spec in test.roots() and ( spec.satisfies("target=x86_64") and not spec.satisfies("callpath") ) assert current_dir.exists() == expected_exists def test_view_link_run( tmp_path: pathlib.Path, mock_fetch, mock_packages, mock_archive, install_mockery ): yaml = str(tmp_path / "spack.yaml") viewdir = str(tmp_path / "view") envdir = str(tmp_path) with open(yaml, "w", encoding="utf-8") as f: f.write( """ spack: specs: - dttop view: combinatorial: root: %s link: run projections: all: '{name}'""" % viewdir ) with ev.Environment(envdir): install("--fake") # make sure transitive run type deps are in the view for pkg in ("dtrun1", "dtrun3"): assert os.path.exists(os.path.join(viewdir, pkg)) # and non-run-type deps are not. for pkg in ( "dtlink1", "dtlink2", "dtlink3", "dtlink4", "dtlink5dtbuild1", "dtbuild2", "dtbuild3", ): assert not os.path.exists(os.path.join(viewdir, pkg)) @pytest.mark.parametrize("link_type", ["hardlink", "copy", "symlink"]) def test_view_link_type(link_type, installed_environment, tmp_path: pathlib.Path): view_dir = tmp_path / "view" with installed_environment( f"""\ spack: specs: - mpileaks view: default: root: {view_dir} link_type: {link_type}""" ) as test: for spec in test.roots(): # Assertions are based on the behavior of the "--fake" install bin_file = pathlib.Path(test.default_view.view()._root) / "bin" / spec.name assert bin_file.exists() assert bin_file.is_symlink() == (link_type == "symlink") def test_view_link_all(installed_environment, template_combinatorial_env, tmp_path: pathlib.Path): view_dir = tmp_path / "view" content = template_combinatorial_env.format( view_config="""select: ['target=x86_64'] exclude: [callpath] link: 'all' """ ) with installed_environment(content) as test: for spec in traverse_nodes(test.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" assert current_dir.exists() == ( spec.satisfies("target=x86_64") and not spec.satisfies("callpath") ) def test_stack_view_activate_from_default( installed_environment, template_combinatorial_env, tmp_path: pathlib.Path ): view_dir = tmp_path / "view" content = template_combinatorial_env.format(view_config="select: ['target=x86_64']") # Replace the name of the view content = content.replace("combinatorial:", "default:") with installed_environment(content): shell = env("activate", "--sh", "test") assert "PATH" in shell, shell assert str(view_dir / "bin") in shell assert "FOOBAR=mpileaks" in shell def test_envvar_set_in_activate(tmp_path: pathlib.Path, mock_packages, install_mockery): spack_yaml = tmp_path / "spack.yaml" env_vars_yaml = tmp_path / "env_vars.yaml" env_vars_yaml.write_text( """ env_vars: set: CONFIG_ENVAR_SET_IN_ENV_LOAD: "True" """ ) spack_yaml.write_text( """ spack: include: - env_vars.yaml specs: - cmake%gcc env_vars: set: SPACK_ENVAR_SET_IN_ENV_LOAD: "True" """ ) env("create", "test", str(spack_yaml)) with ev.read("test"): install("--fake") test_env = ev.read("test") output = env("activate", "--sh", "test") assert "SPACK_ENVAR_SET_IN_ENV_LOAD=True" in output assert "CONFIG_ENVAR_SET_IN_ENV_LOAD=True" in output with test_env: with spack.util.environment.set_env( SPACK_ENVAR_SET_IN_ENV_LOAD="True", CONFIG_ENVAR_SET_IN_ENV_LOAD="True" ): output = env("deactivate", "--sh") assert "unset SPACK_ENVAR_SET_IN_ENV_LOAD" in output assert "unset CONFIG_ENVAR_SET_IN_ENV_LOAD" in output def test_stack_view_no_activate_without_default( installed_environment, template_combinatorial_env, tmp_path: pathlib.Path ): view_dir = tmp_path / "view" content = template_combinatorial_env.format(view_config="select: ['target=x86_64']") with installed_environment(content): shell = env("activate", "--sh", "test") assert "PATH" not in shell assert str(view_dir) not in shell @pytest.mark.parametrize("include_views", [True, False, "split"]) def test_stack_view_multiple_views(installed_environment, tmp_path: pathlib.Path, include_views): """Test multiple views as both included views (True), as both environment views (False), or as one included and the other in the environment. """ # Write the view configuration and or manifest file view_filename = tmp_path / "view.yaml" base_content = """\ definitions: - packages: [mpileaks, cmake] - targets: ['target=x86_64', 'target=core2'] specs: - matrix: - [$packages] - [$targets] """ include_content = f" include:\n - {view_filename}\n" view_line = " view:\n" comb_dir = tmp_path / "combinatorial-view" comb_view = """\ {0}combinatorial: {0} root: {1} {0} exclude: [target=core2] {0} projections: """ projection = " 'all': '{architecture.target}/{name}-{version}'" default_dir = tmp_path / "default-view" default_view = """\ {0}default: {0} root: {1} {0} select: ['target=x86_64'] """ content = "spack:\n" indent = " " if include_views is True: # Include both the gcc and combinatorial views view = "view:\n" + default_view.format(indent, str(default_dir)) view += comb_view.format(indent, str(comb_dir)) + indent + projection view_filename.write_text(view) content += include_content + base_content elif include_views == "split": # Include the gcc view and inline the combinatorial view view = "view:\n" + default_view.format(indent, str(default_dir)) view_filename.write_text(view) content += include_content + base_content + view_line indent += " " content += comb_view.format(indent, str(comb_dir)) + indent + projection else: # Inline both the gcc and combinatorial views in the environment. indent += " " content += base_content + view_line content += default_view.format(indent, str(default_dir)) content += comb_view.format(indent, str(comb_dir)) + indent + projection with installed_environment(content) as e: assert os.path.exists(str(default_dir / "bin")) for spec in traverse_nodes(e.concrete_roots(), deptype=("link", "run")): if spec.name == "gcc-runtime": continue current_dir = comb_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" assert current_dir.exists() is not spec.satisfies("target=core2") def test_env_activate_sh_prints_shell_output(mock_stage, mock_fetch, install_mockery): """Check the shell commands output by ``spack env activate --sh``. This is a cursory check; ``share/spack/qa/setup-env-test.sh`` checks for correctness. """ env("create", "test") out = env("activate", "--sh", "test") assert "export SPACK_ENV=" in out assert "export PS1=" not in out assert "alias despacktivate=" in out out = env("activate", "--sh", "--prompt", "test") assert "export SPACK_ENV=" in out assert "export PS1=" in out assert "alias despacktivate=" in out def test_env_activate_csh_prints_shell_output(mock_stage, mock_fetch, install_mockery): """Check the shell commands output by ``spack env activate --csh``.""" env("create", "test") out = env("activate", "--csh", "test") assert "setenv SPACK_ENV" in out assert "setenv set prompt" not in out assert "alias despacktivate" in out out = env("activate", "--csh", "--prompt", "test") assert "setenv SPACK_ENV" in out assert "set prompt=" in out assert "alias despacktivate" in out @pytest.mark.regression("12719") def test_env_activate_default_view_root_unconditional(mutable_mock_env_path): """Check that the root of the default view in the environment is added to the shell unconditionally.""" env("create", "test") with ev.read("test") as e: viewdir = e.default_view.root out = env("activate", "--sh", "test") viewdir_bin = os.path.join(viewdir, "bin") assert ( "export PATH={0}".format(viewdir_bin) in out or "export PATH='{0}".format(viewdir_bin) in out or 'export PATH="{0}'.format(viewdir_bin) in out ) def test_env_activate_custom_view(tmp_path: pathlib.Path, mock_packages): """Check that an environment can be activated with a non-default view.""" env_template = tmp_path / "spack.yaml" default_dir = tmp_path / "defaultdir" nondefaultdir = tmp_path / "nondefaultdir" with open(env_template, "w", encoding="utf-8") as f: f.write( f"""\ spack: specs: [a] view: default: root: {default_dir} nondefault: root: {nondefaultdir}""" ) env("create", "test", str(env_template)) shell = env("activate", "--sh", "--with-view", "nondefault", "test") assert os.path.join(nondefaultdir, "bin") in shell def test_concretize_user_specs_together(mutable_config): with ev.create("coconcretization") as e: mutable_config.set("concretizer:unify", True) # Concretize a first time using 'mpich' as the MPI provider e.add("mpileaks") e.add("mpich") e.concretize() assert all("mpich" in spec for _, spec in e.concretized_specs()) assert all("mpich2" not in spec for _, spec in e.concretized_specs()) # Concretize a second time using 'mpich2' as the MPI provider e.remove("mpich") e.add("mpich2") exc_cls = spack.error.UnsatisfiableSpecError # Concretizing without invalidating the concrete spec for mpileaks fails with pytest.raises(exc_cls): e.concretize() e.concretize(force=True) assert all("mpich2" in spec for _, spec in e.concretized_specs()) assert all("mpich" not in spec for _, spec in e.concretized_specs()) # Concretize again without changing anything, check everything # stays the same e.concretize() assert all("mpich2" in spec for _, spec in e.concretized_specs()) assert all("mpich" not in spec for _, spec in e.concretized_specs()) def test_duplicate_packages_raise_when_concretizing_together(mutable_config): with ev.create("coconcretization") as e: mutable_config.set("concretizer:unify", True) e.add("mpileaks+opt") e.add("mpileaks~opt") e.add("mpich") exc_cls = spack.error.UnsatisfiableSpecError match = r"You could consider setting `concretizer:unify`" with pytest.raises(exc_cls, match=match): e.concretize() def test_env_write_only_non_default(): env("create", "test") e = ev.read("test") with open(e.manifest_path, "r", encoding="utf-8") as f: yaml = f.read() assert yaml == ev.default_manifest_yaml() @pytest.mark.regression("20526") def test_env_write_only_non_default_nested(tmp_path: pathlib.Path): # setup an environment file # the environment includes configuration because nested configs proved the # most difficult to avoid writing. filename = "spack.yaml" filepath = str(tmp_path / filename) contents = """\ spack: specs: - matrix: - [mpileaks] packages: all: compiler: [gcc] view: true """ # create environment with some structure with open(filepath, "w", encoding="utf-8") as f: f.write(contents) env("create", "test", filepath) # concretize with ev.read("test") as e: concretize() e.write() with open(e.manifest_path, "r", encoding="utf-8") as f: manifest = f.read() assert manifest == contents @pytest.mark.regression("18147") def test_can_update_attributes_with_override(tmp_path: pathlib.Path): spack_yaml = """ spack: mirrors:: test: /foo/bar packages: cmake: paths: cmake@3.18.1: /usr specs: - hdf5 """ abspath = tmp_path / "spack.yaml" abspath.write_text(spack_yaml) # Check that an update does not raise env("update", "-y", str(tmp_path)) @pytest.mark.regression("18338") def test_newline_in_commented_sequence_is_not_an_issue(tmp_path: pathlib.Path): spack_yaml = """ spack: specs: - dyninst packages: libelf: externals: - spec: libelf@0.8.13 modules: - libelf/3.18.1 concretizer: unify: false """ abspath = tmp_path / "spack.yaml" abspath.write_text(spack_yaml) def extract_dag_hash(environment): _, dyninst = next(iter(environment.specs_by_hash.items())) return dyninst["libelf"].dag_hash() # Concretize a first time and create a lockfile with ev.Environment(str(tmp_path)) as e: concretize() libelf_first_hash = extract_dag_hash(e) # Check that a second run won't error with ev.Environment(str(tmp_path)) as e: concretize() libelf_second_hash = extract_dag_hash(e) assert libelf_first_hash == libelf_second_hash @pytest.mark.regression("18441") def test_lockfile_not_deleted_on_write_error(tmp_path: pathlib.Path, monkeypatch): raw_yaml = """ spack: specs: - dyninst packages: libelf: externals: - spec: libelf@0.8.13 prefix: /usr """ spack_yaml = tmp_path / "spack.yaml" spack_yaml.write_text(raw_yaml) spack_lock = tmp_path / "spack.lock" # Concretize a first time and create a lockfile with ev.Environment(str(tmp_path)): concretize() assert os.path.exists(str(spack_lock)) # If I run concretize again and there's an error during write, # the spack.lock file shouldn't disappear from disk def _write_helper_raise(self): raise RuntimeError("some error") monkeypatch.setattr(ev.environment.EnvironmentManifestFile, "flush", _write_helper_raise) with ev.Environment(str(tmp_path)) as e: e.concretize(force=True) with pytest.raises(RuntimeError): e.clear() e.write() assert os.path.exists(str(spack_lock)) def _setup_develop_packages(tmp_path: pathlib.Path): """Sets up a structure ./init_env/spack.yaml, ./build_folder, ./dest_env where spack.yaml has a relative develop path to build_folder""" init_env = tmp_path / "init_env" build_folder = tmp_path / "build_folder" dest_env = tmp_path / "dest_env" init_env.mkdir(parents=True, exist_ok=True) build_folder.mkdir(parents=True, exist_ok=True) dest_env.mkdir(parents=True, exist_ok=True) raw_yaml = """ spack: specs: ['mypkg1', 'mypkg2'] develop: mypkg1: path: ../build_folder spec: mypkg@main mypkg2: path: /some/other/path spec: mypkg@main """ spack_yaml = init_env / "spack.yaml" spack_yaml.write_text(raw_yaml) return init_env, build_folder, dest_env, spack_yaml def test_rewrite_rel_dev_path_new_dir(tmp_path: pathlib.Path): """Relative develop paths should be rewritten for new environments in a different directory from the original manifest file""" _, build_folder, dest_env, spack_yaml = _setup_develop_packages(tmp_path) env("create", "-d", str(dest_env), str(spack_yaml)) with ev.Environment(str(dest_env)) as e: assert e.dev_specs["mypkg1"]["path"] == str(build_folder) assert e.dev_specs["mypkg2"]["path"] == sep + os.path.join("some", "other", "path") def test_rewrite_rel_dev_path_named_env(tmp_path: pathlib.Path): """Relative develop paths should by default be rewritten for new named environment""" _, build_folder, _, spack_yaml = _setup_develop_packages(tmp_path) env("create", "named_env", str(spack_yaml)) with ev.read("named_env") as e: assert e.dev_specs["mypkg1"]["path"] == str(build_folder) assert e.dev_specs["mypkg2"]["path"] == sep + os.path.join("some", "other", "path") def test_does_not_rewrite_rel_dev_path_when_keep_relative_is_set(tmp_path: pathlib.Path): """Relative develop paths should not be rewritten when --keep-relative is passed to create""" _, _, _, spack_yaml = _setup_develop_packages(tmp_path) env("create", "--keep-relative", "named_env", str(spack_yaml)) with ev.read("named_env") as e: assert e.dev_specs["mypkg1"]["path"] == "../build_folder" assert e.dev_specs["mypkg2"]["path"] == "/some/other/path" @pytest.mark.regression("23440") def test_custom_version_concretize_together(mutable_config): # Custom versions should be permitted in specs when # concretizing together with ev.create("custom_version") as e: mutable_config.set("concretizer:unify", True) # Concretize a first time using 'mpich' as the MPI provider e.add("hdf5@=myversion") e.add("mpich") e.concretize() assert any(spec.satisfies("hdf5@myversion") for _, spec in e.concretized_specs()) def test_modules_relative_to_views(environment_from_manifest, install_mockery, mock_fetch): environment_from_manifest( """ spack: specs: - trivial-install-test-package modules: default: enable:: [tcl] use_view: true roots: tcl: modules """ ) with ev.read("test") as e: install("--fake") user_spec_hash = e.concretized_roots[0].hash spec = e.specs_by_hash[user_spec_hash] view_prefix = e.default_view.get_projection_for_spec(spec) modules_glob = "%s/modules/**/*/*" % e.path modules = glob.glob(modules_glob) assert len(modules) == 1 module = modules[0] with open(module, "r", encoding="utf-8") as f: contents = f.read() assert view_prefix in contents assert spec.prefix not in contents def test_modules_exist_after_env_install(installed_environment, monkeypatch): # Some caching issue monkeypatch.setattr(spack.modules.tcl, "configuration_registry", {}) with installed_environment( """ spack: specs: - mpileaks modules: default: enable:: [tcl] use_view: true roots: tcl: uses_view full: enable:: [tcl] roots: tcl: without_view """ ) as e: specs = e.all_specs() for module_set in ("uses_view", "without_view"): modules = glob.glob(f"{e.path}/{module_set}/**/*/*") assert len(modules) == len(specs), "Not all modules were generated" for spec in specs: if spec.external: continue module = next((m for m in modules if os.path.dirname(m).endswith(spec.name)), None) assert module, f"Module for {spec} not found" # Now verify that modules have paths pointing into the view instead of the package # prefix if and only if they set use_view to true. with open(module, "r", encoding="utf-8") as f: contents = f.read() if module_set == "uses_view": assert e.default_view.get_projection_for_spec(spec) in contents assert spec.prefix not in contents else: assert e.default_view.get_projection_for_spec(spec) not in contents assert spec.prefix in contents @pytest.mark.disable_clean_stage_check def test_install_develop_keep_stage( environment_from_manifest, install_mockery, mock_fetch, monkeypatch, tmp_path: pathlib.Path ): """Develop a dependency of a package and make sure that the associated stage for the package is retained after a successful install. """ environment_from_manifest( """ spack: specs: - mpileaks """ ) monkeypatch.setattr(spack.stage.DevelopStage, "destroy", _always_fail) with ev.read("test") as e: libelf_dev_path = tmp_path / "libelf-test-dev-path" libelf_dev_path.mkdir(parents=True) develop(f"--path={libelf_dev_path}", "libelf@0.8.13") concretize() (libelf_spec,) = e.all_matching_specs("libelf") (mpileaks_spec,) = e.all_matching_specs("mpileaks") assert not os.path.exists(libelf_spec.package.stage.path) assert not os.path.exists(mpileaks_spec.package.stage.path) install("--fake") assert os.path.exists(libelf_spec.package.stage.path) assert not os.path.exists(mpileaks_spec.package.stage.path) # Helper method for test_install_develop_keep_stage def _always_fail(cls, *args, **kwargs): raise Exception("Restage or destruction of dev stage detected during install") @pytest.mark.regression("24148") def test_virtual_spec_concretize_together(mutable_config): # An environment should permit to concretize "mpi" with ev.create("virtual_spec") as e: mutable_config.set("concretizer:unify", True) e.add("mpi") e.concretize() assert any(s.package.provides("mpi") for _, s in e.concretized_specs()) def test_query_develop_specs(tmp_path: pathlib.Path): """Test whether a spec is develop'ed or not""" srcdir = tmp_path / "here" srcdir.mkdir() env("create", "test") with ev.read("test") as e: e.add("mpich") e.add("mpileaks") develop("--no-clone", "-p", str(srcdir), "mpich@=1") assert e.is_develop(Spec("mpich")) assert not e.is_develop(Spec("mpileaks")) @pytest.mark.parametrize("method", [spack.cmd.env.env_activate, spack.cmd.env.env_deactivate]) @pytest.mark.parametrize( "env,no_env,env_dir", [("b", False, None), (None, True, None), (None, False, "path/")] ) def test_activation_and_deactivation_ambiguities(method, env, no_env, env_dir, capfd): """spack [-e x | -E | -D x/] env [activate | deactivate] y are ambiguous""" args = Namespace( shell="sh", env_name="a", env=env, no_env=no_env, env_dir=env_dir, keep_relative=False ) with pytest.raises(SystemExit): method(args) _, err = capfd.readouterr() assert "is ambiguous" in err @pytest.mark.regression("26548") def test_custom_store_in_environment(mutable_config, tmp_path: pathlib.Path): spack_yaml = tmp_path / "spack.yaml" install_root = tmp_path / "store" spack_yaml.write_text( """ spack: specs: - libelf config: install_tree: root: {0} """.format(install_root) ) current_store_root = str(spack.store.STORE.root) assert str(current_store_root) != str(install_root) with ev.Environment(str(tmp_path)): assert str(spack.store.STORE.root) == str(install_root) assert str(spack.store.STORE.root) == current_store_root def test_activate_temp(monkeypatch, tmp_path: pathlib.Path): """Tests whether `spack env activate --temp` creates an environment in a temporary directory""" env_dir = lambda: str(tmp_path) monkeypatch.setattr(spack.cmd.env, "create_temp_env_directory", env_dir) shell = env("activate", "--temp", "--sh") active_env_var = next(line for line in shell.splitlines() if ev.spack_env_var in line) assert str(tmp_path) in active_env_var assert ev.is_env_dir(str(tmp_path)) @pytest.mark.parametrize( "conflict_arg", [["--dir"], ["--keep-relative"], ["--with-view", "foo"], ["env"]] ) def test_activate_parser_conflicts_with_temp(conflict_arg): with pytest.raises(SpackCommandError): env("activate", "--sh", "--temp", *conflict_arg) def test_create_and_activate_managed(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): shell = env("activate", "--without-view", "--create", "--sh", "foo") active_env_var = next(line for line in shell.splitlines() if ev.spack_env_var in line) assert str(tmp_path) in active_env_var active_ev = ev.active_environment() assert active_ev and "foo" == active_ev.name env("deactivate") def test_create_and_activate_independent(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): env_dir = os.path.join(str(tmp_path), "foo") shell = env("activate", "--without-view", "--create", "--sh", env_dir) active_env_var = next(line for line in shell.splitlines() if ev.spack_env_var in line) assert str(env_dir) in active_env_var assert ev.is_env_dir(env_dir) env("deactivate") def test_activate_default(monkeypatch): """Tests whether `spack env activate` creates / activates the default environment""" assert not ev.exists("default") # Activating it the first time should create it env("activate", "--sh") env("deactivate", "--sh") assert ev.exists("default") # Activating it while it already exists should work env("activate", "--sh") env("deactivate", "--sh") assert ev.exists("default") env("remove", "-y", "default") assert not ev.exists("default") def test_env_view_fail_if_symlink_points_elsewhere( tmp_path: pathlib.Path, install_mockery, mock_fetch ): view = str(tmp_path / "view") # Put a symlink to an actual directory in view non_view_dir = str(tmp_path / "dont-delete-me") os.mkdir(non_view_dir) os.symlink(non_view_dir, view) with ev.create("env", with_view=view): add("libelf") install("--fake") assert os.path.isdir(non_view_dir) def test_failed_view_cleanup(tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery): """Tests whether Spack cleans up after itself when a view fails to create""" view_dir = tmp_path / "view" with ev.create("env", with_view=str(view_dir)): add("libelf") install("--fake") # Save the current view directory. resolved_view = view_dir.resolve(strict=True) all_views = resolved_view.parent views_before = list(all_views.iterdir()) # Add a spec that results in view clash when creating a view with ev.read("env"): add("libelf cflags=-O3") with pytest.raises(ev.SpackEnvironmentViewError): install("--fake") # Make sure there is no broken view in the views directory, and the current # view is the original view from before the failed regenerate attempt. views_after = list(all_views.iterdir()) assert views_before == views_after assert view_dir.samefile(resolved_view), view_dir def test_environment_view_target_already_exists( tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery ): """When creating a new view, Spack should check whether the new view dir already exists. If so, it should not be removed or modified.""" # Create a new environment view = str(tmp_path / "view") env("create", "--with-view={0}".format(view), "test") with ev.read("test"): add("libelf") install("--fake") # Empty the underlying view real_view = os.path.realpath(view) assert os.listdir(real_view) # make sure it had *some* contents shutil.rmtree(real_view) # Replace it with something new. os.mkdir(real_view) fs.touch(os.path.join(real_view, "file")) # Remove the symlink so Spack can't know about the "previous root" os.unlink(view) # Regenerate the view, which should realize it can't write into the same dir. msg = "Failed to generate environment view" with ev.read("test"): with pytest.raises(ev.SpackEnvironmentViewError, match=msg): env("view", "regenerate") # Make sure the dir was left untouched. assert not os.path.lexists(view) assert os.listdir(real_view) == ["file"] def test_environment_query_spec_by_hash(mock_stage, mock_fetch, install_mockery): env("create", "test") with ev.read("test"): add("libdwarf") concretize() with ev.read("test") as e: spec = e.matching_spec("libelf") install("--fake", f"/{spec.dag_hash()}") with ev.read("test") as e: assert not e.matching_spec("libdwarf").installed assert e.matching_spec("libelf").installed @pytest.mark.parametrize("lockfile", ["v1", "v2", "v3"]) def test_read_old_lock_and_write_new(tmp_path: pathlib.Path, lockfile): # v1 lockfiles stored by a coarse DAG hash that did not include build deps. # They could not represent multiple build deps with different build hashes. # # v2 and v3 lockfiles are keyed by a "build hash", so they can represent specs # with different build deps but the same DAG hash. However, those two specs # could never have been built together, because they cannot coexist in a # Spack DB, which is keyed by DAG hash. The second one would just be a no-op # no-op because its DAG hash was already in the DB. # # Newer Spack uses a fine-grained DAG hash that includes build deps, package hash, # and more. But, we still have to identify old specs by their original DAG hash. # Essentially, the name (hash) we give something in Spack at concretization time is # its name forever (otherwise we'd need to relocate prefixes and disrupt existing # installations). So, we just discard the second conflicting dtbuild1 version when # reading v2 and v3 lockfiles. This is what old Spack would've done when installing # the environment, anyway. # # This test ensures the behavior described above. lockfile_path = os.path.join(spack.paths.test_path, "data", "legacy_env", "%s.lock" % lockfile) # read in the JSON from a legacy lockfile with open(lockfile_path, encoding="utf-8") as f: old_dict = sjson.load(f) # read all DAG hashes from the legacy lockfile and record its shadowed DAG hash. old_hashes = set() shadowed_hash = None for key, spec_dict in old_dict["concrete_specs"].items(): if "hash" not in spec_dict: # v1 and v2 key specs by their name in concrete_specs name, spec_dict = next(iter(spec_dict.items())) else: # v3 lockfiles have a `name` field and key by hash name = spec_dict["name"] # v1 lockfiles do not have a "hash" field -- they use the key. dag_hash = key if lockfile == "v1" else spec_dict["hash"] old_hashes.add(dag_hash) # v1 lockfiles can't store duplicate build dependencies, so they # will not have a shadowed hash. if lockfile != "v1": # v2 and v3 lockfiles store specs by build hash, so they can have multiple # keys for the same DAG hash. We discard the second one (dtbuild@1.0). if name == "dtbuild1" and spec_dict["version"] == "1.0": shadowed_hash = dag_hash # make an env out of the old lockfile -- env should be able to read v1/v2/v3 test_lockfile_path = str(tmp_path / "spack.lock") shutil.copy(lockfile_path, test_lockfile_path) _env_create("test", init_file=test_lockfile_path, with_view=False) # re-read the old env as a new lockfile e = ev.read("test") hashes = set(e._to_lockfile_dict()["concrete_specs"]) # v1 doesn't have duplicate build deps. # in v2 and v3, the shadowed hash will be gone. if shadowed_hash: old_hashes -= set([shadowed_hash]) # make sure we see the same hashes in old and new lockfiles assert old_hashes == hashes def test_read_v1_lock_creates_backup(tmp_path: pathlib.Path): """When reading a version-1 lockfile, make sure that a backup of that file is created. """ v1_lockfile_path = pathlib.Path(spack.paths.test_path) / "data" / "legacy_env" / "v1.lock" test_lockfile_path = tmp_path / "init" / ev.lockfile_name test_lockfile_path.parent.mkdir(parents=True, exist_ok=False) shutil.copy(v1_lockfile_path, test_lockfile_path) e = ev.create_in_dir(tmp_path, init_file=test_lockfile_path) assert os.path.exists(e._lock_backup_v1_path) assert filecmp.cmp(e._lock_backup_v1_path, v1_lockfile_path) @pytest.mark.parametrize("lockfile", ["v1", "v2", "v3"]) def test_read_legacy_lockfile_and_reconcretize( mock_stage, mock_fetch, install_mockery, lockfile, tmp_path: pathlib.Path ): # In legacy lockfiles v2 and v3 (keyed by build hash), there may be multiple # versions of the same spec with different build dependencies, which means # they will have different build hashes but the same DAG hash. # In the case of DAG hash conflicts, we always keep the spec associated with # whichever root spec came first in the "roots" list. # # After reconcretization with the *new*, finer-grained DAG hash, there should no # longer be conflicts, and the previously conflicting specs can coexist in the # same environment. test_path = pathlib.Path(spack.paths.test_path) lockfile_content = test_path / "data" / "legacy_env" / f"{lockfile}.lock" legacy_lockfile_path = tmp_path / ev.lockfile_name shutil.copy(lockfile_content, legacy_lockfile_path) # The order of the root specs in this environment is: # [ # wci7a3a -> dttop ^dtbuild1@0.5, # 5zg6wxw -> dttop ^dtbuild1@1.0 # ] # So in v2 and v3 lockfiles we have two versions of dttop with the same DAG # hash but different build hashes. env("create", "test", str(legacy_lockfile_path)) test = ev.read("test") assert len(test.specs_by_hash) == 1 single_root = next(iter(test.specs_by_hash.values())) # v1 only has version 1.0, because v1 was keyed by DAG hash, and v1.0 overwrote # v0.5 on lockfile creation. v2 only has v0.5, because we specifically prefer # the one that would be installed when we read old lockfiles. if lockfile == "v1": assert single_root["dtbuild1"].version == Version("1.0") else: assert single_root["dtbuild1"].version == Version("0.5") # Now forcefully reconcretize with ev.read("test"): concretize("-f") # After reconcretizing, we should again see two roots, one depending on each # of the dtbuild1 versions specified in the roots of the original lockfile. test = ev.read("test") assert len(test.specs_by_hash) == 2 expected_versions = set([Version("0.5"), Version("1.0")]) current_versions = set(s["dtbuild1"].version for s in test.specs_by_hash.values()) assert current_versions == expected_versions def _parse_dry_run_package_installs(make_output): """Parse `spack install ... # ` output from a make dry run.""" return [ Spec(line.split("# ")[1]).name for line in make_output.splitlines() if line.startswith("spack") ] @pytest.mark.parametrize( "depfile_flags,expected_installs", [ # This installs the full environment ( ["--use-buildcache=never"], [ "dtbuild1", "dtbuild2", "dtbuild3", "dtlink1", "dtlink2", "dtlink3", "dtlink4", "dtlink5", "dtrun1", "dtrun2", "dtrun3", "dttop", ], ), # This prunes build deps at depth > 0 ( ["--use-buildcache=package:never,dependencies:only"], [ "dtbuild1", "dtlink1", "dtlink2", "dtlink3", "dtlink4", "dtlink5", "dtrun1", "dtrun2", "dtrun3", "dttop", ], ), # This prunes all build deps ( ["--use-buildcache=only"], ["dtlink1", "dtlink3", "dtlink4", "dtlink5", "dtrun1", "dtrun3", "dttop"], ), # Test whether pruning of build deps is correct if we explicitly include one # that is also a dependency of a root. ( ["--use-buildcache=only", "dttop", "dtbuild1"], [ "dtbuild1", "dtlink1", "dtlink2", "dtlink3", "dtlink4", "dtlink5", "dtrun1", "dtrun2", "dtrun3", "dttop", ], ), ], ) def test_environment_depfile_makefile( depfile_flags, expected_installs, tmp_path: pathlib.Path, mock_packages ): env("create", "test") make = Executable("make") makefile = str(tmp_path / "Makefile") with ev.read("test"): add("dttop") concretize() # Disable jobserver so we can do a dry run. with ev.read("test"): env( "depfile", "-o", makefile, "--make-disable-jobserver", "--make-prefix=prefix", *depfile_flags, ) # Do make dry run. out = make("-n", "-f", makefile, "SPACK=spack", output=str) specs_that_make_would_install = _parse_dry_run_package_installs(out) # Check that all specs are there (without duplicates) assert set(specs_that_make_would_install) == set(expected_installs) assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install)) def test_depfile_safe_format(): """Test that depfile.MakefileSpec.safe_format() escapes target names.""" class SpecLike: def format(self, _): return "abc@def=ghi" spec = depfile.MakefileSpec(SpecLike()) assert spec.safe_format("{name}") == "abc_def_ghi" assert spec.unsafe_format("{name}") == "abc@def=ghi" def test_depfile_works_with_gitversions(tmp_path: pathlib.Path, mock_packages, monkeypatch): """Git versions may contain = chars, which should be escaped in targets, otherwise they're interpreted as makefile variable assignments.""" monkeypatch.setattr(spack.package_base.PackageBase, "git", "repo.git", raising=False) env("create", "test") make = Executable("make") makefile = str(tmp_path / "Makefile") # Create an environment with dttop and dtlink1 both at a git version, # and generate a depfile with ev.read("test"): add(f"dttop@{'a' * 40}=1.0 ^dtlink1@{'b' * 40}=1.0") concretize() env("depfile", "-o", makefile, "--make-disable-jobserver", "--make-prefix=prefix") # Do a dry run on the generated depfile out = make("-n", "-f", makefile, "SPACK=spack", output=str) # Check that all specs are there (without duplicates) specs_that_make_would_install = _parse_dry_run_package_installs(out) expected_installs = [ "dtbuild1", "dtbuild2", "dtbuild3", "dtlink1", "dtlink2", "dtlink3", "dtlink4", "dtlink5", "dtrun1", "dtrun2", "dtrun3", "dttop", ] assert set(specs_that_make_would_install) == set(expected_installs) assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install)) @pytest.mark.parametrize( "picked_package,expected_installs", [ ( "dttop", [ "dtbuild2", "dtlink2", "dtrun2", "dtbuild1", "dtlink4", "dtlink3", "dtlink1", "dtlink5", "dtbuild3", "dtrun3", "dtrun1", "dttop", ], ), ("dtrun1", ["dtlink5", "dtbuild3", "dtrun3", "dtrun1"]), ], ) def test_depfile_phony_convenience_targets( picked_package, expected_installs: set, tmp_path: pathlib.Path, mock_packages ): """Check whether convenience targets "install/%" and "install-deps/%" are created for each package if "--make-prefix" is absent.""" make = Executable("make") with fs.working_dir(str(tmp_path)): with ev.create_in_dir("."): add("dttop") concretize() with ev.Environment(".") as e: picked_spec = e.matching_spec(picked_package) env("depfile", "-o", "Makefile", "--make-disable-jobserver") # Phony install/* target should install picked package and all its deps specs_that_make_would_install = _parse_dry_run_package_installs( make( "-n", picked_spec.format("install/{name}-{version}-{hash}"), "SPACK=spack", output=str, ) ) assert set(specs_that_make_would_install) == set(expected_installs) assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install)) # Phony install-deps/* target shouldn't install picked package specs_that_make_would_install = _parse_dry_run_package_installs( make( "-n", picked_spec.format("install-deps/{name}-{version}-{hash}"), "SPACK=spack", output=str, ) ) assert set(specs_that_make_would_install) == set(expected_installs) - {picked_package} assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install)) def test_environment_depfile_out(tmp_path: pathlib.Path, mock_packages): env("create", "test") makefile_path = str(tmp_path / "Makefile") with ev.read("test"): add("libdwarf") concretize() with ev.read("test"): env("depfile", "-G", "make", "-o", makefile_path) stdout = env("depfile", "-G", "make") with open(makefile_path, "r", encoding="utf-8") as f: assert stdout == f.read() def test_spack_package_ids_variable(tmp_path: pathlib.Path, mock_packages): # Integration test for post-install hooks through prefix/SPACK_PACKAGE_IDS # variable env("create", "test") makefile_path = str(tmp_path / "Makefile") include_path = str(tmp_path / "include.mk") # Create env and generate depfile in include.mk with prefix example/ with ev.read("test"): add("libdwarf") concretize() with ev.read("test"): env( "depfile", "-G", "make", "--make-disable-jobserver", "--make-prefix=example", "-o", include_path, ) # Include in Makefile and create target that depend on SPACK_PACKAGE_IDS with open(makefile_path, "w", encoding="utf-8") as f: f.write( """ all: post-install include include.mk example/post-install/%: example/install/% \t$(info post-install: $(HASH)) # noqa: W191,E101 post-install: $(addprefix example/post-install/,$(example/SPACK_PACKAGE_IDS)) """ ) make = Executable("make") # Do dry run. out = make("-n", "-C", str(tmp_path), "SPACK=spack", output=str) # post-install: should've been executed with ev.read("test") as test: for s in test.all_specs(): assert "post-install: {}".format(s.dag_hash()) in out def test_depfile_empty_does_not_error(tmp_path: pathlib.Path): # For empty environments Spack should create a depfile that does nothing make = Executable("make") makefile = str(tmp_path / "Makefile") env("create", "test") with ev.read("test"): env("depfile", "-o", makefile) make("-f", makefile) assert make.returncode == 0 def test_unify_when_possible_works_around_conflicts(mutable_config): with ev.create("coconcretization") as e: mutable_config.set("concretizer:unify", "when_possible") e.add("mpileaks+opt") e.add("mpileaks~opt") e.add("mpich") e.concretize() assert len([x for x in e.all_specs() if x.satisfies("mpileaks")]) == 2 assert len([x for x in e.all_specs() if x.satisfies("mpileaks+opt")]) == 1 assert len([x for x in e.all_specs() if x.satisfies("mpileaks~opt")]) == 1 assert len([x for x in e.all_specs() if x.satisfies("mpich")]) == 1 def test_env_include_packages_url( tmp_path: pathlib.Path, mutable_empty_config, mock_fetch_url_text, mock_curl_configs ): """Test inclusion of a (GitHub) URL.""" develop_url = "https://github.com/fake/fake/blob/develop/" default_packages = develop_url + "etc/fake/defaults/packages.yaml" sha256 = "8d428c600b215e3b4a207a08236659dfc2c9ae2782c35943a00ee4204a135702" spack_yaml = tmp_path / "spack.yaml" with open(spack_yaml, "w", encoding="utf-8") as f: f.write( f"""\ spack: include: - path: {default_packages} sha256: {sha256} """ ) with spack.config.override("config:url_fetch_method", "curl"): env = ev.Environment(str(tmp_path)) ev.activate(env) # Make sure a setting from test/data/config/packages.yaml is present cfg = spack.config.get("packages") assert "mpich" in cfg["all"]["providers"]["mpi"] def test_relative_view_path_on_command_line_is_made_absolute(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): env("create", "--with-view", "view", "--dir", "env") environment = ev.Environment(os.path.join(".", "env")) environment.regenerate_views() assert os.path.samefile("view", environment.default_view.root) def test_environment_created_in_users_location(mutable_mock_env_path, tmp_path: pathlib.Path): """Test that an environment is created in a location based on the config""" env_dir = str(mutable_mock_env_path) assert str(tmp_path) in env_dir assert not os.path.isdir(env_dir) dir_name = "user_env" env("create", dir_name) out = env("list") assert dir_name in out assert env_dir in ev.root(dir_name) assert os.path.isdir(os.path.join(env_dir, dir_name)) def test_environment_created_from_lockfile_has_view( mock_packages, temporary_store, tmp_path: pathlib.Path ): """When an env is created from a lockfile, a view should be generated for it""" env_a = str(tmp_path / "a") env_b = str(tmp_path / "b") # Create an environment and install a package in it env("create", "-d", env_a) with ev.Environment(env_a): add("libelf") install("--fake") # Create another environment from the lockfile of the first environment env("create", "-d", env_b, os.path.join(env_a, "spack.lock")) # Make sure the view was created with ev.Environment(env_b) as e: assert os.path.isdir(e.view_path_default) def test_env_view_disabled(tmp_path: pathlib.Path, mutable_mock_env_path): """Ensure an inlined view being disabled means not even the default view is created (since the case doesn't appear to be covered in this module).""" spack_yaml = tmp_path / ev.manifest_name spack_yaml.write_text( """\ spack: specs: - mpileaks view: false """ ) env("create", "disabled", str(spack_yaml)) with ev.read("disabled") as e: e.concretize() assert len(e.views) == 0 assert not os.path.exists(e.view_path_default) @pytest.mark.parametrize("first", ["false", "true", "custom"]) def test_env_include_mixed_views( tmp_path: pathlib.Path, mutable_config, mutable_mock_env_path, first ): """Ensure including path and boolean views in different combinations result in the creation of only the first view if it is not disabled.""" false_yaml = tmp_path / "false-view.yaml" false_yaml.write_text("view: false\n") true_yaml = tmp_path / "true-view.yaml" true_yaml.write_text("view: true\n") custom_name = "my-test-view" custom_view = tmp_path / custom_name custom_yaml = tmp_path / "custom-view.yaml" custom_yaml.write_text( f""" view: {custom_name}: root: {custom_view} """ ) if first == "false": order = [false_yaml, true_yaml, custom_yaml] elif first == "true": order = [true_yaml, custom_yaml, false_yaml] else: order = [custom_yaml, false_yaml, true_yaml] includes = [f" - {yaml}\n" for yaml in order] spack_yaml = tmp_path / ev.manifest_name spack_yaml.write_text( f"""\ spack: include: {"".join(includes)} specs: - mpileaks """ ) env("create", "test", str(spack_yaml)) with ev.read("test") as e: concretize() # Only the first included view should be created if view not disabled by it assert len(e.views) == 0 if first == "false" else 1 if first == "true": assert os.path.exists(e.view_path_default) else: assert not os.path.exists(e.view_path_default) if first == "custom": assert os.path.exists(custom_view) else: assert not os.path.exists(custom_view) def test_stack_view_multiple_views_same_name( installed_environment, template_combinatorial_env, tmp_path: pathlib.Path ): """Test multiple views with the same name combine settings with precedence given to the options in spack.yaml.""" # Write the view configuration and or manifest file view_filename = tmp_path / "view.yaml" default_dir = tmp_path / "default-view" default_view = f"""\ view: default: root: {default_dir} select: ['target=x86_64'] projections: all: '{{architecture.target}}/{{name}}-{{version}}-from-view' """ view_filename.write_text(default_view) view_dir = tmp_path / "view" with installed_environment( f"""\ spack: include: - {view_filename} definitions: - packages: [mpileaks, cmake] - targets: ['target=x86_64', 'target=core2'] specs: - matrix: - [$packages] - [$targets] view: default: root: {view_dir} exclude: ['cmake'] projections: all: '{{architecture.target}}/{{name}}-{{version}}' """ ) as e: # the view root in the included view should NOT exist assert not os.path.exists(str(default_dir)) for spec in traverse_nodes(e.concrete_roots(), deptype=("link", "run")): # no specs will exist in the included view projection base_dir = view_dir / f"{spec.architecture.target}" included_dir = base_dir / f"{spec.name}-{spec.version}-from-view" assert not included_dir.exists() # only target=x86_64 specs (selected in the included view) that # are also not cmake (excluded in the environment view) should exist if spec.name == "gcc-runtime": continue current_dir = view_dir / f"{spec.architecture.target}" / f"{spec.name}-{spec.version}" assert current_dir.exists() is not ( spec.satisfies("cmake") or spec.satisfies("target=core2") ) def test_env_view_resolves_identical_file_conflicts( tmp_path: pathlib.Path, install_mockery, mock_fetch ): """When files clash in a view, but refer to the same file on disk (for example, the dependent symlinks to a file in the dependency at the same relative path), Spack links the first regular file instead of symlinks. This is important for copy type views where we need the underlying file to be copied instead of the symlink (when a symlink would be copied, it would become a self-referencing symlink after relocation). The test uses a symlink type view though, since that keeps track of the original file path.""" with ev.create("env", with_view=tmp_path / "view") as e: add("view-resolve-conflict-top") install() top = e.matching_spec("view-resolve-conflict-top").prefix bottom = e.matching_spec("view-file").prefix # In this example we have `./bin/x` in 3 prefixes, two links, one regular file. We expect the # regular file to be linked into the view. There are also 2 links at `./bin/y`, but no regular # file, so we expect standard behavior: first entry is linked into the view. # view-resolve-conflict-top/bin/ # x -> view-file/bin/x # y -> view-resolve-conflict-middle/bin/y # expect this y to be linked # view-resolve-conflict-middle/bin/ # x -> view-file/bin/x # y -> view-file/bin/x # view-file/bin/ # x # expect this x to be linked assert readlink(tmp_path / "view" / "bin" / "x") == bottom.bin.x assert readlink(tmp_path / "view" / "bin" / "y") == top.bin.y def test_env_view_ignores_different_file_conflicts( tmp_path: pathlib.Path, install_mockery, mock_fetch ): """Test that file-file conflicts for two unique files in environment views are ignored, and that the dependent's file is linked into the view, not the dependency's file.""" with ev.create("env", with_view=tmp_path / "view") as e: add("view-ignore-conflict") install() prefix_dependent = e.matching_spec("view-ignore-conflict").prefix # The dependent's file is linked into the view assert readlink(tmp_path / "view" / "bin" / "x") == prefix_dependent.bin.x @pytest.mark.regression("51054") def test_non_str_repos(installed_environment): with installed_environment( """\ spack: repos: builtin: branch: develop""" ): pass def test_concretized_specs_and_include_concrete(mutable_config): """Tests the consistency of concretized specs when there are either duplicate input specs or duplicate hashes. """ # Create a structure like this one # # Local specs: # - mpileaks -> hash1 # - libelf@0.8.12 -> hash2 # - pkg-a -> hash3 # # Included specs: # - mpileaks -> hash4 # - libelf -> hash2 # - pkg-a -> hash3 env("create", "included-env") with ev.read("included-env") as e: e.add("mpileaks") e.add("libelf") e.add("pkg-a") mutable_config.set( "packages", {"mpileaks": {"require": ["@2.2"]}, "libelf": {"require": ["@0.8.12"]}} ) included_pairs = e.concretize() e.write() env("create", "--include-concrete", "included-env", "main-env") with ev.read("main-env") as e: e.add("mpileaks") e.add("libelf@0.8.12") e.add("pkg-a") mutable_config.set("packages", {"mpileaks": {"require": ["@2.3"]}}) spec_pairs = e.concretize() concretized_specs = list(e.concretized_specs()) assert list(dedupe(spec_pairs + included_pairs)) == concretized_specs assert len(concretized_specs) == 5 def test_view_can_select_group_of_specs(installed_environment, tmp_path: pathlib.Path): """Tests that we can select groups of specs in a view and exclude other groups""" view_dir = tmp_path / "view" with installed_environment( f"""\ spack: specs: - group: apps1 specs: - mpileaks - group: apps2 specs: - cmake - group: apps3 specs: - pkg-a view: default: root: {view_dir} group: [apps1, apps2] """ ) as test: for item in test.concretized_roots: # Assertions are based on the behavior of the "--fake" install bin_file = pathlib.Path(test.default_view.view()._root) / "bin" / item.root.name assert not bin_file.exists() if item.group == "apps3" else bin_file.exists() def test_view_can_select_group_of_specs_using_string( installed_environment, tmp_path: pathlib.Path ): """Tests that we can select groups of specs in a view and exclude other groups""" view_dir = tmp_path / "view" with installed_environment( f"""\ spack: specs: - group: apps1 specs: - mpileaks - group: apps2 specs: - cmake view: default: root: {view_dir} group: apps1 """ ) as test: for item in test.concretized_roots: # Assertions are based on the behavior of the "--fake" install bin_file = pathlib.Path(test.default_view.view()._root) / "bin" / item.root.name assert not bin_file.exists() if item.group == "apps2" else bin_file.exists() def test_env_include_concrete_only(tmp_path, mock_packages, mutable_config): """Confirm that an environment that only includes a concrete environment actually loads it.""" specs = ["libdwarf", "libelf"] include_dir = tmp_path / "includes" include_dir.mkdir() include_manifest = include_dir / ev.manifest_name include_manifest.write_text( f"""\ spack: specs: - {specs[0]} - {specs[1]} """ ) include_env = ev.create("test_include", include_manifest) include_env.concretize() include_env.write() include_lockfile = include_env.lock_path assert os.path.exists(include_lockfile) manifest_file = tmp_path / ev.manifest_name manifest_file.write_text( f"""\ spack: include: - {str(include_lockfile)} """ ) e = ev.create("test", manifest_file) # Confirm the only specs the environment has are those loaded from the # lockfile. assert len(e.user_specs) == 0 all_concrete = [s for s, _ in e.concretized_specs()] for spec in specs: assert Spec(spec) in all_concrete @pytest.mark.parametrize( "concrete,includes", [ (["$HOME/path/to/other/environment"], []), (["$HOME/path/to/another/environment"], ["a/b", "$HOME/includes"]), ], ) def test_env_update_include_concrete(tmp_path: pathlib.Path, concrete, includes): """Confirm update of include_concrete converts it to include.""" config = {"include_concrete": concrete} if includes: config["include"] = includes new_concrete = [os.path.join(p, ev.lockfile_name) for p in concrete] assert spack.schema.env.update(config) assert "include_concrete" not in config assert config["include"] == new_concrete + includes def test_include_concrete_deprecation_warning( tmp_path: pathlib.Path, environment_from_manifest, capfd ): try: environment_from_manifest( """\ spack: include_concrete: - /path/to/some/environment """ ) except ev.SpackEnvironmentError: pass _, err = capfd.readouterr() assert "should be 'include'" in err def test_env_include_concrete_relative_path(tmp_path, mock_packages, mutable_config): """Tests that a relative path in 'include' for a spack.lock is resolved relative to the manifest file, not the current working directory. """ # Create and concretize the included environment. include_dir = tmp_path / "include_env" include_dir.mkdir() (include_dir / ev.manifest_name).write_text( """\ spack: specs: - libdwarf """ ) with ev.Environment(str(include_dir)) as e: e.concretize() e.write() assert os.path.exists(e.lock_path) # Create the main environment in a sibling directory, using a *relative* path main_dir = tmp_path / "main_env" main_dir.mkdir() relative_lockfile = f"../include_env/{ev.lockfile_name}" (main_dir / ev.manifest_name).write_text( f"""\ spack: include: - {relative_lockfile} """ ) with ev.Environment(str(main_dir)) as e: e.concretize() e.write() assert len(e.user_specs) == 0 assert [s for s, _ in e.concretized_specs()] == [Spec("libdwarf")] def test_env_include_concrete_git_lockfile(tmp_path, mock_packages, mutable_config, monkeypatch): """Tests that a spack.lock listed inside a git-based include is resolved using the clone destination as the base, not the manifest directory. """ # Create and concretize the included environment. include_dir = tmp_path / "include_env" include_dir.mkdir() (include_dir / ev.manifest_name).write_text( """\ spack: specs: - libdwarf """ ) with ev.Environment(str(include_dir)) as e: e.concretize() e.write() assert os.path.exists(e.lock_path) # Simulate a cloned git repo: the spack.lock lives at a subpath within the clone. clone_dest = tmp_path / "git_clone" lock_subpath = "envs/staging/spack.lock" lock_in_clone = clone_dest / "envs" / "staging" / ev.lockfile_name lock_in_clone.parent.mkdir(parents=True) shutil.copy(e.lock_path, lock_in_clone) # is_env_dir() requires spack.yaml alongside spack.lock shutil.copy(os.path.join(e.path, ev.manifest_name), lock_in_clone.parent) # Prevent actual git operations; return the pre-built clone destination. monkeypatch.setattr( spack.config.GitIncludePaths, "_clone", lambda self, parent_scope: str(clone_dest) ) main_dir = tmp_path / "main_env" main_dir.mkdir() (main_dir / ev.manifest_name).write_text( f"""\ spack: include: - git: https://example.com/configs.git branch: main paths: - {lock_subpath} """ ) with ev.Environment(str(main_dir)) as e: e.concretize() e.write() assert len(e.user_specs) == 0 assert [s for s, _ in e.concretized_specs()] == [Spec("libdwarf")] @pytest.mark.skipif(sys.platform != "linux", reason="Target is linux-specific") def test_compiler_target_env(mock_packages, environment_from_manifest): """Tests that Spack doesn't drop flag definitions on compilers when a target is required in config. """ cflags = "-Wall" env = environment_from_manifest( f"""\ spack: specs: - libdwarf %c=gcc@12.100.100 packages: all: require: - "target=x86_64_v3" gcc: externals: - spec: gcc@12.100.100 languages:=c,c++ prefix: /fake extra_attributes: compilers: c: /fake/bin/gcc cxx: /fake/bin/g++ flags: cflags: {cflags} require: "gcc" """ ) with env: env.concretize() libdwarf = env.concrete_roots()[0] assert libdwarf.satisfies("cflags=-Wall") # Sanity check: make sure the target we expect was applied to the # compiler entry assert libdwarf["c"].satisfies("gcc@12.100.100 languages:=c,c++ target=x86_64_v3") @pytest.mark.regression("52247") def test_create_with_orphaned_directory(mutable_mock_env_path: pathlib.Path): """Tests that an orphaned environment directory (directory exists, no spack.yaml) must not prevent 'spack env create' from creating a new environment with that name. """ orphaned = mutable_mock_env_path / "test1" orphaned_subdir = orphaned / ".spack-env" orphaned_subdir.mkdir(parents=True) # The orphaned directory must not be seen as an existing environment assert not ev.exists("test1") # Creating an environment over an orphaned directory must succeed env("create", "test1") assert ev.exists("test1") assert "test1" in env("list") @pytest.mark.parametrize( "setup", [ # valid environment: spack.yaml is a regular file pytest.param("valid", id="valid"), # orphaned directory: no spack.yaml at all pytest.param("orphaned", id="orphaned"), # broken manifest symlink: spack.yaml points to a non-existent target pytest.param("broken_symlink", id="broken_symlink"), ], ) @pytest.mark.regression("52247") def test_exists_consistent_with_all_environment_names( mutable_mock_env_path: pathlib.Path, setup: str ): """Tests that exists() and all_environment_names() agree on whether an environment exists.""" env_dir = mutable_mock_env_path / "myenv" env_dir.mkdir(parents=True) manifest = env_dir / ev.manifest_name if setup == "valid": manifest.write_text(ev.default_manifest_yaml()) elif setup == "orphaned": pass # no manifest elif setup == "broken_symlink": manifest.symlink_to("/nonexistent/spack.yaml") listed = "myenv" in ev.all_environment_names() assert ev.exists("myenv") == listed ================================================ FILE: lib/spack/spack/test/cmd/extensions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.concretize from spack.installer import PackageInstaller from spack.main import SpackCommand, SpackCommandError extensions = SpackCommand("extensions") @pytest.fixture def python_database(mock_packages, mutable_database): specs = [ spack.concretize.concretize_one(s) for s in ["python", "py-extension1", "py-extension2"] ] PackageInstaller([s.package for s in specs], explicit=True, fake=True).install() yield @pytest.mark.not_on_windows("All Fetchers Failed") @pytest.mark.db def test_extensions(mock_packages, python_database): ext2 = spack.concretize.concretize_one("py-extension2") def check_output(ni): output = extensions("python") packages = extensions("-s", "packages", "python") installed = extensions("-s", "installed", "python") assert "==> python@2.7.11" in output assert "==> 4 extensions" in output assert "py-extension1" in output assert "py-extension2" in output assert "python-venv" in output assert "==> 4 extensions" in packages assert "py-extension1" in packages assert "py-extension2" in packages assert "python-venv" in packages assert "installed" not in packages assert f"{ni if ni else 'None'} installed" in output assert f"{ni if ni else 'None'} installed" in installed check_output(3) ext2.package.do_uninstall(force=True) check_output(2) def test_extensions_no_arguments(mock_packages): out = extensions() assert "python" in out def test_extensions_raises_if_not_extendable(mock_packages): with pytest.raises(SpackCommandError): extensions("flake8") def test_extensions_raises_if_multiple_specs(mock_packages): with pytest.raises(SpackCommandError): extensions("python", "flake8") ================================================ FILE: lib/spack/spack/test/cmd/external.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import sys import pytest import spack.cmd.external import spack.config import spack.cray_manifest import spack.detection import spack.detection.path import spack.repo from spack.llnl.util.filesystem import getuid, touch from spack.main import SpackCommand from spack.spec import Spec pytestmark = [pytest.mark.usefixtures("mock_packages")] @pytest.fixture def executables_found(monkeypatch): def _factory(result): def _mock_search(path_hints=None): return result monkeypatch.setattr(spack.detection.path, "executables_in_path", _mock_search) return _factory def define_plat_exe(exe): if sys.platform == "win32": exe += ".bat" return exe def test_find_external_update_config(mutable_config): entries = [ Spec.from_detection("cmake@1.foo", external_path="/x/y1"), Spec.from_detection("cmake@3.17.2", external_path="/x/y2"), ] pkg_to_entries = {"cmake": entries} scope = spack.config.default_modify_scope("packages") spack.detection.update_configuration(pkg_to_entries, scope=scope, buildable=True) pkgs_cfg = spack.config.get("packages") cmake_cfg = pkgs_cfg["cmake"] cmake_externals = cmake_cfg["externals"] assert {"spec": "cmake@1.foo", "prefix": "/x/y1"} in cmake_externals assert {"spec": "cmake@3.17.2", "prefix": "/x/y2"} in cmake_externals def test_get_executables(working_env, mock_executable): cmake_path1 = mock_executable("cmake", output="echo cmake version 1.foo") path_to_exe = spack.detection.executables_in_path([os.path.dirname(cmake_path1)]) cmake_exe = define_plat_exe("cmake") assert path_to_exe[str(cmake_path1)] == cmake_exe external = SpackCommand("external") # TODO: this test should be made to work, but in the meantime it is # causing intermittent (spurious) CI failures on all PRs @pytest.mark.not_on_windows("Test fails intermittently on Windows") def test_find_external_cmd_not_buildable( mutable_config, working_env, mock_executable, monkeypatch ): """When the user invokes 'spack external find --not-buildable', the config for any package where Spack finds an external version should be marked as not buildable. """ version = "1.foo" @classmethod def _determine_version(cls, exe): return version cmake_cls = spack.repo.PATH.get_pkg_class("cmake") monkeypatch.setattr(cmake_cls, "determine_version", _determine_version) cmake_path = mock_executable("cmake", output=f"echo cmake version {version}") os.environ["PATH"] = str(cmake_path.parent) external("find", "--not-buildable", "cmake") pkgs_cfg = spack.config.get("packages") assert "cmake" in pkgs_cfg assert not pkgs_cfg["cmake"]["buildable"] @pytest.mark.parametrize( "names,tags,exclude,expected", [ # find -all ( None, ["detectable"], [], [ "builtin_mock.cmake", "builtin_mock.find-externals1", "builtin_mock.gcc", "builtin_mock.intel-oneapi-compilers", "builtin_mock.llvm", "builtin_mock.mpich", ], ), # find --all --exclude find-externals1 ( None, ["detectable"], ["builtin_mock.find-externals1"], [ "builtin_mock.cmake", "builtin_mock.gcc", "builtin_mock.intel-oneapi-compilers", "builtin_mock.llvm", "builtin_mock.mpich", ], ), ( None, ["detectable"], ["find-externals1"], [ "builtin_mock.cmake", "builtin_mock.gcc", "builtin_mock.intel-oneapi-compilers", "builtin_mock.llvm", "builtin_mock.mpich", ], ), # find hwloc (and mock hwloc is not detectable) (["hwloc"], ["detectable"], [], []), ], ) def test_package_selection(names, tags, exclude, expected): """Tests various cases of selecting packages""" # In the mock repo we only have 'find-externals1' that is detectable result = spack.cmd.external.packages_to_search_for(names=names, tags=tags, exclude=exclude) assert set(result) == set(expected) def test_find_external_no_manifest(mutable_config, working_env, monkeypatch): """The user runs 'spack external find'; the default path for storing manifest files does not exist. Ensure that the command does not fail. """ monkeypatch.setenv("PATH", "") monkeypatch.setattr( spack.cray_manifest, "default_path", os.path.join("a", "path", "that", "doesnt", "exist") ) external("find") def test_find_external_empty_default_manifest_dir( mutable_config, working_env, tmp_path: pathlib.Path, monkeypatch ): """The user runs 'spack external find'; the default path for storing manifest files exists but is empty. Ensure that the command does not fail. """ empty_manifest_dir = str(tmp_path / "manifest_dir") (tmp_path / "manifest_dir").mkdir() monkeypatch.setenv("PATH", "") monkeypatch.setattr(spack.cray_manifest, "default_path", empty_manifest_dir) external("find") @pytest.mark.not_on_windows("Can't chmod on Windows") @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_find_external_manifest_with_bad_permissions( mutable_config, working_env, tmp_path: pathlib.Path, monkeypatch ): """The user runs 'spack external find'; the default path for storing manifest files exists but with insufficient permissions. Check that the command does not fail. """ test_manifest_dir = str(tmp_path / "manifest_dir") (tmp_path / "manifest_dir").mkdir() test_manifest_file_path = os.path.join(test_manifest_dir, "badperms.json") touch(test_manifest_file_path) monkeypatch.setenv("PATH", "") monkeypatch.setattr(spack.cray_manifest, "default_path", test_manifest_dir) try: os.chmod(test_manifest_file_path, 0) output = external("find") assert "insufficient permissions" in output assert "Skipping manifest and continuing" in output finally: os.chmod(test_manifest_file_path, 0o700) def test_find_external_manifest_failure(mutable_config, tmp_path: pathlib.Path, monkeypatch): """The user runs 'spack external find'; the manifest parsing fails with some exception. Ensure that the command still succeeds (i.e. moves on to other external detection mechanisms). """ # First, create an empty manifest file (without a file to read, the # manifest parsing is skipped) test_manifest_dir = str(tmp_path / "manifest_dir") (tmp_path / "manifest_dir").mkdir() test_manifest_file_path = os.path.join(test_manifest_dir, "test.json") touch(test_manifest_file_path) def fail(): raise Exception() monkeypatch.setattr(spack.cmd.external, "_collect_and_consume_cray_manifest_files", fail) monkeypatch.setenv("PATH", "") output = external("find") assert "Skipping manifest and continuing" in output def test_find_external_merge(mutable_config): """Checks that 'spack find external' doesn't overwrite an existing spec in packages.yaml.""" pkgs_cfg_init = { "find-externals1": { "externals": [{"spec": "find-externals1@1.1", "prefix": "/preexisting-prefix"}], "buildable": False, } } mutable_config.update_config("packages", pkgs_cfg_init) entries = [ Spec.from_detection("find-externals1@1.1", external_path="/x/y1"), Spec.from_detection("find-externals1@1.2", external_path="/x/y2"), ] pkg_to_entries = {"find-externals1": entries} scope = spack.config.default_modify_scope("packages") spack.detection.update_configuration(pkg_to_entries, scope=scope, buildable=True) pkgs_cfg = spack.config.get("packages") pkg_cfg = pkgs_cfg["find-externals1"] pkg_externals = pkg_cfg["externals"] assert {"spec": "find-externals1@1.1", "prefix": "/preexisting-prefix"} in pkg_externals assert {"spec": "find-externals1@1.2", "prefix": "/x/y2"} in pkg_externals def test_list_detectable_packages(mutable_config): external("list") assert external.returncode == 0 def test_overriding_prefix(mock_executable, mutable_config, monkeypatch): gcc_exe = mock_executable("gcc", output="echo 4.2.1") search_dir = gcc_exe.parent @classmethod def _determine_variants(cls, exes, version_str): return "languages=c", {"prefix": "/opt/gcc/bin", "compilers": {"c": exes[0]}} gcc_cls = spack.repo.PATH.get_pkg_class("gcc") monkeypatch.setattr(gcc_cls, "determine_variants", _determine_variants) finder = spack.detection.path.ExecutablesFinder() detected_specs = finder.find( pkg_name="gcc", initial_guess=[str(search_dir)], repository=spack.repo.PATH ) assert len(detected_specs) == 1 gcc = detected_specs[0] assert gcc.name == "gcc" assert gcc.external_path == os.path.sep + os.path.join("opt", "gcc", "bin") @pytest.mark.not_on_windows("Fails spuriously on Windows") def test_new_entries_are_reported_correctly(mock_executable, mutable_config, monkeypatch): # Prepare an environment to detect a fake gcc gcc_exe = mock_executable("gcc", output="echo 4.2.1") prefix = os.path.dirname(gcc_exe) monkeypatch.setenv("PATH", prefix) # The first run will find and add the external gcc output = external("find", "gcc") assert "The following specs have been" in output # The second run should report that no new external # has been found output = external("find", "gcc") assert "No new external packages detected" in output @pytest.mark.parametrize("command_args", [("-t", "build-tools"), ("-t", "build-tools", "cmake")]) @pytest.mark.not_on_windows("the test uses bash scripts") def test_use_tags_for_detection(command_args, mock_executable, mutable_config, monkeypatch): versions = {"cmake": "3.19.1", "openssl": "2.8.3"} @classmethod def _determine_version(cls, exe): return versions[os.path.basename(exe)] cmake_cls = spack.repo.PATH.get_pkg_class("cmake") monkeypatch.setattr(cmake_cls, "determine_version", _determine_version) # Prepare an environment to detect a fake cmake cmake_exe = mock_executable("cmake", output=f"echo cmake version {versions['cmake']}") prefix = os.path.dirname(cmake_exe) monkeypatch.setenv("PATH", prefix) openssl_exe = mock_executable("openssl", output=f"OpenSSL {versions['openssl']}") prefix = os.path.dirname(openssl_exe) monkeypatch.setenv("PATH", prefix) # Test that we detect specs output = external("find", *command_args) assert "The following specs have been" in output assert "cmake" in output assert "openssl" not in output @pytest.mark.regression("38733") @pytest.mark.not_on_windows("the test uses bash scripts") def test_failures_in_scanning_do_not_result_in_an_error( mock_executable, monkeypatch, mutable_config ): """Tests that scanning paths with wrong permissions, won't cause `external find` to error.""" cmake_exe1 = mock_executable( "cmake", output="echo cmake version 3.19.1", subdir=("first", "bin") ) cmake_exe2 = mock_executable( "cmake", output="echo cmake version 3.23.3", subdir=("second", "bin") ) @classmethod def _determine_version(cls, exe): name = pathlib.Path(exe).parent.parent.name if name == "first": return "3.19.1" elif name == "second": return "3.23.3" assert False, f"Unexpected exe path {exe}" cmake_cls = spack.repo.PATH.get_pkg_class("cmake") monkeypatch.setattr(cmake_cls, "determine_version", _determine_version) monkeypatch.setenv("PATH", f"{cmake_exe1.parent}{os.pathsep}{cmake_exe2.parent}") try: # Remove access from the first directory executable cmake_exe1.parent.chmod(0o600) output = external("find", "cmake") finally: cmake_exe1.parent.chmod(0o700) assert external.returncode == 0 assert "The following specs have been" in output assert "cmake" in output assert "3.19.1" in output assert "3.23.3" in output def test_detect_virtuals(mock_executable, mutable_config, monkeypatch): """Test whether external find --not-buildable sets virtuals as non-buildable (unless user config sets them to buildable)""" version = "4.0.2" @classmethod def _determine_version(cls, exe): return version cmake_cls = spack.repo.PATH.get_pkg_class("mpich") monkeypatch.setattr(cmake_cls, "determine_version", _determine_version) mpich = mock_executable("mpichversion", output=f"echo MPICH Version: {version}") prefix = os.path.dirname(mpich) external("find", "--path", prefix, "--not-buildable", "mpich") # Check that mpich was correctly detected mpich = mutable_config.get("packages:mpich") assert mpich["buildable"] is False assert Spec(mpich["externals"][0]["spec"]).satisfies(f"mpich@{version}") # Check that the virtual package mpi was marked as non-buildable assert mutable_config.get("packages:mpi:buildable") is False # Delete the mpich entry, and set mpi explicitly to buildable mutable_config.set("packages:mpich", {}) mutable_config.set("packages:mpi:buildable", True) # Run the detection again external("find", "--path", prefix, "--not-buildable", "mpich") # Check that the mpi:buildable entry was not overwritten assert mutable_config.get("packages:mpi:buildable") is True ================================================ FILE: lib/spack/spack/test/cmd/fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.environment as ev from spack.main import SpackCommand, SpackCommandError # everything here uses the mock_env_path pytestmark = pytest.mark.usefixtures( "mutable_mock_env_path", "mutable_config", "mutable_mock_repo" ) @pytest.mark.disable_clean_stage_check def test_fetch_in_env(mock_archive, mock_stage, mock_fetch, install_mockery): SpackCommand("env")("create", "test") with ev.read("test"): SpackCommand("add")("python") with pytest.raises(SpackCommandError): SpackCommand("fetch")() SpackCommand("concretize")() SpackCommand("fetch")() @pytest.mark.disable_clean_stage_check def test_fetch_single_spec(mock_archive, mock_stage, mock_fetch, install_mockery): SpackCommand("fetch")("mpileaks") @pytest.mark.disable_clean_stage_check def test_fetch_multiple_specs(mock_archive, mock_stage, mock_fetch, install_mockery): SpackCommand("fetch")("mpileaks", "gcc@3.0", "python") def test_fetch_no_argument(): with pytest.raises(SpackCommandError): SpackCommand("fetch")() ================================================ FILE: lib/spack/spack/test/cmd/find.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import io import json import os import pathlib import pytest import spack.cmd import spack.cmd.find import spack.concretize import spack.environment as ev import spack.package_base import spack.paths import spack.repo import spack.spec import spack.store import spack.user_environment as uenv from spack.enums import InstallRecordStatus from spack.llnl.util.filesystem import working_dir from spack.main import SpackCommand from spack.test.utilities import SpackCommandArgs from spack.util.pattern import Bunch find = SpackCommand("find") env = SpackCommand("env") install = SpackCommand("install") base32_alphabet = "abcdefghijklmnopqrstuvwxyz234567" @pytest.fixture(scope="module") def parser(): """Returns the parser for the module command""" prs = argparse.ArgumentParser() spack.cmd.find.setup_parser(prs) return prs @pytest.fixture() def specs(): s = [] return s @pytest.fixture() def mock_display(monkeypatch, specs): """Monkeypatches the display function to return its first argument""" def display(x, *args, **kwargs): specs.extend(x) monkeypatch.setattr(spack.cmd, "display_specs", display) def test_query_arguments(): query_arguments = spack.cmd.find.query_arguments # Default arguments args = Bunch( only_missing=False, missing=False, only_deprecated=False, deprecated=False, unknown=False, explicit=False, implicit=False, start_date="2018-02-23", end_date=None, install_tree="all", ) q_args = query_arguments(args) assert "installed" in q_args assert "predicate_fn" in q_args assert "explicit" in q_args assert q_args["installed"] == InstallRecordStatus.INSTALLED assert q_args["predicate_fn"] is None assert q_args["explicit"] is None assert "start_date" in q_args assert "end_date" not in q_args assert q_args["install_tree"] == "all" # Check that explicit works correctly args.explicit = True q_args = query_arguments(args) assert q_args["explicit"] is True args.explicit = False args.implicit = True q_args = query_arguments(args) assert q_args["explicit"] is False @pytest.mark.db @pytest.mark.usefixtures("database", "mock_display") def test_tag1(parser, specs): args = parser.parse_args(["--tag", "tag1"]) spack.cmd.find.find(parser, args) assert len(specs) == 2 assert "mpich" in [x.name for x in specs] assert "mpich2" in [x.name for x in specs] @pytest.mark.db @pytest.mark.usefixtures("database", "mock_display") def test_tag2(parser, specs): args = parser.parse_args(["--tag", "tag2"]) spack.cmd.find.find(parser, args) assert len(specs) == 1 assert "mpich" in [x.name for x in specs] @pytest.mark.db @pytest.mark.usefixtures("database", "mock_display") def test_tag2_tag3(parser, specs): args = parser.parse_args(["--tag", "tag2", "--tag", "tag3"]) spack.cmd.find.find(parser, args) assert len(specs) == 0 @pytest.mark.parametrize( "args,with_namespace", [([], False), (["--namespace"], True), (["--namespaces"], True)] ) @pytest.mark.db def test_namespaces_shown_correctly(args, with_namespace, database): """Test that --namespace(s) works. Old syntax is --namespace""" assert ("builtin_mock.zmpi" in find(*args)) == with_namespace @pytest.mark.db def test_find_cli_output_format(database, mock_tty_stdout): assert find("zmpi").endswith( """\ zmpi@1.0 ==> 1 installed package """ ) def _check_json_output(spec_list): assert len(spec_list) == 3 assert all(spec["name"] == "mpileaks" for spec in spec_list) assert all(spec["hash"] for spec in spec_list) deps = [spec["dependencies"] for spec in spec_list] assert sum(["zmpi" in [node["name"] for d in deps for node in d]]) == 1 assert sum(["mpich" in [node["name"] for d in deps for node in d]]) == 1 assert sum(["mpich2" in [node["name"] for d in deps for node in d]]) == 1 def _check_json_output_deps(spec_list): assert len(spec_list) == 16 names = [spec["name"] for spec in spec_list] assert names.count("mpileaks") == 3 assert names.count("callpath") == 3 assert names.count("zmpi") == 1 assert names.count("mpich") == 1 assert names.count("mpich2") == 1 assert names.count("fake") == 1 assert names.count("dyninst") == 1 assert names.count("libdwarf") == 1 assert names.count("libelf") == 1 @pytest.mark.db def test_find_json(database): output = find("--json", "mpileaks") spec_list = json.loads(output) _check_json_output(spec_list) @pytest.mark.db def test_find_json_deps(database): output = find("-d", "--json", "mpileaks") spec_list = json.loads(output) _check_json_output_deps(spec_list) @pytest.mark.db def test_display_json(database, capfd): specs = [ spack.concretize.concretize_one(s) for s in ["mpileaks ^zmpi", "mpileaks ^mpich", "mpileaks ^mpich2"] ] spack.cmd.display_specs_as_json(specs) spec_list = json.loads(capfd.readouterr()[0]) _check_json_output(spec_list) spack.cmd.display_specs_as_json(specs + specs + specs) spec_list = json.loads(capfd.readouterr()[0]) _check_json_output(spec_list) @pytest.mark.db def test_display_json_deps(database, capfd): specs = [ spack.concretize.concretize_one(s) for s in ["mpileaks ^zmpi", "mpileaks ^mpich", "mpileaks ^mpich2"] ] spack.cmd.display_specs_as_json(specs, deps=True) spec_list = json.loads(capfd.readouterr()[0]) _check_json_output_deps(spec_list) spack.cmd.display_specs_as_json(specs + specs + specs, deps=True) spec_list = json.loads(capfd.readouterr()[0]) _check_json_output_deps(spec_list) @pytest.mark.regression("52219") def test_display_abstract_hash(): spec = spack.spec.Spec("/foobar") out = io.StringIO() spack.cmd.display_specs([spec], output=out) # errors on failure assert "/foobar" in out.getvalue() @pytest.mark.db def test_find_format(database, config): output = find("--format", "{name}-{^mpi.name}", "mpileaks") assert set(output.strip().split("\n")) == { "mpileaks-zmpi", "mpileaks-mpich", "mpileaks-mpich2", } output = find("--format", "{name}-{version}-{compiler.name}-{^mpi.name}", "mpileaks") assert "installed package" not in output assert set(output.strip().split("\n")) == { "mpileaks-2.3-gcc-zmpi", "mpileaks-2.3-gcc-mpich", "mpileaks-2.3-gcc-mpich2", } output = find("--format", "{name}-{^mpi.name}-{hash:7}", "mpileaks") elements = output.strip().split("\n") assert set(e[:-7] for e in elements) == { "mpileaks-zmpi-", "mpileaks-mpich-", "mpileaks-mpich2-", } # hashes are in base32 for e in elements: for c in e[-7:]: assert c in base32_alphabet @pytest.mark.db def test_find_format_deps(database, config): output = find("-d", "--format", "{name}-{version}", "mpileaks", "^zmpi") assert ( output == """\ mpileaks-2.3 callpath-1.0 dyninst-8.2 libdwarf-20130729 libelf-0.8.13 compiler-wrapper-1.0 gcc-10.2.1 gcc-runtime-10.2.1 zmpi-1.0 fake-1.0 """ ) @pytest.mark.db def test_find_format_deps_paths(database, config): output = find("-dp", "--format", "{name}-{version}", "mpileaks", "^zmpi") mpileaks = spack.concretize.concretize_one("mpileaks ^zmpi") assert ( output == f"""\ mpileaks-2.3 {mpileaks.prefix} callpath-1.0 {mpileaks["callpath"].prefix} dyninst-8.2 {mpileaks["dyninst"].prefix} libdwarf-20130729 {mpileaks["libdwarf"].prefix} libelf-0.8.13 {mpileaks["libelf"].prefix} compiler-wrapper-1.0 {mpileaks["compiler-wrapper"].prefix} gcc-10.2.1 {mpileaks["gcc"].prefix} gcc-runtime-10.2.1 {mpileaks["gcc-runtime"].prefix} zmpi-1.0 {mpileaks["zmpi"].prefix} fake-1.0 {mpileaks["fake"].prefix} """ ) @pytest.mark.db def test_find_very_long(database, config): output = find("-L", "--no-groups", "mpileaks") specs = [ spack.concretize.concretize_one(s) for s in ["mpileaks ^zmpi", "mpileaks ^mpich", "mpileaks ^mpich2"] ] assert set(output.strip().split("\n")) == set( [("%s mpileaks@2.3" % s.dag_hash()) for s in specs] ) @pytest.mark.db def test_find_not_found(database, config): output = find("foobarbaz", fail_on_error=False) assert "No package matches the query: foobarbaz" in output assert find.returncode == 1 @pytest.mark.db def test_find_no_sections(database, config): output = find() assert "-----------" in output output = find("--no-groups") assert "-----------" not in output assert "==>" not in output @pytest.mark.db def test_find_command_basic_usage(database): output = find() assert "mpileaks" in output @pytest.mark.regression("9875") def test_find_prefix_in_env( mutable_mock_env_path, install_mockery, mock_fetch, mock_packages, mock_archive ): """Test `find` formats requiring concrete specs work in environments.""" env("create", "test") with ev.read("test"): install("--fake", "--add", "mpileaks") find("-p") find("-l") find("-L") # Would throw error on regression def test_find_specs_include_concrete_env( mutable_mock_env_path, mutable_mock_repo, tmp_path: pathlib.Path ): path = tmp_path / "spack.yaml" with working_dir(str(tmp_path)): with open(str(path), "w", encoding="utf-8") as f: f.write( """\ spack: specs: - mpileaks """ ) env("create", "test1", "spack.yaml") test1 = ev.read("test1") test1.concretize() test1.write() with working_dir(str(tmp_path)): with open(str(path), "w", encoding="utf-8") as f: f.write( """\ spack: specs: - libelf """ ) env("create", "test2", "spack.yaml") test2 = ev.read("test2") test2.concretize() test2.write() env("create", "--include-concrete", "test1", "--include-concrete", "test2", "combined_env") with ev.read("combined_env"): output = find() assert "no root specs" in output assert "Included specs" in output assert "mpileaks" in output assert "libelf" in output def test_find_specs_nested_include_concrete_env( mutable_mock_env_path, mutable_mock_repo, tmp_path: pathlib.Path ): path = tmp_path / "spack.yaml" with working_dir(str(tmp_path)): with open(str(path), "w", encoding="utf-8") as f: f.write( """\ spack: specs: - mpileaks """ ) env("create", "test1", "spack.yaml") test1 = ev.read("test1") test1.concretize() test1.write() env("create", "--include-concrete", "test1", "test2") test2 = ev.read("test2") test2.add("libelf") test2.concretize() test2.write() env("create", "--include-concrete", "test2", "test3") with ev.read("test3"): output = find() assert "no root specs" in output assert "Included specs" in output assert "mpileaks" in output assert "libelf" in output def test_find_loaded(database, working_env): output = find("--loaded", "--group") assert output == "" os.environ[uenv.spack_loaded_hashes_var] = os.pathsep.join( [x.dag_hash() for x in spack.store.STORE.db.query()] ) output = find("--loaded") expected = find() assert output == expected @pytest.mark.regression("37712") def test_environment_with_version_range_in_compiler_doesnt_fail( tmp_path: pathlib.Path, mock_packages ): """Tests that having an active environment with a root spec containing a compiler constrained by a version range (i.e. @X.Y rather the single version than @=X.Y) doesn't result in an error when invoking "spack find". """ test_environment = ev.create_in_dir(tmp_path) test_environment.add("zlib %gcc@12.1.0") test_environment.write() with test_environment: output = find() assert "zlib" in output # a0 d0 # / \ / \ # b0 c0 e0 @pytest.fixture def test_repo(mock_stage): with spack.repo.use_repositories( os.path.join(spack.paths.test_repos_path, "spack_repo", "find") ) as mock_packages_repo: yield mock_packages_repo def test_find_concretized_not_installed( mutable_mock_env_path, install_mockery, mock_fetch, test_repo, mock_archive ): """Test queries against installs of specs against fake repo. Given A, B, C, D, E, create an environment and install A. Add and concretize (but do not install) D. Test a few queries after force uninstalling a dependency of A (but not A itself). """ add = SpackCommand("add") concretize = SpackCommand("concretize") uninstall = SpackCommand("uninstall") def _query(_e, *args): return spack.cmd.find._find_query(SpackCommandArgs("find")(*args), _e) def _nresults(_qresult): return len(_qresult[0]), len(_qresult[1]) env("create", "test") with ev.read("test") as e: install("--fake", "--add", "a0") assert _nresults(_query(e)) == (3, 0) assert _nresults(_query(e, "--explicit")) == (1, 0) add("d0") concretize("--reuse") # At this point d0 should use existing c0, but d/e # are not installed in the env # --explicit, --deprecated, --start-date, etc. are all # filters on records, and therefore don't apply to # concretized-but-not-installed results assert _nresults(_query(e, "--explicit")) == (1, 2) assert _nresults(_query(e)) == (3, 2) assert _nresults(_query(e, "-c", "d0")) == (0, 1) uninstall("-f", "-y", "b0") # b0 is now missing (it is not installed, but has an # installed parent) assert _nresults(_query(e)) == (2, 3) # b0 is "double-counted" here: it meets the --missing # criteria, and also now qualifies as a # concretized-but-not-installed spec assert _nresults(_query(e, "--missing")) == (3, 3) assert _nresults(_query(e, "--only-missing")) == (1, 3) # Tags are not attached to install records, so they # can modify the concretized-but-not-installed results assert _nresults(_query(e, "--tag=tag0")) == (1, 0) assert _nresults(_query(e, "--tag=tag1")) == (1, 1) assert _nresults(_query(e, "--tag=tag2")) == (0, 1) @pytest.mark.usefixtures("install_mockery", "mock_fetch") def test_find_based_on_commit_sha(mock_git_version_info, monkeypatch): repo_path, filename, commits = mock_git_version_info file_url = pathlib.Path(repo_path).as_uri() monkeypatch.setattr(spack.package_base.PackageBase, "git", file_url, raising=False) install("--fake", f"git-test-commit commit={commits[0]}") output = find(f"commit={commits[0]}") assert "git-test-commit" in output @pytest.mark.usefixtures("mock_packages") @pytest.mark.parametrize( "spack_yaml,expected,not_expected", [ ( """ spack: specs: - mpileaks - group: extras specs: - libelf """, [ "2 root specs", # Group names "extras", "default", # root specs "mpileaks", "libelf", ], [], ), ( """ spack: specs: - group: tools specs: - libelf """, ["1 root spec", "tools", "libelf"], ["1 root specs", "default"], ), ], ) def test_find_env_with_groups(spack_yaml, expected, not_expected, tmp_path: pathlib.Path): """Tests that the output of spack find contains expected matches when using an environment with groups. """ (tmp_path / "spack.yaml").write_text(spack_yaml) with ev.Environment(tmp_path): output = find() assert all(x in output for x in expected) assert all(x not in output for x in not_expected) ================================================ FILE: lib/spack/spack/test/cmd/gc.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.concretize import spack.deptypes as dt import spack.environment as ev import spack.main import spack.spec import spack.traverse from spack.installer import PackageInstaller gc = spack.main.SpackCommand("gc") add = spack.main.SpackCommand("add") install = spack.main.SpackCommand("install") @pytest.mark.db def test_gc_without_build_dependency(mutable_database): assert "There are no unused specs." in gc("-yb") # 'gcc' is a pure build dependency in the DB assert "There are no unused specs." not in gc("-y") @pytest.mark.db def test_gc_with_build_dependency(mutable_database): s = spack.concretize.concretize_one("simple-inheritance") PackageInstaller([s.package], explicit=True, fake=True).install() assert "There are no unused specs." in gc("-yb") assert "Successfully uninstalled cmake" in gc("-y") assert "There are no unused specs." in gc("-y") @pytest.mark.db def test_gc_with_constraints(mutable_database): s_cmake1 = spack.concretize.concretize_one("simple-inheritance ^cmake@3.4.3") s_cmake2 = spack.concretize.concretize_one("simple-inheritance ^cmake@3.23.1") PackageInstaller([s_cmake1.package], explicit=True, fake=True).install() PackageInstaller([s_cmake2.package], explicit=True, fake=True).install() assert "There are no unused specs." in gc("python") assert "Successfully uninstalled cmake@3.4.3" in gc("-y", "cmake@3.4.3") assert "There are no unused specs." in gc("-y", "cmake@3.4.3") assert "Successfully uninstalled cmake" in gc("-y", "cmake@3.23.1") assert "There are no unused specs." in gc("-y", "cmake") @pytest.mark.db def test_gc_with_environment(mutable_database, mutable_mock_env_path): s = spack.concretize.concretize_one("simple-inheritance") PackageInstaller([s.package], explicit=True, fake=True).install() e = ev.create("test_gc") with e: add("cmake") install() assert mutable_database.query_local("cmake") output = gc("-by") assert "Restricting garbage collection" in output assert "There are no unused specs" in output @pytest.mark.db def test_gc_with_build_dependency_in_environment(mutable_database, mutable_mock_env_path): s = spack.concretize.concretize_one("simple-inheritance") PackageInstaller([s.package], explicit=True, fake=True).install() e = ev.create("test_gc") with e: add("simple-inheritance") install() assert mutable_database.query_local("simple-inheritance") output = gc("-yb") assert "Restricting garbage collection" in output assert "There are no unused specs" in output with e: assert mutable_database.query_local("simple-inheritance") fst = gc("-y") assert "Restricting garbage collection" in fst assert "Successfully uninstalled cmake" in fst snd = gc("-y") assert "Restricting garbage collection" in snd assert "There are no unused specs" in snd @pytest.mark.db def test_gc_except_any_environments(mutable_database, mutable_mock_env_path): """Tests whether the garbage collector can remove all specs except those still needed in some environment (needed in the sense of roots + link/run deps).""" assert mutable_database.query_local("zmpi") e = ev.create("test_gc") e.add("simple-inheritance") e.concretize() e.install_all(fake=True) e.write() assert mutable_database.query_local("simple-inheritance") assert not e.all_matching_specs(spack.spec.Spec("zmpi")) output = gc("-yE") assert "Restricting garbage collection" not in output assert "Successfully uninstalled zmpi" in output assert not mutable_database.query_local("zmpi") # All runtime specs in this env should still be installed. assert all( s.installed for s in spack.traverse.traverse_nodes(e.concrete_roots(), deptype=dt.LINK | dt.RUN) ) @pytest.mark.db def test_gc_except_specific_environments(mutable_database, mutable_mock_env_path): s = spack.concretize.concretize_one("simple-inheritance") PackageInstaller([s.package], explicit=True, fake=True).install() assert mutable_database.query_local("zmpi") e = ev.create("test_gc") with e: add("simple-inheritance") install() assert mutable_database.query_local("simple-inheritance") output = gc("-ye", "test_gc") assert "Restricting garbage collection" not in output assert "Successfully uninstalled zmpi" in output assert not mutable_database.query_local("zmpi") @pytest.mark.db def test_gc_except_nonexisting_dir_env( mutable_database, mutable_mock_env_path, tmp_path: pathlib.Path ): output = gc("-ye", str(tmp_path), fail_on_error=False) assert "No such environment" in output assert gc.returncode == 1 @pytest.mark.db def test_gc_except_specific_dir_env( mutable_database, mutable_mock_env_path, tmp_path: pathlib.Path ): s = spack.concretize.concretize_one("simple-inheritance") PackageInstaller([s.package], explicit=True, fake=True).install() assert mutable_database.query_local("zmpi") e = ev.create_in_dir(str(tmp_path)) with e: add("simple-inheritance") install() assert mutable_database.query_local("simple-inheritance") output = gc("-ye", str(tmp_path)) assert "Restricting garbage collection" not in output assert "Successfully uninstalled zmpi" in output assert not mutable_database.query_local("zmpi") @pytest.fixture def mock_installed_environment(mutable_database, mutable_mock_env_path): def _create_environment(name, spack_yaml): tmp_env = ev.create(name) spack_yaml_path = pathlib.Path(tmp_env.path) / "spack.yaml" spack_yaml_path.write_text(spack_yaml) e = ev.read(name) with ev.read(name): e.concretize() e.install_all(fake=True) e.write() return e return _create_environment @pytest.mark.db @pytest.mark.parametrize( "explicit,expected_explicit,expected_implicit", [ (True, ["gcc@14.0.1", "openblas", "dyninst"], []), (False, ["dyninst"], ["gcc@14.0.1", "openblas"]), ], ) def test_gc_with_explicit_groups( explicit, expected_explicit, expected_implicit, mutable_database, mock_installed_environment ): """Tests the semantics of the "explicit" attribute of environment groups""" e = mock_installed_environment( "test_gc_explicit", f""" spack: config: installer: new specs: - group: base explicit: {explicit} specs: - gcc@14.0.1 - openblas - group: apps needs: [base] specs: - dyninst %c=gcc@14.0.1 """, ) # Test DB status for query in expected_explicit: assert mutable_database.query_local(query, explicit=True) for query in expected_implicit: assert mutable_database.query_local(query, explicit=False) with e: output = gc("-y") # Test gc behavior for query in expected_implicit: assert f"Successfully uninstalled {query}" in output for query in expected_explicit: assert f"Successfully uninstalled {query}" not in output ================================================ FILE: lib/spack/spack/test/cmd/gpg.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.binary_distribution import spack.llnl.util.filesystem as fs import spack.util.gpg from spack.main import SpackCommand from spack.paths import mock_gpg_data_path, mock_gpg_keys_path from spack.util.executable import ProcessError #: spack command used by tests below gpg = SpackCommand("gpg") bootstrap = SpackCommand("bootstrap") mirror = SpackCommand("mirror") pytestmark = pytest.mark.not_on_windows("does not run on windows") # test gpg command detection @pytest.mark.parametrize( "cmd_name,version", [ ("gpg", "undetectable"), # undetectable version ("gpg", "gpg (GnuPG) 1.3.4"), # insufficient version ("gpg", "gpg (GnuPG) 2.2.19"), # sufficient version ("gpg2", "gpg (GnuPG) 2.2.19"), # gpg2 command ], ) def test_find_gpg(cmd_name, version, tmp_path: pathlib.Path, mock_gnupghome, monkeypatch): TEMPLATE = '#!/bin/sh\necho "{version}"\n' with fs.working_dir(str(tmp_path)): for fname in (cmd_name, "gpgconf"): with open(fname, "w", encoding="utf-8") as f: f.write(TEMPLATE.format(version=version)) fs.set_executable(fname) monkeypatch.setenv("PATH", str(tmp_path)) if version == "undetectable" or version.endswith("1.3.4"): with pytest.raises(spack.util.gpg.SpackGPGError): spack.util.gpg.init(force=True) else: spack.util.gpg.init(force=True) assert spack.util.gpg.GPG is not None assert spack.util.gpg.GPGCONF is not None def test_no_gpg_in_path(tmp_path: pathlib.Path, mock_gnupghome, monkeypatch, mutable_config): monkeypatch.setenv("PATH", str(tmp_path)) bootstrap("disable") with pytest.raises(RuntimeError): spack.util.gpg.init(force=True) @pytest.mark.maybeslow def test_gpg(tmp_path: pathlib.Path, mutable_config, mock_gnupghome): # Verify a file with an empty keyring. with pytest.raises(ProcessError): gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) # Import the default key. gpg("init", "--from", mock_gpg_keys_path) # List the keys. # TODO: Test the output here. gpg("list", "--trusted") gpg("list", "--signing") # Verify the file now that the key has been trusted. gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) # Untrust the default key. gpg("untrust", "Spack testing") # Now that the key is untrusted, verification should fail. with pytest.raises(ProcessError): gpg("verify", os.path.join(mock_gpg_data_path, "content.txt")) # Create a file to test signing. test_path = tmp_path / "to-sign.txt" with open(str(test_path), "w+", encoding="utf-8") as fout: fout.write("Test content for signing.\n") # Signing without a private key should fail. with pytest.raises(RuntimeError) as exc_info: gpg("sign", str(test_path)) assert exc_info.value.args[0] == "no signing keys are available" # Create a key for use in the tests. keypath = tmp_path / "testing-1.key" gpg( "create", "--comment", "Spack testing key", "--export", str(keypath), "Spack testing 1", "spack@googlegroups.com", ) keyfp = spack.util.gpg.signing_keys()[0] # List the keys. # TODO: Test the output here. gpg("list") gpg("list", "--trusted") gpg("list", "--signing") # Signing with the default (only) key. gpg("sign", str(test_path)) # Verify the file we just verified. gpg("verify", str(test_path)) # Export the key for future use. export_path = tmp_path / "export.testing.key" gpg("export", str(export_path)) # Test exporting the private key private_export_path = tmp_path / "export-secret.testing.key" gpg("export", "--secret", str(private_export_path)) # Ensure we exported the right content! with open(str(private_export_path), "r", encoding="utf-8") as fd: content = fd.read() assert "BEGIN PGP PRIVATE KEY BLOCK" in content # and for the public key with open(str(export_path), "r", encoding="utf-8") as fd: content = fd.read() assert "BEGIN PGP PUBLIC KEY BLOCK" in content # Create a second key for use in the tests. gpg("create", "--comment", "Spack testing key", "Spack testing 2", "spack@googlegroups.com") # List the keys. # TODO: Test the output here. gpg("list", "--trusted") gpg("list", "--signing") test_path = tmp_path / "to-sign-2.txt" with open(str(test_path), "w+", encoding="utf-8") as fout: fout.write("Test content for signing.\n") # Signing with multiple signing keys is ambiguous. with pytest.raises(RuntimeError) as exc_info: gpg("sign", str(test_path)) assert exc_info.value.args[0] == "multiple signing keys are available; please choose one" # Signing with a specified key. gpg("sign", "--key", keyfp, str(test_path)) # Untrusting signing keys needs a flag. with pytest.raises(ProcessError): gpg("untrust", "Spack testing 1") # Untrust the key we created. gpg("untrust", "--signing", keyfp) # Verification should now fail. with pytest.raises(ProcessError): gpg("verify", str(test_path)) # Trust the exported key. gpg("trust", str(export_path)) # Verification should now succeed again. gpg("verify", str(test_path)) relative_keys_path = spack.binary_distribution.buildcache_relative_keys_path() # Publish the keys using a directory path test_path = tmp_path / "dir_cache" os.makedirs(f"{test_path}") gpg("publish", "--rebuild-index", "-d", str(test_path)) assert os.path.exists(f"{test_path}/{relative_keys_path}/keys.manifest.json") # Publish the keys using a mirror url test_path = tmp_path / "url_cache" os.makedirs(f"{test_path}") test_url = test_path.as_uri() gpg("publish", "--rebuild-index", "--mirror-url", test_url) assert os.path.exists(f"{test_path}/{relative_keys_path}/keys.manifest.json") # Publish the keys using a mirror name test_path = tmp_path / "named_cache" os.makedirs(f"{test_path}") mirror_url = test_path.as_uri() mirror("add", "gpg", mirror_url) gpg("publish", "--rebuild-index", "-m", "gpg") assert os.path.exists(f"{test_path}/{relative_keys_path}/keys.manifest.json") ================================================ FILE: lib/spack/spack/test/cmd/graph.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest from spack.main import SpackCommand, SpackCommandError graph = SpackCommand("graph") @pytest.mark.db @pytest.mark.usefixtures("mock_packages", "database") def test_graph_ascii(): """Tests spack graph --ascii""" graph("--ascii", "dt-diamond") @pytest.mark.db @pytest.mark.usefixtures("mock_packages", "database") def test_graph_dot(): """Tests spack graph --dot""" graph("--dot", "dt-diamond") @pytest.mark.db @pytest.mark.usefixtures("mock_packages", "database") def test_graph_static(): """Tests spack graph --static""" graph("--static", "dt-diamond") @pytest.mark.db @pytest.mark.usefixtures("mock_packages", "database") def test_graph_installed(): """Tests spack graph --installed""" graph("--installed") with pytest.raises(SpackCommandError): graph("--installed", "dt-diamond") @pytest.mark.db @pytest.mark.usefixtures("mock_packages", "database") def test_graph_deptype(): """Tests spack graph --deptype""" graph("--deptype", "all", "dt-diamond") def test_graph_no_specs(): """Tests spack graph with no arguments""" with pytest.raises(SpackCommandError): graph() ================================================ FILE: lib/spack/spack/test/cmd/help.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.main import SpackCommand def test_reuse_after_help(): """Test `spack help` can be called twice with the same SpackCommand.""" help_cmd = SpackCommand("help") help_cmd() help_cmd() def test_help(): """Sanity check the help command to make sure it works.""" help_cmd = SpackCommand("help") out = help_cmd() assert "Common spack commands:" in out assert "Options:" in out def test_help_all(): """Test the spack help --all flag""" help_cmd = SpackCommand("help") out = help_cmd("--all") assert "Commands:" in out assert "Options:" in out def test_help_spec(): """Test the spack help --spec flag""" help_cmd = SpackCommand("help") out = help_cmd("--spec") assert "spec expression syntax:" in out def test_help_subcommand(): """Test the spack help subcommand argument""" help_cmd = SpackCommand("help") out = help_cmd("help") assert "get help on spack and its commands" in out ================================================ FILE: lib/spack/spack/test/cmd/info.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import pytest from spack.main import SpackCommand, SpackCommandError from spack.repo import UnknownPackageError pytestmark = [pytest.mark.usefixtures("mock_packages")] info = SpackCommand("info") def test_package_suggestion(): with pytest.raises(UnknownPackageError) as exc_info: info("vtk") assert "Did you mean one of the following packages?" in str(exc_info.value) def test_deprecated_option_warns(): info("--variants-by-name", "vtk-m") assert "--variants-by-name is deprecated" in info.output # no specs, more than one spec @pytest.mark.parametrize("args", [[], ["vtk-m", "zmpi"]]) def test_info_failures(args): with pytest.raises(SpackCommandError): info(*args) def test_info_noversion(): """Check that a mock package with no versions outputs None.""" output = info("noversion") assert "Preferred\n None" not in output assert "Safe\n None" not in output assert "Deprecated\n None" not in output @pytest.mark.parametrize( "pkg_query,expected", [("zlib", "False"), ("find-externals1", "True (version)")] ) def test_is_externally_detectable(pkg_query, expected): output = info("--detectable", pkg_query) assert f"Externally Detectable:\n {expected}" in output @pytest.mark.parametrize( "pkg_query", ["vtk-m", "gcc"], # This should ensure --test's c_names processing loop covered ) @pytest.mark.parametrize("extra_args", [[], ["--by-name"]]) def test_info_fields(pkg_query, extra_args): expected_fields = ( "Description:", "Homepage:", "Externally Detectable:", "Safe versions:", "Variants:", "Installation Phases:", "Virtual Packages:", "Tags:", "Licenses:", ) output = info("--all", *extra_args, pkg_query) assert all(field in output for field in expected_fields) @pytest.mark.parametrize( "args,in_output,not_in_output", [ # no variants (["package-base-extendee"], [r"Variants:\n\s*None"], []), # test that long lines wrap around ( ["long-boost-dependency+longdep"], [ r"boost\+atomic\+chrono\+date_time\+filesystem\+graph\+iostreams\+locale\n" r"\s*build, link" ], [], ), ( ["long-boost-dependency~longdep"], [], [ r"boost\+atomic\+chrono\+date_time\+filesystem\+graph\+iostreams\+locale\n" r"\s*build, link" ], ), # conditional licenses change output (["licenses-1 +foo"], ["MIT"], ["Apache-2.0"]), (["licenses-1 ~foo"], ["Apache-2.0"], ["MIT"]), # filtering bowtie versions (["bowtie"], ["1.4.0", "1.3.0", "1.2.2", "1.2.0"], []), (["bowtie@1.2:"], ["1.4.0", "1.3.0", "1.2.2", "1.2.0"], []), (["bowtie@1.3:"], ["1.4.0", "1.3.0"], ["1.2.2", "1.2.0"]), (["bowtie@1.2"], ["1.2.2", "1.2.0"], ["1.3.0"]), # 1.4.0 still shown as preferred # many dependencies with suggestion to filter ( ["many-conditional-deps"], ["consider this for a simpler view:\n spack info many-conditional-deps~cuda~rocm"], [], ), ( ["many-conditional-deps ~cuda"], ["consider this for a simpler view:\n spack info many-conditional-deps~cuda~rocm"], [], ), ( ["many-conditional-deps ~rocm"], ["consider this for a simpler view:\n spack info many-conditional-deps~cuda~rocm"], [], ), (["many-conditional-deps ~cuda ~rocm"], [], ["consider this for a simpler view:"]), # Ensure spack info knows that build_system is a single value variant ( ["dual-cmake-autotools"], [r"when\s*build_system=cmake", r"when\s*build_system=autotools"], [], ), ( ["dual-cmake-autotools build_system=cmake"], [r"when\s*build_system=cmake"], [r"when\s*build_system=autotools"], ), # Ensure that gemerator=make implies build_system=cmake and therefore no autotools ( ["dual-cmake-autotools generator=make"], [r"when\s*build_system=cmake"], [r"when\s*build_system=autotools"], ), ( ["optional-dep-test"], [ r"when \^pkg-g", r"when \%intel", r"when \%intel\@64\.1", r"when \%clang@34\:40", r"when \^pkg\-f", ], [], ), ], ) @pytest.mark.parametrize("by_name", [True, False]) def test_info_output(by_name, args, in_output, not_in_output, monkeypatch): monkeypatch.setenv("COLUMNS", "80") by_name_arg = ["--by-name"] if by_name else ["--by-when"] output = info(*(by_name_arg + args)) for io in in_output: assert re.search(io, output) for nio in not_in_output: assert not re.search(nio, output) ================================================ FILE: lib/spack/spack/test/cmd/init_py_functions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.config import spack.environment as ev import spack.error import spack.solver.asp as asp import spack.store from spack.cmd import ( CommandNameError, PythonNameError, cmd_name, matching_specs_from_env, parse_specs, python_name, require_cmd_name, require_python_name, ) def test_require_python_name(): """Python module names should not contain dashes---ensure that require_python_name() raises the appropriate exception if one is detected. """ require_python_name("okey_dokey") with pytest.raises(PythonNameError): require_python_name("okey-dokey") require_python_name(python_name("okey-dokey")) def test_require_cmd_name(): """By convention, Spack command names should contain dashes rather than underscores---ensure that require_cmd_name() raises the appropriate exception if underscores are detected. """ require_cmd_name("okey-dokey") with pytest.raises(CommandNameError): require_cmd_name("okey_dokey") require_cmd_name(cmd_name("okey_dokey")) @pytest.mark.parametrize( "unify,spec_strs,error", [ # single spec (True, ["zmpi"], None), (False, ["mpileaks"], None), # multiple specs, some from hash some from file (True, ["zmpi", "mpileaks^zmpi", "libelf"], None), (True, ["mpileaks^zmpi", "mpileaks^mpich", "libelf"], spack.error.SpecError), (False, ["mpileaks^zmpi", "mpileaks^mpich", "libelf"], None), ], ) def test_special_cases_concretization_parse_specs( unify, spec_strs, error, monkeypatch, mutable_config, mutable_database, tmp_path: pathlib.Path ): """Test that special cases in parse_specs(concretize=True) bypass solver""" # monkeypatch to ensure we do not call the actual concretizer def _fail(*args, **kwargs): assert False monkeypatch.setattr(asp.SpackSolverSetup, "setup", _fail) spack.config.set("concretizer:unify", unify) args = [f"/{spack.store.STORE.db.query(s)[0].dag_hash()}" for s in spec_strs] if len(args) > 1: # We convert the last one to a specfile input filename = tmp_path / "spec.json" spec = parse_specs(args[-1], concretize=True)[0] with open(filename, "w", encoding="utf-8") as f: spec.to_json(f) args[-1] = str(filename) if error: with pytest.raises(error): parse_specs(args, concretize=True) else: # assertion error from monkeypatch above if test fails parse_specs(args, concretize=True) @pytest.mark.parametrize( "unify,spec_strs,error", [ # single spec (True, ["zmpi"], None), (False, ["mpileaks"], None), # multiple specs, some from hash some from file (True, ["zmpi", "mpileaks^zmpi", "libelf"], None), (True, ["mpileaks^zmpi", "mpileaks^mpich", "libelf"], spack.error.SpecError), (False, ["mpileaks^zmpi", "mpileaks^mpich", "libelf"], None), ], ) def test_special_cases_concretization_matching_specs_from_env( unify, spec_strs, error, monkeypatch, mutable_config, mutable_database, tmp_path: pathlib.Path, mutable_mock_env_path, ): """Test that special cases in parse_specs(concretize=True) bypass solver""" # monkeypatch to ensure we do not call the actual concretizer def _fail(*args, **kwargs): assert False monkeypatch.setattr(asp.SpackSolverSetup, "setup", _fail) spack.config.set("concretizer:unify", unify) ev.create("test") env = ev.read("test") args = [f"/{spack.store.STORE.db.query(s)[0].dag_hash()}" for s in spec_strs] if len(args) > 1: # We convert the last one to a specfile input filename = tmp_path / "spec.json" spec = parse_specs(args[-1], concretize=True)[0] with open(filename, "w", encoding="utf-8") as f: spec.to_json(f) args[-1] = str(filename) with env: specs = parse_specs(args, concretize=False) if error: with pytest.raises(error): matching_specs_from_env(specs) else: # assertion error from monkeypatch above if test fails matching_specs_from_env(specs) ================================================ FILE: lib/spack/spack/test/cmd/install.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import builtins import filecmp import gzip import itertools import os import pathlib import re import sys import time import pytest import spack.build_environment import spack.cmd.common.arguments import spack.cmd.install import spack.concretize import spack.config import spack.environment as ev import spack.error import spack.hash_types as ht import spack.installer import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.package_base import spack.store from spack.error import SpackError, SpecSyntaxError from spack.installer import PackageInstaller from spack.main import SpackCommand from spack.spec import Spec install = SpackCommand("install") env = SpackCommand("env") add = SpackCommand("add") mirror = SpackCommand("mirror") uninstall = SpackCommand("uninstall") buildcache = SpackCommand("buildcache") find = SpackCommand("find") @pytest.fixture() def noop_install(monkeypatch): def noop(*args, **kwargs): pass monkeypatch.setattr(spack.installer.PackageInstaller, "install", noop) def test_install_package_and_dependency( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant, ): log = "test" with fs.working_dir(str(tmp_path)): install("--fake", "--log-format=junit", f"--log-file={log}", "libdwarf") files = list(tmp_path.iterdir()) filename = tmp_path / f"{log}.xml" assert filename in files content = filename.read_text() assert 'tests="5"' in content assert 'failures="0"' in content assert 'errors="0"' in content def _check_runtests_none(pkg): assert not pkg.run_tests def _check_runtests_dttop(pkg): assert pkg.run_tests == (pkg.name == "dttop") def _check_runtests_all(pkg): assert pkg.run_tests @pytest.mark.disable_clean_stage_check def test_install_runtests_notests(monkeypatch, mock_packages, install_mockery): monkeypatch.setattr(spack.package_base.PackageBase, "_unit_test_check", _check_runtests_none) install("-v", "dttop") @pytest.mark.disable_clean_stage_check def test_install_runtests_root(monkeypatch, mock_packages, install_mockery): monkeypatch.setattr(spack.package_base.PackageBase, "_unit_test_check", _check_runtests_dttop) install("--test=root", "dttop") @pytest.mark.disable_clean_stage_check def test_install_runtests_all(monkeypatch, mock_packages, install_mockery): monkeypatch.setattr(spack.package_base.PackageBase, "_unit_test_check", _check_runtests_all) install("--test=all", "pkg-a") def test_install_package_already_installed( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant, ): with fs.working_dir(str(tmp_path)): install("--fake", "libdwarf") install("--fake", "--log-format=junit", "--log-file=test.xml", "libdwarf") files = list(tmp_path.iterdir()) filename = tmp_path / "test.xml" assert filename in files content = filename.read_text() assert 'tests="5"' in content assert 'failures="0"' in content assert 'errors="0"' in content skipped = [line for line in content.split("\n") if "skipped" in line] assert len(skipped) == 5 @pytest.mark.parametrize( "arguments,expected", [ ([], spack.config.get("config:dirty")), # default from config file (["--clean"], False), (["--dirty"], True), ], ) def test_install_dirty_flag(arguments, expected): parser = argparse.ArgumentParser() spack.cmd.install.setup_parser(parser) args = parser.parse_args(arguments) assert args.dirty == expected def test_package_output(install_mockery, mock_fetch): """ Ensure output printed from pkgs is captured by output redirection. """ # we can't use output capture here because it interferes with Spack's # logging. TODO: see whether we can get multiple log_outputs to work # when nested AND in pytest spec = spack.concretize.concretize_one("printing-package") pkg = spec.package PackageInstaller([pkg], explicit=True, verbose=True, tests=sys.platform != "win32").install() with gzip.open(pkg.install_log_path, "rt") as f: out = f.read() # make sure that output from the actual package file appears in the # right place in the build log. assert "BEFORE INSTALL" in out assert "AFTER INSTALL" in out if not sys.platform == "win32": # Check that install-time test log contains check and installcheck output log_path = pkg.tester.archived_install_test_log assert os.path.exists(log_path), f"Missing install-time test log at {log_path}" with open(log_path, "r", encoding="utf-8") as f: test_log_contents = f.read() assert "PRINTING PACKAGE CHECK" in test_log_contents assert "PRINTING PACKAGE INSTALLCHECK" in test_log_contents @pytest.mark.disable_clean_stage_check def test_install_output_on_build_error(mock_packages, mock_archive, mock_fetch, install_mockery): """ This test used to assume receiving full output, but since we've updated spack to generate logs on the level of phases, it will only return the last phase, install. """ # capfd interferes with Spack's capturing out = install("-v", "build-error", fail_on_error=False) assert "Installing build-error" in out @pytest.mark.disable_clean_stage_check def test_install_output_on_python_error(mock_packages, mock_archive, mock_fetch, install_mockery): out = install("failing-build", fail_on_error=False) assert isinstance(install.error, spack.build_environment.ChildError) assert install.error.name == "InstallError" assert 'raise InstallError("Expected failure.")' in out @pytest.mark.disable_clean_stage_check def test_install_with_source( mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant ): """Verify that source has been copied into place.""" install("--source", "--keep-stage", "trivial-install-test-package") spec = spack.concretize.concretize_one("trivial-install-test-package") src = os.path.join(spec.prefix.share, "trivial-install-test-package", "src") assert filecmp.cmp( os.path.join(mock_archive.path, "configure"), os.path.join(src, "configure") ) def test_install_env_variables( mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant ): spec = spack.concretize.concretize_one("pkg-c") install("pkg-c") assert os.path.isfile(spec.package.install_env_path) @pytest.mark.disable_clean_stage_check def test_show_log_on_error(mock_packages, mock_archive, mock_fetch, install_mockery): """ Make sure --show-log-on-error works. """ out = install("--show-log-on-error", "build-error", fail_on_error=False) assert isinstance(install.error, spack.build_environment.ChildError) assert install.error.pkg.name == "build-error" assert "Installing build-error" in out assert "See build log for details:" in out def test_install_overwrite( mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant ): """Tests installing a spec, and then re-installing it in the same prefix.""" spec = spack.concretize.concretize_one("pkg-c") install("pkg-c") # Ignore manifest and install times manifest = os.path.join( spec.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name, ) ignores = [manifest, spec.package.times_log_path] assert os.path.exists(spec.prefix) expected_md5 = fs.hash_directory(spec.prefix, ignore=ignores) # Modify the first installation to be sure the content is not the same # as the one after we reinstalled with open(os.path.join(spec.prefix, "only_in_old"), "w", encoding="utf-8") as f: f.write("This content is here to differentiate installations.") bad_md5 = fs.hash_directory(spec.prefix, ignore=ignores) assert bad_md5 != expected_md5 install("--overwrite", "-y", "pkg-c") assert os.path.exists(spec.prefix) assert fs.hash_directory(spec.prefix, ignore=ignores) == expected_md5 assert fs.hash_directory(spec.prefix, ignore=ignores) != bad_md5 def test_install_overwrite_not_installed( mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant ): """Tests that overwrite doesn't fail if the package is not installed""" spec = spack.concretize.concretize_one("pkg-c") assert not os.path.exists(spec.prefix) install("--overwrite", "-y", "pkg-c") assert os.path.exists(spec.prefix) def test_install_commit(mock_git_version_info, install_mockery, mock_packages, monkeypatch): """Test installing a git package from a commit. This ensures Spack associates commit versions with their packages in time to do version lookups. Details of version lookup tested elsewhere. """ repo_path, filename, commits = mock_git_version_info file_url = pathlib.Path(repo_path).as_uri() monkeypatch.setattr(spack.package_base.PackageBase, "git", file_url, raising=False) # Use the earliest commit in the repository spec = spack.concretize.concretize_one(f"git-test-commit@{commits[-1]}") PackageInstaller([spec.package], explicit=True).install() # Ensure first commit file contents were written installed = os.listdir(spec.prefix.bin) assert filename in installed with open(spec.prefix.bin.join(filename), "r", encoding="utf-8") as f: content = f.read().strip() assert content == "[0]" # contents are weird for another test def test_install_overwrite_multiple( mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant ): # Try to install a spec and then to reinstall it. libdwarf = spack.concretize.concretize_one("libdwarf") cmake = spack.concretize.concretize_one("cmake") install("--fake", "libdwarf") install("--fake", "cmake") ld_manifest = os.path.join( libdwarf.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name, ) ld_ignores = [ld_manifest, libdwarf.package.times_log_path] assert os.path.exists(libdwarf.prefix) expected_libdwarf_md5 = fs.hash_directory(libdwarf.prefix, ignore=ld_ignores) cm_manifest = os.path.join( cmake.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name, ) cm_ignores = [cm_manifest, cmake.package.times_log_path] assert os.path.exists(cmake.prefix) expected_cmake_md5 = fs.hash_directory(cmake.prefix, ignore=cm_ignores) # Modify the first installation to be sure the content is not the same # as the one after we reinstalled with open(os.path.join(libdwarf.prefix, "only_in_old"), "w", encoding="utf-8") as f: f.write("This content is here to differentiate installations.") with open(os.path.join(cmake.prefix, "only_in_old"), "w", encoding="utf-8") as f: f.write("This content is here to differentiate installations.") bad_libdwarf_md5 = fs.hash_directory(libdwarf.prefix, ignore=ld_ignores) bad_cmake_md5 = fs.hash_directory(cmake.prefix, ignore=cm_ignores) assert bad_libdwarf_md5 != expected_libdwarf_md5 assert bad_cmake_md5 != expected_cmake_md5 install("--fake", "--overwrite", "-y", "libdwarf", "cmake") assert os.path.exists(libdwarf.prefix) assert os.path.exists(cmake.prefix) ld_hash = fs.hash_directory(libdwarf.prefix, ignore=ld_ignores) cm_hash = fs.hash_directory(cmake.prefix, ignore=cm_ignores) assert ld_hash == expected_libdwarf_md5 assert cm_hash == expected_cmake_md5 assert ld_hash != bad_libdwarf_md5 assert cm_hash != bad_cmake_md5 @pytest.mark.usefixtures("mock_packages", "mock_archive", "mock_fetch", "install_mockery") def test_install_conflicts(conflict_spec): # Make sure that spec with conflicts raises a SpackError with pytest.raises(SpackError): install(conflict_spec) @pytest.mark.usefixtures("mock_packages", "mock_archive", "mock_fetch", "install_mockery") def test_install_invalid_spec(): # Make sure that invalid specs raise a SpackError with pytest.raises(SpecSyntaxError, match="unexpected characters"): install("conflict%~") @pytest.mark.disable_clean_stage_check @pytest.mark.usefixtures("mock_packages", "mock_archive", "mock_fetch", "install_mockery") @pytest.mark.parametrize( "exc_typename,msg", [("RuntimeError", "something weird happened"), ("ValueError", "spec is not concrete")], ) def test_junit_output_with_failures(tmp_path: pathlib.Path, exc_typename, msg, installer_variant): with fs.working_dir(str(tmp_path)): install( "--verbose", "--log-format=junit", "--log-file=test.xml", "raiser", "exc_type={0}".format(exc_typename), 'msg="{0}"'.format(msg), fail_on_error=False, ) # New installer considers Python exceptions ordinary build failures. if installer_variant == "old": assert isinstance(install.error, spack.build_environment.ChildError) assert install.error.name == exc_typename assert install.error.pkg.name == "raiser" files = list(tmp_path.iterdir()) filename = tmp_path / "test.xml" assert filename in files content = filename.read_text() # Count failures and errors correctly assert 'tests="1"' in content assert 'failures="1"' in content assert 'errors="0"' in content # Nothing should have succeeded assert 'tests="0"' not in content assert 'failures="0"' not in content # We want to have both stdout and stderr assert "" in content assert msg in content def _throw(task, exc_typename, exc_type, msg): # Self is a spack.installer.Task exc_type = getattr(builtins, exc_typename) exc = exc_type(msg) task.fail(exc) def _runtime_error(task, *args, **kwargs): _throw(task, "RuntimeError", spack.error.InstallError, "something weird happened") def _keyboard_error(task, *args, **kwargs): _throw(task, "KeyboardInterrupt", KeyboardInterrupt, "Ctrl-C strikes again") @pytest.mark.disable_clean_stage_check @pytest.mark.parametrize( "exc_typename,expected_exc,msg", [ ("RuntimeError", spack.error.InstallError, "something weird happened"), ("KeyboardInterrupt", KeyboardInterrupt, "Ctrl-C strikes again"), ], ) def test_junit_output_with_errors( exc_typename, expected_exc, msg, mock_packages, mock_archive, mock_fetch, install_mockery, tmp_path: pathlib.Path, monkeypatch, ): throw = _keyboard_error if expected_exc is KeyboardInterrupt else _runtime_error monkeypatch.setattr(spack.installer.BuildTask, "complete", throw) with fs.working_dir(str(tmp_path)): install( "--verbose", "--log-format=junit", "--log-file=test.xml", "trivial-install-test-dependent", fail_on_error=False, ) assert isinstance(install.error, expected_exc) files = list(tmp_path.iterdir()) filename = tmp_path / "test.xml" assert filename in files content = filename.read_text() # Only original error is reported, dependent # install is skipped and it is not an error. assert 'tests="0"' not in content assert 'failures="0"' in content assert 'errors="0"' not in content # Nothing should have succeeded assert 'errors="0"' not in content # We want to have both stdout and stderr assert "" in content assert f'error message="{msg}"' in content @pytest.fixture(params=["yaml", "json"]) def spec_format(request): return request.param @pytest.mark.usefixtures("noop_install", "mock_packages", "config") @pytest.mark.parametrize( "clispecs,filespecs", [ [[], ["mpi"]], [[], ["mpi", "boost"]], [["cmake"], ["mpi"]], [["cmake", "libelf"], []], [["cmake", "libelf"], ["mpi", "boost"]], ], ) def test_install_mix_cli_and_files(spec_format, clispecs, filespecs, tmp_path: pathlib.Path): args = clispecs for spec in filespecs: filepath = tmp_path / (spec + f".{spec_format}") args = [str(filepath)] + args s = spack.concretize.concretize_one(spec) with filepath.open("w") as f: s.to_yaml(f) if spec_format == "yaml" else s.to_json(f) install(*args, fail_on_error=False) assert install.returncode == 0 def test_extra_files_are_archived( mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant ): s = spack.concretize.concretize_one("archive-files") install("archive-files") archive_dir = os.path.join(spack.store.STORE.layout.metadata_path(s), "archived-files") config_log = os.path.join(archive_dir, mock_archive.expanded_archive_basedir, "config.log") assert os.path.exists(config_log) errors_txt = os.path.join(archive_dir, "errors.txt") assert os.path.exists(errors_txt) @pytest.mark.disable_clean_stage_check def test_cdash_report_concretization_error( tmp_path: pathlib.Path, mock_fetch, install_mockery, conflict_spec, installer_variant ): with fs.working_dir(str(tmp_path)): with pytest.raises(SpackError): install("--log-format=cdash", "--log-file=cdash_reports", conflict_spec) report_dir = tmp_path / "cdash_reports" assert report_dir in list(tmp_path.iterdir()) report_file = report_dir / "Update.xml" assert report_file in list(report_dir.iterdir()) content = report_file.read_text() assert "" in content # The message is different based on using the # new or the old concretizer expected_messages = ("Conflicts in concretized spec", "conflicts with") assert any(x in content for x in expected_messages) @pytest.mark.not_on_windows("Windows log_output logs phase header out of order") @pytest.mark.disable_clean_stage_check def test_cdash_upload_build_error( capfd, tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant ): with fs.working_dir(str(tmp_path)): with pytest.raises(SpackError): install( "--log-format=cdash", "--log-file=cdash_reports", "--cdash-upload-url=http://localhost/fakeurl/submit.php?project=Spack", "build-error", ) report_dir = tmp_path / "cdash_reports" assert report_dir in list(tmp_path.iterdir()) report_file = report_dir / "Build.xml" assert report_file in list(report_dir.iterdir()) content = report_file.read_text() assert "configure: error: in /path/to/some/file:" in content @pytest.mark.disable_clean_stage_check def test_cdash_upload_clean_build( tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant ): with fs.working_dir(str(tmp_path)): install("--log-file=cdash_reports", "--log-format=cdash", "pkg-c") report_dir = tmp_path / "cdash_reports" assert report_dir in list(tmp_path.iterdir()) report_file = report_dir / "Build.xml" assert report_file in list(report_dir.iterdir()) content = report_file.read_text() assert "" in content assert "" not in content @pytest.mark.disable_clean_stage_check def test_cdash_upload_extra_params( tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant ): with fs.working_dir(str(tmp_path)): install( "--log-file=cdash_reports", "--log-format=cdash", "--cdash-build=my_custom_build", "--cdash-site=my_custom_site", "--cdash-track=my_custom_track", "pkg-c", ) report_dir = tmp_path / "cdash_reports" assert report_dir in list(tmp_path.iterdir()) report_file = report_dir / "Build.xml" assert report_file in list(report_dir.iterdir()) content = report_file.read_text() assert 'Site BuildName="my_custom_build"' in content assert 'Name="my_custom_site"' in content assert "-my_custom_track" in content @pytest.mark.disable_clean_stage_check def test_cdash_buildstamp_param( tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant ): with fs.working_dir(str(tmp_path)): cdash_track = "some_mocked_track" buildstamp_format = f"%Y%m%d-%H%M-{cdash_track}" buildstamp = time.strftime(buildstamp_format, time.localtime(int(time.time()))) install( "--log-file=cdash_reports", "--log-format=cdash", f"--cdash-buildstamp={buildstamp}", "pkg-c", ) report_dir = tmp_path / "cdash_reports" assert report_dir in list(tmp_path.iterdir()) report_file = report_dir / "Build.xml" assert report_file in list(report_dir.iterdir()) content = report_file.read_text() assert buildstamp in content @pytest.mark.disable_clean_stage_check def test_cdash_install_from_spec_json( tmp_path: pathlib.Path, mock_fetch, install_mockery, mock_packages, mock_archive, installer_variant, ): with fs.working_dir(str(tmp_path)): spec_json_path = str(tmp_path / "spec.json") pkg_spec = spack.concretize.concretize_one("pkg-c") with open(spec_json_path, "w", encoding="utf-8") as fd: fd.write(pkg_spec.to_json(hash=ht.dag_hash)) install( "--log-format=cdash", "--log-file=cdash_reports", "--cdash-build=my_custom_build", "--cdash-site=my_custom_site", "--cdash-track=my_custom_track", spec_json_path, ) report_dir = tmp_path / "cdash_reports" assert report_dir in list(tmp_path.iterdir()) report_file = report_dir / "Configure.xml" assert report_file in list(report_dir.iterdir()) content = report_file.read_text() install_command_regex = re.compile( r"(.+)", re.MULTILINE | re.DOTALL ) m = install_command_regex.search(content) assert m install_command = m.group(1) assert "pkg-c@" in install_command @pytest.mark.disable_clean_stage_check def test_build_error_output(capfd, mock_fetch, install_mockery): with pytest.raises(spack.build_environment.ChildError) as e: install("build-error") assert "configure: error: in /path/to/some/file:" in install.output assert "configure: error: in /path/to/some/file:" in e.value.long_message assert "configure: error: cannot run C compiled programs." in install.output assert "configure: error: cannot run C compiled programs." in e.value.long_message @pytest.mark.disable_clean_stage_check def test_build_warning_output(mock_fetch, install_mockery): with pytest.raises(spack.build_environment.ChildError) as e: install("build-warnings") assert "WARNING: ALL CAPITAL WARNING!" in install.output assert "WARNING: ALL CAPITAL WARNING!" in e.value.long_message assert "foo.c:89: warning: some weird warning!" in install.output assert "foo.c:89: warning: some weird warning!" in e.value.long_message def test_cache_only_fails(mock_fetch, install_mockery): # libelf from cache fails to install, which automatically removes the # the libdwarf build task out = install("--cache-only", "libdwarf", fail_on_error=False) assert "Failed to install gcc-runtime" in out assert "Skipping build of libdwarf" in out assert "was not installed" in out # Check that failure prefix locks are still cached failed_packages = [ pkg_name for dag_hash, pkg_name in spack.store.STORE.failure_tracker.locker.locks.keys() ] assert "libelf" in failed_packages assert "libdwarf" in failed_packages def test_install_only_dependencies(mock_fetch, install_mockery, installer_variant): dep = spack.concretize.concretize_one("dependency-install") root = spack.concretize.concretize_one("dependent-install") install("--only", "dependencies", "dependent-install") assert os.path.exists(dep.prefix) assert not os.path.exists(root.prefix) def test_install_only_package(mock_fetch, install_mockery): msg = "" try: install("--only", "package", "dependent-install") except spack.error.InstallError as e: msg = str(e) assert "Cannot proceed with dependent-install" in msg assert "1 uninstalled dependency" in msg def test_install_deps_then_package(mock_fetch, install_mockery, installer_variant): dep = spack.concretize.concretize_one("dependency-install") root = spack.concretize.concretize_one("dependent-install") install("--only", "dependencies", "dependent-install") assert os.path.exists(dep.prefix) assert not os.path.exists(root.prefix) install("--only", "package", "dependent-install") assert os.path.exists(root.prefix) # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") @pytest.mark.regression("12002") def test_install_only_dependencies_in_env( mutable_mock_env_path, mock_fetch, install_mockery, installer_variant ): env("create", "test") with ev.read("test"): dep = spack.concretize.concretize_one("dependency-install") root = spack.concretize.concretize_one("dependent-install") install("-v", "--only", "dependencies", "--add", "dependent-install") assert os.path.exists(dep.prefix) assert not os.path.exists(root.prefix) # Unit tests should not be affected by the user's managed environments @pytest.mark.regression("12002") def test_install_only_dependencies_of_all_in_env( mutable_mock_env_path, mock_fetch, install_mockery, installer_variant ): env("create", "--without-view", "test") with ev.read("test"): roots = [ spack.concretize.concretize_one("dependent-install@1.0"), spack.concretize.concretize_one("dependent-install@2.0"), ] add("dependent-install@1.0") add("dependent-install@2.0") install("--only", "dependencies") for root in roots: assert not os.path.exists(root.prefix) for dep in root.traverse(root=False): assert os.path.exists(dep.prefix) # Unit tests should not be affected by the user's managed environments def test_install_no_add_in_env( tmp_path: pathlib.Path, mutable_mock_env_path, mock_fetch, install_mockery, installer_variant ): # To test behavior of --add option, we create the following environment: # # mpileaks # ^callpath # ^dyninst # ^libelf@0.8.13 # or latest, really # ^libdwarf # ^mpich # libelf@0.8.10 # pkg-a~bvv # ^pkg-b # pkg-a # ^pkg-b e = ev.create("test", with_view=False) e.add("mpileaks") e.add("libelf@0.8.10") # so env has both root and dep libelf specs e.add("pkg-a") e.add("pkg-a ~bvv") e.concretize() e.write() env_specs = e.all_specs() a_spec = None b_spec = None mpi_spec = None # First find and remember some target concrete specs in the environment for e_spec in env_specs: if e_spec.satisfies(Spec("pkg-a ~bvv")): a_spec = e_spec elif e_spec.name == "pkg-b": b_spec = e_spec elif e_spec.satisfies(Spec("mpi")): mpi_spec = e_spec assert a_spec assert a_spec.concrete assert b_spec assert b_spec.concrete assert b_spec not in e.roots() assert mpi_spec assert mpi_spec.concrete # Activate the environment with e: # Assert using --no-add with a spec not in the env fails inst_out = install("--fake", "--no-add", "boost", fail_on_error=False) assert "Specs can be added to the environment with 'spack add " in inst_out # Without --add, ensure that two packages "a" get installed inst_out = install("--fake", "pkg-a") assert len([x for x in e.all_specs() if x.installed and x.name == "pkg-a"]) == 2 # Install an unambiguous dependency spec (that already exists as a dep # in the environment) and make sure it gets installed (w/ deps), # but is not added to the environment. install("dyninst") find_output = find("-l") assert "dyninst" in find_output assert "libdwarf" in find_output assert "libelf" in find_output assert "callpath" not in find_output post_install_specs = e.all_specs() assert all([s in env_specs for s in post_install_specs]) # Make sure we can install a concrete dependency spec from a spec.json # file on disk, and the spec is installed but not added as a root mpi_spec_json_path = tmp_path / f"{mpi_spec.name}.json" with open(mpi_spec_json_path, "w", encoding="utf-8") as fd: fd.write(mpi_spec.to_json(hash=ht.dag_hash)) install(str(mpi_spec_json_path)) assert mpi_spec not in e.roots() find_output = find("-l") assert mpi_spec.name in find_output # Install an unambiguous depependency spec (that already exists as a # dep in the environment) with --add and make sure it is added as a # root of the environment as well as installed. assert b_spec not in e.roots() install("--fake", "--add", "pkg-b") assert b_spec in e.roots() assert b_spec not in e.uninstalled_specs() # Install a novel spec with --add and make sure it is added as a root # and installed. install("--fake", "--add", "bowtie") assert any([s.name == "bowtie" for s in e.roots()]) assert not any([s.name == "bowtie" for s in e.uninstalled_specs()]) def test_install_help_does_not_show_cdash_options(): """ Make sure `spack install --help` does not describe CDash arguments """ assert "CDash URL" not in install("--help") def test_install_help_cdash(): """Make sure `spack install --help-cdash` describes CDash arguments""" install_cmd = SpackCommand("install") out = install_cmd("--help-cdash") assert "CDash URL" in out @pytest.mark.disable_clean_stage_check def test_cdash_auth_token( tmp_path: pathlib.Path, mock_fetch, install_mockery, monkeypatch, installer_variant ): with fs.working_dir(str(tmp_path)): monkeypatch.setenv("SPACK_CDASH_AUTH_TOKEN", "asdf") out = install("--fake", "-v", "--log-file=cdash_reports", "--log-format=cdash", "pkg-a") assert "Using CDash auth token from environment" in out @pytest.mark.not_on_windows("Windows log_output logs phase header out of order") @pytest.mark.disable_clean_stage_check def test_cdash_configure_warning( tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant ): with fs.working_dir(str(tmp_path)): # Test would fail if install raised an error. # Ensure that even on non-x86_64 architectures, there are no # dependencies installed spec = spack.concretize.concretize_one("configure-warning") spec.clear_dependencies() specfile = "./spec.json" with open(specfile, "w", encoding="utf-8") as f: f.write(spec.to_json()) install("--log-file=cdash_reports", "--log-format=cdash", specfile) # Verify Configure.xml exists with expected contents. report_dir = tmp_path / "cdash_reports" assert report_dir.exists() report_file = report_dir / "Configure.xml" assert report_file.exists() content = report_file.read_text() assert "foo: No such file or directory" in content def test_install_fails_no_args(tmp_path: pathlib.Path): # ensure no spack.yaml in directory with fs.working_dir(str(tmp_path)): output = install(fail_on_error=False) # check we got the short version of the error message with no spack.yaml assert "requires a package argument or active environment" in output assert "spack env activate ." not in output assert "using the `spack.yaml` in this directory" not in output def test_install_fails_no_args_suggests_env_activation(tmp_path: pathlib.Path): # ensure spack.yaml in directory (tmp_path / "spack.yaml").touch() with fs.working_dir(str(tmp_path)): output = install(fail_on_error=False) # check we got the long version of the error message with spack.yaml assert "requires a package argument or active environment" in output assert "spack env activate ." in output assert "using the `spack.yaml` in this directory" in output # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") def test_install_env_with_tests_all( mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant ): env("create", "test") with ev.read("test"): test_dep = spack.concretize.concretize_one("test-dependency") add("depb") install("--fake", "--test", "all") assert os.path.exists(test_dep.prefix) # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") def test_install_env_with_tests_root( mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant ): env("create", "test") with ev.read("test"): test_dep = spack.concretize.concretize_one("test-dependency") add("depb") install("--fake", "--test", "root") assert not os.path.exists(test_dep.prefix) # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701") def test_install_empty_env( mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant ): env_name = "empty" env("create", env_name) with ev.read(env_name): out = install(fail_on_error=False) assert env_name in out assert "environment" in out assert "no specs to install" in out @pytest.mark.not_on_windows("Windows logger I/O operation on closed file when install fails") @pytest.mark.disable_clean_stage_check @pytest.mark.parametrize( "name,method", [ ("test-build-callbacks", "undefined-build-test"), ("test-install-callbacks", "undefined-install-test"), ], ) def test_installation_fail_tests(install_mockery, mock_fetch, name, method): """Confirm build-time tests with unknown methods fail.""" output = install("--test=root", "--no-cache", name, fail_on_error=False) # Check that there is a single test failure reported assert output.count("TestFailure: 1 test failed") == 1 # Check that the method appears twice: no attribute error and in message assert output.count(method) == 2 assert output.count("method not implemented") == 1 # Check that the path to the test log file is also output assert "See test log for details" in output # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("Buildcache not supported on windows") def test_install_use_buildcache( mutable_mock_env_path, mock_packages, mock_fetch, mock_archive, mock_binary_index, tmp_path: pathlib.Path, install_mockery, ): """ Make sure installing with use-buildcache behaves correctly. """ package_name = "dependent-install" dependency_name = "dependency-install" def validate(mode, out, pkg): def assert_auto(pkg, out): assert "==> Extracting {0}".format(pkg) in out def assert_only(pkg, out): assert "==> Extracting {0}".format(pkg) in out def assert_never(pkg, out): assert "==> {0}: Executing phase: 'install'".format(pkg) in out if mode == "auto": assert_auto(pkg, out) elif mode == "only": assert_only(pkg, out) else: assert_never(pkg, out) def install_use_buildcache(opt): out = install( "--no-check-signature", "--use-buildcache", opt, package_name, fail_on_error=True ) pkg_opt, dep_opt = spack.cmd.common.arguments.use_buildcache(opt) validate(dep_opt, out, dependency_name) validate(pkg_opt, out, package_name) # Clean up installed packages uninstall("-y", "-a") # Setup the mirror # Create a temp mirror directory for buildcache usage mirror_dir = tmp_path / "mirror_dir" mirror_url = mirror_dir.as_uri() # Populate the buildcache install(package_name) buildcache("push", "-u", "-f", str(mirror_dir), package_name, dependency_name) # Uninstall the all of the packages for clean slate uninstall("-y", "-a") # Configure the mirror where we put that buildcache w/ the compiler mirror("add", "test-mirror", mirror_url) # Install using the matrix of possible combinations with --use-buildcache for pkg, deps in itertools.product(["auto", "only", "never"], repeat=2): tty.debug( "Testing `spack install --use-buildcache package:{0},dependencies:{1}`".format( pkg, deps ) ) install_use_buildcache("package:{0},dependencies:{1}".format(pkg, deps)) install_use_buildcache("dependencies:{0},package:{1}".format(deps, pkg)) # Install using a default override option # Alternative to --cache-only (always) or --no-cache (never) for opt in ["auto", "only", "never"]: install_use_buildcache(opt) @pytest.mark.not_on_windows("Windows logger I/O operation on closed file when install fails") @pytest.mark.regression("34006") @pytest.mark.disable_clean_stage_check def test_padded_install_runtests_root(install_mockery, mock_fetch): spack.config.set("config:install_tree:padded_length", 255) output = install( "--verbose", "--test=root", "--no-cache", "test-build-callbacks", fail_on_error=False ) assert "method not implemented [undefined-build-test]" in output @pytest.mark.regression("35337") def test_report_filename_for_cdash(install_mockery, mock_fetch): """Test that the temporary file used to write the XML for CDash is not the upload URL""" parser = argparse.ArgumentParser() spack.cmd.install.setup_parser(parser) args = parser.parse_args( ["--cdash-upload-url", "https://blahblah/submit.php?project=debugging", "pkg-a"] ) specs = spack.cmd.install.concrete_specs_from_cli(args, {}) filename = spack.cmd.install.report_filename(args, specs) assert filename != "https://blahblah/submit.php?project=debugging" def test_setting_concurrent_packages_flag(mutable_config): """Ensure that the number of concurrent packages is properly set from the command-line flag""" install = SpackCommand("install") install("--concurrent-packages", "8", fail_on_error=False) assert spack.config.get("config:concurrent_packages", scope="command_line") == 8 def test_invalid_concurrent_packages_flag(mutable_config): """Test that an invalid value for --concurrent-packages CLI flag raises a ValueError""" install = SpackCommand("install") with pytest.raises(ValueError, match="expected a positive integer"): install("--concurrent-packages", "-2") @pytest.mark.skipif(sys.platform == "win32", reason="Feature disabled on windows due to locking") def test_concurrent_packages_set_in_config(mutable_config, mock_packages): """Ensure that the number of concurrent packages is properly set from adding to config""" spack.config.set("config:concurrent_packages", 3) spec = spack.concretize.concretize_one("pkg-a") installer = spack.installer.PackageInstaller([spec.package]) assert installer.concurrent_packages == 3 ================================================ FILE: lib/spack/spack/test/cmd/is_git_repo.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import sys import pytest import spack.cmd import spack.fetch_strategy from spack.llnl.util.filesystem import mkdirp, working_dir from spack.version import ver @pytest.fixture(scope="function") def git_tmp_worktree(git, tmp_path: pathlib.Path, mock_git_version_info): """Create new worktree in a temporary folder and monkeypatch spack.paths.prefix to point to it. """ # We need `git worktree remove` for this fixture, which was added in 2.17.0. # See https://github.com/git/git/commit/cc73385cf6c5c229458775bc92e7dbbe24d11611 git_version = spack.fetch_strategy.GitFetchStrategy.version_from_git(git) if git_version < ver("2.17.0"): pytest.skip("git_tmp_worktree requires git v2.17.0") with working_dir(mock_git_version_info[0]): # TODO: This is fragile and should be high priority for # follow up fixes. 27021 # Path length is occasionally too long on Windows # the following reduces the path length to acceptable levels if sys.platform == "win32": long_pth = str(tmp_path).split(os.path.sep) tmp_worktree = os.path.sep.join(long_pth[:-1]) else: tmp_worktree = str(tmp_path) worktree_root = os.path.sep.join([tmp_worktree, "wrktree"]) mkdirp(worktree_root) git("worktree", "add", "--detach", worktree_root, "HEAD") yield worktree_root git("worktree", "remove", "--force", worktree_root) def test_is_git_repo_in_worktree(git_tmp_worktree): """Verify that spack.cmd.spack_is_git_repo() can identify a git repository in a worktree. """ assert spack.cmd.is_git_repo(git_tmp_worktree) def test_spack_is_git_repo_nongit(tmp_path: pathlib.Path, monkeypatch): """Verify that spack.cmd.spack_is_git_repo() correctly returns False if we are in a non-git directory. """ assert not spack.cmd.is_git_repo(str(tmp_path)) ================================================ FILE: lib/spack/spack/test/cmd/license.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import re import pytest import spack.paths from spack.llnl.util.filesystem import mkdirp, touch from spack.main import SpackCommand license = SpackCommand("license") pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_list_files(): files = license("list-files").strip().split("\n") assert all(f.startswith(spack.paths.prefix) for f in files) assert os.path.join(spack.paths.bin_path, "spack") in files assert os.path.abspath(__file__) in files def test_verify(tmp_path: pathlib.Path): source_dir = tmp_path / "lib" / "spack" / "spack" mkdirp(str(source_dir)) no_header = source_dir / "no_header.py" touch(str(no_header)) lgpl_header = source_dir / "lgpl_header.py" with lgpl_header.open("w") as f: f.write( """\ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: LGPL-2.1-only """ ) not_in_first_n_lines = source_dir / "not_in_first_n_lines.py" with not_in_first_n_lines.open("w") as f: f.write( """\ # # # # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ ) correct_header = source_dir / "correct_header.py" with correct_header.open("w") as f: f.write( """\ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ ) out = license("--root", str(tmp_path), "verify", fail_on_error=False) assert str(no_header) in out assert str(lgpl_header) in out assert str(not_in_first_n_lines) in out assert str(correct_header) not in out assert "3 improperly licensed files" in out assert re.search(r"files not containing expected license:\s*1", out) assert re.search(r"files with wrong SPDX-License-Identifier:\s*1", out) assert re.search(r"files without license in first 6 lines:\s*1", out) assert license.returncode == 1 ================================================ FILE: lib/spack/spack/test/cmd/list.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.cmd.list import spack.paths import spack.repo import spack.util.git from spack.main import SpackCommand from spack.test.conftest import RepoBuilder pytestmark = [pytest.mark.usefixtures("mock_packages")] list = SpackCommand("list") def test_list(): output = list() assert "bzip2" in output assert "hdf5" in output def test_list_cli_output_format(mock_tty_stdout): assert ( list("mpileaks") == """\ mpileaks ==> 1 packages """ ) def test_list_filter(): output = list("py-*") assert "py-extension1" in output assert "py-extension2" in output assert "py-extension3" in output assert "python" not in output assert "mpich" not in output output = list("py") assert "py-extension1" in output assert "py-extension2" in output assert "py-extension3" in output assert "python" in output assert "mpich" not in output def test_list_search_description(): output = list("--search-description", "one build dependency") assert "depb" in output def test_list_format_name_only(): output = list("--format", "name_only") assert "zmpi" in output assert "hdf5" in output def test_list_format_version_json(): output = list("--format", "version_json") assert '{"name": "zmpi",' in output assert '{"name": "dyninst",' in output assert "packages/zmpi/package.py" in output import json json.loads(output) def test_list_format_html(): output = list("--format", "html") assert '
' in output assert "

zmpi" in output assert '
' in output assert "

hdf5" in output assert "packages/hdf5/package.py" in output @pytest.mark.parametrize( "url", [ "git@github.com:username/spack-packages.git", "https://github.com/username/spack-packages.git", "git@github.com:username/spack.git", "https://github.com/username/spack.git", ], ) def test_list_url_schemes(mock_util_executable, url): """Confirm the command handles supported repository URLs.""" pkg_name = "hdf5" _, _, registered_responses = mock_util_executable registered_responses["config"] = url registered_responses["rev-parse"] = f"path/to/builtin/packages/{pkg_name}/" output = list("--format", "version_json", pkg_name) assert f"{registered_responses['rev-parse']}package.py" in output assert os.path.basename(url).replace(".git", "") in output def test_list_format_local_repo(tmp_path: pathlib.Path): """Confirm a file path is returned for local repository.""" pkg_name = "mypkg" repo_root = tmp_path / "repos" / "spack_repo" / "builtin" repo_root.mkdir(parents=True) (repo_root / "repo.yaml").write_text("repo:\n namespace: builtin\n api: v2.2\n") package_root = repo_root / "packages" / pkg_name package_root.mkdir(parents=True) (package_root / "package.py").write_text( """\ from spack.package import * class Mypkg(Package): pass """ ) test_repo = spack.repo.from_path(str(repo_root)) with spack.repo.use_repositories(test_repo): # Confirm a path is returned when fail to retrieve the remote origin URL output = list("--format", "version_json", pkg_name) assert "github.com" not in output assert f"packages/{pkg_name}/package.py" in output def test_list_format_non_github_repo(tmp_path: pathlib.Path, mock_util_executable): """Confirm a file path is returned for a non-github repository.""" pkg_name = "mypkg" repo_root = tmp_path / "my" / "project" / "spack_repo" / "builtin" repo_root.mkdir(parents=True) (repo_root / "repo.yaml").write_text("repo:\n namespace: builtin\n api: v2.2\n") package_root = repo_root / "packages" / pkg_name package_root.mkdir(parents=True) package_path = package_root / "package.py" package_path.write_text( """\ from spack.package import * class Mypkg(Package): pass """ ) test_repo = spack.repo.from_path(str(repo_root)) with spack.repo.use_repositories(test_repo): # Confirm a path is returned for a non-standard spack repository _, _, registered_responses = mock_util_executable registered_responses["config"] = "https://gitlab.com/username/my-packages.git" registered_responses["rev-parse"] = str(package_root) + os.sep output = list("--format", "version_json", pkg_name) assert package_path.as_uri() in output def test_list_update(tmp_path: pathlib.Path): update_file = tmp_path / "output" # not yet created when list is run list("--update", str(update_file)) assert update_file.exists() with update_file.open() as f: assert f.read() # created but older than any package with update_file.open("w") as f: f.write("empty\n") os.utime(str(update_file), (0, 0)) # Set mtime to 0 list("--update", str(update_file)) assert update_file.exists() with update_file.open() as f: assert f.read() != "empty\n" # newer than any packages with update_file.open("w") as f: f.write("empty\n") list("--update", str(update_file)) assert update_file.exists() with update_file.open() as f: assert f.read() == "empty\n" def test_list_tags(): output = list("--tag", "tag1") assert "mpich" in output assert "mpich2" in output output = list("--tag", "tag2") assert "mpich\n" in output assert "mpich2" not in output output = list("--tag", "tag3") assert "mpich\n" not in output assert "mpich2" in output def test_list_count(): output = list("--count") assert int(output.strip()) == len(spack.repo.all_package_names()) output = list("--count", "py-") assert int(output.strip()) == len( [name for name in spack.repo.all_package_names() if "py-" in name] ) def test_list_repos(): with spack.repo.use_repositories( os.path.join(spack.paths.test_repos_path, "spack_repo", "builtin_mock"), os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test"), ): total_pkgs = len(list().strip().split()) mock_pkgs = len(list("-r", "builtin_mock").strip().split()) builder_pkgs = len(list("-r", "builder_test").strip().split()) both_repos = len(list("-r", "builtin_mock", "-r", "builder_test").strip().split()) assert total_pkgs > mock_pkgs > builder_pkgs assert both_repos == total_pkgs @pytest.mark.usefixtures("config") def test_list_github_url_fails(repo_builder: RepoBuilder, monkeypatch): with spack.repo.use_repositories(repo_builder.root): repo_builder.add_package("pkg-a") repo = spack.repo.PATH.repos[0] pkg = repo.get_pkg_class("pkg-a") old_path = repo.python_path try: # Check that a repository with no python path has no URL monkeypatch.setattr(repo, "python_path", None) assert spack.cmd.list.github_url(pkg) is None, ( "Expected no python path means unable to determine the repo URL" ) # Check that a repository path that doesn't exist has no URL monkeypatch.setattr(repo, "python_path", "/repo/root/does/not/exists") assert spack.cmd.list.github_url(pkg) is None, ( "Expected bad repo path means unable to determine the repo URL" ) finally: monkeypatch.setattr(repo, "python_path", old_path) # Check that missing git results in the file path monkeypatch.setattr(spack.util.git, "git", lambda: None) filepath = spack.cmd.list.github_url(pkg) assert filepath and filepath.startswith("file://"), ( "Expected missing 'git' results in a file URI" ) ================================================ FILE: lib/spack/spack/test/cmd/load.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import re import sys import pytest import spack.concretize import spack.user_environment as uenv from spack.main import SpackCommand load = SpackCommand("load") unload = SpackCommand("unload") install = SpackCommand("install") location = SpackCommand("location") def test_manpath_trailing_colon( install_mockery, mock_fetch, mock_archive, mock_packages, working_env ): (shell, set_command, commandsep) = ( ("--bat", 'set "%s=%s"', "\n") if sys.platform == "win32" else ("--sh", "export %s=%s", ";") ) # Test that the commands generated by load add the MANPATH prefix # inspections. Also test that Spack correctly preserves the default/existing # manpath search path via a trailing colon install("--fake", "mpileaks") sh_out = load(shell, "mpileaks") lines = [line.strip("\n") for line in sh_out.split(commandsep)] assert any(re.match(set_command % ("MANPATH", ".*" + os.pathsep), ln) for ln in lines) os.environ["MANPATH"] = "/tmp/man" + os.pathsep sh_out = load(shell, "mpileaks") lines = [line.strip("\n") for line in sh_out.split(commandsep)] assert any( re.match(set_command % ("MANPATH", ".*" + os.pathsep + "/tmp/man" + os.pathsep), ln) for ln in lines ) def test_load_recursive(install_mockery, mock_fetch, mock_archive, mock_packages, working_env): def test_load_shell(shell, set_command): """Test that `spack load` applies prefix inspections of its required runtime deps in topo-order""" install("--fake", "mpileaks") mpileaks_spec = spack.concretize.concretize_one("mpileaks") # Ensure our reference variable is clean. os.environ["CMAKE_PREFIX_PATH"] = "/hello" + os.pathsep + "/world" shell_out = load(shell, "mpileaks") def extract_value(output, variable): match = re.search(set_command % variable, output, flags=re.MULTILINE) value = match.group(1) return value.split(os.pathsep) # Map a prefix found in CMAKE_PREFIX_PATH back to a package name in mpileaks' DAG. prefix_to_pkg = lambda prefix: next( s.name for s in mpileaks_spec.traverse() if s.prefix == prefix ) paths_shell = extract_value(shell_out, "CMAKE_PREFIX_PATH") # We should've prepended new paths, and keep old ones. assert paths_shell[-2:] == ["/hello", "/world"] # All but the last two paths are added by spack load; lookup what packages they're from. pkgs = [prefix_to_pkg(p) for p in paths_shell[:-2]] # Do we have all the runtime packages? assert set(pkgs) == set( s.name for s in mpileaks_spec.traverse(deptype=("link", "run"), root=True) ) # Finally, do we list them in topo order? for i, pkg in enumerate(pkgs): assert {s.name for s in mpileaks_spec[pkg].traverse(direction="parents")}.issubset( pkgs[: i + 1] ) # Lastly, do we keep track that mpileaks was loaded? assert ( extract_value(shell_out, uenv.spack_loaded_hashes_var)[0] == mpileaks_spec.dag_hash() ) return paths_shell if sys.platform == "win32": shell, set_command = ("--bat", r'set "%s=(.*)"') test_load_shell(shell, set_command) else: params = [("--sh", r"export %s=([^;]*)"), ("--csh", r"setenv %s ([^;]*)")] shell, set_command = params[0] paths_sh = test_load_shell(shell, set_command) shell, set_command = params[1] paths_csh = test_load_shell(shell, set_command) assert paths_sh == paths_csh @pytest.mark.parametrize( "shell,set_command", ( [("--bat", 'set "%s=%s"')] if sys.platform == "win32" else [("--sh", "export %s=%s"), ("--csh", "setenv %s %s")] ), ) def test_load_includes_run_env( shell, set_command, install_mockery, mock_fetch, mock_archive, mock_packages ): """Tests that environment changes from the package's `setup_run_environment` method are added to the user environment in addition to the prefix inspections""" install("--fake", "mpileaks") shell_out = load(shell, "mpileaks") assert set_command % ("FOOBAR", "mpileaks") in shell_out def test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages): """Test with and without the --first option""" shell = "--bat" if sys.platform == "win32" else "--sh" install("--fake", "libelf@0.8.12") install("--fake", "libelf@0.8.13") # Now there are two versions of libelf, which should cause an error out = load(shell, "libelf", fail_on_error=False) assert "matches multiple packages" in out assert "Use a more specific spec" in out # Using --first should avoid the error condition load(shell, "--first", "libelf") def test_load_fails_no_shell(install_mockery, mock_fetch, mock_archive, mock_packages): """Test that spack load prints an error message without a shell.""" install("--fake", "mpileaks") out = load("mpileaks", fail_on_error=False) assert "To set up shell support" in out @pytest.mark.parametrize( "shell,set_command,unset_command", ( [("--bat", 'set "%s=%s"', 'set "%s="')] if sys.platform == "win32" else [("--sh", "export %s=%s", "unset %s"), ("--csh", "setenv %s %s", "unsetenv %s")] ), ) def test_unload( shell, set_command, unset_command, install_mockery, mock_fetch, mock_archive, mock_packages, working_env, ): """Tests that any variables set in the user environment are undone by the unload command""" install("--fake", "mpileaks") mpileaks_spec = spack.concretize.concretize_one("mpileaks") # Set so unload has something to do os.environ["FOOBAR"] = "mpileaks" os.environ[uenv.spack_loaded_hashes_var] = ("%s" + os.pathsep + "%s") % ( mpileaks_spec.dag_hash(), "garbage", ) shell_out = unload(shell, "mpileaks") assert (unset_command % "FOOBAR") in shell_out assert set_command % (uenv.spack_loaded_hashes_var, "garbage") in shell_out def test_unload_fails_no_shell( install_mockery, mock_fetch, mock_archive, mock_packages, working_env ): """Test that spack unload prints an error message without a shell.""" install("--fake", "mpileaks") mpileaks_spec = spack.concretize.concretize_one("mpileaks") os.environ[uenv.spack_loaded_hashes_var] = mpileaks_spec.dag_hash() out = unload("mpileaks", fail_on_error=False) assert "To set up shell support" in out ================================================ FILE: lib/spack/spack/test/cmd/location.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import shutil import pytest import spack.concretize import spack.environment as ev import spack.main import spack.paths import spack.repo import spack.stage from spack.llnl.util.filesystem import mkdirp from spack.main import SpackCommand # Everything here uses (or can use) the mock config and database. pytestmark = [pytest.mark.usefixtures("mutable_config", "mutable_database")] # location prints out "locations of packages and spack directories" location = SpackCommand("location") env = SpackCommand("env") @pytest.fixture def mock_spec(): # Make it look like the source was actually expanded. s = spack.concretize.concretize_one("externaltest") source_path = s.package.stage.source_path mkdirp(source_path) yield s, s.package # Remove the spec from the mock stage area. shutil.rmtree(s.package.stage.path) def test_location_first(install_mockery, mock_fetch, mock_archive, mock_packages): """Test with and without the --first option""" install = SpackCommand("install") install("--fake", "libelf@0.8.12") install("--fake", "libelf@0.8.13") # This would normally return an error without --first assert location("--first", "--install-dir", "libelf") def test_location_build_dir(mock_spec): """Tests spack location --build-dir.""" spec, pkg = mock_spec assert location("--build-dir", spec.name).strip() == pkg.stage.source_path @pytest.mark.regression("22738") def test_location_source_dir(mock_spec): """Tests spack location --source-dir.""" spec, pkg = mock_spec assert location("--source-dir", spec.name).strip() == pkg.stage.source_path assert location(spec.name).strip() == pkg.stage.source_path def test_location_source_dir_missing(): """Tests spack location --source-dir with a missing source directory.""" spec = "mpileaks" prefix = "==> Error: " expected = ( "%sSource directory does not exist yet. Run this to create it:" "%s spack stage %s" % (prefix, "\n", spec) ) out = location("--source-dir", spec, fail_on_error=False).strip() assert out == expected @pytest.mark.parametrize( "options", [([]), (["--source-dir", "mpileaks"]), (["--env", "missing-env"]), (["spec1", "spec2"])], ) def test_location_cmd_error(options): """Ensure the proper error is raised with problematic location options.""" with pytest.raises(spack.main.SpackCommandError) as e: location(*options) assert e.value.code == 1 def test_location_env_exists(mutable_mock_env_path): """Tests spack location --env for an existing environment.""" e = ev.create("example") e.write() assert location("--env", "example").strip() == e.path def test_location_with_active_env(mutable_mock_env_path): """Tests spack location --env with active env""" e = ev.create("example") e.write() with e: assert location("--env").strip() == e.path def test_location_env_missing(): """Tests spack location --env.""" missing_env_name = "missing-env" error = "==> Error: no such environment: '%s'" % missing_env_name out = location("--env", missing_env_name, fail_on_error=False).strip() assert out == error @pytest.mark.db @pytest.mark.not_on_windows("Broken on Windows") def test_location_install_dir(mock_spec): """Tests spack location --install-dir.""" spec, _ = mock_spec assert location("--install-dir", spec.name).strip() == spec.prefix @pytest.mark.db def test_location_package_dir(mock_spec): """Tests spack location --package-dir.""" spec, pkg = mock_spec assert location("--package-dir", spec.name).strip() == pkg.package_dir @pytest.mark.db @pytest.mark.parametrize( "option,expected", [ ("--module-dir", spack.paths.module_path), ("--packages", spack.paths.mock_packages_path), ("--spack-root", spack.paths.prefix), ], ) def test_location_paths_options(option, expected): """Tests basic spack.paths location command options.""" assert location(option).strip() == expected @pytest.mark.parametrize( "specs,expected", [([], "You must supply a spec."), (["spec1", "spec2"], "Too many specs. Supply only one.")], ) def test_location_spec_errors(specs, expected): """Tests spack location with bad spec options.""" error = "==> Error: %s" % expected assert location(*specs, fail_on_error=False).strip() == error @pytest.mark.db def test_location_stage_dir(mock_spec): """Tests spack location --stage-dir.""" spec, pkg = mock_spec assert location("--stage-dir", spec.name).strip() == pkg.stage.path @pytest.mark.db def test_location_stages(mock_spec): """Tests spack location --stages.""" assert location("--stages").strip() == spack.stage.get_stage_root() def test_location_specified_repo(): """Tests spack location --repo .""" with spack.repo.use_repositories( os.path.join(spack.paths.test_repos_path, "spack_repo", "builtin_mock"), os.path.join(spack.paths.test_repos_path, "spack_repo", "builder_test"), ): assert location("--repo").strip() == spack.repo.PATH.get_repo("builtin_mock").root assert ( location("--repo", "builtin_mock").strip() == spack.repo.PATH.get_repo("builtin_mock").root ) assert ( location("--packages", "builder_test").strip() == spack.repo.PATH.get_repo("builder_test").root ) assert ( location("--repo", "nonexistent", fail_on_error=False).strip() == "==> Error: no such repository: 'nonexistent'" ) ================================================ FILE: lib/spack/spack/test/cmd/logs.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import gzip import os import sys import tempfile from contextlib import contextmanager from io import BytesIO, TextIOWrapper import pytest import spack.cmd.logs import spack.concretize import spack.error import spack.spec from spack.main import SpackCommand logs = SpackCommand("logs") install = SpackCommand("install") @contextmanager def stdout_as_buffered_text_stream(): """Attempt to simulate "typical" interface for stdout when user is running Spack/Python from terminal. "spack log" should not be run for all possible cases of what stdout might look like, in particular some programmatic redirections of stdout like StringIO are not meant to be supported by this command; more-generally, mechanisms that depend on decoding binary output prior to write are not supported for "spack log". """ original_stdout = sys.stdout with tempfile.TemporaryFile(mode="w+b") as tf: sys.stdout = TextIOWrapper(tf, encoding="utf-8") try: yield tf finally: sys.stdout = original_stdout def _rewind_collect_and_decode(rw_stream): rw_stream.seek(0) return rw_stream.read().decode("utf-8") def test_logs_cmd_errors(install_mockery, mock_fetch, mock_archive, mock_packages): spec = spack.concretize.concretize_one("pkg-c") assert not spec.installed with pytest.raises(spack.error.SpackError, match="is not installed or staged"): logs("pkg-c") with pytest.raises(spack.error.SpackError, match="Too many specs"): logs("pkg-c mpi") install("pkg-c") os.remove(spec.package.install_log_path) with pytest.raises(spack.error.SpackError, match="No logs are available"): logs("pkg-c") def _write_string_to_path(string, path): """Write a string to a file, preserving newline format in the string.""" with open(path, "wb") as f: f.write(string.encode("utf-8")) def test_dump_logs(install_mockery, mock_fetch, mock_archive, mock_packages): """Test that ``spack log`` can find (and print) the logs for partial builds and completed installs. Also make sure that for compressed logs, that we automatically decompress them. """ cmdline_spec = spack.spec.Spec("libelf") concrete_spec = spack.concretize.concretize_one(cmdline_spec) # Sanity check, make sure this test is checking what we want: to # start with assert not concrete_spec.installed stage_log_content = "test_log stage output\nanother line" installed_log_content = "test_log install output\nhere to test multiple lines" with concrete_spec.package.stage: _write_string_to_path(stage_log_content, concrete_spec.package.log_path) with stdout_as_buffered_text_stream() as redirected_stdout: spack.cmd.logs._logs(cmdline_spec, concrete_spec) assert _rewind_collect_and_decode(redirected_stdout) == stage_log_content install("--fake", "libelf") # Sanity check: make sure a path is recorded, regardless of whether # it exists (if it does exist, we will overwrite it with content # in this test) assert concrete_spec.package.install_log_path with gzip.open(concrete_spec.package.install_log_path, "wb") as compressed_file: bstream = BytesIO(installed_log_content.encode("utf-8")) compressed_file.writelines(bstream) with stdout_as_buffered_text_stream() as redirected_stdout: spack.cmd.logs._logs(cmdline_spec, concrete_spec) assert _rewind_collect_and_decode(redirected_stdout) == installed_log_content with concrete_spec.package.stage: _write_string_to_path(stage_log_content, concrete_spec.package.log_path) # We re-create the stage, but "spack log" should ignore that # if the package is installed with stdout_as_buffered_text_stream() as redirected_stdout: spack.cmd.logs._logs(cmdline_spec, concrete_spec) assert _rewind_collect_and_decode(redirected_stdout) == installed_log_content ================================================ FILE: lib/spack/spack/test/cmd/maintainers.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import pytest import spack.main import spack.repo pytestmark = [pytest.mark.usefixtures("mock_packages")] maintainers = spack.main.SpackCommand("maintainers") MAINTAINED_PACKAGES = [ "gcc-runtime", "maintainers-1", "maintainers-2", "maintainers-3", "py-extension1", ] def split(output): """Split command line output into an array.""" output = output.strip() return re.split(r"\s+", output) if output else [] def test_maintained(): out = split(maintainers("--maintained")) assert out == MAINTAINED_PACKAGES def test_unmaintained(): out = split(maintainers("--unmaintained")) assert out == sorted(set(spack.repo.all_package_names()) - set(MAINTAINED_PACKAGES)) def test_all(): out = split(maintainers("--all")) assert out == [ "gcc-runtime:", "haampie", "maintainers-1:", "user1,", "user2", "maintainers-2:", "user2,", "user3", "maintainers-3:", "user0,", "user1,", "user2,", "user3", "py-extension1:", "user1,", "user2", ] out = split(maintainers("--all", "maintainers-1")) assert out == ["maintainers-1:", "user1,", "user2"] def test_all_by_user(): out = split(maintainers("--all", "--by-user")) assert out == [ "haampie:", "gcc-runtime", "user0:", "maintainers-3", "user1:", "maintainers-1,", "maintainers-3,", "py-extension1", "user2:", "maintainers-1,", "maintainers-2,", "maintainers-3,", "py-extension1", "user3:", "maintainers-2,", "maintainers-3", ] out = split(maintainers("--all", "--by-user", "user1", "user2")) assert out == [ "user1:", "maintainers-1,", "maintainers-3,", "py-extension1", "user2:", "maintainers-1,", "maintainers-2,", "maintainers-3,", "py-extension1", ] def test_no_args(): with pytest.raises(spack.main.SpackCommandError): maintainers() def test_no_args_by_user(): with pytest.raises(spack.main.SpackCommandError): maintainers("--by-user") def test_mutex_args_fail(): with pytest.raises(spack.main.SpackCommandError): maintainers("--maintained", "--unmaintained") def test_maintainers_list_packages(): out = split(maintainers("maintainers-1")) assert out == ["user1", "user2"] out = split(maintainers("maintainers-1", "maintainers-2")) assert out == ["user1", "user2", "user3"] out = split(maintainers("maintainers-2")) assert out == ["user2", "user3"] def test_maintainers_list_fails(): out = maintainers("pkg-a", fail_on_error=False) assert not out assert maintainers.returncode == 1 def test_maintainers_list_by_user(): out = split(maintainers("--by-user", "user1")) assert out == ["maintainers-1", "maintainers-3", "py-extension1"] out = split(maintainers("--by-user", "user1", "user2")) assert out == ["maintainers-1", "maintainers-2", "maintainers-3", "py-extension1"] out = split(maintainers("--by-user", "user2")) assert out == ["maintainers-1", "maintainers-2", "maintainers-3", "py-extension1"] out = split(maintainers("--by-user", "user3")) assert out == ["maintainers-2", "maintainers-3"] ================================================ FILE: lib/spack/spack/test/cmd/mark.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.store from spack.main import SpackCommand, SpackCommandError gc = SpackCommand("gc") mark = SpackCommand("mark") install = SpackCommand("install") uninstall = SpackCommand("uninstall") # Unit tests should not be affected by the user's managed environments pytestmark = pytest.mark.usefixtures("mutable_mock_env_path") @pytest.mark.db def test_mark_mode_required(mutable_database): with pytest.raises(SpackCommandError): mark("-a") @pytest.mark.db def test_mark_spec_required(mutable_database): with pytest.raises(SpackCommandError): mark("-i") @pytest.mark.db def test_mark_all_explicit(mutable_database): mark("-e", "-a") gc("-y") all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 17 @pytest.mark.db def test_mark_all_implicit(mutable_database): mark("-i", "-a") gc("-y") all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 0 @pytest.mark.db def test_mark_one_explicit(mutable_database): mark("-e", "libelf") uninstall("-y", "-a", "mpileaks") gc("-y") all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 4 @pytest.mark.db def test_mark_one_implicit(mutable_database): mark("-i", "externaltest") gc("-y") all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 15 @pytest.mark.db def test_mark_all_implicit_then_explicit(mutable_database): mark("-i", "-a") mark("-e", "-a") gc("-y") all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 17 ================================================ FILE: lib/spack/spack/test/cmd/mirror.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.binary_distribution import spack.cmd.mirror import spack.concretize import spack.config import spack.environment as ev import spack.error import spack.mirrors.utils import spack.package_base import spack.spec import spack.util.git import spack.util.url as url_util import spack.version from spack.main import SpackCommand, SpackCommandError from spack.mirrors.utils import MirrorStatsForAllSpecs, MirrorStatsForOneSpec config = SpackCommand("config") mirror = SpackCommand("mirror") env = SpackCommand("env") add = SpackCommand("add") concretize = SpackCommand("concretize") install = SpackCommand("install") buildcache = SpackCommand("buildcache") uninstall = SpackCommand("uninstall") pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.disable_clean_stage_check @pytest.mark.regression("8083") def test_regression_8083(tmp_path: pathlib.Path, mock_packages, mock_fetch, config): output = mirror("create", "-d", str(tmp_path), "externaltool") assert "Skipping" in output assert "as it is an external spec" in output # Unit tests should not be affected by the user's managed environments @pytest.mark.regression("12345") def test_mirror_from_env(mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mock_fetch): mirror_dir = str(tmp_path / "mirror") env_name = "test" env("create", env_name) with ev.read(env_name): add("trivial-install-test-package") add("git-test") concretize() with spack.config.override("config:checksum", False): mirror("create", "-d", mirror_dir, "--all") e = ev.read(env_name) assert set(os.listdir(mirror_dir)) == set([s.name for s in e.user_specs]) for spec in e.specs_by_hash.values(): mirror_res = os.listdir(os.path.join(mirror_dir, spec.name)) expected = ["%s.tar.gz" % spec.format("{name}-{version}")] assert mirror_res == expected def test_mirror_cli_parallel_args( tmp_path, mock_packages, mock_fetch, mutable_mock_env_path, monkeypatch ): """Test the CLI parallel args""" mirror_dir = str(tmp_path / "mirror") env_name = "test-parallel" def mock_create_mirror_for_all_specs(mirror_specs, path, skip_unstable_versions, workers): assert path == mirror_dir assert workers == 2 monkeypatch.setattr( spack.cmd.mirror, "create_mirror_for_all_specs", mock_create_mirror_for_all_specs ) env("create", env_name) with ev.read(env_name): add("trivial-install-test-package") add("git-test") concretize() with spack.config.override("config:checksum", False): mirror("create", "-d", mirror_dir, "--all", "-j", "2") def test_mirror_from_env_parallel(tmp_path, mock_packages, mock_fetch, mutable_mock_env_path): """Directly test create_mirror_for_all_specs with parallel option""" mirror_dir = str(tmp_path / "mirror") env_name = "test-parallel" env("create", env_name) with ev.read(env_name): add("trivial-install-test-package") add("git-test") concretize() e = ev.read(env_name) specs = list(e.specs_by_hash.values()) with spack.config.override("config:checksum", False): mirror_stats = spack.cmd.mirror.create_mirror_for_all_specs( specs, mirror_dir, False, workers=2 ) assert len(mirror_stats.errors) == 0 assert set(os.listdir(mirror_dir)) == set([s.name for s in e.user_specs]) for spec in e.specs_by_hash.values(): mirror_res = os.listdir(os.path.join(mirror_dir, spec.name)) expected = ["%s.tar.gz" % spec.format("{name}-{version}")] assert mirror_res == expected def test_mirror_stats_merge(): """Test MirrorStats merge functionality""" spec1 = "package@1.0" spec2 = "package@2.0" spec3 = "package@3.0" s1 = MirrorStatsForOneSpec(spec1) s1.added("/test/path/1") s1.added("/test/path/2") s1.finalize() s2 = MirrorStatsForOneSpec(spec2) s2.already_existed("/test/path/3") s2.finalize() all_stats = MirrorStatsForAllSpecs() # Check before merge, should be empty present, mirrored, errors = all_stats.stats() assert len(present) == 0 assert len(mirrored) == 0 assert len(errors) == 0 # Merge package 1 and 2 all_stats.merge(s1) all_stats.merge(s2) # Check after merge present, mirrored, errors = all_stats.stats() assert present.count(spec2) == 1 assert mirrored.count(spec1) == 1 assert len(present) == 1 assert len(mirrored) == 1 assert len(errors) == 0 # Merge package 3 s3 = MirrorStatsForOneSpec(spec3) s3.already_existed("/test/path/4") s3.added("/test/path/5") s3.finalize() all_stats.merge(s3) present, mirrored, errors = all_stats.stats() assert present.count(spec3) == 1 assert mirrored.count(spec3) == 1 assert len(present) == 2 assert len(mirrored) == 2 assert len(errors) == 0 # Test for command line-specified spec in concretized environment def test_mirror_spec_from_env( mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mock_fetch ): mirror_dir = str(tmp_path / "mirror-B") env_name = "test" env("create", env_name) with ev.read(env_name): add("simple-standalone-test@0.9") concretize() with spack.config.override("config:checksum", False): mirror("create", "-d", mirror_dir, "simple-standalone-test") e = ev.read(env_name) assert set(os.listdir(mirror_dir)) == set([s.name for s in e.user_specs]) spec = e.concrete_roots()[0] mirror_res = os.listdir(os.path.join(mirror_dir, spec.name)) expected = ["%s.tar.gz" % spec.format("{name}-{version}")] assert mirror_res == expected @pytest.fixture def source_for_pkg_with_hash(mock_packages, tmp_path: pathlib.Path): s = spack.concretize.concretize_one("trivial-pkg-with-valid-hash") local_url_basename = os.path.basename(s.package.url) local_path = tmp_path / local_url_basename local_path.write_text(s.package.hashed_content, encoding="utf-8") local_url = url_util.path_to_file_url(str(local_path)) s.package.versions[spack.version.Version("1.0")]["url"] = local_url def test_mirror_skip_unstable( tmp_path_factory: pytest.TempPathFactory, mock_packages, config, source_for_pkg_with_hash ): mirror_dir = str(tmp_path_factory.mktemp("mirror-dir")) specs = [ spack.concretize.concretize_one(x) for x in ["git-test", "trivial-pkg-with-valid-hash"] ] spack.cmd.mirror.create(mirror_dir, specs, skip_unstable_versions=True) assert set(os.listdir(mirror_dir)) - set(["_source-cache"]) == set( ["trivial-pkg-with-valid-hash"] ) class MockMirrorArgs: def __init__( self, specs=None, all=False, file=None, versions_per_spec=None, dependencies=False, exclude_file=None, exclude_specs=None, directory=None, private=False, ): self.specs = specs or [] self.all = all self.file = file self.versions_per_spec = versions_per_spec self.dependencies = dependencies self.exclude_file = exclude_file self.exclude_specs = exclude_specs self.private = private self.directory = directory def test_exclude_specs(mock_packages, config): args = MockMirrorArgs( specs=["mpich"], versions_per_spec="all", exclude_specs="mpich@3.0.1:3.0.2 mpich@1.0" ) mirror_specs = spack.cmd.mirror._specs_to_mirror(args) expected_include = set( spack.concretize.concretize_one(x) for x in ["mpich@3.0.3", "mpich@3.0.4", "mpich@3.0"] ) expected_exclude = set(spack.spec.Spec(x) for x in ["mpich@3.0.1", "mpich@3.0.2", "mpich@1.0"]) assert expected_include <= set(mirror_specs) assert not any(spec.satisfies(y) for spec in mirror_specs for y in expected_exclude) def test_exclude_specs_public_mirror(mock_packages, config): args = MockMirrorArgs( specs=["no-redistribute-dependent"], versions_per_spec="all", dependencies=True, private=False, ) mirror_specs = spack.cmd.mirror._specs_to_mirror(args) assert not any(s.name == "no-redistribute" for s in mirror_specs) assert any(s.name == "no-redistribute-dependent" for s in mirror_specs) def test_exclude_file(mock_packages, tmp_path: pathlib.Path, config): exclude_path = tmp_path / "test-exclude.txt" exclude_path.write_text( """\ mpich@3.0.1:3.0.2 mpich@1.0 """, encoding="utf-8", ) args = MockMirrorArgs(specs=["mpich"], versions_per_spec="all", exclude_file=str(exclude_path)) mirror_specs = spack.cmd.mirror._specs_to_mirror(args) expected_include = set( spack.concretize.concretize_one(x) for x in ["mpich@3.0.3", "mpich@3.0.4", "mpich@3.0"] ) expected_exclude = set(spack.spec.Spec(x) for x in ["mpich@3.0.1", "mpich@3.0.2", "mpich@1.0"]) assert expected_include <= set(mirror_specs) assert not any(spec.satisfies(y) for spec in mirror_specs for y in expected_exclude) def test_mirror_remove_by_scope(mutable_config, tmp_path: pathlib.Path): # add a new mirror to two scopes mirror("add", "--scope=site", "mock", str(tmp_path / "mock_mirror")) mirror("add", "--scope=system", "mock", str(tmp_path / "mock_mirror")) # Confirm that it is not removed when the scope is incorrect with pytest.raises(SpackCommandError): mirror("remove", "--scope=user", "mock") output = mirror("list") assert "mock" in output # Confirm that when the scope is specified, it is only removed from that scope mirror("remove", "--scope=site", "mock") site_output = mirror("list", "--scope=site") system_output = mirror("list", "--scope=system") assert "mock" not in site_output assert "mock" in system_output # Confirm that when the scope is not specified, it is removed from top scope mirror("add", "--scope=site", "mock", str(tmp_path / "mock_mirror")) mirror("remove", "mock") site_output = mirror("list", "--scope=site") system_output = mirror("list", "--scope=system") assert "mock" not in site_output assert "mock" in system_output # Check that the `--all-scopes` option works mirror("add", "--scope=site", "mock", str(tmp_path / "mockrepo")) mirror("remove", "--all-scopes", "mock") output = mirror("list") assert "mock" not in output def test_mirror_crud(mutable_config): mirror("add", "mirror", "http://spack.io") output = mirror("remove", "mirror") assert "Removed mirror" in output mirror("add", "mirror", "http://spack.io") # no-op output = mirror("set-url", "mirror", "http://spack.io") assert "No changes made" in output output = mirror("set-url", "--push", "mirror", "s3://spack-public") assert not output # no-op output = mirror("set-url", "--push", "mirror", "s3://spack-public") assert "No changes made" in output output = mirror("remove", "mirror") assert "Removed mirror" in output # Test S3 connection info token as variable mirror("add", "--s3-access-token-variable", "aaaaaazzzzz", "mirror", "s3://spack-public") output = mirror("remove", "mirror") assert "Removed mirror" in output def do_add_set_seturl_access_pair( id_arg, secret_arg, mirror_name="mirror", mirror_url="s3://spack-public" ): # Test connection info id/key output = mirror("add", id_arg, "foo", secret_arg, "bar", mirror_name, mirror_url) output = config("blame", "mirrors") assert all([x in output for x in ("foo", "bar", mirror_name, mirror_url)]) output = mirror("set", id_arg, "foo_set", secret_arg, "bar_set", mirror_name) output = config("blame", "mirrors") assert all([x in output for x in ("foo_set", "bar_set", mirror_name, mirror_url)]) if "variable" not in secret_arg: output = mirror( "set", id_arg, "foo_set", secret_arg + "-variable", "bar_set_var", mirror_name ) assert "support for plain text secrets" not in output output = config("blame", "mirrors") assert all([x in output for x in ("foo_set", "bar_set_var", mirror_name, mirror_url)]) output = mirror( "set-url", id_arg, "foo_set_url", secret_arg, "bar_set_url", "--push", mirror_name, mirror_url + "-push", ) output = config("blame", "mirrors") assert all( [ x in output for x in ("foo_set_url", "bar_set_url", mirror_name, mirror_url + "-push") ] ) output = mirror("set", id_arg, "a", mirror_name) assert "No changes made to mirror" not in output output = mirror("set", secret_arg, "b", mirror_name) assert "No changes made to mirror" not in output output = mirror("set-url", id_arg, "c", mirror_name, mirror_url) assert "No changes made to mirror" not in output output = mirror("set-url", secret_arg, "d", mirror_name, mirror_url) assert "No changes made to mirror" not in output output = mirror("remove", mirror_name) assert "Removed mirror" in output output = mirror("add", id_arg, "foo", mirror_name, mirror_url) assert "Expected both parts of the access pair to be specified. " in output output = mirror("set-url", id_arg, "bar", mirror_name, mirror_url) assert "Expected both parts of the access pair to be specified. " in output output = mirror("set", id_arg, "bar", mirror_name) assert "Expected both parts of the access pair to be specified. " in output output = mirror("remove", mirror_name) assert "Removed mirror" in output output = mirror("add", secret_arg, "bar", mirror_name, mirror_url) assert "Expected both parts of the access pair to be specified. " in output output = mirror("set-url", secret_arg, "bar", mirror_name, mirror_url) assert "Expected both parts of the access pair to be specified. " in output output = mirror("set", secret_arg, "bar", mirror_name) assert "Expected both parts of the access pair to be specified. " in output output = mirror("remove", mirror_name) assert "Removed mirror" in output output = mirror("list") assert "No mirrors configured" in output do_add_set_seturl_access_pair("--s3-access-key-id", "--s3-access-key-secret-variable") do_add_set_seturl_access_pair("--s3-access-key-id-variable", "--s3-access-key-secret-variable") # Test OCI connection info user/password do_add_set_seturl_access_pair("--oci-username", "--oci-password-variable") do_add_set_seturl_access_pair("--oci-username-variable", "--oci-password-variable") # Test S3 connection info with endpoint URL mirror( "add", "--s3-access-token-variable", "aaaaaazzzzz", "--s3-endpoint-url", "http://localhost/", "mirror", "s3://spack-public", ) output = mirror("remove", "mirror") assert "Removed mirror" in output output = mirror("list") assert "No mirrors configured" in output # Test GCS Mirror mirror("add", "mirror", "gs://spack-test") output = mirror("remove", "mirror") assert "Removed mirror" in output output = mirror("list") assert "No mirrors configured" in output def test_mirror_nonexisting(mutable_config): with pytest.raises(SpackCommandError): mirror("remove", "not-a-mirror") with pytest.raises(SpackCommandError): mirror("set-url", "not-a-mirror", "http://spack.io") def test_mirror_name_collision(mutable_config): mirror("add", "first", "1") with pytest.raises(SpackCommandError): mirror("add", "first", "1") # Unit tests should not be affected by the user's managed environments def test_mirror_destroy( mutable_mock_env_path, install_mockery, mock_packages, mock_fetch, mock_archive, mutable_config, monkeypatch, tmp_path: pathlib.Path, ): # Create a temp mirror directory for buildcache usage mirror_dir = tmp_path / "mirror_dir" mirror_url = mirror_dir.as_uri() mirror("add", "atest", mirror_url) spec_name = "libdwarf" # Put a binary package in a buildcache install("--fake", "--no-cache", spec_name) buildcache("push", "-u", "-f", str(mirror_dir), spec_name) blobs_path = spack.binary_distribution.buildcache_relative_blobs_path() contents = os.listdir(str(mirror_dir)) assert blobs_path in contents # Destroy mirror by name mirror("destroy", "-m", "atest") assert not os.path.exists(str(mirror_dir)) uninstall("-y", spec_name) mirror("remove", "atest") @pytest.mark.usefixtures("mock_packages") class TestMirrorCreate: @pytest.mark.regression("31736", "31985") def test_all_specs_with_all_versions_dont_concretize(self): args = MockMirrorArgs(all=True, exclude_file=None, exclude_specs=None) mirror_specs = spack.cmd.mirror._specs_to_mirror(args) assert all(not s.concrete for s in mirror_specs) @pytest.mark.parametrize( "cli_args,error_str", [ # Passed more than one among -f --all ( {"specs": None, "file": "input.txt", "all": True}, "cannot specify specs with a file if", ), ( {"specs": "hdf5", "file": "input.txt", "all": False}, "cannot specify specs with a file AND", ), ({"specs": None, "file": None, "all": False}, "no packages were specified"), # Passed -n along with --all ( {"specs": None, "file": None, "all": True, "versions_per_spec": 2}, "cannot specify '--versions_per-spec'", ), ], ) def test_error_conditions(self, cli_args, error_str): args = MockMirrorArgs(**cli_args) with pytest.raises(spack.error.SpackError, match=error_str): spack.cmd.mirror.mirror_create(args) @pytest.mark.parametrize( "cli_args,not_expected", [ ( { "specs": "boost bowtie callpath", "exclude_specs": "bowtie", "dependencies": False, }, ["bowtie"], ), ( { "specs": "boost bowtie callpath", "exclude_specs": "bowtie callpath", "dependencies": False, }, ["bowtie", "callpath"], ), ( { "specs": "boost bowtie callpath", "exclude_specs": "bowtie", "dependencies": True, }, ["bowtie"], ), ], ) def test_exclude_specs_from_user(self, cli_args, not_expected, config): mirror_specs = spack.cmd.mirror._specs_to_mirror(MockMirrorArgs(**cli_args)) assert not any(s.satisfies(y) for s in mirror_specs for y in not_expected) @pytest.mark.parametrize("abstract_specs", [("bowtie", "callpath")]) def test_specs_from_cli_are_the_same_as_from_file( self, abstract_specs, config, tmp_path: pathlib.Path ): args = MockMirrorArgs(specs=" ".join(abstract_specs)) specs_from_cli = spack.cmd.mirror.concrete_specs_from_user(args) input_file = tmp_path / "input.txt" input_file.write_text("\n".join(abstract_specs)) args = MockMirrorArgs(file=str(input_file)) specs_from_file = spack.cmd.mirror.concrete_specs_from_user(args) assert specs_from_cli == specs_from_file @pytest.mark.parametrize( "input_specs,nversions", [("callpath", 1), ("mpich", 4), ("callpath mpich", 3), ("callpath mpich", "all")], ) def test_versions_per_spec_produces_concrete_specs(self, input_specs, nversions, config): args = MockMirrorArgs(specs=input_specs, versions_per_spec=nversions) specs = spack.cmd.mirror.concrete_specs_from_user(args) assert all(s.concrete for s in specs) def test_mirror_type(mutable_config): """Test the mirror set command""" mirror("add", "example", "--type", "binary", "http://example.com") assert spack.config.get("mirrors:example") == { "url": "http://example.com", "source": False, "binary": True, } mirror("set", "example", "--type", "source") assert spack.config.get("mirrors:example") == { "url": "http://example.com", "source": True, "binary": False, } mirror("set", "example", "--type", "binary") assert spack.config.get("mirrors:example") == { "url": "http://example.com", "source": False, "binary": True, } mirror("set", "example", "--type", "binary", "--type", "source") assert spack.config.get("mirrors:example") == { "url": "http://example.com", "source": True, "binary": True, } def test_mirror_set_2(mutable_config): """Test the mirror set command""" mirror("add", "example", "http://example.com") mirror( "set", "example", "--push", "--url", "http://example2.com", "--s3-access-key-id", "username", "--s3-access-key-secret-variable", "password", ) assert spack.config.get("mirrors:example") == { "url": "http://example.com", "push": { "url": "http://example2.com", "access_pair": {"id": "username", "secret_variable": "password"}, }, } def test_mirror_add_set_signed(mutable_config): mirror("add", "--signed", "example", "http://example.com") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": True} mirror("set", "--unsigned", "example") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": False} mirror("set", "--signed", "example") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": True} def test_mirror_add_set_autopush(mutable_config): # Add mirror without autopush mirror("add", "example", "http://example.com") assert spack.config.get("mirrors:example") == "http://example.com" mirror("set", "--no-autopush", "example") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False} mirror("set", "--autopush", "example") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True} mirror("set", "--no-autopush", "example") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False} mirror("remove", "example") # Add mirror with autopush mirror("add", "--autopush", "example", "http://example.com") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True} mirror("set", "--autopush", "example") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True} mirror("set", "--no-autopush", "example") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False} mirror("set", "--autopush", "example") assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True} mirror("remove", "example") @pytest.mark.require_provenance @pytest.mark.disable_clean_stage_check @pytest.mark.parametrize("mirror_knows_commit", (True, False)) def test_git_provenance_url_fails_mirror_resolves_commit( git, mock_git_repository, mock_packages, monkeypatch, tmp_path: pathlib.Path, mutable_config, mirror_knows_commit, ): """Extract git commit from a source mirror since other methods failed""" repo_path = mock_git_repository.path monkeypatch.setattr( spack.package_base.PackageBase, "git", f"file://{repo_path}", raising=False ) monkeypatch.setattr(spack.util.git, "get_commit_sha", lambda x, y: None, raising=False) gold_commit = git("-C", repo_path, "rev-parse", "main", output=str).strip() # create a fake mirror mirror_path = str(tmp_path / "test-mirror") if mirror_knows_commit: mirror("create", "-d", mirror_path, f"git-test-commit@main commit={gold_commit}") else: mirror("create", "-d", mirror_path, "git-test-commit@main") mirror("add", "--type", "source", "test-mirror", mirror_path) spec = spack.concretize.concretize_one("git-test-commit@main") assert spec.package.fetcher.source_id() == gold_commit assert "commit" in spec.variants assert spec.variants["commit"].value == gold_commit @pytest.mark.require_provenance @pytest.mark.disable_clean_stage_check def test_git_provenance_relative_to_mirror( git, mock_git_version_info, mock_packages, monkeypatch, tmp_path: pathlib.Path, mutable_config ): """Integration test to evaluate how commit resolution should behave with a mirror We want to confirm that the mirror doesn't break users ability to get a more recent commit Use `mock_git_version_info` repo because it has function scope and we can mess with the git history. """ repo_path, _, _ = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", f"file://{repo_path}", raising=False ) # create a fake mirror mirror_path = str(tmp_path / "test-mirror") mirror("create", "-d", mirror_path, "git-test-commit@main") mirror("add", "--type", "source", "test-mirror", mirror_path) mirror_commit = git("-C", repo_path, "rev-parse", "main", output=str).strip() # push the commit past mirror git("-C", repo_path, "checkout", "main", output=str) git("-C", repo_path, "commit", "--no-gpg-sign", "--allow-empty", "-m", "bump sha") head_commit = git("-C", repo_path, "rev-parse", "main", output=str).strip() spec_mirror = spack.concretize.concretize_one("git-test-commit@main") assert spec_mirror.variants["commit"].value == mirror_commit spec_head = spack.concretize.concretize_one(f"git-test-commit@main commit={head_commit}") assert spec_head.variants["commit"].value == head_commit @pytest.mark.usefixtures("mock_packages") def test_mirror_skip_placeholder_pkg(tmp_path: pathlib.Path): """Test a placeholder package which should skip during mirror all""" from spack.repo import PATH spec = spack.spec.Spec("placeholder@1.5") pkg_cls = PATH.get_pkg_class(spec.name) pkg_obj = pkg_cls(spec) mirror_cache = spack.mirrors.utils.get_mirror_cache(str(tmp_path)) mirror_stats = spack.mirrors.utils.MirrorStatsForOneSpec(spec) result = spack.mirrors.utils.create_mirror_from_package_object( pkg_obj, mirror_cache, mirror_stats ) assert result is False assert not mirror_stats.errors ================================================ FILE: lib/spack/spack/test/cmd/module.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import re import pytest import spack.concretize import spack.config import spack.main import spack.modules import spack.modules.lmod import spack.repo import spack.store from spack.installer import PackageInstaller module = spack.main.SpackCommand("module") pytestmark = pytest.mark.not_on_windows("does not run on windows") #: make sure module files are generated for all the tests here @pytest.fixture(scope="module", autouse=True) def ensure_module_files_are_there(mock_packages_repo, mock_store, mock_configuration_scopes): """Generate module files for module tests.""" module = spack.main.SpackCommand("module") with spack.store.use_store(str(mock_store)): with spack.config.use_configuration(*mock_configuration_scopes): with spack.repo.use_repositories(mock_packages_repo): module("tcl", "refresh", "-y") def _module_files(module_type, *specs): specs = [spack.concretize.concretize_one(x) for x in specs] writer_cls = spack.modules.module_types[module_type] return [writer_cls(spec, "default").layout.filename for spec in specs] @pytest.fixture( params=[ ["rm", "doesnotexist"], # Try to remove a non existing module ["find", "mpileaks"], # Try to find a module with multiple matches ["find", "doesnotexist"], # Try to find a module with no matches ["find", "--unknown_args"], # Try to give an unknown argument ] ) def failure_args(request): """A list of arguments that will cause a failure""" return request.param @pytest.fixture(params=["tcl", "lmod"]) def module_type(request): return request.param # TODO : test the --delete-tree option # TODO : this requires having a separate directory for test modules # TODO : add tests for loads and find to check the prompt format @pytest.mark.db def test_exit_with_failure(database, module_type, failure_args): with pytest.raises(spack.main.SpackCommandError): module(module_type, *failure_args) @pytest.mark.db def test_remove_and_add(database, module_type): """Tests adding and removing a tcl module file.""" if module_type == "lmod": # TODO: Testing this with lmod requires mocking # TODO: the core compilers return rm_cli_args = ["rm", "-y", "mpileaks"] module_files = _module_files(module_type, "mpileaks") for item in module_files: assert os.path.exists(item) module(module_type, *rm_cli_args) for item in module_files: assert not os.path.exists(item) module(module_type, "refresh", "-y", "mpileaks") for item in module_files: assert os.path.exists(item) @pytest.mark.db @pytest.mark.parametrize("cli_args", [["libelf"], ["--full-path", "libelf"]]) def test_find(database, cli_args, module_type): if module_type == "lmod": # TODO: Testing this with lmod requires mocking # TODO: the core compilers return module(module_type, *(["find"] + cli_args)) @pytest.mark.db @pytest.mark.usefixtures("database") @pytest.mark.regression("2215") def test_find_fails_on_multiple_matches(): # As we installed multiple versions of mpileaks, the command will # fail because of multiple matches out = module("tcl", "find", "mpileaks", fail_on_error=False) assert module.returncode == 1 assert "matches multiple packages" in out # Passing multiple packages from the command line also results in the # same failure out = module("tcl", "find", "mpileaks ^mpich", "libelf", fail_on_error=False) assert module.returncode == 1 assert "matches multiple packages" in out @pytest.mark.db @pytest.mark.usefixtures("database") @pytest.mark.regression("2570") def test_find_fails_on_non_existing_packages(): # Another way the command might fail is if the package does not exist out = module("tcl", "find", "doesnotexist", fail_on_error=False) assert module.returncode == 1 assert "matches no package" in out @pytest.mark.db @pytest.mark.usefixtures("database") def test_find_recursive(): # If we call find without options it should return only one module out = module("tcl", "find", "mpileaks ^zmpi") assert len(out.split()) == 1 # If instead we call it with the recursive option the length should # be greater out = module("tcl", "find", "-r", "mpileaks ^zmpi") assert len(out.split()) > 1 @pytest.mark.db def test_find_recursive_excluded(mutable_database, module_configuration): module_configuration("exclude") module("lmod", "refresh", "-y", "--delete-tree") module("lmod", "find", "-r", "mpileaks ^mpich") @pytest.mark.db def test_loads_recursive_excluded(mutable_database, module_configuration): module_configuration("exclude") module("lmod", "refresh", "-y", "--delete-tree") output = module("lmod", "loads", "-r", "mpileaks ^mpich") lines = output.split("\n") assert any(re.match(r"[^#]*module load.*mpileaks", ln) for ln in lines) assert not any(re.match(r"[^#]module load.*callpath", ln) for ln in lines) assert any(re.match(r"## excluded or missing.*callpath", ln) for ln in lines) # TODO: currently there is no way to separate stdout and stderr when # invoking a SpackCommand. Supporting this requires refactoring # SpackCommand, or log_output, or both. # start_of_warning = spack.cmd.modules._missing_modules_warning[:10] # assert start_of_warning not in output # Needed to make the 'module_configuration' fixture below work writer_cls = spack.modules.lmod.LmodModulefileWriter @pytest.mark.db def test_setdefault_command(mutable_database, mutable_config): data = { "default": { "enable": ["lmod"], "lmod": {"core_compilers": ["clang@3.3"], "hierarchy": ["mpi"]}, } } spack.config.set("modules", data) # Install two different versions of pkg-a other_spec, preferred = "pkg-a@1.0", "pkg-a@2.0" specs = [ spack.concretize.concretize_one(other_spec), spack.concretize.concretize_one(preferred), ] PackageInstaller([s.package for s in specs], explicit=True, fake=True).install() writers = { preferred: writer_cls(specs[1], "default"), other_spec: writer_cls(specs[0], "default"), } # Create two module files for the same software module("lmod", "refresh", "-y", "--delete-tree", preferred, other_spec) # Assert initial directory state: no link and all module files present link_name = os.path.join(os.path.dirname(writers[preferred].layout.filename), "default") for k in preferred, other_spec: assert os.path.exists(writers[k].layout.filename) assert not os.path.exists(link_name) # Set the default to be the other spec module("lmod", "setdefault", other_spec) # Check that a link named 'default' exists, and points to the right file for k in preferred, other_spec: assert os.path.exists(writers[k].layout.filename) assert os.path.exists(link_name) and os.path.islink(link_name) assert os.path.realpath(link_name) == os.path.realpath(writers[other_spec].layout.filename) # Reset the default to be the preferred spec module("lmod", "setdefault", preferred) # Check that a link named 'default' exists, and points to the right file for k in preferred, other_spec: assert os.path.exists(writers[k].layout.filename) assert os.path.exists(link_name) and os.path.islink(link_name) assert os.path.realpath(link_name) == os.path.realpath(writers[preferred].layout.filename) ================================================ FILE: lib/spack/spack/test/cmd/pkg.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import shutil import pytest import spack.cmd import spack.cmd.pkg import spack.main import spack.paths import spack.repo import spack.util.executable import spack.util.file_cache from spack.llnl.util.filesystem import mkdirp, working_dir pkg = spack.main.SpackCommand("pkg") #: new fake package template pkg_template = """\ from spack.package import * class {name}(Package): homepage = "http://www.example.com" url = "http://www.example.com/test-1.0.tar.gz" version("1.0", md5="0123456789abcdef0123456789abcdef") def install(self, spec, prefix): pass """ # Force all tests to use a git repository *in* the mock packages repo. @pytest.fixture(scope="module") def _builtin_mock_copy( git: spack.util.executable.Executable, tmp_path_factory: pytest.TempPathFactory ): """Copy the builtin_mock repo and make a mutable git repo inside it.""" root_dir: pathlib.Path = tmp_path_factory.mktemp("builtin_mock_copy") # create spack_repo subdir (root_dir / "spack_repo").mkdir() repo_dir = root_dir / "spack_repo" / "builtin_mock" shutil.copytree(spack.paths.mock_packages_path, str(repo_dir)) repo_cache = spack.util.file_cache.FileCache(root_dir / "cache") mock_repo = spack.repo.Repo(str(repo_dir), cache=repo_cache) with working_dir(mock_repo.packages_path): git("init") # initial commit with mock packages # the -f is necessary in case people ignore build-* in their ignores git("add", "-f", ".") git("config", "user.email", "testing@spack.io") git("config", "user.name", "Spack Testing") git("-c", "commit.gpgsign=false", "commit", "-m", "initial mock repo commit") # add commit with mockpkg-a, mockpkg-b, mockpkg-c packages mkdirp("mockpkg_a", "mockpkg_b", "mockpkg_c") with open("mockpkg_a/package.py", "w", encoding="utf-8") as f: f.write(pkg_template.format(name="PkgA")) with open("mockpkg_b/package.py", "w", encoding="utf-8") as f: f.write(pkg_template.format(name="PkgB")) with open("mockpkg_c/package.py", "w", encoding="utf-8") as f: f.write(pkg_template.format(name="PkgC")) git("add", "mockpkg_a", "mockpkg_b", "mockpkg_c") git("-c", "commit.gpgsign=false", "commit", "-m", "add mockpkg-a, mockpkg-b, mockpkg-c") # remove mockpkg-c, add mockpkg-d with open("mockpkg_b/package.py", "a", encoding="utf-8") as f: f.write("\n# change mockpkg-b") git("add", "mockpkg_b") mkdirp("mockpkg_d") with open("mockpkg_d/package.py", "w", encoding="utf-8") as f: f.write(pkg_template.format(name="PkgD")) git("add", "mockpkg_d") git("rm", "-rf", "mockpkg_c") git( "-c", "commit.gpgsign=false", "commit", "-m", "change mockpkg-b, remove mockpkg-c, add mockpkg-d", ) yield mock_repo @pytest.fixture def builtin_mock_copy(_builtin_mock_copy: spack.repo.Repo): """Fixture that enables a copy of the builtin_mock repo.""" with spack.repo.use_repositories(_builtin_mock_copy): yield _builtin_mock_copy def test_pkg_add(git, builtin_mock_copy: spack.repo.Repo): with working_dir(builtin_mock_copy.packages_path): mkdirp("mockpkg_e") with open("mockpkg_e/package.py", "w", encoding="utf-8") as f: f.write(pkg_template.format(name="PkgE")) pkg("add", "mockpkg-e") with working_dir(builtin_mock_copy.packages_path): try: assert "A mockpkg_e/package.py" in git("status", "--short", output=str) finally: shutil.rmtree("mockpkg_e") # Removing a package mid-run disrupts Spack's caching if spack.repo.PATH.repos[0]._fast_package_checker: spack.repo.PATH.repos[0]._fast_package_checker.invalidate() with pytest.raises(spack.main.SpackCommandError): pkg("add", "does-not-exist") @pytest.mark.not_on_windows("stdout format conflict") def test_pkg_list(builtin_mock_copy: spack.repo.Repo): # Be sure to include virtual packages since packages with stand-alone # tests may inherit additional tests from the virtuals they provide, # such as packages that implement `mpi`. mock_pkg_names = { name for name in builtin_mock_copy.all_package_names(include_virtuals=True) if not name.startswith("mockpkg-") } out = pkg("list", "HEAD^^").split() assert sorted(mock_pkg_names) == sorted(out) out = pkg("list", "HEAD^").split() assert sorted(mock_pkg_names.union(["mockpkg-a", "mockpkg-b", "mockpkg-c"])) == sorted(out) out = pkg("list", "HEAD").split() assert sorted(mock_pkg_names.union(["mockpkg-a", "mockpkg-b", "mockpkg-d"])) == sorted(out) # test with three dots to make sure pkg calls `git merge-base` out = pkg("list", "HEAD^^...").split() assert sorted(mock_pkg_names) == sorted(out) @pytest.mark.not_on_windows("stdout format conflict") def test_pkg_diff(builtin_mock_copy: spack.repo.Repo): out = pkg("diff", "HEAD^^", "HEAD^").split() assert out == ["HEAD^:", "mockpkg-a", "mockpkg-b", "mockpkg-c"] out = pkg("diff", "HEAD^^", "HEAD").split() assert out == ["HEAD:", "mockpkg-a", "mockpkg-b", "mockpkg-d"] out = pkg("diff", "HEAD^", "HEAD").split() assert out == ["HEAD^:", "mockpkg-c", "HEAD:", "mockpkg-d"] @pytest.mark.not_on_windows("stdout format conflict") def test_pkg_added(builtin_mock_copy: spack.repo.Repo): out = pkg("added", "HEAD^^", "HEAD^").split() assert ["mockpkg-a", "mockpkg-b", "mockpkg-c"] == out out = pkg("added", "HEAD^^", "HEAD").split() assert ["mockpkg-a", "mockpkg-b", "mockpkg-d"] == out out = pkg("added", "HEAD^", "HEAD").split() assert ["mockpkg-d"] == out out = pkg("added", "HEAD", "HEAD").split() assert out == [] @pytest.mark.not_on_windows("stdout format conflict") def test_pkg_removed(builtin_mock_copy: spack.repo.Repo): out = pkg("removed", "HEAD^^", "HEAD^").split() assert out == [] out = pkg("removed", "HEAD^^", "HEAD").split() assert out == [] out = pkg("removed", "HEAD^", "HEAD").split() assert out == ["mockpkg-c"] @pytest.mark.not_on_windows("stdout format conflict") def test_pkg_changed(builtin_mock_copy: spack.repo.Repo): out = pkg("changed", "HEAD^^", "HEAD^").split() assert out == [] out = pkg("changed", "--type", "c", "HEAD^^", "HEAD^").split() assert out == [] out = pkg("changed", "--type", "a", "HEAD^^", "HEAD^").split() assert out == ["mockpkg-a", "mockpkg-b", "mockpkg-c"] out = pkg("changed", "--type", "r", "HEAD^^", "HEAD^").split() assert out == [] out = pkg("changed", "--type", "ar", "HEAD^^", "HEAD^").split() assert out == ["mockpkg-a", "mockpkg-b", "mockpkg-c"] out = pkg("changed", "--type", "arc", "HEAD^^", "HEAD^").split() assert out == ["mockpkg-a", "mockpkg-b", "mockpkg-c"] out = pkg("changed", "HEAD^", "HEAD").split() assert out == ["mockpkg-b"] out = pkg("changed", "--type", "c", "HEAD^", "HEAD").split() assert out == ["mockpkg-b"] out = pkg("changed", "--type", "a", "HEAD^", "HEAD").split() assert out == ["mockpkg-d"] out = pkg("changed", "--type", "r", "HEAD^", "HEAD").split() assert out == ["mockpkg-c"] out = pkg("changed", "--type", "ar", "HEAD^", "HEAD").split() assert out == ["mockpkg-c", "mockpkg-d"] out = pkg("changed", "--type", "arc", "HEAD^", "HEAD").split() assert out == ["mockpkg-b", "mockpkg-c", "mockpkg-d"] # invalid type argument with pytest.raises(spack.main.SpackCommandError): pkg("changed", "--type", "foo") def test_pkg_fails_when_not_git_repo(monkeypatch): monkeypatch.setattr(spack.cmd, "spack_is_git_repo", lambda: False) with pytest.raises(spack.main.SpackCommandError): pkg("added") def test_pkg_source_requires_one_arg(mock_packages): with pytest.raises(spack.main.SpackCommandError): pkg("source", "a", "b") with pytest.raises(spack.main.SpackCommandError): pkg("source", "--canonical", "a", "b") def test_pkg_source(mock_packages): fake_source = pkg("source", "fake") fake_file = spack.repo.PATH.filename_for_package_name("fake") with open(fake_file, encoding="utf-8") as f: contents = f.read() assert fake_source == contents def test_pkg_canonical_source(mock_packages): source = pkg("source", "multimethod") assert '@when("@2.0")' in source assert "Check that multimethods work with boolean values" in source canonical_1 = pkg("source", "--canonical", "multimethod@1.0") assert "@when" not in canonical_1 assert "should_not_be_reached by diamond inheritance test" not in canonical_1 assert "return 'base@1.0'" in canonical_1 assert "return 'base@2.0'" not in canonical_1 assert "return 'first_parent'" not in canonical_1 assert "'should_not_be_reached by diamond inheritance test'" not in canonical_1 canonical_2 = pkg("source", "--canonical", "multimethod@2.0") assert "@when" not in canonical_2 assert "return 'base@1.0'" not in canonical_2 assert "return 'base@2.0'" in canonical_2 assert "return 'first_parent'" in canonical_2 assert "'should_not_be_reached by diamond inheritance test'" not in canonical_2 canonical_3 = pkg("source", "--canonical", "multimethod@3.0") assert "@when" not in canonical_3 assert "return 'base@1.0'" not in canonical_3 assert "return 'base@2.0'" not in canonical_3 assert "return 'first_parent'" not in canonical_3 assert "'should_not_be_reached by diamond inheritance test'" not in canonical_3 canonical_4 = pkg("source", "--canonical", "multimethod@4.0") assert "@when" not in canonical_4 assert "return 'base@1.0'" not in canonical_4 assert "return 'base@2.0'" not in canonical_4 assert "return 'first_parent'" not in canonical_4 assert "'should_not_be_reached by diamond inheritance test'" in canonical_4 def test_pkg_hash(mock_packages): output = pkg("hash", "pkg-a", "pkg-b").split() assert len(output) == 2 and all(len(elt) == 32 for elt in output) output = pkg("hash", "multimethod").split() assert len(output) == 1 and all(len(elt) == 32 for elt in output) group_args = [ "/path/one.py", # 12 "/path/two.py", # 12 "/path/three.py", # 14 "/path/four.py", # 13 "/path/five.py", # 13 "/path/six.py", # 12 "/path/seven.py", # 14 "/path/eight.py", # 14 "/path/nine.py", # 13 "/path/ten.py", # 12 ] @pytest.mark.parametrize( ["args", "max_group_size", "prefix_length", "max_group_length", "lengths", "error"], [ (group_args, 3, 0, 1, None, ValueError), # element too long (group_args, 3, 0, 13, None, ValueError), # element too long (group_args, 3, 12, 25, None, ValueError), # prefix and words too long (group_args, 3, 0, 25, [2, 1, 1, 1, 1, 1, 1, 1, 1], None), (group_args, 3, 0, 26, [2, 1, 1, 2, 1, 1, 2], None), (group_args, 3, 0, 40, [3, 3, 2, 2], None), (group_args, 3, 0, 43, [3, 3, 3, 1], None), (group_args, 4, 0, 54, [4, 3, 3], None), (group_args, 4, 0, 56, [4, 4, 2], None), ([], 500, 0, None, [], None), ], ) def test_group_arguments( mock_packages, args, max_group_size, prefix_length, max_group_length, lengths, error ): generator = spack.cmd.group_arguments( args, max_group_size=max_group_size, prefix_length=prefix_length, max_group_length=max_group_length, ) # just check that error cases raise if error: with pytest.raises(ValueError): list(generator) return groups = list(generator) assert sum(groups, []) == args assert [len(group) for group in groups] == lengths assert all( sum(len(elt) for elt in group) + (len(group) - 1) <= max_group_length for group in groups ) @pytest.mark.skipif(not spack.cmd.pkg.get_grep(), reason="grep is not installed") def test_pkg_grep(mock_packages): # only splice-* mock packages have the string "splice" in them pkg("grep", "-l", "splice") assert pkg.output.strip() == "\n".join( spack.repo.PATH.get_pkg_class(name).module.__file__ for name in [ "depends-on-manyvariants", "manyvariants", "splice-a", "splice-depends-on-t", "splice-h", "splice-t", "splice-vh", "splice-vt", "splice-z", "virtual-abi-1", "virtual-abi-2", "virtual-abi-multi", ] ) # ensure that this string isn't found with pytest.raises(spack.main.SpackCommandError): pkg("grep", "abcdefghijklmnopqrstuvwxyz") assert pkg.returncode == 1 assert pkg.output.strip() == "" # ensure that we return > 1 for an error with pytest.raises(spack.main.SpackCommandError): pkg("grep", "--foobarbaz-not-an-option") assert pkg.returncode == 2 ================================================ FILE: lib/spack/spack/test/cmd/print_shell_vars.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.main import print_setup_info def test_print_shell_vars_sh(capfd): print_setup_info("sh") out, _ = capfd.readouterr() assert "_sp_sys_type=" in out assert "_sp_tcl_roots=" in out assert "_sp_lmod_roots=" in out assert "_sp_module_prefix" not in out def test_print_shell_vars_csh(capfd): print_setup_info("csh") out, _ = capfd.readouterr() assert "set _sp_sys_type = " in out assert "set _sp_tcl_roots = " in out assert "set _sp_lmod_roots = " in out assert "set _sp_module_prefix = " not in out ================================================ FILE: lib/spack/spack/test/cmd/providers.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest from spack.main import SpackCommand pytestmark = [pytest.mark.usefixtures("mock_packages")] providers = SpackCommand("providers") @pytest.mark.parametrize( "pkg", [("mpi",), ("mpi@2",), ("mpi", "lapack"), ("",)], # Lists all the available virtual packages ) def test_it_just_runs(pkg): providers(*pkg) @pytest.mark.parametrize( "vpkg,provider_list", [ ( ("mpi",), [ "intel-parallel-studio", "low-priority-provider", "mpich@3:", "mpich2", "multi-provider-mpi@1.10.0", "multi-provider-mpi@2.0.0", "zmpi", ], ), ( ("lapack", "something"), [ "intel-parallel-studio", "low-priority-provider", "netlib-lapack", "openblas-with-lapack", "simple-inheritance", "splice-a", "splice-h", "splice-vh", ], ), # Call 2 virtual packages at once ], ) def test_provider_lists(vpkg, provider_list): output = providers(*vpkg) for item in provider_list: assert item in output @pytest.mark.parametrize( "pkg,error_cls", [ ("zlib", ValueError), ("foo", ValueError), # Trying to call with a package that does not exist ], ) def test_it_just_fails(pkg, error_cls): with pytest.raises(error_cls): providers(pkg) ================================================ FILE: lib/spack/spack/test/cmd/python.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform import sys import pytest import spack from spack.main import SpackCommand python = SpackCommand("python") def test_python(): out = python("-c", "import spack; print(spack.spack_version)") assert out.strip() == spack.spack_version def test_python_interpreter_path(): out = python("--path") assert out.strip() == sys.executable def test_python_version(): out = python("-V") assert platform.python_version() in out def test_python_with_module(): # pytest rewrites a lot of modules, which interferes with runpy, so # it's hard to test this. Trying to import a module like sys, that # has no code associated with it, raises an error reliably in python # 2 and 3, which indicates we successfully ran runpy.run_module. with pytest.raises(ImportError, match="No code object"): python("-m", "sys") def test_python_raises(): out = python("--foobar", fail_on_error=False) assert "Error: Unknown arguments" in out ================================================ FILE: lib/spack/spack/test/cmd/reindex.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import shutil import spack.store from spack.database import Database from spack.enums import InstallRecordStatus from spack.main import SpackCommand install = SpackCommand("install") deprecate = SpackCommand("deprecate") reindex = SpackCommand("reindex") def test_reindex_basic(mock_packages, mock_archive, mock_fetch, install_mockery): install("--fake", "libelf@0.8.13") install("--fake", "libelf@0.8.12") all_installed = spack.store.STORE.db.query() reindex() assert spack.store.STORE.db.query() == all_installed def _clear_db(tmp_path: pathlib.Path): empty_db = Database(str(tmp_path)) with empty_db.write_transaction(): pass shutil.rmtree(spack.store.STORE.db.database_directory) shutil.copytree(empty_db.database_directory, spack.store.STORE.db.database_directory) # force a re-read of the database assert len(spack.store.STORE.db.query()) == 0 def test_reindex_db_deleted( mock_packages, mock_archive, mock_fetch, install_mockery, tmp_path: pathlib.Path ): install("--fake", "libelf@0.8.13") install("--fake", "libelf@0.8.12") all_installed = spack.store.STORE.db.query() _clear_db(tmp_path) reindex() assert spack.store.STORE.db.query() == all_installed def test_reindex_with_deprecated_packages( mock_packages, mock_archive, mock_fetch, install_mockery, tmp_path: pathlib.Path ): install("--fake", "libelf@0.8.13") install("--fake", "libelf@0.8.12") deprecate("-y", "libelf@0.8.12", "libelf@0.8.13") db = spack.store.STORE.db all_installed = db.query(installed=InstallRecordStatus.ANY) non_deprecated = db.query(installed=True) _clear_db(tmp_path) reindex() assert db.query(installed=InstallRecordStatus.ANY) == all_installed assert db.query(installed=True) == non_deprecated old_libelf = db.query_local_by_spec_hash( db.query_local("libelf@0.8.12", installed=InstallRecordStatus.ANY)[0].dag_hash() ) new_libelf = db.query_local_by_spec_hash( db.query_local("libelf@0.8.13", installed=True)[0].dag_hash() ) assert old_libelf is not None and new_libelf is not None assert old_libelf.deprecated_for == new_libelf.spec.dag_hash() assert new_libelf.deprecated_for is None assert new_libelf.ref_count == 1 ================================================ FILE: lib/spack/spack/test/cmd/repo.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import os import pathlib import sys from typing import Dict, Optional, Union import pytest import spack.cmd.repo import spack.config import spack.environment as ev import spack.main import spack.repo import spack.repo_migrate from spack.error import SpackError from spack.llnl.util.filesystem import working_dir from spack.util.executable import Executable repo = spack.main.SpackCommand("repo") env = spack.main.SpackCommand("env") def test_help_option(): # Test 'spack repo --help' to check basic import works # and the command exits successfully repo("--help") assert repo.returncode in (None, 0) def test_create_add_list_remove(mutable_config, tmp_path: pathlib.Path): # Create a new repository and check that the expected # files are there repo("create", str(tmp_path), "mockrepo") assert (tmp_path / "spack_repo" / "mockrepo" / "repo.yaml").exists() # Add the new repository and check it appears in the list output repo("add", "--scope=site", str(tmp_path / "spack_repo" / "mockrepo")) output = repo("list", "--scope=site") assert "mockrepo" in output # Then remove it and check it's not there repo("remove", "--scope=site", str(tmp_path / "spack_repo" / "mockrepo")) output = repo("list", "--scope=site") assert "mockrepo" not in output def test_repo_remove_by_scope(mutable_config, tmp_path: pathlib.Path): # Create and add a new repo repo("create", str(tmp_path), "mockrepo") repo("add", "--scope=site", str(tmp_path / "spack_repo" / "mockrepo")) repo("add", "--scope=system", str(tmp_path / "spack_repo" / "mockrepo")) # Confirm that it is not removed when the scope is incorrect with pytest.raises(spack.main.SpackCommandError): repo("remove", "--scope=user", "mockrepo") output = repo("list") assert "mockrepo" in output # Confirm that when the scope is specified, it is only removed from that scope repo("remove", "--scope=site", "mockrepo") site_output = repo("list", "--scope=site") system_output = repo("list", "--scope=system") assert "mockrepo" not in site_output assert "mockrepo" in system_output # Confirm that when the scope is not specified, it is removed from top scope with it present repo("add", "--scope=site", str(tmp_path / "spack_repo" / "mockrepo")) repo("remove", "mockrepo") site_output = repo("list", "--scope=site") system_output = repo("list", "--scope=system") assert "mockrepo" not in site_output assert "mockrepo" in system_output # Check that the `--all-scopes` option removes from all scopes repo("add", "--scope=site", str(tmp_path / "spack_repo" / "mockrepo")) repo("remove", "--all-scopes", "mockrepo") output = repo("list") assert "mockrepo" not in output def test_env_repo_path_vars_substitution( tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, monkeypatch ): """Test Spack correctly substitutes repo paths with environment variables when creating an environment from a manifest file.""" monkeypatch.setenv("CUSTOM_REPO_PATH", ".") # setup environment from spack.yaml envdir = tmp_path / "env" envdir.mkdir() with working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( """\ spack: specs: [] repos: current_dir: $CUSTOM_REPO_PATH """ ) # creating env from manifest file env("create", "test", "./spack.yaml") # check that repo path was correctly substituted with the environment variable current_dir = os.getcwd() with ev.read("test") as newenv: repos_specs = spack.config.get("repos", default={}, scope=newenv.scope_name) assert current_dir in repos_specs.values() OLD_7ZIP = b"""\ # some comment from spack.package import * from .blt import linker_helpers class _7zip(Package): pass """ NEW_7ZIP = b"""\ # some comment from spack_repo.builtin.build_systems.generic import Package from spack.package import * from ..blt.package import linker_helpers class _7zip(Package): pass """ # this is written like this to be explicit about line endings and indentation OLD_NUMPY = ( b"# some comment\r\n" b"\r\n" b"import spack.pkg.builtin.foo, spack.pkg.builtin.bar\r\n" b"from spack.package import *\r\n" b"from something.unrelated import AutotoolsPackage\r\n" b"from spack.pkg.builtin.num7zip import _7zip\r\n" b"\r\n" b"\r\n" b"class PyNumpy(CMakePackage, AutotoolsPackage):\r\n" b"\tgenerator('ninja')\r\n" b"\r\n" b"\tdef example(self):\r\n" b"\t\t# unchanged comment: spack.pkg.builtin.foo.something\r\n" b"\t\treturn spack.pkg.builtin.foo.example(), foo, baz\r\n" ) NEW_NUMPY = ( b"# some comment\r\n" b"\r\n" b"import spack_repo.builtin.packages.foo.package, spack_repo.builtin.packages.bar.package\r\n" b"from spack_repo.builtin.build_systems.cmake import CMakePackage, generator\r\n" b"from spack.package import *\r\n" b"from something.unrelated import AutotoolsPackage\r\n" b"from spack_repo.builtin.packages._7zip.package import _7zip\r\n" b"\r\n" b"\r\n" b"class PyNumpy(CMakePackage, AutotoolsPackage):\r\n" b"\tgenerator('ninja')\r\n" b"\r\n" b"\tdef example(self):\r\n" b"\t\t# unchanged comment: spack.pkg.builtin.foo.something\r\n" b"\t\treturn spack_repo.builtin.packages.foo.package.example(), foo, baz\r\n" ) OLD_NONTRIVIAL_IMPORT = b"""\ if True: from spack.pkg.builtin import ( foo, bar as baz, num7zip as my7zip ) class NonTrivialImport(Package): pass """ NEW_NONTRIVIAL_IMPORT = b"""\ from spack_repo.builtin.build_systems.generic import Package if True: import spack_repo.builtin.packages.foo.package as foo import spack_repo.builtin.packages.bar.package as baz import spack_repo.builtin.packages._7zip.package as my7zip class NonTrivialImport(Package): pass """ def test_repo_migrate(tmp_path: pathlib.Path, config): old_root, _ = spack.repo.create_repo(str(tmp_path), "org.repo", package_api=(1, 0)) pkgs_path = pathlib.Path(spack.repo.from_path(old_root).packages_path) new_root = pathlib.Path(old_root) / "spack_repo" / "org" / "repo" pkg_7zip_old = pkgs_path / "7zip" / "package.py" pkg_numpy_old = pkgs_path / "py-numpy" / "package.py" pkg_py_7zip_new = new_root / "packages" / "_7zip" / "package.py" pkg_py_numpy_new = new_root / "packages" / "py_numpy" / "package.py" pkg_7zip_old.parent.mkdir(parents=True) pkg_numpy_old.parent.mkdir(parents=True) pkg_7zip_old.write_bytes(OLD_7ZIP) pkg_numpy_old.write_bytes(OLD_NUMPY) repo("migrate", "--fix", old_root) # old files are not touched since they are moved assert pkg_7zip_old.read_bytes() == OLD_7ZIP assert pkg_numpy_old.read_bytes() == OLD_NUMPY # new files are created and have updated contents assert pkg_py_7zip_new.read_bytes() == NEW_7ZIP assert pkg_py_numpy_new.read_bytes() == NEW_NUMPY def test_migrate_diff(git: Executable, tmp_path: pathlib.Path): root, _ = spack.repo.create_repo(str(tmp_path), "foo", package_api=(2, 0)) r = pathlib.Path(root) pkg_7zip = r / "packages" / "_7zip" / "package.py" pkg_py_numpy = r / "packages" / "py_numpy" / "package.py" pkg_broken = r / "packages" / "broken" / "package.py" pkg_nontrivial_import = r / "packages" / "non_trivial_import" / "package.py" pkg_7zip.parent.mkdir(parents=True) pkg_py_numpy.parent.mkdir(parents=True) pkg_broken.parent.mkdir(parents=True) pkg_nontrivial_import.parent.mkdir(parents=True) pkg_7zip.write_bytes(OLD_7ZIP) pkg_py_numpy.write_bytes(OLD_NUMPY) pkg_broken.write_bytes(b"syntax(error") pkg_nontrivial_import.write_bytes(OLD_NONTRIVIAL_IMPORT) stderr = io.StringIO() with open(tmp_path / "imports.patch", "wb") as patch_file: spack.repo_migrate.migrate_v2_imports( str(r / "packages"), str(r), patch_file=patch_file, err=stderr ) err_output = stderr.getvalue() assert f"Skipping {pkg_broken}" in err_output # apply the patch and verify the changes with working_dir(str(r)): git("apply", str(tmp_path / "imports.patch")) # Git may change line endings upon applying the patch, so let Python normalize in TextIOWrapper # and compare strings instead of bytes. assert ( pkg_7zip.read_text(encoding="utf-8") == io.TextIOWrapper(io.BytesIO(NEW_7ZIP), encoding="utf-8").read() ) assert ( pkg_py_numpy.read_text(encoding="utf-8") == io.TextIOWrapper(io.BytesIO(NEW_NUMPY), encoding="utf-8").read() ) # the multi-line non-trivial import rewrite cannot be done in Python < 3.8 because it doesn't # support end_lineno in ast.ImportFrom. So here we check that it's either warned about or # modified correctly. if sys.version_info >= (3, 8): assert ( pkg_nontrivial_import.read_text(encoding="utf-8") == io.TextIOWrapper(io.BytesIO(NEW_NONTRIVIAL_IMPORT), encoding="utf-8").read() ) else: assert ( f"{pkg_nontrivial_import}:2: cannot rewrite spack.pkg.builtin import statement" in err_output ) class MockRepo(spack.repo.Repo): def __init__(self, namespace: str): self.namespace = namespace class MockDescriptor(spack.repo.RepoDescriptor): def __init__(self, to_construct: Dict[str, Union[spack.repo.Repo, Exception]]): self.to_construct = to_construct self.initialized = False def initialize(self, fetch=True, git=None) -> None: self.initialized = True def get_commit(self, git: Optional[Executable] = None): pass def update(self, git: Optional[Executable] = None, remote: Optional[str] = "origin") -> None: pass def construct(self, cache, overrides=None): assert self.initialized, "MockDescriptor must be initialized before construction" return self.to_construct def make_repo_config(repo_config: Optional[dict] = None) -> spack.config.Configuration: """Create a Configuration instance with writable scope and optional repo configuration.""" scope = spack.config.InternalConfigScope("test", {"repos": repo_config or {}}) scope.writable = True config = spack.config.Configuration() config.push_scope(scope) return config def test_add_repo_name_already_exists(tmp_path: pathlib.Path): """Test _add_repo raises error when name already exists in config.""" # Set up existing config with the same name config = make_repo_config({"test_name": "/some/path"}) # Should raise error when name already exists with pytest.raises(SpackError, match="A repository with the name 'test_name' already exists"): spack.cmd.repo._add_repo( str(tmp_path), name="test_name", scope=None, paths=[], destination=None, config=config ) def test_add_repo_destination_with_local_path(tmp_path: pathlib.Path): """Test _add_repo raises error when args are added that do not apply to local paths.""" # Should raise error when destination is provided with local path with pytest.raises( SpackError, match="The 'destination' argument is only valid for git repositories" ): spack.cmd.repo._add_repo( str(tmp_path), name="test_name", scope=None, paths=[], destination="/some/destination", config=make_repo_config(), ) with pytest.raises(SpackError, match="The --paths flag is only valid for git repositories"): spack.cmd.repo._add_repo( str(tmp_path), name="test_name", scope=None, paths=["path1", "path2"], destination=None, config=make_repo_config(), ) def test_add_repo_computed_key_already_exists(tmp_path: pathlib.Path, monkeypatch): """Test _add_repo raises error when computed key already exists in config.""" def mock_parse_config_descriptor(name, entry, lock): return MockDescriptor({str(tmp_path): MockRepo("test_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) # Should raise error when computed key already exists with pytest.raises(SpackError, match="A repository with the name 'test_repo' already exists"): spack.cmd.repo._add_repo( str(tmp_path), name=None, # Will use namespace as key scope=None, paths=[], destination=None, config=make_repo_config({"test_repo": "/some/path"}), ) def test_add_repo_git_url_with_paths(monkeypatch): """Test _add_repo correctly handles git URL with multiple paths.""" config = make_repo_config({"test_repo": "/some/path"}) def mock_parse_config_descriptor(name, entry, lock): # Verify the entry has the expected git structure assert "git" in entry assert entry["git"] == "https://example.com/repo.git" assert entry["paths"] == ["path1", "path2"] return MockDescriptor({"/some/path": MockRepo("git_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) # Should succeed with git URL and multiple paths key = spack.cmd.repo._add_repo( "https://example.com/repo.git", name="git_test", scope=None, paths=["path1", "path2"], destination=None, config=config, ) assert key == "git_test" repos = config.get("repos", scope=None) assert "git_test" in repos assert repos["git_test"]["git"] == "https://example.com/repo.git" assert repos["git_test"]["paths"] == ["path1", "path2"] def test_add_repo_git_url_with_destination(monkeypatch): """Test _add_repo correctly handles git URL with destination.""" config = make_repo_config({"test_repo": "/some/path"}) def mock_parse_config_descriptor(name, entry, lock): # Verify the entry has the expected git structure assert "git" in entry assert entry["git"] == "https://example.com/repo.git" assert entry["destination"] == "/custom/destination" return MockDescriptor({"/some/path": MockRepo("git_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) # Should succeed with git URL and destination key = spack.cmd.repo._add_repo( "https://example.com/repo.git", name="git_test", scope=None, paths=[], destination="/custom/destination", config=config, ) assert key == "git_test" repos = config.get("repos", scope=None) assert "git_test" in repos assert repos["git_test"]["git"] == "https://example.com/repo.git" assert repos["git_test"]["destination"] == "/custom/destination" def test_add_repo_ssh_git_url_detection(monkeypatch): """Test _add_repo correctly detects SSH git URLs.""" config = make_repo_config({"test_repo": "/some/path"}) def mock_parse_config_descriptor(name, entry, lock): # Verify the entry has the expected git structure assert "git" in entry assert entry["git"] == "git@github.com:user/repo.git" return MockDescriptor({"/some/path": MockRepo("git_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) # Should detect SSH URL as git URL (colon not preceded by forward slash) key = spack.cmd.repo._add_repo( "git@github.com:user/repo.git", name="ssh_git_test", scope=None, paths=[], destination=None, config=config, ) assert key == "ssh_git_test" repos = config.get("repos", scope=None) assert "ssh_git_test" in repos assert repos["ssh_git_test"]["git"] == "git@github.com:user/repo.git" def test_add_repo_no_usable_repositories_error(monkeypatch): """Test that _add_repo raises SpackError when no usable repositories can be constructed.""" config = make_repo_config() def mock_parse_config_descriptor(name, entry, lock): return MockDescriptor( {"/path1": Exception("Invalid repo"), "/path2": Exception("Another error")} ) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) with pytest.raises( SpackError, match="No package repository could be constructed from /invalid/path" ): spack.cmd.repo._add_repo( "/invalid/path", name="test_repo", scope=None, paths=[], destination=None, config=config, ) def test_add_repo_multiple_repos_no_name_error(monkeypatch): """Test that _add_repo raises SpackError when multiple repositories found without specifying --name.""" def mock_parse_config_descriptor(name, entry, lock): return MockDescriptor({"/path1": MockRepo("repo1"), "/path2": MockRepo("repo2")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) with pytest.raises( SpackError, match="Multiple package repositories found, please specify a name" ): spack.cmd.repo._add_repo( "/path/with/multiple/repos", name=None, # No name specified scope=None, paths=[], destination=None, config=make_repo_config(), ) def test_add_repo_git_url_basic_success(monkeypatch): """Test successful addition of a git repository.""" config = make_repo_config() def mock_parse_config_descriptor(name, entry, lock): # Verify git entry structure assert isinstance(entry, dict) assert entry["git"] == "https://github.com/example/repo.git" return MockDescriptor({"/git/path": MockRepo("git_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) key = spack.cmd.repo._add_repo( "https://github.com/example/repo.git", name="test_git_repo", scope=None, paths=[], destination=None, config=config, ) assert key == "test_git_repo" repos_config = config.get("repos", scope=None) assert "test_git_repo" in repos_config assert "git" in repos_config["test_git_repo"] def test_add_repo_git_url_with_custom_destination(monkeypatch): """Test successful addition of a git repository with destination.""" config = make_repo_config() def mock_parse_config_descriptor(name, entry, lock): # Verify git entry structure with destination assert isinstance(entry, dict) assert "git" in entry assert "destination" in entry assert entry["destination"] == "/custom/destination" return MockDescriptor({"/git/path": MockRepo("git_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) key = spack.cmd.repo._add_repo( "git@github.com:example/repo.git", name="test_git_repo", scope=None, paths=[], destination="/custom/destination", config=config, ) assert key == "test_git_repo" def test_add_repo_git_url_with_single_repo_path_new(monkeypatch): """Test successful addition of a git repository with repo_path.""" config = make_repo_config() def mock_parse_config_descriptor(name, entry, lock): # Verify git entry structure with repo_path assert isinstance(entry, dict) assert "git" in entry assert "paths" in entry assert entry["paths"] == ["subdirectory/repo"] return MockDescriptor({"/git/path": MockRepo("git_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) key = spack.cmd.repo._add_repo( "https://github.com/example/repo.git", name="test_git_repo", scope=None, paths=["subdirectory/repo"], destination=None, config=config, ) assert key == "test_git_repo" def test_add_repo_local_path_success(monkeypatch, tmp_path: pathlib.Path): """Test successful addition of a local repository.""" config = make_repo_config() def mock_parse_config_descriptor(name, entry, lock): # Verify local path entry assert isinstance(entry, str) return MockDescriptor({str(tmp_path): MockRepo("test_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) key = spack.cmd.repo._add_repo( str(tmp_path), name="test_local_repo", scope=None, paths=[], destination=None, config=config, ) assert key == "test_local_repo" # Verify the local path was added repos_config = config.get("repos") assert "test_local_repo" in repos_config assert repos_config["test_local_repo"] == str(tmp_path) def test_add_repo_auto_name_from_namespace(monkeypatch, tmp_path: pathlib.Path): """Test successful addition of a repository with auto-generated name from namespace.""" config = make_repo_config() def mock_parse_config_descriptor(name, entry, lock): return MockDescriptor({str(tmp_path): MockRepo("auto_name_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) key = spack.cmd.repo._add_repo( str(tmp_path), name=None, # No name specified, should use namespace scope=None, paths=[], destination=None, config=config, ) assert key == "auto_name_repo" # Verify the repo was added with the namespace as key repos_config = config.get("repos", scope=None) assert "auto_name_repo" in repos_config assert repos_config["auto_name_repo"] == str(tmp_path) def test_add_repo_partial_repo_construction_warning(monkeypatch, capfd): """Test that _add_repo issues warnings for repos that can't be constructed but succeeds if at least one can be.""" def mock_parse_config_descriptor(name, entry, lock): return MockDescriptor( { "/good/path": MockRepo("good_repo"), "/bad/path": Exception("Failed to construct repo"), } ) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) key = spack.cmd.repo._add_repo( "/mixed/path", name="test_mixed_repo", scope=None, paths=[], destination=None, config=make_repo_config(), ) assert key == "test_mixed_repo" # Check that a warning was issued for the failed repo captured = capfd.readouterr() assert "Skipping package repository" in captured.err @pytest.mark.parametrize( "test_url,expected_type", [ ("ssh://git@github.com/user/repo.git", "git"), # ssh URL ("git://github.com/user/repo.git", "git"), # git protocol ("user@host:repo.git", "git"), # SSH short form ("file:///local/path", "git"), # file URL ("/local/path", "local"), # local path ("./relative/path", "local"), # relative path ("C:\\Windows\\Path", "local"), # Windows path ], ) def test_add_repo_git_url_detection_edge_cases(monkeypatch, test_url, expected_type): """Test edge cases for git URL detection.""" config = make_repo_config() def mock_parse_config_descriptor(name, entry, lock): return MockDescriptor({"/path": MockRepo("test_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) spack.cmd.repo._add_repo( test_url, name=None, scope=None, paths=[], destination=None, config=config ) entry = config.get("repos").get("test_repo") if expected_type == "git": assert entry == {"git": test_url} else: assert isinstance(entry, str) def test_repo_set_git_config(mutable_config): """Test that 'spack repo set' properly modifies git repository configurations.""" # Set up initial git repository config in defaults scope git_url = "https://github.com/example/test-repo.git" initial_config = {"repos": {"test-repo": {"git": git_url}}} spack.config.set("repos", initial_config["repos"], scope="site") # Test setting destination and paths repo("set", "--scope=user", "--destination", "/custom/path", "test-repo") repo("set", "--scope=user", "--path", "subdir1", "--path", "subdir2", "test-repo") # Check that the user config has the updated entry user_repos = spack.config.get("repos", scope="user") assert user_repos["test-repo"]["paths"] == ["subdir1", "subdir2"] assert user_repos["test-repo"]["destination"] == "/custom/path" # Check that site scope is unchanged site_repos = spack.config.get("repos", scope="site") assert "destination" not in site_repos["test-repo"] def test_repo_set_nonexistent_repo(mutable_config): with pytest.raises(SpackError, match="No repository with namespace 'nonexistent'"): repo("set", "--destination", "/some/path", "nonexistent") def test_repo_set_does_not_work_on_local_path(mutable_config): spack.config.set("repos", {"local-repo": "/local/path"}, scope="site") with pytest.raises(SpackError, match="is not a git repository"): repo("set", "--destination", "/some/path", "local-repo") def test_add_repo_prepends_instead_of_appends(monkeypatch, tmp_path: pathlib.Path): """Test that newly added repositories are prepended to the configuration, giving them higher priority than existing repositories.""" existing_path = str(tmp_path / "existing_repo") new_path = str(tmp_path / "new_repo") config = make_repo_config({"existing_repo": existing_path}) def mock_parse_config_descriptor(name, entry, lock): return MockDescriptor({new_path: MockRepo("new_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) # Add a new repository key = spack.cmd.repo._add_repo( path_or_repo=new_path, name="new_repo", scope=None, paths=[], destination=None, config=config, ) assert key == "new_repo" # Check that the new repository is first in the configuration repos_config = config.get("repos", scope=None) repo_names = list(repos_config.keys()) # The new repository should be first (highest priority) assert repo_names == ["new_repo", "existing_repo"] assert repos_config["new_repo"] == new_path assert repos_config["existing_repo"] == existing_path def test_repo_list_format_flags( mutable_config: spack.config.Configuration, tmp_path: pathlib.Path ): """Test the --config-names and --namespaces flags for repo list command""" # Fake a git monorepo with two package repositories (tmp_path / "monorepo" / ".git").mkdir(parents=True) repo("create", str(tmp_path / "monorepo"), "repo_one") repo("create", str(tmp_path / "monorepo"), "repo_two") mutable_config.set( "repos", { # git repo that provides two package repositories "monorepo": { "git": "https://example.com/monorepo.git", "destination": str(tmp_path / "monorepo"), "paths": ["spack_repo/repo_one", "spack_repo/repo_two"], }, # git repo that is not yet cloned "uninitialized": { "git": "https://example.com/uninitialized.git", "destination": str(tmp_path / "uninitialized"), }, # invalid local repository "misconfigured": str(tmp_path / "misconfigured"), }, scope="site", ) # Test default table format, which shows one line per package repository table_output = repo("list") assert "[+] repo_one" in table_output assert "[+] repo_two" in table_output assert " - uninitialized" in table_output assert "[-] misconfigured" in table_output # Test --namespaces flag namespaces_output = repo("list", "--namespaces") assert namespaces_output.strip().split("\n") == ["repo_one", "repo_two"] # Test --names flag config_names_output = repo("list", "--names") config_names_lines = config_names_output.strip().split("\n") assert config_names_lines == ["monorepo", "uninitialized", "misconfigured"] def test_repo_list_json_output(mutable_config: spack.config.Configuration, tmp_path: pathlib.Path): """Test the --json flag for repo list command. This test verifies that: 1. The --json flag produces valid JSON output 2. The output contains the expected repository information 3. Different repository types (installed, uninitialized, error) are correctly represented """ import json # Fake a git monorepo with two package repositories monorepo_path = tmp_path / "monorepo" (monorepo_path / ".git").mkdir(parents=True) repo("create", str(monorepo_path), "repo_one") repo("create", str(monorepo_path), "repo_two") # Configure repositories in Spack test_repos = { # git repo that provides two package repositories "monorepo": { "git": "https://example.com/monorepo.git", "destination": str(monorepo_path), "paths": ["spack_repo/repo_one", "spack_repo/repo_two"], }, # git repo that is not yet cloned "uninitialized": { "git": "https://example.com/uninitialized.git", "destination": str(tmp_path / "uninitialized"), }, # invalid local repository "misconfigured": str(tmp_path / "misconfigured"), } mutable_config.set("repos", test_repos, scope="site") # Get and parse JSON output json_output = repo("list", "--json") repo_data = json.loads(json_output) # Verify we got a list of repositories assert isinstance(repo_data, list), "Expected JSON output to be a list" # Index repositories by namespace for easier validation repos_by_namespace = {} for item in repo_data: # Check all required fields are present required_fields = ["name", "namespace", "path", "api_version", "status", "error"] for field in required_fields: assert field in item, f"Repository missing required field: {field}" # Store by namespace for later validation repos_by_namespace[item["namespace"]] = item # Verify installed repositories (repo_one and repo_two) for namespace in ["repo_one", "repo_two"]: assert namespace in repos_by_namespace, f"Missing repository: {namespace}" repo_info = repos_by_namespace[namespace] assert repo_info["name"] == "monorepo", f"Incorrect name for {namespace}" assert repo_info["status"] == "installed", f"Incorrect status for {namespace}" assert repo_info["error"] is None, f"Unexpected error for {namespace}" assert repo_info["api_version"], f"Missing API version for {namespace}" # Verify uninitialized repository assert "uninitialized" in repos_by_namespace, "Missing uninitialized repository" uninit_repo = repos_by_namespace["uninitialized"] assert uninit_repo["name"] == "uninitialized", "Incorrect name for uninitialized repo" assert uninit_repo["status"] == "uninitialized", "Incorrect status for uninitialized repo" # Verify misconfigured repository assert "misconfigured" in repos_by_namespace, "Missing misconfigured repository" misc_repo = repos_by_namespace["misconfigured"] assert misc_repo["name"] == "misconfigured", "Incorrect name for misconfigured repo" assert misc_repo["status"] == "error", "Incorrect status for misconfigured repo" assert misc_repo["error"] is not None, "Missing error message for misconfigured repo" @pytest.mark.parametrize( "repo_name,flags", [ ("new_repo", []), ("new_repo", ["--branch", "develop"]), ("new_repo", ["--branch", "develop", "--remote", "upstream"]), ("new_repo", ["--tag", "v1.0"]), ("new_repo", ["--commit", "abc123"]), ], ) def test_repo_update_successful_flags(monkeypatch, mutable_config, repo_name, flags): """Test repo update with flags.""" def mock_parse_config_descriptor(name, entry, lock): return MockDescriptor({"/path": MockRepo("new_repo")}) monkeypatch.setattr(spack.repo, "parse_config_descriptor", mock_parse_config_descriptor) monkeypatch.setattr(spack.repo, "RemoteRepoDescriptor", MockDescriptor) repos_config = spack.config.get("repos") repos_config[repo_name] = {"git": "https://github.com/example/repo.git"} spack.config.set("repos", repos_config) repo("update", repo_name, *flags) # check that the branch,tag,commit was updated in the configuration repos_config = spack.config.get("repos") if "--branch" in flags: assert repos_config[repo_name]["branch"] == "develop" if "--tag" in flags: assert repos_config[repo_name]["tag"] == "v1.0" if "--commit" in flags: assert repos_config[repo_name]["commit"] == "abc123" @pytest.mark.parametrize( "flags", [ ["--branch", "develop"], ["--branch", "develop", "new_repo_1", "new_repo_2"], ["--branch", "develop", "unknown_repo"], ], ) def test_repo_update_invalid_flags(monkeypatch, mutable_config, flags): """Test repo update with invalid flags.""" with pytest.raises(SpackError): repo("update", *flags) def test_repo_show_version_updates_no_changes(mock_git_package_changes): """Test that show-version-updates handles empty results gracefully""" test_repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(test_repo): # Use the same commit for both refs - no changes output = repo("show-version-updates", test_repo.root, commits[-1], commits[-1]) # Should have warning message assert "No packages were added or changed" in output # Should not have any specs assert "diff-test@" not in output def test_repo_show_version_updates_success(mock_git_package_changes): """Test that show-version-updates successfully outputs the correct specs""" test_repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(test_repo): # commits are ordered from newest to oldest after reversal # commits[-2] = add v2.1.5, commits[-4] = add v2.1.7 and v2.1.8 # Find versions added between these commits # Includes v2.1.6 (git version), v2.1.7, and v2.1.8 (sha256 versions) output = repo("show-version-updates", test_repo.root, commits[-2], commits[-4]) # Verify all three versions are included assert "diff-test@" in output assert "2.1.6" in output assert "2.1.7" in output assert "2.1.8" in output # Should have three specs lines = [ line.strip() for line in output.strip().split("\n") if line.strip() and "Warning" not in line ] assert len(lines) == 3 def test_repo_show_version_updates_excludes_manual_packages(monkeypatch, mock_git_package_changes): """Test --no-manual-packages flag excludes packages with manual_download=True""" test_repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(test_repo): # Set manual_download=True on the package pkg_class = spack.repo.PATH.get_pkg_class("diff-test") monkeypatch.setattr(pkg_class, "manual_download", True) # Run show-version-updates with --no-manual-packages flag output = repo( "show-version-updates", "--no-manual-packages", test_repo.root, commits[-2], commits[-4], ) # Package should be excluded assert "diff-test@" not in output assert "No packages were added or changed" in output def test_repo_show_version_updates_excludes_non_redistributable( monkeypatch, mock_git_package_changes ): """Test --only-redistributable flag excludes packages if redistribute_source returns False""" test_repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(test_repo): # Set redistribute_source to return False pkg_class = spack.repo.PATH.get_pkg_class("diff-test") monkeypatch.setattr(pkg_class, "redistribute_source", classmethod(lambda cls, spec: False)) # Run show-version-updates with --only-redistributable flag output = repo( "show-version-updates", "--only-redistributable", test_repo.root, commits[-2], commits[-4], ) # Package should be excluded assert "diff-test@" not in output assert "No new package versions found" in output def test_repo_show_version_updates_excludes_git_versions(mock_git_package_changes): """Test --no-git-versions flag excludes versions from git (tag/commit)""" test_repo, _, commits = mock_git_package_changes with spack.repo.use_repositories(test_repo): # commits[-3] = add v2.1.6 (git version), commits[-4] = add v2.1.7 and v2.1.8 (sha256) # Without --no-git-versions, v2.1.6 would be included output = repo( "show-version-updates", "--no-git-versions", test_repo.root, commits[-3], commits[-4] ) # v2.1.6 (git version) should be excluded assert "2.1.6" not in output # v2.1.7 and v2.1.8 (sha256 versions) should be included assert "diff-test@" in output assert "2.1.7" in output assert "2.1.8" in output ================================================ FILE: lib/spack/spack/test/cmd/resource.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import sys from spack.main import SpackCommand resource = SpackCommand("resource") #: these are hashes used in mock packages mock_hashes = ( [ "abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234", "1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd", "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c", "c45c1564f70def3fc1a6e22139f62cb21cd190cc3a7dbe6f4120fa59ce33dcb8", "24eceabef5fe8f575ff4b438313dc3e7b30f6a2d1c78841fbbe3b9293a589277", "689b8f9b32cb1d2f9271d29ea3fca2e1de5df665e121fca14e1364b711450deb", "208fcfb50e5a965d5757d151b675ca4af4ce2dfd56401721b6168fae60ab798f", "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c", "7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730", ] if sys.platform != "win32" else [ "abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234", "1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd", "d0df7988457ec999c148a4a2af25ce831bfaad13954ba18a4446374cb0aef55e", "aeb16c4dec1087e39f2330542d59d9b456dd26d791338ae6d80b6ffd10c89dfa", "mid21234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234", "ff34cb21271d16dbf928374f610bb5dd593d293d311036ddae86c4846ff79070", "bf874c7dd3a83cf370fdc17e496e341de06cd596b5c66dbf3c9bb7f6c139e3ee", "3c5b65abcd6a3b2c714dbf7c31ff65fe3748a1adc371f030c283007ca5534f11", ] ) def test_resource_list(mock_packages): out = resource("list") for h in mock_hashes: assert h in out assert "url:" in out assert "applies to:" in out assert "patched by:" in out assert "path:" in out assert ( os.path.join( "spack_repo", "builtin_mock", "packages", "patch_a_dependency", "libelf.patch" ) in out ) assert "applies to: builtin_mock.libelf" in out assert "patched by: builtin_mock.patch-a-dependency" in out def test_resource_list_only_hashes(mock_packages): out = resource("list", "--only-hashes") for h in mock_hashes: assert h in out def test_resource_show(mock_packages): test_hash = ( "c45c1564f70def3fc1a6e22139f62cb21cd190cc3a7dbe6f4120fa59ce33dcb8" if sys.platform != "win32" else "3c5b65abcd6a3b2c714dbf7c31ff65fe3748a1adc371f030c283007ca5534f11" ) out = resource("show", test_hash) assert out.startswith(test_hash) assert ( os.path.join( "spack_repo", "builtin_mock", "packages", "patch_a_dependency", "libelf.patch" ) in out ) assert "applies to: builtin_mock.libelf" in out assert "patched by: builtin_mock.patch-a-dependency" in out assert len(out.strip().split("\n")) == 4 ================================================ FILE: lib/spack/spack/test/cmd/spec.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import pytest import spack.config import spack.environment as ev import spack.error import spack.spec import spack.store from spack.main import SpackCommand, SpackCommandError # Unit tests should not be affected by the user's managed environments pytestmark = pytest.mark.usefixtures( "mutable_mock_env_path", "mutable_config", "mutable_mock_repo" ) spec = SpackCommand("spec") def test_spec(): output = spec("mpileaks") assert "mpileaks@2.3" in output assert "callpath@1.0" in output assert "dyninst@8.2" in output assert "libdwarf@20130729" in output assert "libelf@0.8.1" in output assert "mpich@3.0.4" in output def test_spec_concretizer_args(mutable_database): """End-to-end test of CLI concretizer prefs. It's here to make sure that everything works from CLI options to `solver.py`, and that config options are not lost along the way. """ # remove two non-preferred mpileaks installations # so that reuse will pick up the zmpi one uninstall = SpackCommand("uninstall") uninstall("-y", "mpileaks^mpich") uninstall("-y", "mpileaks^mpich2") # get the hash of mpileaks^zmpi mpileaks_zmpi = spack.store.STORE.db.query_one("mpileaks^zmpi") h = mpileaks_zmpi.dag_hash()[:7] output = spec("--fresh", "-l", "mpileaks") assert h not in output output = spec("--reuse", "-l", "mpileaks") assert h in output def test_spec_parse_dependency_variant_value(): """Verify that we can provide multiple key=value variants to multiple separate packages within a spec string.""" output = spec("multivalue-variant fee=barbaz ^ pkg-a foobar=baz") assert "fee=barbaz" in output assert "foobar=baz" in output def test_spec_parse_cflags_quoting(): """Verify that compiler flags can be provided to a spec from the command line.""" output = spec("--yaml", 'gcc cflags="-Os -pipe" cxxflags="-flto -Os"') gh_flagged = spack.spec.Spec.from_yaml(output) assert ["-Os", "-pipe"] == gh_flagged.compiler_flags["cflags"] assert ["-flto", "-Os"] == gh_flagged.compiler_flags["cxxflags"] def test_spec_yaml(): output = spec("--yaml", "mpileaks") mpileaks = spack.spec.Spec.from_yaml(output) assert "mpileaks" in mpileaks assert "callpath" in mpileaks assert "dyninst" in mpileaks assert "libdwarf" in mpileaks assert "libelf" in mpileaks assert "mpich" in mpileaks def test_spec_json(): output = spec("--json", "mpileaks") mpileaks = spack.spec.Spec.from_json(output) assert "mpileaks" in mpileaks assert "callpath" in mpileaks assert "dyninst" in mpileaks assert "libdwarf" in mpileaks assert "libelf" in mpileaks assert "mpich" in mpileaks def test_spec_format(mutable_database): output = spec("--format", "{name}-{^mpi.name}", "mpileaks^mpich") assert output.rstrip("\n") == "mpileaks-mpich" def _parse_types(string): """Parse deptypes for specs from `spack spec -t` output.""" lines = string.strip().split("\n") result = {} for line in lines: match = re.match(r"\[([^]]*)\]\s*\^?([^@]*)@", line) if match: types, name = match.groups() result.setdefault(name, []).append(types) result[name] = sorted(result[name]) return result def test_spec_deptypes_nodes(): output = spec("--types", "--cover", "nodes", "--no-install-status", "dt-diamond") types = _parse_types(output) assert types["dt-diamond"] == [" "] assert types["dt-diamond-left"] == ["bl "] assert types["dt-diamond-right"] == ["bl "] assert types["dt-diamond-bottom"] == ["blr "] def test_spec_deptypes_edges(): output = spec("--types", "--cover", "edges", "--no-install-status", "dt-diamond") types = _parse_types(output) assert types["dt-diamond"] == [" "] assert types["dt-diamond-left"] == ["bl "] assert types["dt-diamond-right"] == ["bl "] assert types["dt-diamond-bottom"] == ["b ", "blr "] def test_spec_returncode(): with pytest.raises(SpackCommandError): spec() assert spec.returncode == 1 def test_spec_parse_error(): with pytest.raises(spack.error.SpecSyntaxError) as e: spec("1.15:") # make sure the error is formatted properly error_msg = "unexpected characters in the spec string\n1.15:\n ^" assert error_msg in str(e.value) def test_env_aware_spec(mutable_mock_env_path): env = ev.create("test") env.add("mpileaks") with env: output = spec() assert "mpileaks@2.3" in output assert "callpath@1.0" in output assert "dyninst@8.2" in output assert "libdwarf@20130729" in output assert "libelf@0.8.1" in output assert "mpich@3.0.4" in output @pytest.mark.parametrize( "name, version, error", [ ("develop-branch-version", "f3c7206350ac8ee364af687deaae5c574dcfca2c=develop", None), ("develop-branch-version", "git." + "a" * 40 + "=develop", None), ("callpath", "f3c7206350ac8ee364af687deaae5c574dcfca2c=1.0", spack.error.PackageError), ("develop-branch-version", "git.foo=0.2.15", None), ], ) @pytest.mark.use_package_hash def test_spec_version_assigned_git_ref_as_version(name, version, error): if error: with pytest.raises(error): output = spec(name + "@" + version) else: output = spec(name + "@" + version) assert version in output @pytest.mark.parametrize( "unify, spec_hash_args, match, error", [ # success cases with unfiy:true (True, ["mpileaks_mpich"], "mpich", None), (True, ["mpileaks_zmpi"], "zmpi", None), (True, ["mpileaks_mpich", "dyninst"], "mpich", None), (True, ["mpileaks_zmpi", "dyninst"], "zmpi", None), # same success cases with unfiy:false (False, ["mpileaks_mpich"], "mpich", None), (False, ["mpileaks_zmpi"], "zmpi", None), (False, ["mpileaks_mpich", "dyninst"], "mpich", None), (False, ["mpileaks_zmpi", "dyninst"], "zmpi", None), # cases with unfiy:false (True, ["mpileaks_mpich", "mpileaks_zmpi"], "callpath, mpileaks", spack.error.SpecError), (False, ["mpileaks_mpich", "mpileaks_zmpi"], "zmpi", None), ], ) def test_spec_unification_from_cli( install_mockery, mutable_config, mutable_database, unify, spec_hash_args, match, error ): """Ensure specs grouped together on the CLI are concretized together when unify:true.""" spack.config.set("concretizer:unify", unify) db = spack.store.STORE.db spec_lookup = { "mpileaks_mpich": db.query_one("mpileaks ^mpich").dag_hash(), "mpileaks_zmpi": db.query_one("mpileaks ^zmpi").dag_hash(), "dyninst": db.query_one("dyninst").dag_hash(), } hashes = [f"/{spec_lookup[name]}" for name in spec_hash_args] if error: with pytest.raises(error, match=match): output = spec(*hashes) else: output = spec(*hashes) assert match in output ================================================ FILE: lib/spack/spack/test/cmd/stage.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.config import spack.environment as ev import spack.package_base import spack.traverse from spack.cmd.stage import StageFilter from spack.main import SpackCommand, SpackCommandError from spack.spec import Spec from spack.version import Version stage = SpackCommand("stage") env = SpackCommand("env") pytestmark = pytest.mark.usefixtures("install_mockery", "mock_packages") @pytest.mark.disable_clean_stage_check def test_stage_spec(monkeypatch): """Verify that staging specs works.""" expected = set(["trivial-install-test-package", "mpileaks"]) def fake_stage(pkg, mirror_only=False): expected.remove(pkg.name) monkeypatch.setattr(spack.package_base.PackageBase, "do_stage", fake_stage) stage("trivial-install-test-package", "mpileaks") assert len(expected) == 0 @pytest.fixture(scope="function") def check_stage_path(monkeypatch, tmp_path: pathlib.Path): expected_path = str(tmp_path / "x") def fake_stage(pkg, mirror_only=False): assert pkg.path == expected_path monkeypatch.setattr(spack.package_base.PackageBase, "do_stage", fake_stage) return expected_path def test_stage_path(check_stage_path): """Verify that --path only works with single specs.""" stage("--path={0}".format(check_stage_path), "trivial-install-test-package") def test_stage_path_errors_multiple_specs(check_stage_path): """Verify that --path only works with single specs.""" with pytest.raises(SpackCommandError): stage(f"--path={check_stage_path}", "trivial-install-test-package", "mpileaks") @pytest.mark.disable_clean_stage_check def test_stage_with_env_outside_env(mutable_mock_env_path, monkeypatch): """Verify that stage concretizes specs not in environment instead of erroring.""" def fake_stage(pkg, mirror_only=False): assert pkg.name == "trivial-install-test-package" assert pkg.path is None monkeypatch.setattr(spack.package_base.PackageBase, "do_stage", fake_stage) e = ev.create("test") e.add("mpileaks") e.concretize() with e: stage("trivial-install-test-package") @pytest.mark.disable_clean_stage_check def test_stage_with_env_inside_env(mutable_mock_env_path, monkeypatch): """Verify that stage filters specs in environment instead of reconcretizing.""" def fake_stage(pkg, mirror_only=False): assert pkg.name == "mpileaks" assert pkg.version == Version("100.100") monkeypatch.setattr(spack.package_base.PackageBase, "do_stage", fake_stage) e = ev.create("test") e.add("mpileaks@=100.100") e.concretize() with e: stage("mpileaks") @pytest.mark.disable_clean_stage_check def test_stage_full_env(mutable_mock_env_path, monkeypatch): """Verify that stage filters specs in environment.""" e = ev.create("test") e.add("mpileaks@=100.100") e.concretize() # list all the package names that should be staged expected, externals = set(), set() for dep in spack.traverse.traverse_nodes(e.concrete_roots()): expected.add(dep.name) if dep.external: externals.add(dep.name) # pop the package name from the list instead of actually staging def fake_stage(pkg, mirror_only=False): expected.remove(pkg.name) monkeypatch.setattr(spack.package_base.PackageBase, "do_stage", fake_stage) with e: stage() assert expected == externals @pytest.mark.disable_clean_stage_check def test_concretizer_arguments(mock_packages, mock_fetch): """Make sure stage also has --reuse and --fresh flags.""" stage("--reuse", "trivial-install-test-package") assert spack.config.get("concretizer:reuse", None) is True stage("--fresh", "trivial-install-test-package") assert spack.config.get("concretizer:reuse", None) is False @pytest.mark.maybeslow @pytest.mark.parametrize("externals", [["libelf"], []]) @pytest.mark.parametrize( "installed, skip_installed", [(["libdwarf"], False), (["libdwarf"], True)] ) @pytest.mark.parametrize("exclusions", [["mpich", "callpath"], []]) def test_stage_spec_filters( mutable_mock_env_path, mock_packages, mock_fetch, externals, installed, skip_installed, exclusions, monkeypatch, ): e = ev.create("test") e.add("mpileaks@=100.100") e.concretize() all_specs = e.all_specs() def is_installed(self): return self.name in installed if skip_installed: monkeypatch.setattr(Spec, "installed", is_installed) should_be_filtered = [] for spec in all_specs: for ext in externals: if spec.satisfies(Spec(ext)): spec.external_path = "/usr" assert spec.external should_be_filtered.append(spec) for ins in installed: if skip_installed and spec.satisfies(Spec(ins)): assert spec.installed should_be_filtered.append(spec) for exc in exclusions: if spec.satisfies(Spec(exc)): should_be_filtered.append(spec) filter = StageFilter(exclusions, skip_installed=skip_installed) specs_to_stage = [s for s in all_specs if not filter(s)] specs_were_filtered = [skip not in specs_to_stage for skip in should_be_filtered] assert all(specs_were_filtered), ( f"Packages associated with bools: {[s.name for s in should_be_filtered]}" ) ================================================ FILE: lib/spack/spack/test/cmd/style.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp import io import os import pathlib import shutil import sys import pytest import spack.cmd.style import spack.main import spack.paths import spack.repo from spack.cmd.style import _run_import_check, changed_files from spack.llnl.util.filesystem import FileFilter, working_dir from spack.util.executable import which #: directory with sample style files style_data = os.path.join(spack.paths.test_path, "data", "style") style = spack.main.SpackCommand("style") pytestmark = pytest.mark.skipif( sys.platform == "win32", reason="CI uses cross drive paths that raise errors with relpath" ) RUFF = which("ruff") MYPY = which("mypy") @pytest.fixture(autouse=True) def has_develop_branch(git): """spack style requires git and a develop branch to run -- skip if we're missing either.""" git("show-ref", "--verify", "--quiet", "refs/heads/develop", fail_on_error=False) if git.returncode != 0: pytest.skip("requires git and a develop branch") @pytest.fixture(scope="function") def ruff_package(tmp_path: pathlib.Path): """Style only checks files that have been modified. This fixture makes a small change to the ``ruff`` mock package, yields the filename, then undoes the change on cleanup. """ repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("ruff") rel_path = os.path.dirname(os.path.relpath(filename, spack.paths.prefix)) tmp = tmp_path / rel_path / "ruff-ci-package.py" tmp.parent.mkdir(parents=True, exist_ok=True) tmp.touch() tmp = str(tmp) shutil.copy(filename, tmp) package = FileFilter(tmp) package.filter("state = 'unmodified'", "state = 'modified'", string=True) yield tmp @pytest.fixture def ruff_package_with_errors(scope="function"): """A ruff package with errors.""" repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("ruff") tmp = filename + ".tmp" shutil.copy(filename, tmp) package = FileFilter(tmp) # this is a ruff error (quote style and spacing before/after operator) package.filter('state = "unmodified"', "state = 'modified'", string=True) # this is two ruff errors (unused import) (orderign) package.filter( "from spack.package import *", "from spack.package import *\nimport os", string=True ) yield tmp def test_changed_files_from_git_rev_base(git, tmp_path: pathlib.Path): """Test arbitrary git ref as base.""" with working_dir(str(tmp_path)): git("init") git("checkout", "-b", "main") git("config", "user.name", "test user") git("config", "user.email", "test@user.com") git("commit", "--no-gpg-sign", "--allow-empty", "-m", "initial commit") (tmp_path / "bin").mkdir(parents=True, exist_ok=True) (tmp_path / "bin" / "spack").touch() assert changed_files(base="HEAD") == [pathlib.Path("bin/spack")] assert changed_files(base="main") == [pathlib.Path("bin/spack")] git("add", "bin/spack") git("commit", "--no-gpg-sign", "-m", "v1") assert changed_files(base="HEAD") == [] assert changed_files(base="HEAD~") == [pathlib.Path("bin/spack")] def test_changed_no_base(git, tmp_path: pathlib.Path, capfd): """Ensure that we fail gracefully with no base branch.""" (tmp_path / "bin").mkdir(parents=True, exist_ok=True) (tmp_path / "bin" / "spack").touch() with working_dir(str(tmp_path)): git("init") git("config", "user.name", "test user") git("config", "user.email", "test@user.com") git("add", ".") git("commit", "--no-gpg-sign", "-m", "initial commit") with pytest.raises(SystemExit): changed_files(base="foobar") out, err = capfd.readouterr() assert "This repository does not have a 'foobar'" in err def test_changed_files_all_files(mock_packages): # it's hard to guarantee "all files", so do some sanity checks. files = { os.path.join(spack.paths.prefix, os.path.normpath(path)) for path in changed_files(all_files=True) } # spack has a lot of files -- check that we're in the right ballpark assert len(files) > 500 # a builtin package zlib = spack.repo.PATH.get_pkg_class("zlib") zlib_file = zlib.module.__file__ if zlib_file.endswith("pyc"): zlib_file = zlib_file[:-1] assert zlib_file in files # a core spack file assert os.path.join(spack.paths.module_path, "spec.py") in files # a mock package repo = spack.repo.from_path(spack.paths.mock_packages_path) filename = repo.filename_for_package_name("ruff") assert filename in files # this test assert __file__ in files # ensure externals are excluded assert not any(f.startswith(spack.paths.vendor_path) for f in files) def test_bad_root(tmp_path: pathlib.Path): """Ensure that `spack style` doesn't run on non-spack directories.""" output = style("--root", str(tmp_path), fail_on_error=False) assert "This does not look like a valid spack root" in output assert style.returncode != 0 @pytest.fixture def external_style_root(git, ruff_package_with_errors, tmp_path: pathlib.Path): """Create a mock repository for running spack style.""" # create a sort-of spack-looking directory script = tmp_path / "bin" / "spack" script.parent.mkdir(parents=True, exist_ok=True) script.touch() spack_dir = tmp_path / "lib" / "spack" / "spack" spack_dir.mkdir(parents=True, exist_ok=True) (spack_dir / "__init__.py").touch() llnl_dir = tmp_path / "lib" / "spack" / "llnl" llnl_dir.mkdir(parents=True, exist_ok=True) (llnl_dir / "__init__.py").touch() # create a base develop branch with working_dir(str(tmp_path)): git("init") git("config", "user.name", "test user") git("config", "user.email", "test@user.com") git("add", ".") git("commit", "--no-gpg-sign", "-m", "initial commit") git("branch", "-m", "develop") git("checkout", "-b", "feature") # copy the buggy package in py_file = spack_dir / "dummy.py" py_file.touch() shutil.copy(ruff_package_with_errors, str(py_file)) yield tmp_path, py_file @pytest.mark.skipif(not RUFF, reason="ruff is not installed.") def test_fix_style(external_style_root): """Make sure spack style --fix works.""" tmp_path, py_file = external_style_root broken_dummy = os.path.join(style_data, "broken.dummy") broken_py = str(tmp_path / "lib" / "spack" / "spack" / "broken.py") fixed_py = os.path.join(style_data, "fixed.py") shutil.copy(broken_dummy, broken_py) assert not filecmp.cmp(broken_py, fixed_py) # dummy.py is in the same directory and will raise errors unrelated to this # check, don't fail on those errors, just check to make sure # we fixed the intended file correctly # Note: can't just specify the correct file due to cross drive issues on Windows style( "--root", str(tmp_path), "--tool", "ruff-check,ruff-format", "--fix", fail_on_error=False ) assert filecmp.cmp(broken_py, fixed_py) @pytest.mark.skipif(not RUFF, reason="ruff is not installed.") @pytest.mark.skipif(not MYPY, reason="mypy is not installed.") def test_external_root(external_style_root): """Ensure we can run in a separate root directory w/o configuration files.""" tmp_path, py_file = external_style_root # make sure tools are finding issues with external root, # not the real one. output = style("--root-relative", "--root", str(tmp_path), fail_on_error=False) # make sure it failed assert style.returncode != 0 # ruff-check error assert "Import block is un-sorted or un-formatted\n --> lib/spack/spack/dummy.py" in output # mypy error assert 'lib/spack/spack/dummy.py:45: error: Name "version" is not defined' in output # ruff-format error assert "--- lib/spack/spack/dummy.py" in output assert "+++ lib/spack/spack/dummy.py" in output # ruff-check error assert "`os` imported but unused\n --> lib/spack/spack/dummy.py" in output @pytest.mark.skipif(not RUFF, reason="ruff is not installed.") def test_style(ruff_package, tmp_path: pathlib.Path): root_relative = os.path.relpath(ruff_package, spack.paths.prefix) # use a working directory to test cwd-relative paths, as tests run in # the spack prefix by default with working_dir(str(tmp_path)): relative = os.path.relpath(ruff_package) # one specific arg output = style("--tool", "ruff-check", ruff_package, fail_on_error=False) assert relative in output assert "spack style checks were clean" in output # specific file that isn't changed output = style("--tool", "ruff-check", __file__, fail_on_error=False) assert relative not in output assert __file__ in output assert "spack style checks were clean" in output # root-relative paths output = style("--tool", "ruff-check", "--root-relative", ruff_package) assert root_relative in output assert "spack style checks were clean" in output @pytest.mark.skipif(not RUFF, reason="ruff is not installed.") def test_style_with_errors(ruff_package_with_errors): root_relative = os.path.relpath(ruff_package_with_errors, spack.paths.prefix) output = style( "--tool", "ruff-check", "--root-relative", ruff_package_with_errors, fail_on_error=False ) assert root_relative in output assert style.returncode != 0 assert "spack style found errors" in output @pytest.mark.skipif(not RUFF, reason="ruff is not installed.") def test_style_with_ruff_format(ruff_package_with_errors): output = style("--tool", "ruff-format", ruff_package_with_errors, fail_on_error=False) assert "ruff-format found errors" in output assert style.returncode != 0 assert "spack style found errors" in output def test_skip_tools(): output = style("--skip", "import,ruff-check,ruff-format,mypy") assert "Nothing to run" in output @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires Python 3.9+") def test_run_import_check(tmp_path: pathlib.Path): file = tmp_path / "issues.py" contents = ''' import spack.cmd import spack.config # do not drop this import because of this comment import spack.repo import spack.repo_utils from spack_repo.builtin_mock.build_systems import autotools # this comment about spack.error should not be removed class Example(autotools.AutotoolsPackage): """this is a docstring referencing unused spack.error.SpackError, which is fine""" pass def foo(config: "spack.error.SpackError"): # the type hint is quoted, so it should not be removed spack.util.executable.Executable("example") print(spack.__version__) print(spack.repo_utils.__file__) import spack.enums from spack.enums import ConfigScopePriority import spack.util.url as url_util def something(y: spack.util.url.Url): ... ''' file.write_text(contents) root = str(tmp_path) output_buf = io.StringIO() exit_code = _run_import_check( [file], fix=False, out=output_buf, root_relative=False, root=pathlib.Path(spack.paths.prefix), working_dir=pathlib.Path(root), ) output = output_buf.getvalue() assert "issues.py: redundant import: spack.cmd" in output assert "issues.py: redundant import: spack.repo" in output assert "issues.py: redundant import: spack.config" not in output # comment prevents removal assert "issues.py: redundant import: spack.enums" in output # imported via from-import assert "issues.py: missing import: spack" in output # used by spack.__version__ assert "issues.py: missing import: spack.util.executable" in output assert "issues.py: missing import: spack.util.url" in output # used in type hint assert "issues.py: missing import: spack.error" not in output # not directly used assert exit_code == 1 assert file.read_text() == contents # fix=False should not change the file # run it with --fix, should have the same output. output_buf = io.StringIO() exit_code = _run_import_check( [file], fix=True, out=output_buf, root_relative=False, root=pathlib.Path(spack.paths.prefix), working_dir=pathlib.Path(root), ) output = output_buf.getvalue() assert exit_code == 1 assert "issues.py: redundant import: spack.cmd" in output assert "issues.py: redundant import: spack.enums" in output assert "issues.py: missing import: spack" in output assert "issues.py: missing import: spack.util.executable" in output assert "issues.py: missing import: spack.util.url" in output # after fix a second fix is idempotent output_buf = io.StringIO() exit_code = _run_import_check( [file], fix=True, out=output_buf, root_relative=False, root=pathlib.Path(spack.paths.prefix), working_dir=pathlib.Path(root), ) output = output_buf.getvalue() assert exit_code == 0 assert not output # check that the file was fixed new_contents = file.read_text() assert "import spack.cmd" not in new_contents assert "import spack.enums" not in new_contents assert "import spack\n" in new_contents assert "import spack.util.executable\n" in new_contents assert "import spack.util.url\n" in new_contents @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires Python 3.9+") def test_run_import_check_syntax_error_and_missing(tmp_path: pathlib.Path): (tmp_path / "syntax-error.py").write_text("""this 'is n(ot python code""") output_buf = io.StringIO() exit_code = _run_import_check( [tmp_path / "syntax-error.py", tmp_path / "missing.py"], fix=False, out=output_buf, root_relative=True, root=tmp_path, working_dir=tmp_path / "does-not-matter", ) output = output_buf.getvalue() assert "syntax-error.py: could not parse" in output assert "missing.py: could not parse" in output assert exit_code == 1 def test_case_sensitive_imports(tmp_path: pathlib.Path): # example.Example is a name, while example.example is a module. (tmp_path / "lib" / "spack" / "example").mkdir(parents=True) (tmp_path / "lib" / "spack" / "example" / "__init__.py").write_text("class Example:\n pass") (tmp_path / "lib" / "spack" / "example" / "example.py").write_text("foo = 1") assert spack.cmd.style._module_part(tmp_path, "example.Example") == "example" def test_pkg_imports(): assert ( spack.cmd.style._module_part(pathlib.Path(spack.paths.prefix), "spack.pkg.builtin.boost") is None ) assert spack.cmd.style._module_part(pathlib.Path(spack.paths.prefix), "spack.pkg") is None ================================================ FILE: lib/spack/spack/test/cmd/tags.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import spack.concretize import spack.main import spack.repo from spack.installer import PackageInstaller tags = spack.main.SpackCommand("tags") def test_tags_bad_options(mock_packages): out = tags("-a", "tag1", fail_on_error=False) assert "option OR provide" in out def test_tags_no_installed(install_mockery, mock_fetch): out = tags("-i") assert "No installed" in out def test_tags_invalid_tag(mock_packages): out = tags("nosuchtag") assert "None" in out def test_tags_all_mock_tags(mock_packages): out = tags() for tag in ["tag1", "tag2", "tag3"]: assert tag in out def test_tags_all_mock_tag_packages(mock_packages): out = tags("-a") for pkg in ["mpich\n", "mpich2\n"]: assert pkg in out def test_tags_no_tags(repo_builder): repo_builder.add_package("pkg-a") with spack.repo.use_repositories(repo_builder.root): out = tags() assert "No tagged" in out def test_tags_installed(install_mockery, mock_fetch): s = spack.concretize.concretize_one("mpich") PackageInstaller([s.package], explicit=True, fake=True).install() out = tags("-i") for tag in ["tag1", "tag2"]: assert tag in out out = tags("-i", "tag1") assert "mpich" in out out = tags("-i", "tag3") assert "No installed" in out ================================================ FILE: lib/spack/spack/test/cmd/test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import argparse import os import pathlib import pytest import spack.cmd.common.arguments import spack.cmd.test import spack.concretize import spack.config import spack.install_test import spack.paths from spack.install_test import TestStatus from spack.llnl.util.filesystem import copy_tree, working_dir from spack.main import SpackCommand install = SpackCommand("install") spack_test = SpackCommand("test") pytestmark = pytest.mark.not_on_windows("does not run on windows") def test_test_package_not_installed( mock_packages, mock_archive, mock_fetch, install_mockery, mock_test_stage ): output = spack_test("run", "libdwarf") assert "No installed packages match spec libdwarf" in output @pytest.mark.parametrize( "arguments,expected", [ (["run"], spack.config.get("config:dirty")), # default from config file (["run", "--clean"], False), (["run", "--dirty"], True), ], ) def test_test_dirty_flag(arguments, expected): parser = argparse.ArgumentParser() spack.cmd.test.setup_parser(parser) args = parser.parse_args(arguments) assert args.dirty == expected def test_test_dup_alias(mock_test_stage, mock_packages, mock_archive, mock_fetch, install_mockery): """Ensure re-using an alias fails with suggestion to change.""" install("--fake", "libdwarf") # Run the (no) tests with the alias once spack_test("run", "--alias", "libdwarf", "libdwarf") # Try again with the alias but don't let it fail on the error out = spack_test("run", "--alias", "libdwarf", "libdwarf", fail_on_error=False) assert "already exists" in out and "Try another alias" in out def test_test_output(mock_test_stage, mock_packages, mock_archive, mock_fetch, install_mockery): """Ensure output printed from pkgs is captured by output redirection.""" install("printing-package") spack_test("run", "--alias", "printpkg", "printing-package") stage_files = os.listdir(mock_test_stage) assert len(stage_files) == 1 # Grab test stage directory contents testdir = os.path.join(mock_test_stage, stage_files[0]) testdir_files = os.listdir(testdir) testlogs = [name for name in testdir_files if str(name).endswith("out.txt")] assert len(testlogs) == 1 # Grab the output from the test log to confirm expected result outfile = os.path.join(testdir, testlogs[0]) with open(outfile, "r", encoding="utf-8") as f: output = f.read() assert "test_print" in output assert "PASSED" in output @pytest.mark.parametrize( "pkg_name,failure", [("test-error", "exited with status 1"), ("test-fail", "not callable")] ) def test_test_output_fails( mock_packages, mock_archive, mock_fetch, install_mockery, mock_test_stage, pkg_name, failure ): """Confirm stand-alone test failure with expected outputs.""" install(pkg_name) out = spack_test("run", pkg_name, fail_on_error=False) # Confirm package-specific failure is in the output assert failure in out # Confirm standard failure tagging AND test log reference also output assert "TestFailure" in out assert "See test log for details" in out @pytest.mark.usefixtures("mock_packages", "mock_archive", "mock_fetch", "install_mockery") @pytest.mark.parametrize( "pkg_name,msgs", [ ("test-error", ["exited with status 1", "TestFailure"]), ("test-fail", ["not callable", "TestFailure"]), ], ) def test_junit_output_with_failures(tmp_path: pathlib.Path, mock_test_stage, pkg_name, msgs): """Confirm stand-alone test failure expected outputs in JUnit reporting.""" install(pkg_name) with working_dir(str(tmp_path)): spack_test( "run", "--log-format=junit", "--log-file=test.xml", pkg_name, fail_on_error=False ) files = list(tmp_path.iterdir()) filename = tmp_path / "test.xml" assert filename in files content = filename.read_text() # Count failures and errors correctly assert 'tests="1"' in content assert 'failures="1"' in content assert 'errors="0"' in content # We want to have both stdout and stderr assert "" in content for msg in msgs: assert msg in content def test_cdash_output_test_error( tmp_path: pathlib.Path, mock_fetch, install_mockery, mock_packages, mock_archive, mock_test_stage, ): """Confirm stand-alone test error expected outputs in CDash reporting.""" install("test-error") with working_dir(str(tmp_path)): spack_test( "run", "--log-format=cdash", "--log-file=cdash_reports", "test-error", fail_on_error=False, ) report_dir = tmp_path / "cdash_reports" reports = [name for name in report_dir.iterdir() if str(name).endswith("Testing.xml")] assert len(reports) == 1 content = reports[0].read_text() assert "Command exited with status 1" in content def test_cdash_upload_clean_test( tmp_path: pathlib.Path, mock_fetch, install_mockery, mock_packages, mock_archive, mock_test_stage, ): install("printing-package") with working_dir(str(tmp_path)): spack_test("run", "--log-file=cdash_reports", "--log-format=cdash", "printing-package") report_dir = tmp_path / "cdash_reports" reports = [name for name in report_dir.iterdir() if str(name).endswith("Testing.xml")] assert len(reports) == 1 content = reports[0].read_text() assert "passed" in content assert "Running test_print" in content, "Expected first command output" assert "second command" in content, "Expected second command output" assert "" in content assert "" not in content def test_test_help_does_not_show_cdash_options(mock_test_stage): """Make sure `spack test --help` does not describe CDash arguments""" spack_test("run", "--help") assert "CDash URL" not in spack_test.output def test_test_help_cdash(mock_test_stage): """Make sure `spack test --help-cdash` describes CDash arguments""" out = spack_test("run", "--help-cdash") assert "CDash URL" in out def test_test_list_all(mock_packages): """Confirm `spack test list --all` returns all packages with test methods""" pkgs = spack_test("list", "--all").strip().split() assert set(pkgs) == { "py-numpy", "fail-test-audit", "fail-test-audit-docstring", "fail-test-audit-impl", "mpich", "perl-extension", "printing-package", "py-extension1", "py-extension2", "py-test-callback", "simple-standalone-test", "test-error", "test-fail", } def test_test_list(mock_packages, mock_archive, mock_fetch, install_mockery): pkg_with_tests = "printing-package" install(pkg_with_tests) output = spack_test("list") assert pkg_with_tests in output def test_read_old_results(mock_packages, mock_test_stage): """Take test data generated before the switch to full hash everywhere and make sure we can still read it in""" # Test data was generated with: # spack install printing-package # spack test run --alias printpkg printing-package test_data_src = os.path.join(spack.paths.test_path, "data", "test", "test_stage") # Copy the old test data into the mock stage directory copy_tree(test_data_src, mock_test_stage) # The find command should print info about the old test, under # the alias used at test generation time find_output = spack_test("find") assert "printpkg" in find_output # The results command should still print the old test results results_output = spack_test("results") assert str(TestStatus.PASSED) in results_output def test_test_results_none(mock_packages, mock_test_stage): name = "trivial" spec = spack.concretize.concretize_one("trivial-smoke-test") suite = spack.install_test.TestSuite([spec], name) suite.ensure_stage() spack.install_test.write_test_suite_file(suite) results = spack_test("results", name) assert "has no results" in results assert "if it is running" in results @pytest.mark.parametrize( "status", [TestStatus.FAILED, TestStatus.NO_TESTS, TestStatus.SKIPPED, TestStatus.PASSED] ) def test_test_results_status(mock_packages, mock_test_stage, status): """Confirm 'spack test results' returns expected status.""" name = "trivial" spec = spack.concretize.concretize_one("trivial-smoke-test") suite = spack.install_test.TestSuite([spec], name) suite.ensure_stage() spack.install_test.write_test_suite_file(suite) suite.write_test_result(spec, status) for opt in ["", "--failed", "--log"]: args = ["results", name] if opt: args.insert(1, opt) results = spack_test(*args) if opt == "--failed" and status != TestStatus.FAILED: assert str(status) not in results else: assert str(status) in results assert "1 {0}".format(status.lower()) in results @pytest.mark.regression("35337") def test_report_filename_for_cdash(install_mockery, mock_fetch): """Test that the temporary file used to write Testing.xml for CDash is not the upload URL""" name = "trivial" spec = spack.concretize.concretize_one("trivial-smoke-test") suite = spack.install_test.TestSuite([spec], name) suite.ensure_stage() parser = argparse.ArgumentParser() spack.cmd.test.setup_parser(parser) args = parser.parse_args( [ "run", "--cdash-upload-url=https://blahblah/submit.php?project=debugging", "trivial-smoke-test", ] ) spack.cmd.common.arguments.sanitize_reporter_options(args) filename = spack.cmd.test.report_filename(args, suite) assert filename != "https://blahblah/submit.php?project=debugging" def test_test_output_multiple_specs( mock_test_stage, mock_packages, mock_archive, mock_fetch, install_mockery ): """Ensure proper reporting for suite with skipped, failing, and passed tests.""" install("test-error", "simple-standalone-test@0.9", "simple-standalone-test@1.0") out = spack_test("run", "test-error", "simple-standalone-test", fail_on_error=False) # Note that a spec with passing *and* skipped tests is still considered # to have passed at this level. If you want to see the spec-specific # part result summaries, you'll have to look at the "test-out.txt" files # for each spec. assert "1 failed, 2 passed of 3 specs" in out ================================================ FILE: lib/spack/spack/test/cmd/undevelop.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import spack.concretize import spack.environment as ev from spack.llnl.util.filesystem import working_dir from spack.main import SpackCommand undevelop = SpackCommand("undevelop") env = SpackCommand("env") concretize = SpackCommand("concretize") def test_undevelop(tmp_path: pathlib.Path, mutable_config, mock_packages, mutable_mock_env_path): # setup environment envdir = tmp_path / "env" envdir.mkdir() with working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( """\ spack: specs: - mpich develop: mpich: spec: mpich@1.0 path: /fake/path """ ) env("create", "test", "./spack.yaml") with ev.read("test"): before = spack.concretize.concretize_one("mpich") undevelop("mpich") after = spack.concretize.concretize_one("mpich") # Removing dev spec from environment changes concretization assert before.satisfies("dev_path=*") assert not after.satisfies("dev_path=*") def test_undevelop_all( tmp_path: pathlib.Path, mutable_config, mock_packages, mutable_mock_env_path ): # setup environment envdir = tmp_path / "env" envdir.mkdir() with working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( """\ spack: specs: - mpich develop: mpich: spec: mpich@1.0 path: /fake/path """ ) env("create", "test", "./spack.yaml") with ev.read("test"): before = spack.concretize.concretize_one("mpich") undevelop("--all") after = spack.concretize.concretize_one("mpich") # Removing dev spec from environment changes concretization assert before.satisfies("dev_path=*") assert not after.satisfies("dev_path=*") def test_undevelop_nonexistent( tmp_path: pathlib.Path, mutable_config, mock_packages, mutable_mock_env_path ): # setup environment envdir = tmp_path / "env" envdir.mkdir() with working_dir(str(envdir)): with open("spack.yaml", "w", encoding="utf-8") as f: f.write( """\ spack: specs: - mpich develop: mpich: spec: mpich@1.0 path: /fake/path """ ) env("create", "test", "./spack.yaml") with ev.read("test") as e: concretize() before = e.specs_by_hash undevelop("package-not-in-develop") # does nothing concretize("-f") after = e.specs_by_hash # nothing should have changed assert before == after ================================================ FILE: lib/spack/spack/test/cmd/uninstall.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.cmd.uninstall import spack.environment import spack.llnl.util.tty as tty import spack.store from spack.enums import InstallRecordStatus from spack.main import SpackCommand, SpackCommandError uninstall = SpackCommand("uninstall") install = SpackCommand("install") # Unit tests should not be affected by the user's managed environments pytestmark = pytest.mark.usefixtures("mutable_mock_env_path") class MockArgs: def __init__(self, packages, all=False, force=False, dependents=False): self.packages = packages self.all = all self.force = force self.dependents = dependents self.yes_to_all = True @pytest.mark.db def test_multiple_matches(mutable_database): """Test unable to uninstall when multiple matches.""" with pytest.raises(SpackCommandError): uninstall("-y", "mpileaks") @pytest.mark.db def test_installed_dependents(mutable_database): """Test can't uninstall when there are installed dependents.""" with pytest.raises(SpackCommandError): uninstall("-y", "libelf") @pytest.mark.db def test_correct_installed_dependents(mutable_database): # Test whether we return the right dependents. # Take callpath from the database callpath = spack.store.STORE.db.query_local("callpath")[0] # Ensure it still has dependents and dependencies dependents = callpath.dependents(deptype=("run", "link")) dependencies = callpath.dependencies(deptype=("run", "link")) assert dependents and dependencies # Uninstall it, so it's missing. callpath.package.do_uninstall(force=True) # Retrieve all dependent hashes dependents = spack.cmd.uninstall.installed_dependents(dependencies) assert dependents dependent_hashes = [s.dag_hash() for s in dependents] set_dependent_hashes = set(dependent_hashes) # Assert uniqueness assert len(dependent_hashes) == len(set_dependent_hashes) # Ensure parents of callpath are listed assert all(s.dag_hash() in set_dependent_hashes for s in dependents) # Ensure callpath itself is not, since it was missing. assert callpath.dag_hash() not in set_dependent_hashes @pytest.mark.db def test_recursive_uninstall(mutable_database): """Test recursive uninstall.""" uninstall("-y", "-a", "--dependents", "callpath") # query specs with multiple configurations all_specs = spack.store.STORE.layout.all_specs() mpileaks_specs = [s for s in all_specs if s.satisfies("mpileaks")] callpath_specs = [s for s in all_specs if s.satisfies("callpath")] mpi_specs = [s for s in all_specs if s.satisfies("mpi")] assert len(mpileaks_specs) == 0 assert len(callpath_specs) == 0 assert len(mpi_specs) == 3 @pytest.mark.db @pytest.mark.regression("3690") @pytest.mark.parametrize("constraint,expected_number_of_specs", [("dyninst", 10), ("libelf", 8)]) def test_uninstall_spec_with_multiple_roots( constraint, expected_number_of_specs, mutable_database ): uninstall("-y", "-a", "--dependents", constraint) all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == expected_number_of_specs @pytest.mark.db @pytest.mark.parametrize("constraint,expected_number_of_specs", [("dyninst", 16), ("libelf", 16)]) def test_force_uninstall_spec_with_ref_count_not_zero( constraint, expected_number_of_specs, mutable_database ): uninstall("-f", "-y", constraint) all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == expected_number_of_specs @pytest.mark.db def test_force_uninstall_and_reinstall_by_hash(mutable_database): """Test forced uninstall and reinstall of old specs.""" # this is the spec to be removed callpath_spec = spack.store.STORE.db.query_one("callpath ^mpich") dag_hash = callpath_spec.dag_hash() # ensure can look up by hash and that it's a dependent of mpileaks def validate_callpath_spec(installed): assert installed is True or installed is False specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=installed) assert len(specs) == 1 and specs[0] == callpath_spec specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=installed) assert len(specs) == 1 and specs[0] == callpath_spec specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=InstallRecordStatus.ANY) assert len(specs) == 1 and specs[0] == callpath_spec specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=InstallRecordStatus.ANY) assert len(specs) == 1 and specs[0] == callpath_spec specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=not installed) assert specs is None specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=not installed) assert specs is None mpileaks_spec = spack.store.STORE.db.query_one("mpileaks ^mpich") assert callpath_spec in mpileaks_spec spec = spack.store.STORE.db.query_one("callpath ^mpich", installed=installed) assert spec == callpath_spec spec = spack.store.STORE.db.query_one("callpath ^mpich", installed=InstallRecordStatus.ANY) assert spec == callpath_spec spec = spack.store.STORE.db.query_one("callpath ^mpich", installed=not installed) assert spec is None validate_callpath_spec(True) uninstall("-y", "-f", "callpath ^mpich") # ensure that you can still look up by hash and see deps, EVEN though # the callpath spec is missing. validate_callpath_spec(False) # BUT, make sure that the removed callpath spec is not in queries def db_specs(): all_specs = spack.store.STORE.layout.all_specs() return ( all_specs, [s for s in all_specs if s.satisfies("mpileaks")], [s for s in all_specs if s.satisfies("callpath")], [s for s in all_specs if s.satisfies("mpi")], ) all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs() total_specs = len(all_specs) assert total_specs == 16 assert len(mpileaks_specs) == 3 assert len(callpath_specs) == 2 assert len(mpi_specs) == 3 # Now, REINSTALL the spec and make sure everything still holds install("--fake", "/%s" % dag_hash[:7]) validate_callpath_spec(True) all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs() assert len(all_specs) == total_specs + 1 # back to total_specs+1 assert len(mpileaks_specs) == 3 assert len(callpath_specs) == 3 # back to 3 assert len(mpi_specs) == 3 @pytest.mark.db @pytest.mark.regression("15773") def test_in_memory_consistency_when_uninstalling(mutable_database, monkeypatch): """Test that uninstalling doesn't raise warnings""" def _warn(*args, **kwargs): raise RuntimeError("a warning was triggered!") monkeypatch.setattr(tty, "warn", _warn) # Now try to uninstall and check this doesn't trigger warnings uninstall("-y", "-a") # Note: I want to use https://docs.pytest.org/en/7.1.x/how-to/skipping.html#skip-all-test-functions-of-a-class-or-module # the style formatter insists on separating these two lines. class TestUninstallFromEnv: """Tests an installation with two environments e1 and e2, which each have shared package installations: e1 has diamond-link-left -> diamond-link-bottom e2 has diamond-link-right -> diamond-link-bottom """ env = SpackCommand("env") add = SpackCommand("add") concretize = SpackCommand("concretize") find = SpackCommand("find") @pytest.fixture(scope="function") def environment_setup(self, mock_packages, mutable_database, install_mockery): TestUninstallFromEnv.env("create", "e1") e1 = spack.environment.read("e1") with e1: TestUninstallFromEnv.add("diamond-link-left") TestUninstallFromEnv.add("diamond-link-bottom") TestUninstallFromEnv.concretize() install("--fake") TestUninstallFromEnv.env("create", "e2") e2 = spack.environment.read("e2") with e2: TestUninstallFromEnv.add("diamond-link-right") TestUninstallFromEnv.add("diamond-link-bottom") TestUninstallFromEnv.concretize() install("--fake") yield "environment_setup" TestUninstallFromEnv.env("rm", "e1", "-y") TestUninstallFromEnv.env("rm", "e2", "-y") def test_basic_env_sanity(self, environment_setup): for env_name in ["e1", "e2"]: e = spack.environment.read(env_name) with e: for _, concretized_spec in e.concretized_specs(): assert concretized_spec.installed def test_uninstall_force_dependency_shared_between_envs(self, environment_setup): """If you "spack uninstall -f --dependents diamond-link-bottom" from e1, then all packages should be uninstalled (but not removed) from both e1 and e2. """ e1 = spack.environment.read("e1") with e1: uninstall("-f", "-y", "--dependents", "diamond-link-bottom") # The specs should still be in the environment, since # --remove was not specified assert set(root.name for (root, _) in e1.concretized_specs()) == set( ["diamond-link-left", "diamond-link-bottom"] ) for _, concretized_spec in e1.concretized_specs(): assert not concretized_spec.installed # Everything in e2 depended on diamond-link-bottom, so should also # have been uninstalled. The roots should be unchanged though. e2 = spack.environment.read("e2") with e2: assert set(root.name for (root, _) in e2.concretized_specs()) == set( ["diamond-link-right", "diamond-link-bottom"] ) for _, concretized_spec in e2.concretized_specs(): assert not concretized_spec.installed def test_uninstall_remove_dependency_shared_between_envs(self, environment_setup): """If you "spack uninstall --dependents --remove diamond-link-bottom" from e1, then all packages are removed from e1 (it is now empty); diamond-link-left is also uninstalled (since only e1 needs it) but diamond-link-bottom is not uninstalled (since e2 needs it). """ e1 = spack.environment.read("e1") with e1: dtdiamondleft = next( concrete for (_, concrete) in e1.concretized_specs() if concrete.name == "diamond-link-left" ) output = uninstall("-y", "--dependents", "--remove", "diamond-link-bottom") assert "The following specs will be removed but not uninstalled" in output assert not list(e1.roots()) assert not dtdiamondleft.installed # Since -f was not specified, all specs in e2 should still be installed # (and e2 should be unchanged) e2 = spack.environment.read("e2") with e2: assert set(root.name for (root, _) in e2.concretized_specs()) == set( ["diamond-link-right", "diamond-link-bottom"] ) for _, concretized_spec in e2.concretized_specs(): assert concretized_spec.installed def test_uninstall_dependency_shared_between_envs_fail(self, environment_setup): """If you "spack uninstall --dependents diamond-link-bottom" from e1 (without --remove or -f), then this should fail (this is needed by e2). """ e1 = spack.environment.read("e1") with e1: output = uninstall("-y", "--dependents", "diamond-link-bottom", fail_on_error=False) assert "There are still dependents." in output assert "use `spack env remove`" in output # The environment should be unchanged and nothing should have been # uninstalled assert set(root.name for (root, _) in e1.concretized_specs()) == set( ["diamond-link-left", "diamond-link-bottom"] ) for _, concretized_spec in e1.concretized_specs(): assert concretized_spec.installed def test_uninstall_force_and_remove_dependency_shared_between_envs(self, environment_setup): """If you "spack uninstall -f --dependents --remove diamond-link-bottom" from e1, then all packages should be uninstalled and removed from e1. All packages will also be uninstalled from e2, but the roots will remain unchanged. """ e1 = spack.environment.read("e1") with e1: dtdiamondleft = next( concrete for (_, concrete) in e1.concretized_specs() if concrete.name == "diamond-link-left" ) uninstall("-f", "-y", "--dependents", "--remove", "diamond-link-bottom") assert not list(e1.roots()) assert not dtdiamondleft.installed e2 = spack.environment.read("e2") with e2: assert set(root.name for (root, _) in e2.concretized_specs()) == set( ["diamond-link-right", "diamond-link-bottom"] ) for _, concretized_spec in e2.concretized_specs(): assert not concretized_spec.installed def test_uninstall_keep_dependents_dependency_shared_between_envs(self, environment_setup): """If you "spack uninstall -f --remove diamond-link-bottom" from e1, then diamond-link-bottom should be uninstalled, which leaves "dangling" references in both environments, since diamond-link-left and diamond-link-right both need it. """ e1 = spack.environment.read("e1") with e1: dtdiamondleft = next( concrete for (_, concrete) in e1.concretized_specs() if concrete.name == "diamond-link-left" ) uninstall("-f", "-y", "--remove", "diamond-link-bottom") # diamond-link-bottom was removed from the list of roots (note that # it would still be installed since diamond-link-left depends on it) assert set(x.name for x in e1.roots()) == set(["diamond-link-left"]) assert dtdiamondleft.installed e2 = spack.environment.read("e2") with e2: assert set(root.name for (root, _) in e2.concretized_specs()) == set( ["diamond-link-right", "diamond-link-bottom"] ) dtdiamondright = next( concrete for (_, concrete) in e2.concretized_specs() if concrete.name == "diamond-link-right" ) assert dtdiamondright.installed dtdiamondbottom = next( concrete for (_, concrete) in e2.concretized_specs() if concrete.name == "diamond-link-bottom" ) assert not dtdiamondbottom.installed ================================================ FILE: lib/spack/spack/test/cmd/unit_test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pytest from spack.main import SpackCommand pytest.skip("Recursive pytest is brittle", allow_module_level=True) spack_test = SpackCommand("unit-test") cmd_test_py = os.path.join("lib", "spack", "spack", "test", "cmd", "unit_test.py") def test_list(): output = spack_test("--list") assert "unit_test.py" in output assert "spec_semantics.py" in output assert "test_list" not in output def test_list_with_pytest_arg(): output = spack_test("--list", cmd_test_py) assert cmd_test_py in output.strip() def test_list_with_keywords(): # Here we removed querying with a "/" to separate directories # since the behavior is inconsistent across different pytest # versions, see https://stackoverflow.com/a/48814787/771663 output = spack_test("--list", "-k", "unit_test.py") assert cmd_test_py in output.strip() def test_list_long(): output = spack_test("--list-long") assert "unit_test.py::\n" in output assert "test_list" in output assert "test_list_with_pytest_arg" in output assert "test_list_with_keywords" in output assert "test_list_long" in output assert "test_list_long_with_pytest_arg" in output assert "test_list_names" in output assert "test_list_names_with_pytest_arg" in output assert "spec_dag.py::\n" in output assert "test_installed_deps" in output assert "test_test_deptype" in output def test_list_long_with_pytest_arg(): output = spack_test("--list-long", cmd_test_py) assert "unit_test.py::\n" in output assert "test_list" in output assert "test_list_with_pytest_arg" in output assert "test_list_with_keywords" in output assert "test_list_long" in output assert "test_list_long_with_pytest_arg" in output assert "test_list_names" in output assert "test_list_names_with_pytest_arg" in output assert "spec_dag.py::\n" not in output assert "test_installed_deps" not in output assert "test_test_deptype" not in output def test_list_names(): output = spack_test("--list-names") assert "unit_test.py::test_list\n" in output assert "unit_test.py::test_list_with_pytest_arg\n" in output assert "unit_test.py::test_list_with_keywords\n" in output assert "unit_test.py::test_list_long\n" in output assert "unit_test.py::test_list_long_with_pytest_arg\n" in output assert "unit_test.py::test_list_names\n" in output assert "unit_test.py::test_list_names_with_pytest_arg\n" in output assert "spec_dag.py::test_installed_deps\n" in output assert "spec_dag.py::test_test_deptype\n" in output def test_list_names_with_pytest_arg(): output = spack_test("--list-names", cmd_test_py) assert "unit_test.py::test_list\n" in output assert "unit_test.py::test_list_with_pytest_arg\n" in output assert "unit_test.py::test_list_with_keywords\n" in output assert "unit_test.py::test_list_long\n" in output assert "unit_test.py::test_list_long_with_pytest_arg\n" in output assert "unit_test.py::test_list_names\n" in output assert "unit_test.py::test_list_names_with_pytest_arg\n" in output assert "spec_dag.py::test_installed_deps\n" not in output assert "spec_dag.py::test_test_deptype\n" not in output def test_pytest_help(): output = spack_test("--pytest-help") assert "-k EXPRESSION" in output assert "pytest-warnings:" in output assert "--collect-only" in output ================================================ FILE: lib/spack/spack/test/cmd/url.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import pytest import spack.repo from spack.cmd.url import name_parsed_correctly, url_summary, version_parsed_correctly from spack.main import SpackCommand from spack.url import UndetectableVersionError url = SpackCommand("url") class MyPackage: def __init__(self, name, versions): self.name = name self.versions = versions def test_name_parsed_correctly(): # Expected True assert name_parsed_correctly(MyPackage("netcdf", []), "netcdf") assert name_parsed_correctly(MyPackage("r-devtools", []), "devtools") assert name_parsed_correctly(MyPackage("py-numpy", []), "numpy") assert name_parsed_correctly(MyPackage("octave-splines", []), "splines") assert name_parsed_correctly(MyPackage("th-data", []), "TH.data") assert name_parsed_correctly(MyPackage("imagemagick", []), "ImageMagick") # Expected False assert not name_parsed_correctly(MyPackage("", []), "hdf5") assert not name_parsed_correctly(MyPackage("hdf5", []), "") assert not name_parsed_correctly(MyPackage("yaml-cpp", []), "yamlcpp") assert not name_parsed_correctly(MyPackage("yamlcpp", []), "yaml-cpp") assert not name_parsed_correctly(MyPackage("r-py-parser", []), "parser") assert not name_parsed_correctly(MyPackage("oce", []), "oce-0.18.0") def test_version_parsed_correctly(): # Expected True assert version_parsed_correctly(MyPackage("", ["1.2.3"]), "1.2.3") assert version_parsed_correctly(MyPackage("", ["5.4a", "5.4b"]), "5.4a") assert version_parsed_correctly(MyPackage("", ["5.4a", "5.4b"]), "5.4b") assert version_parsed_correctly(MyPackage("", ["1.63.0"]), "1_63_0") assert version_parsed_correctly(MyPackage("", ["0.94h"]), "094h") # Expected False assert not version_parsed_correctly(MyPackage("", []), "1.2.3") assert not version_parsed_correctly(MyPackage("", ["1.2.3"]), "") assert not version_parsed_correctly(MyPackage("", ["1.2.3"]), "1.2.4") assert not version_parsed_correctly(MyPackage("", ["3.4a"]), "3.4") assert not version_parsed_correctly(MyPackage("", ["3.4"]), "3.4b") assert not version_parsed_correctly(MyPackage("", ["0.18.0"]), "oce-0.18.0") def test_url_parse(): url("parse", "http://zlib.net/fossils/zlib-1.2.10.tar.gz") def test_url_with_no_version_fails(): # No version in URL with pytest.raises(UndetectableVersionError): url("parse", "http://www.netlib.org/voronoi/triangle.zip") def test_url_list(mock_packages): out = url("list") total_urls = len(out.split("\n")) # The following two options should not change the number of URLs printed. out = url("list", "--color", "--extrapolation") colored_urls = len(out.split("\n")) assert colored_urls == total_urls # The following options should print fewer URLs than the default. # If they print the same number of URLs, something is horribly broken. # If they say we missed 0 URLs, something is probably broken too. out = url("list", "--incorrect-name") incorrect_name_urls = len(out.split("\n")) assert 0 < incorrect_name_urls < total_urls out = url("list", "--incorrect-version") incorrect_version_urls = len(out.split("\n")) assert 0 < incorrect_version_urls < total_urls out = url("list", "--correct-name") correct_name_urls = len(out.split("\n")) assert 0 < correct_name_urls < total_urls out = url("list", "--correct-version") correct_version_urls = len(out.split("\n")) assert 0 < correct_version_urls < total_urls def test_url_summary(mock_packages): """Test the URL summary command.""" # test url_summary, the internal function that does the work (total_urls, correct_names, correct_versions, name_count_dict, version_count_dict) = ( url_summary(None) ) assert 0 < correct_names <= sum(name_count_dict.values()) <= total_urls assert 0 < correct_versions <= sum(version_count_dict.values()) <= total_urls # make sure it agrees with the actual command. out = url("summary") out_total_urls = int(re.search(r"Total URLs found:\s*(\d+)", out).group(1)) assert out_total_urls == total_urls out_correct_names = int(re.search(r"Names correctly parsed:\s*(\d+)", out).group(1)) assert out_correct_names == correct_names out_correct_versions = int(re.search(r"Versions correctly parsed:\s*(\d+)", out).group(1)) assert out_correct_versions == correct_versions def test_url_stats(mock_packages): output = url("stats") npkgs = "%d packages" % len(spack.repo.all_package_names()) assert npkgs in output assert "url" in output assert "git" in output assert "schemes" in output assert "versions" in output assert "resources" in output output = url("stats", "--show-issues") npkgs = "%d packages" % len(spack.repo.all_package_names()) assert npkgs in output assert "url" in output assert "git" in output assert "schemes" in output assert "versions" in output assert "resources" in output assert "Package URLs with md5 hashes" in output assert "needs-relocation" in output assert "https://cmake.org/files/v3.4/cmake-0.0.0.tar.gz" in output assert "Package URLs with http urls" in output assert "zmpi" in output assert "http://www.spack-fake-zmpi.org/downloads/zmpi-1.0.tar.gz" in output ================================================ FILE: lib/spack/spack/test/cmd/verify.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for the `spack verify` command""" import os import pathlib import platform import pytest import spack.cmd.verify import spack.concretize import spack.installer import spack.llnl.util.filesystem as fs import spack.store import spack.util.executable import spack.util.spack_json as sjson import spack.verify from spack.main import SpackCommand, SpackCommandError from spack.spec import Spec verify = SpackCommand("verify") install = SpackCommand("install") def skip_unless_linux(f): return pytest.mark.skipif( str(platform.system()) != "Linux", reason="only tested on linux for now" )(f) def test_single_file_verify_cmd(tmp_path: pathlib.Path): # Test the verify command interface to verifying a single file. filedir = tmp_path / "a" / "b" / "c" / "d" filepath = filedir / "file" metadir = tmp_path / spack.store.STORE.layout.metadata_dir fs.mkdirp(str(filedir)) fs.mkdirp(str(metadir)) with open(str(filepath), "w", encoding="utf-8") as f: f.write("I'm a file") data = spack.verify.create_manifest_entry(str(filepath)) manifest_file = metadir / spack.store.STORE.layout.manifest_file_name with open(str(manifest_file), "w", encoding="utf-8") as f: sjson.dump({str(filepath): data}, f) results = verify("manifest", "-f", str(filepath), fail_on_error=False) assert not results os.utime(str(filepath), (0, 0)) with open(str(filepath), "w", encoding="utf-8") as f: f.write("I changed.") results = verify("manifest", "-f", str(filepath), fail_on_error=False) expected = ["hash"] mtime = os.stat(str(filepath)).st_mtime if mtime != data["time"]: expected.append("mtime") assert results assert str(filepath) in results assert all(x in results for x in expected) results = verify("manifest", "-fj", str(filepath), fail_on_error=False) res = sjson.load(results) assert len(res) == 1 errors = res.pop(str(filepath)) assert sorted(errors) == sorted(expected) def test_single_spec_verify_cmd(mock_packages, mock_archive, mock_fetch, install_mockery): # Test the verify command interface to verify a single spec install("--fake", "libelf") s = spack.concretize.concretize_one("libelf") prefix = s.prefix hash = s.dag_hash() results = verify("manifest", "/%s" % hash, fail_on_error=False) assert not results new_file = os.path.join(prefix, "new_file_for_verify_test") with open(new_file, "w", encoding="utf-8") as f: f.write("New file") results = verify("manifest", "/%s" % hash, fail_on_error=False) assert new_file in results assert "added" in results results = verify("manifest", "-j", "/%s" % hash, fail_on_error=False) res = sjson.load(results) assert len(res) == 1 assert res[new_file] == ["added"] @pytest.mark.requires_executables("gcc") @skip_unless_linux def test_libraries(tmp_path: pathlib.Path, install_mockery, mock_fetch): gcc = spack.util.executable.which("gcc", required=True) s = spack.concretize.concretize_one("libelf") spack.installer.PackageInstaller([s.package], fake=True).install() # There are no ELF files so the verification should pass verify("libraries", f"/{s.dag_hash()}") # Now put main_with_rpath linking to libf.so inside the prefix and verify again. This should # work because libf.so can be located in the rpath. (tmp_path / "f.c").write_text("void f(void){return;}") (tmp_path / "main.c").write_text("void f(void); int main(void){f();return 0;}") gcc("-shared", "-fPIC", "-o", str(tmp_path / "libf.so"), str(tmp_path / "f.c")) gcc( "-o", str(s.prefix.bin.main_with_rpath), str(tmp_path / "main.c"), "-L", str(tmp_path), f"-Wl,-rpath,{tmp_path}", "-lf", ) verify("libraries", f"/{s.dag_hash()}") # Now put main_without_rpath linking to libf.so inside the prefix and verify again. This should # fail because libf.so cannot be located in the rpath. gcc( "-o", str(s.prefix.bin.main_without_rpath), str(tmp_path / "main.c"), "-L", str(tmp_path), "-lf", ) with pytest.raises(SpackCommandError): verify("libraries", f"/{s.dag_hash()}") # Check the error message msg = spack.cmd.verify._verify_libraries(s, []) assert msg is not None and "libf.so => not found" in msg # And check that we can make it pass by ignoring it. assert spack.cmd.verify._verify_libraries(s, ["libf.so"]) is None def test_verify_versions(mock_packages): missing = "thisisnotapackage" unknown = "deprecated-versions@=thisisnotaversion" deprecated = "deprecated-versions@=1.1.0" good = "deprecated-versions@=1.0.0" strs = (missing, unknown, deprecated, good) specs = [Spec(c) for c in strs] + [Spec(f"deprecated-client@=1.1.0^{c}") for c in strs] for spec in specs: spec._mark_concrete() msg_lines = spack.cmd.verify._verify_version(specs) assert "3 installed packages have unknown/deprecated" in msg_lines[0] assert "thisisnotapackage" in msg_lines[1] assert "Cannot load package" in msg_lines[1] assert "version thisisnotaversion unknown to Spack" in msg_lines[2] assert "deprecated version 1.1.0" in msg_lines[3] ================================================ FILE: lib/spack/spack/test/cmd/versions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.url from spack.main import SpackCommand from spack.version import Version versions = SpackCommand("versions") pytestmark = [pytest.mark.usefixtures("mock_packages")] def _mock_find_versions_of_archive(*args, **kwargs): return { Version("1.3.1"): "https://zlib.net/zlib-1.3.1.tar.gz", Version("1.3"): "https://zlib.net/zlib-1.3.tar.gz", Version("1.2.13"): "https://zlib.net/zlib-1.2.13.tar.gz", } def test_safe_versions(): """Only test the safe versions of a package.""" assert versions("--safe", "zlib") == " 1.2.11\n 1.2.8\n 1.2.3\n" def test_remote_versions(monkeypatch): """Test a package for which remote versions should be available.""" monkeypatch.setattr(spack.url, "find_versions_of_archive", _mock_find_versions_of_archive) assert versions("zlib") == " 1.2.11\n 1.2.8\n 1.2.3\n 1.3.1\n 1.3\n 1.2.13\n" def test_remote_versions_only(monkeypatch): """Test a package for which remote versions should be available.""" monkeypatch.setattr(spack.url, "find_versions_of_archive", _mock_find_versions_of_archive) assert versions("--remote", "zlib") == " 1.3.1\n 1.3\n 1.2.13\n" def test_new_versions_only(monkeypatch): """Test a package for which new versions should be available.""" from spack_repo.builtin_mock.packages.brillig.package import Brillig # type: ignore[import] def mock_fetch_remote_versions(*args, **kwargs): mock_remote_versions = { # new version, we expect this to be in output: Version("99.99.99"): {}, # some packages use '3.2' equivalently to '3.2.0' # thus '3.2.1' is considered to be a new version # and expected in the output also Version("3.2.1"): {}, # new version, we expect this to be in output Version("3.2"): {}, Version("1.0.0"): {}, } return mock_remote_versions mock_versions = { # already checksummed versions: Version("3.2"): {}, Version("1.0.0"): {}, } monkeypatch.setattr(Brillig, "versions", mock_versions) monkeypatch.setattr(Brillig, "fetch_remote_versions", mock_fetch_remote_versions) v = versions("--new", "brillig") assert v.strip(" \n\t") == "99.99.99\n 3.2.1" def test_no_unchecksummed_versions(monkeypatch): """Test a package for which no unchecksummed versions are available.""" def mock_find_versions_of_archive(*args, **kwargs): """Mock find_versions_of_archive to avoid network calls.""" # Return some fake versions for bzip2 return { Version("1.0.8"): "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz", Version("1.0.7"): "https://sourceware.org/pub/bzip2/bzip2-1.0.7.tar.gz", } monkeypatch.setattr(spack.url, "find_versions_of_archive", mock_find_versions_of_archive) versions("bzip2") def test_versions_no_url(): """Test a package with versions but without a ``url`` attribute.""" assert versions("attributes-foo-app") == " 1.0\n" def test_no_versions_no_url(): """Test a package without versions or a ``url`` attribute.""" assert versions("no-url-or-version") == "" ================================================ FILE: lib/spack/spack/test/cmd/view.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import sys import pytest import spack.concretize import spack.main import spack.util.spack_yaml as s_yaml from spack.installer import PackageInstaller from spack.llnl.util.filesystem import _windows_can_symlink from spack.main import SpackCommand extensions = SpackCommand("extensions") install = SpackCommand("install") view = SpackCommand("view") if sys.platform == "win32": if not _windows_can_symlink(): pytest.skip( "Windows must be able to create symlinks to run tests.", allow_module_level=True ) # TODO: Skipping hardlink command testing on windows until robust checks can be added. # See https://github.com/spack/spack/pull/46335#discussion_r1757411915 commands = ["symlink", "add", "copy", "relocate"] else: commands = ["hardlink", "symlink", "hard", "add", "copy", "relocate"] def create_projection_file(tmp_path: pathlib.Path, projection): if "projections" not in projection: projection = {"projections": projection} projection_file = tmp_path / "projection" / "projection.yaml" projection_file.parent.mkdir(parents=True, exist_ok=True) projection_file.write_text(s_yaml.dump(projection)) return projection_file @pytest.mark.parametrize("cmd", commands) def test_view_link_type( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, cmd ): install("--fake", "libdwarf") view_dir = tmp_path / f"view_{cmd}" view(cmd, str(view_dir), "libdwarf") package_bin = view_dir / "bin" / "libdwarf" assert package_bin.exists() # Check that we use symlinks for and only for the appropriate subcommands is_link_cmd = cmd in ("symlink", "add") assert os.path.islink(str(package_bin)) == is_link_cmd @pytest.mark.parametrize("add_cmd", commands) def test_view_link_type_remove( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, add_cmd ): install("needs-relocation") viewpath = str(tmp_path / "view_{0}".format(add_cmd)) (tmp_path / "view_{0}".format(add_cmd)).mkdir() view(add_cmd, viewpath, "needs-relocation") bindir = os.path.join(viewpath, "bin") assert os.path.exists(bindir) view("remove", viewpath, "needs-relocation") assert not os.path.exists(bindir) @pytest.mark.parametrize("cmd", commands) def test_view_projections( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, cmd ): install("--fake", "libdwarf@20130207") view_dir = tmp_path / f"view_{cmd}" view_projection = {"projections": {"all": "{name}-{version}"}} projection_file = create_projection_file(tmp_path, view_projection) view(cmd, str(view_dir), f"--projection-file={projection_file}", "libdwarf") package_bin = view_dir / "libdwarf-20130207" / "bin" / "libdwarf" assert package_bin.exists() # Check that we use symlinks for and only for the appropriate subcommands is_symlink_cmd = cmd in ("symlink", "add") assert package_bin.is_symlink() == is_symlink_cmd def test_view_multiple_projections( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery ): install("--fake", "libdwarf@20130207") install("--fake", "extendee@1.0") view_dir = tmp_path / "view" view_projection = s_yaml.syaml_dict( [("extendee", "{name}-{architecture.platform}"), ("all", "{name}-{version}")] ) projection_file = create_projection_file(tmp_path, view_projection) view("add", str(view_dir), f"--projection-file={projection_file}", "libdwarf", "extendee") libdwarf_prefix = view_dir / "libdwarf-20130207" / "bin" extendee_prefix = view_dir / "extendee-test" / "bin" assert libdwarf_prefix.exists() assert extendee_prefix.exists() def test_view_multiple_projections_all_first( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery ): install("--fake", "libdwarf@20130207") install("--fake", "extendee@1.0") view_dir = tmp_path / "view" view_projection = s_yaml.syaml_dict( [("all", "{name}-{version}"), ("extendee", "{name}-{architecture.platform}")] ) projection_file = create_projection_file(tmp_path, view_projection) view("add", str(view_dir), f"--projection-file={projection_file}", "libdwarf", "extendee") libdwarf_prefix = view_dir / "libdwarf-20130207" / "bin" extendee_prefix = view_dir / "extendee-test" / "bin" assert libdwarf_prefix.exists() assert extendee_prefix.exists() def test_view_external( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery ): install("externaltool") viewpath = str(tmp_path / "view") (tmp_path / "view").mkdir() output = view("symlink", viewpath, "externaltool") assert "Skipping external package: externaltool" in output def test_view_extension( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery ): install("extendee") install("extension1@1.0") install("extension1@2.0") install("extension2@1.0") viewpath = str(tmp_path / "view") (tmp_path / "view").mkdir() view("symlink", viewpath, "extension1@1.0") all_installed = extensions("--show", "installed", "extendee") assert "extension1@1.0" in all_installed assert "extension1@2.0" in all_installed assert "extension2@1.0" in all_installed assert os.path.exists(os.path.join(viewpath, "bin", "extension1")) def test_view_extension_remove( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery ): install("extendee") install("extension1@1.0") viewpath = str(tmp_path / "view") (tmp_path / "view").mkdir() view("symlink", viewpath, "extension1@1.0") view("remove", viewpath, "extension1@1.0") all_installed = extensions("--show", "installed", "extendee") assert "extension1@1.0" in all_installed assert not os.path.exists(os.path.join(viewpath, "bin", "extension1")) def test_view_extension_conflict( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery ): install("extendee") install("extension1@1.0") install("extension1@2.0") viewpath = str(tmp_path / "view") (tmp_path / "view").mkdir() view("symlink", viewpath, "extension1@1.0") output = view("symlink", viewpath, "extension1@2.0") assert "Package conflict detected" in output def test_view_extension_conflict_ignored( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery ): install("extendee") install("extension1@1.0") install("extension1@2.0") viewpath = str(tmp_path / "view") (tmp_path / "view").mkdir() view("symlink", viewpath, "extension1@1.0") view("symlink", viewpath, "-i", "extension1@2.0") with open(os.path.join(viewpath, "bin", "extension1"), "r", encoding="utf-8") as fin: assert fin.read() == "1.0" def test_view_fails_with_missing_projections_file(tmp_path: pathlib.Path): viewpath = str(tmp_path / "view") (tmp_path / "view").mkdir() projection_file = str(tmp_path / "nonexistent") with pytest.raises(spack.main.SpackCommandError): view("symlink", "--projection-file", projection_file, viewpath, "foo") @pytest.mark.parametrize("with_projection", [False, True]) @pytest.mark.parametrize("cmd", ["symlink", "copy"]) def test_view_files_not_ignored( tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, cmd, with_projection, ): spec = spack.concretize.concretize_one("view-not-ignored") pkg = spec.package PackageInstaller([pkg], explicit=True).install() pkg.assert_installed(spec.prefix) install("view-file") # Arbitrary package to add noise viewpath = str(tmp_path / "view_{0}".format(cmd)) (tmp_path / "view_{0}".format(cmd)).mkdir() if with_projection: proj = str(tmp_path / "proj.yaml") with open(proj, "w", encoding="utf-8") as f: f.write('{"projections":{"all":"{name}"}}') prefix_in_view = os.path.join(viewpath, "view-not-ignored") args = ["--projection-file", proj] else: prefix_in_view = viewpath args = [] view(cmd, *(args + [viewpath, "view-not-ignored", "view-file"])) pkg.assert_installed(prefix_in_view) view("remove", viewpath, "view-not-ignored") pkg.assert_not_installed(prefix_in_view) ================================================ FILE: lib/spack/spack/test/cmd_extensions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import contextlib import os import pathlib import sys import pytest import spack.cmd import spack.config import spack.extensions import spack.main class Extension: """Helper class to simplify the creation of simple command extension directory structures with a conventional format for testing. """ def __init__(self, name, root: pathlib.Path): """Create a command extension. Args: name (str): The name of the command extension. root (path object): The temporary root for the command extension (e.g. from tmp_path.mkdir()). """ self.name = name self.pname = spack.cmd.python_name(name) self.root = root self.main = self.root / self.pname self.main.mkdir(parents=True, exist_ok=True) self.cmd = self.main / "cmd" self.cmd.mkdir(parents=True, exist_ok=True) def add_command(self, command_name, contents): """Add a command to this command extension. Args: command_name (str): The name of the command. contents (str): the desired contents of the new command module file.""" spack.cmd.require_cmd_name(command_name) python_name = spack.cmd.python_name(command_name) cmd = self.cmd / (python_name + ".py") cmd.write_text(contents) @pytest.fixture(scope="function") def extension_creator(tmp_path: pathlib.Path, config): """Create a basic extension command directory structure""" @contextlib.contextmanager def _ce(extension_name="testcommand"): root = tmp_path / ("spack-" + extension_name) root.mkdir() extension = Extension(extension_name, root) with spack.config.override("config:extensions", [str(extension.root)]): yield extension list_of_modules = list(sys.modules.keys()) try: yield _ce finally: to_be_deleted = [x for x in sys.modules if x not in list_of_modules] for module_name in to_be_deleted: del sys.modules[module_name] @pytest.fixture(scope="function") def hello_world_extension(extension_creator): """Create an extension with a hello-world command.""" with extension_creator() as extension: extension.add_command( "hello-world", """ description = "hello world extension command" section = "test command" level = "long" def setup_parser(subparser): pass def hello_world(parser, args): print('Hello world!') """, ) yield extension @pytest.fixture(scope="function") def hello_world_cmd(hello_world_extension): """Create and return an invocable "hello-world" extension command.""" yield spack.main.SpackCommand("hello-world") @pytest.fixture(scope="function") def hello_world_with_module_in_root(extension_creator): """Create a "hello-world" extension command with additional code in the root folder. """ @contextlib.contextmanager def _hwwmir(extension_name=None): with ( extension_creator(extension_name) if extension_name else extension_creator() ) as extension: # Note that the namespace of the extension is derived from the # fixture. extension.add_command( "hello", """ # Test an absolute import from spack.extensions.{ext_pname}.implementation import hello_world # Test a relative import from ..implementation import hello_folks description = "hello world extension command" section = "test command" level = "long" # Test setting a global variable in setup_parser and retrieving # it in the command global_message = 'foo' def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='subcommand') global global_message sp.add_parser('world', help='Print Hello world!') sp.add_parser('folks', help='Print Hello folks!') sp.add_parser('global', help='Print Hello folks!') global_message = 'bar' def hello(parser, args): if args.subcommand == 'world': hello_world() elif args.subcommand == 'folks': hello_folks() elif args.subcommand == 'global': print(global_message) """.format(ext_pname=extension.pname), ) init_file = extension.main / "__init__.py" init_file.touch() implementation = extension.main / "implementation.py" implementation.write_text( """ def hello_world(): print('Hello world!') def hello_folks(): print('Hello folks!') """ ) yield spack.main.SpackCommand("hello") yield _hwwmir def test_simple_command_extension(hello_world_cmd): """Basic test of a functioning command.""" output = hello_world_cmd() assert "Hello world!" in output def test_multi_extension_search(hello_world_extension, extension_creator): """Ensure we can find an extension command even if it's not in the first place we look. """ with extension_creator("testcommand2"): assert ("Hello world") in spack.main.SpackCommand("hello-world")() def test_duplicate_module_load(hello_world_cmd, capfd): """Ensure duplicate module load attempts are successful. The command module will already have been loaded once by the hello_world_cmd fixture. """ parser = spack.main.make_argument_parser() args = [] hw_cmd = spack.cmd.get_command(hello_world_cmd.command_name) hw_cmd(parser, args) captured = capfd.readouterr() assert captured == ("Hello world!\n", "") @pytest.mark.parametrize( "extension_name", [None, "hyphenated-extension"], ids=["simple", "hyphenated_extension_name"] ) def test_command_with_import(extension_name, hello_world_with_module_in_root): """Ensure we can write a functioning command with multiple imported subcommands, including where the extension name contains a hyphen. """ with hello_world_with_module_in_root(extension_name) as hello_world: output = hello_world("world") assert "Hello world!" in output output = hello_world("folks") assert "Hello folks!" in output output = hello_world("global") assert "bar" in output def test_missing_command(): """Ensure that we raise the expected exception if the desired command is not present. """ with pytest.raises(spack.cmd.CommandNotFoundError): spack.cmd.get_module("no-such-command") @pytest.mark.parametrize( "extension_path,expected_exception", [ ("/my/bad/extension", spack.extensions.ExtensionNamingError), ("", spack.extensions.ExtensionNamingError), ("/my/bad/spack--extra-hyphen", spack.extensions.ExtensionNamingError), ("/my/good/spack-extension", spack.cmd.CommandNotFoundError), ("/my/still/good/spack-extension/", spack.cmd.CommandNotFoundError), ("/my/spack-hyphenated-extension", spack.cmd.CommandNotFoundError), ], ids=["no_stem", "vacuous", "leading_hyphen", "basic_good", "trailing_slash", "hyphenated"], ) def test_extension_naming(tmp_path: pathlib.Path, extension_path, expected_exception, config): """Ensure that we are correctly validating configured extension paths for conformity with the rules: the basename should match ``spack-``; may have embedded hyphens but not begin with one. """ # NOTE: if the directory is a valid extension directory name the "vacuous" test will # fail because it resolves to current working directory import spack.llnl.util.filesystem as fs with fs.working_dir(str(tmp_path)): with spack.config.override("config:extensions", [extension_path]): with pytest.raises(expected_exception): spack.cmd.get_module("no-such-command") def test_missing_command_function(extension_creator, capfd): """Ensure we die as expected if a command module does not have the expected command function defined. """ with extension_creator() as extension: extension.add_command("bad-cmd", """\ndescription = "Empty command implementation"\n""") with pytest.raises(SystemExit): spack.cmd.get_module("bad-cmd") capture = capfd.readouterr() assert "must define function 'bad_cmd'." in capture[1] def test_get_command_paths(config): """Exercise the construction of extension command search paths.""" extensions = ("extension-1", "extension-2") ext_paths = [] expected_cmd_paths = [] for ext in extensions: ext_path = os.path.join("my", "path", "to", "spack-" + ext) ext_paths.append(ext_path) path = os.path.join(ext_path, spack.cmd.python_name(ext), "cmd") path = os.path.abspath(path) expected_cmd_paths.append(path) with spack.config.override("config:extensions", ext_paths): assert spack.extensions.get_command_paths() == expected_cmd_paths def test_variable_in_extension_path(config, working_env): """Test variables in extension paths.""" os.environ["_MY_VAR"] = os.path.join("my", "var") ext_paths = [os.path.join("~", "${_MY_VAR}", "spack-extension-1")] # Home env variable is USERPROFILE on Windows home_env = "USERPROFILE" if sys.platform == "win32" else "HOME" expected_ext_paths = [ os.path.join(os.environ[home_env], os.environ["_MY_VAR"], "spack-extension-1") ] with spack.config.override("config:extensions", ext_paths): assert spack.extensions.get_extension_paths() == expected_ext_paths @pytest.mark.parametrize( "command_name,contents,exception", [ ("bad-cmd", "from oopsie.daisy import bad\n", ImportError), ("bad-cmd", """var = bad_function_call('blech')\n""", NameError), ("bad-cmd", ")\n", SyntaxError), ], ids=["ImportError", "NameError", "SyntaxError"], ) def test_failing_command(command_name, contents, exception, extension_creator): """Ensure that the configured command fails to import with the specified error. """ with extension_creator() as extension: extension.add_command(command_name, contents) with pytest.raises(exception): spack.extensions.get_module(command_name) ================================================ FILE: lib/spack/spack/test/compilers/conversion.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests conversions from compilers.yaml""" import pathlib import pytest from spack.compilers.config import CompilerFactory pytestmark = [pytest.mark.usefixtures("config", "mock_packages")] @pytest.fixture() def mock_compiler(mock_executable): gcc = mock_executable("gcc", "echo 13.2.0") gxx = mock_executable("g++", "echo 13.2.0") gfortran = mock_executable("gfortran", "echo 13.2.0") return { "spec": "gcc@13.2.0", "paths": {"cc": str(gcc), "cxx": str(gxx), "f77": str(gfortran), "fc": str(gfortran)}, } # - compiler: # spec: clang@=10.0.0 # paths: # cc: /usr/bin/clang # cxx: /usr/bin/clang++ # f77: null # fc: null # flags: {} # operating_system: ubuntu20.04 # target: x86_64 # modules: [] # environment: {} # extra_rpaths: [] def test_basic_compiler_conversion(mock_compiler, tmp_path: pathlib.Path): """Tests the conversion of a compiler using a single toolchain, with default options.""" compilers = CompilerFactory.from_legacy_yaml(mock_compiler) compiler_spec = compilers[0] assert compiler_spec.satisfies("gcc@13.2.0 languages=c,c++,fortran") assert compiler_spec.external assert compiler_spec.external_path == str(tmp_path) for language in ("c", "cxx", "fortran"): assert language in compiler_spec.extra_attributes["compilers"] def test_compiler_conversion_with_flags(mock_compiler): """Tests that flags are converted appropriately for external compilers""" mock_compiler["flags"] = {"cflags": "-O3", "cxxflags": "-O2 -g"} compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0] assert compiler_spec.external assert "flags" in compiler_spec.extra_attributes assert compiler_spec.extra_attributes["flags"]["cflags"] == "-O3" assert compiler_spec.extra_attributes["flags"]["cxxflags"] == "-O2 -g" def test_compiler_conversion_with_environment(mock_compiler): """Tests that custom environment modifications are converted appropriately for external compilers """ mods = {"set": {"FOO": "foo", "BAR": "bar"}, "unset": ["BAZ"]} mock_compiler["environment"] = mods compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0] assert compiler_spec.external assert "environment" in compiler_spec.extra_attributes assert compiler_spec.extra_attributes["environment"] == mods def test_compiler_conversion_extra_rpaths(mock_compiler): """Tests that extra rpaths are converted appropriately for external compilers""" mock_compiler["extra_rpaths"] = ["/foo/bar"] compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0] assert compiler_spec.external assert "extra_rpaths" in compiler_spec.extra_attributes assert compiler_spec.extra_attributes["extra_rpaths"] == ["/foo/bar"] def test_compiler_conversion_modules(mock_compiler): """Tests that modules are converted appropriately for external compilers""" modules = ["foo/4.1.2", "bar/5.1.4"] mock_compiler["modules"] = modules compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0] assert compiler_spec.external assert compiler_spec.external_modules == modules @pytest.mark.regression("49717") def test_compiler_conversion_corrupted_paths(mock_compiler): """Tests that compiler entries with corrupted path do not raise""" mock_compiler["paths"] = {"cc": "gcc", "cxx": "g++", "fc": "gfortran", "f77": "gfortran"} # Test this call doesn't raise compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler) assert compiler_spec == [] ================================================ FILE: lib/spack/spack/test/compilers/libraries.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import copy import os import pathlib import pytest import spack.compilers.config import spack.compilers.libraries import spack.llnl.util.filesystem as fs import spack.util.executable import spack.util.module_cmd without_flag_output = "ld -L/path/to/first/lib -L/path/to/second/lib64" with_flag_output = "ld -L/path/to/first/with/flag/lib -L/path/to/second/lib64" def call_compiler(exe, *args, **kwargs): # This method can replace Executable.__call__ to emulate a compiler that # changes libraries depending on a flag. if "--correct-flag" in exe.exe: return with_flag_output return without_flag_output @pytest.fixture() def mock_gcc(config): compilers = spack.compilers.config.all_compilers_from(configuration=config) assert compilers, "No compilers available" compilers.sort(key=lambda x: (x.name == "gcc", x.version)) # Deepcopy is used to avoid more boilerplate when changing the "extra_attributes" return copy.deepcopy(compilers[-1]) @pytest.mark.usefixtures("mock_packages") class TestCompilerPropertyDetector: @pytest.mark.parametrize( "language,flagname", [ ("cxx", "cxxflags"), ("cxx", "cppflags"), ("cxx", "ldflags"), ("c", "cflags"), ("c", "cppflags"), ], ) @pytest.mark.not_on_windows("Not supported on Windows") def test_compile_dummy_c_source(self, mock_gcc, monkeypatch, language, flagname): monkeypatch.setattr(spack.util.executable.Executable, "__call__", call_compiler) for key in list(mock_gcc.extra_attributes["compilers"]): if key == language: continue mock_gcc.extra_attributes["compilers"].pop(key) detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) # Test without flags assert detector._compile_dummy_c_source() == without_flag_output # Set flags and test if flagname: mock_gcc.extra_attributes.setdefault("flags", {}) monkeypatch.setitem(mock_gcc.extra_attributes["flags"], flagname, "--correct-flag") assert detector._compile_dummy_c_source() == with_flag_output def test_compile_dummy_c_source_no_path(self, mock_gcc): mock_gcc.extra_attributes["compilers"] = {} detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) assert detector._compile_dummy_c_source() is None def test_compile_dummy_c_source_no_verbose_flags(self, mock_gcc, monkeypatch): monkeypatch.setattr(mock_gcc.package, "verbose_flags", "") detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) assert detector._compile_dummy_c_source() is None @pytest.mark.not_on_windows("Module files are not supported on Windows") def test_compile_dummy_c_source_load_env(self, mock_gcc, monkeypatch, tmp_path: pathlib.Path): gcc = tmp_path / "gcc" gcc.write_text( f"""#!/bin/sh if [ "$ENV_SET" = "1" ] && [ "$MODULE_LOADED" = "1" ]; then printf '{without_flag_output}' fi """ ) fs.set_executable(str(gcc)) # Set module load to turn compiler on def module(*args): if args[0] == "show": return "" elif args[0] == "load": monkeypatch.setenv("MODULE_LOADED", "1") monkeypatch.setenv("LOADEDMODULES", "turn_on") monkeypatch.setattr(spack.util.module_cmd, "module", module) mock_gcc.extra_attributes["compilers"]["c"] = str(gcc) mock_gcc.extra_attributes["environment"] = {"set": {"ENV_SET": "1"}} mock_gcc.external_modules = ["turn_on"] detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) assert detector._compile_dummy_c_source() == without_flag_output @pytest.mark.not_on_windows("Not supported on Windows") def test_implicit_rpaths(self, mock_gcc, dirs_with_libfiles, monkeypatch): lib_to_dirs, all_dirs = dirs_with_libfiles detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) monkeypatch.setattr( spack.compilers.libraries.CompilerPropertyDetector, "_compile_dummy_c_source", lambda self: "ld " + " ".join(f"-L{d}" for d in all_dirs), ) retrieved_rpaths = detector.implicit_rpaths() assert set(retrieved_rpaths) == set(lib_to_dirs["libstdc++"] + lib_to_dirs["libgfortran"]) def test_compiler_environment(self, working_env, mock_gcc, monkeypatch): """Test whether environment modifications are applied in compiler_environment""" monkeypatch.delenv("TEST", raising=False) mock_gcc.extra_attributes["environment"] = {"set": {"TEST": "yes"}} detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) with detector.compiler_environment(): assert os.environ["TEST"] == "yes" @pytest.mark.not_on_windows("Module files are not supported on Windows") def test_compiler_invalid_module_raises(self, working_env, mock_gcc, monkeypatch): """Test if an exception is raised when a module cannot be loaded""" def mock_load_module(module_name): # Simulate module load failure raise spack.util.module_cmd.ModuleLoadError(module_name) monkeypatch.setattr(spack.util.module_cmd, "load_module", mock_load_module) mock_gcc.external_modules = ["non_existent"] detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc) with pytest.raises(spack.util.module_cmd.ModuleLoadError): with detector.compiler_environment(): pass ================================================ FILE: lib/spack/spack/test/concretization/compiler_runtimes.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.vendor.archspec.cpu import spack.concretize import spack.config import spack.paths import spack.repo import spack.solver.asp import spack.spec from spack.environment.environment import ViewDescriptor from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals from spack.version import Version def _concretize_with_reuse(*, root_str, reused_str, config): reused_spec = spack.concretize.concretize_one(reused_str) packages_with_externals = external_config_with_implicit_externals(config) completion_mode = config.get("concretizer:externals:completion") external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], exclude=[], ).selected_specs() setup = spack.solver.asp.SpackSolverSetup(tests=False) driver = spack.solver.asp.PyclingoDriver() result, _, _ = driver.solve( setup, [spack.spec.Spec(f"{root_str}")], reuse=[reused_spec] + external_specs ) root = result.specs[0] return root, reused_spec @pytest.fixture def runtime_repo(mutable_config): repo = os.path.join(spack.paths.test_repos_path, "spack_repo", "compiler_runtime_test") with spack.repo.use_repositories(repo) as mock_repo: yield mock_repo def test_correct_gcc_runtime_is_injected_as_dependency(runtime_repo): s = spack.concretize.concretize_one("pkg-a%gcc@10.2.1 ^pkg-b%gcc@9.4.0") a, b = s["pkg-a"], s["pkg-b"] # Both a and b should depend on the same gcc-runtime directly assert a.dependencies("gcc-runtime") == b.dependencies("gcc-runtime") # And the gcc-runtime version should be that of the newest gcc used in the dag. assert a["gcc-runtime"].version == Version("10.2.1") @pytest.mark.regression("41972") def test_external_nodes_do_not_have_runtimes(runtime_repo, mutable_config, tmp_path: pathlib.Path): """Tests that external nodes don't have runtime dependencies.""" packages_yaml = {"pkg-b": {"externals": [{"spec": "pkg-b@1.0", "prefix": f"{str(tmp_path)}"}]}} spack.config.set("packages", packages_yaml) s = spack.concretize.concretize_one("pkg-a%gcc@10.2.1") a, b = s["pkg-a"], s["pkg-b"] # Since b is an external, it doesn't depend on gcc-runtime assert a.dependencies("gcc-runtime") assert a.dependencies("pkg-b") assert not b.dependencies("gcc-runtime") @pytest.mark.parametrize( "root_str,reused_str,expected,nruntime", [ # The reused runtime is older than we need, thus we'll add a more recent one for a ( "pkg-a%gcc@10.2.1", "pkg-b%gcc@9.4.0", {"pkg-a": "gcc-runtime@10.2.1", "pkg-b": "gcc-runtime@9.4.0"}, 2, ), # The root is compiled with an older compiler, thus we'll NOT reuse the runtime from b ( "pkg-a%gcc@9.4.0", "pkg-b%gcc@10.2.1", {"pkg-a": "gcc-runtime@9.4.0", "pkg-b": "gcc-runtime@9.4.0"}, 1, ), # Same as before, but tests that we can reuse from a more generic target pytest.param( "pkg-a%gcc@9.4.0", "pkg-b target=x86_64 %gcc@10.2.1", {"pkg-a": "gcc-runtime@9.4.0", "pkg-b": "gcc-runtime@9.4.0"}, 1, marks=pytest.mark.skipif( str(spack.vendor.archspec.cpu.host().family) != "x86_64", reason="test data is x86_64 specific", ), ), pytest.param( "pkg-a%gcc@10.2.1", "pkg-b target=x86_64 %gcc@9.4.0", { "pkg-a": "gcc-runtime@10.2.1 target=core2", "pkg-b": "gcc-runtime@9.4.0 target=x86_64", }, 2, marks=pytest.mark.skipif( str(spack.vendor.archspec.cpu.host().family) != "x86_64", reason="test data is x86_64 specific", ), ), ], ) @pytest.mark.regression("44444") def test_reusing_specs_with_gcc_runtime( root_str, reused_str, expected, nruntime, runtime_repo, mutable_config ): """Tests that we can reuse specs with a "gcc-runtime" leaf node. In particular, checks that the semantic for gcc-runtimes versions accounts for reused packages too. Reusable runtime versions should be lower, or equal, to that of parent nodes. """ root, reused_spec = _concretize_with_reuse( root_str=root_str, reused_str=reused_str, config=mutable_config ) runtime_a = root.dependencies("gcc-runtime")[0] assert runtime_a.satisfies(expected["pkg-a"]), runtime_a.tree() runtime_b = root["pkg-b"].dependencies("gcc-runtime")[0] assert runtime_b.satisfies(expected["pkg-b"]) runtimes = [x for x in root.traverse() if x.name == "gcc-runtime"] assert len(runtimes) == nruntime @pytest.mark.parametrize( "root_str,reused_str,expected,not_expected", [ # Ensure that, whether we have multiple runtimes in the DAG or not, # we always link only the latest version ("pkg-a%gcc@10.2.1", "pkg-b%gcc@9.4.0", ["gcc-runtime@10.2.1"], ["gcc-runtime@9.4.0"]) ], ) def test_views_can_handle_duplicate_runtime_nodes( root_str, reused_str, expected, not_expected, runtime_repo, tmp_path: pathlib.Path, monkeypatch, mutable_config, ): """Tests that an environment is able to select the latest version of a runtime node to be linked in a view, in case more than one compatible version is in the DAG. """ root, reused_spec = _concretize_with_reuse( root_str=root_str, reused_str=reused_str, config=mutable_config ) # Mock the installation status to allow selecting nodes for the view monkeypatch.setattr(spack.spec.Spec, "installed", True) nodes = list(root.traverse()) view = ViewDescriptor(str(tmp_path), str(tmp_path)) candidate_specs = view.specs_for_view(nodes) for x in expected: assert any(node.satisfies(x) for node in candidate_specs) for x in not_expected: assert all(not node.satisfies(x) for node in candidate_specs) def test_runtimes_can_be_concretized_as_standalone(runtime_repo): """Tests that we can concretize a runtime as a standalone""" gcc_runtime = spack.concretize.concretize_one("gcc-runtime") deps = gcc_runtime.dependencies() assert len(deps) == 1 gcc = deps[0] assert gcc_runtime.version == gcc.version def test_runtimes_are_not_reused_if_compiler_not_used(runtime_repo, mutable_config): """Tests that, if we can reuse specs with a more recent runtime version than the compiler we asked for, we will not end-up with a DAG using the recent runtime, and the old compiler. """ root, reused = _concretize_with_reuse( root_str="pkg-a %gcc@9", reused_str="pkg-a %gcc@10", config=mutable_config ) assert "gcc-runtime" in root gcc_runtime, gcc = root["gcc-runtime"], root["gcc"] assert gcc_runtime.satisfies("@9") and not gcc_runtime.satisfies("@10") assert gcc.satisfies("@9") and not gcc.satisfies("@10") # Same gcc used for both languages assert root["c"] == root["cxx"] ================================================ FILE: lib/spack/spack/test/concretization/conditional_dependencies.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.concretize import spack.spec @pytest.mark.parametrize( "abstract_spec,expected,not_expected", [ # Set +mpi explicitly ( "hdf5+mpi ^[when='^mpi' virtuals=mpi] zmpi", ["%[virtuals=mpi] zmpi", "^mpi", "%mpi"], ["%[virtuals=mpi] mpich"], ), ( "hdf5+mpi %[when='%mpi' virtuals=mpi] zmpi", ["%[virtuals=mpi] zmpi", "^mpi", "%mpi"], ["%[virtuals=mpi] mpich"], ), ( "hdf5+mpi %[when='+mpi' virtuals=mpi] zmpi", ["%[virtuals=mpi] zmpi", "^mpi", "%mpi"], ["%[virtuals=mpi] mpich"], ), ( "hdf5+mpi ^[when='^mpi' virtuals=mpi] mpich", ["%[virtuals=mpi] mpich", "^mpi", "%mpi"], ["%[virtuals=mpi] zmpi"], ), ( "hdf5+mpi %[when='%mpi' virtuals=mpi] mpich", ["%[virtuals=mpi] mpich", "^mpi", "%mpi"], ["%[virtuals=mpi] zmpi"], ), # Use the default, which is to have +mpi ( "hdf5 ^[when='^mpi' virtuals=mpi] zmpi", ["%[virtuals=mpi] zmpi", "^mpi", "%mpi"], ["%[virtuals=mpi] mpich"], ), ( "hdf5 %[when='%mpi' virtuals=mpi] zmpi", ["%[virtuals=mpi] zmpi", "^mpi", "%mpi"], ["%[virtuals=mpi] mpich"], ), ( "hdf5 %[when='+mpi' virtuals=mpi] zmpi", ["%[virtuals=mpi] zmpi", "^mpi", "%mpi"], ["%[virtuals=mpi] mpich"], ), # Set ~mpi explicitly ("hdf5~mpi ^[when='^mpi' virtuals=mpi] zmpi", [], ["%[virtuals=mpi] zmpi", "^mpi"]), ("hdf5~mpi %[when='%mpi' virtuals=mpi] zmpi", [], ["%[virtuals=mpi] zmpi", "^mpi"]), ("hdf5~mpi %[when='+mpi' virtuals=mpi] zmpi", [], ["%[virtuals=mpi] zmpi", "^mpi"]), ], ) def test_conditional_mpi_dependency( abstract_spec, expected, not_expected, default_mock_concretization ): """Test concretizing conditional mpi dependencies.""" concrete = default_mock_concretization(abstract_spec) for x in expected: assert concrete.satisfies(x), x for x in not_expected: assert not concrete.satisfies(x), x assert concrete.satisfies(abstract_spec) @pytest.mark.parametrize("c", [True, False]) @pytest.mark.parametrize("cxx", [True, False]) @pytest.mark.parametrize("fortran", [True, False]) def test_conditional_compilers(c, cxx, fortran, mutable_config, mock_packages, config_two_gccs): """Test concretizing with conditional compilers, using every combination of +~c, +~cxx, and +~fortran. """ # Abstract spec parametrized to depend/not on c/cxx/fortran # and with conditional dependencies for each on the less preferred gcc abstract = spack.spec.Spec(f"conditional-languages c={c} cxx={cxx} fortran={fortran}") concrete_unconstrained = spack.concretize.concretize_one(abstract) abstract.constrain( "^[when='%c' virtuals=c]gcc@10.3.1 " "^[when='%cxx' virtuals=cxx]gcc@10.3.1 " "^[when='%fortran' virtuals=fortran]gcc@10.3.1" ) concrete = spack.concretize.concretize_one(abstract) # We should get the dependency we specified for each language we enabled assert concrete.satisfies("%[virtuals=c]gcc@10.3.1") == c assert concrete.satisfies("%[virtuals=cxx]gcc@10.3.1") == cxx assert concrete.satisfies("%[virtuals=fortran]gcc@10.3.1") == fortran # The only time the two concrete specs are the same is if we don't use gcc at all assert (concrete == concrete_unconstrained) == (not any((c, cxx, fortran))) ================================================ FILE: lib/spack/spack/test/concretization/core.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import difflib import json import os import pathlib import platform import re import sys from typing import Any, Dict import pytest import spack.vendor.archspec.cpu import spack.vendor.jinja2 import spack.archspec import spack.binary_distribution import spack.cmd import spack.compilers.config import spack.compilers.libraries import spack.concretize import spack.config import spack.deptypes as dt import spack.environment as ev import spack.error import spack.hash_types as ht import spack.llnl.util.lang import spack.package_base import spack.paths import spack.platforms import spack.platforms.test import spack.repo import spack.solver.asp import spack.solver.core import spack.solver.input_analysis import spack.solver.reuse import spack.solver.runtimes import spack.spec import spack.spec_filter import spack.util.file_cache import spack.util.hash import spack.util.spack_yaml as syaml import spack.variant as vt from spack.externals import ExternalDependencyError from spack.installer import PackageInstaller from spack.solver.asp import Result from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals from spack.spec import Spec from spack.test.conftest import RepoBuilder from spack.version import Version, VersionList, ver def check_spec(abstract, concrete): if abstract.versions.concrete: assert abstract.versions == concrete.versions if abstract.variants: for name in abstract.variants: avariant = abstract.variants[name] cvariant = concrete.variants[name] assert avariant.value == cvariant.value if abstract.compiler_flags: for flag in abstract.compiler_flags: aflag = abstract.compiler_flags[flag] cflag = concrete.compiler_flags[flag] assert set(aflag) <= set(cflag) for name in spack.repo.PATH.get_pkg_class(abstract.name).variant_names(): assert name in concrete.variants for flag in concrete.compiler_flags.valid_compiler_flags(): assert flag in concrete.compiler_flags if abstract.architecture and abstract.architecture.concrete: assert abstract.architecture == concrete.architecture def check_concretize(abstract_spec): abstract = Spec(abstract_spec) concrete = spack.concretize.concretize_one(abstract) assert not abstract.concrete assert concrete.concrete check_spec(abstract, concrete) return concrete def _true(): return True @pytest.fixture(scope="function", autouse=True) def binary_compatibility(monkeypatch, request): """Selects whether we use OS compatibility for binaries, or libc compatibility.""" if spack.platforms.real_host().name != "linux": return if "mock_packages" not in request.fixturenames: # Only builtin_mock has a mock glibc package return if "database" in request.fixturenames or "mutable_database" in request.fixturenames: # Databases have been created without glibc support return monkeypatch.setattr(spack.solver.core, "using_libc_compatibility", _true) monkeypatch.setattr(spack.solver.runtimes, "using_libc_compatibility", _true) monkeypatch.setattr(spack.solver.asp, "using_libc_compatibility", _true) @pytest.fixture( params=[ # no_deps "libelf", "libelf@0.8.13", # dag "callpath", "mpileaks", "libelf", # variant "mpich+debug", "mpich~debug", "mpich debug=True", "mpich", # compiler flags 'mpich cppflags="-O3"', 'mpich cppflags=="-O3"', # with virtual "mpileaks ^mpi", "mpileaks ^mpi@:1.1", "mpileaks ^mpi@2:", "mpileaks ^mpi@2.1", "mpileaks ^mpi@2.2", "mpileaks ^mpi@2.2", "mpileaks ^mpi@:1", "mpileaks ^mpi@1.2:2", # conflict not triggered "conflict", "conflict~foo%clang", "conflict-parent%gcc", # Direct dependency with different deptypes "mpileaks %[deptypes=link] mpich", ] ) def spec(request): """Spec to be concretized""" return request.param @pytest.fixture( params=[ # Mocking the host detection "haswell", "broadwell", "skylake", "icelake", # Using preferred targets from packages.yaml "icelake-preference", "cannonlake-preference", ] ) def current_host(request, monkeypatch): # is_preference is not empty if we want to supply the # preferred target via packages.yaml cpu, _, is_preference = request.param.partition("-") monkeypatch.setattr(spack.platforms.Test, "default", cpu) monkeypatch.setattr( spack.archspec, "HOST_TARGET_FAMILY", spack.vendor.archspec.cpu.TARGETS["x86_64"] ) if not is_preference: target = spack.vendor.archspec.cpu.TARGETS[cpu] monkeypatch.setattr(spack.vendor.archspec.cpu, "host", lambda: target) yield target else: target = spack.vendor.archspec.cpu.TARGETS["sapphirerapids"] monkeypatch.setattr(spack.vendor.archspec.cpu, "host", lambda: target) with spack.config.override("packages:all", {"target": [cpu]}): yield target @pytest.fixture(scope="function", params=[True, False]) def fuzz_dep_order(request, monkeypatch): """Meta-function that tweaks the order of iteration over dependencies in a package.""" def reverser(pkg_name): if request.param: pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) reversed_dict = dict(reversed(list(pkg_cls.dependencies.items()))) monkeypatch.setattr(pkg_cls, "dependencies", reversed_dict) return reverser @pytest.fixture() def repo_with_changing_recipe(tmp_path_factory: pytest.TempPathFactory, mutable_mock_repo): repos_dir: pathlib.Path = tmp_path_factory.mktemp("repos_dir") root, _ = spack.repo.create_repo(str(repos_dir), "changing") packages_dir = pathlib.Path(root, "packages") root_pkg_str = """ from spack_repo.builtin_mock.build_systems.generic import Package from spack.package import * class Root(Package): homepage = "http://www.example.com" url = "http://www.example.com/root-1.0.tar.gz" version("1.0", sha256="abcde") depends_on("middle") depends_on("changing") conflicts("^changing~foo") """ package_py = packages_dir / "root" / "package.py" package_py.parent.mkdir(parents=True) package_py.write_text(root_pkg_str) middle_pkg_str = """ from spack_repo.builtin_mock.build_systems.generic import Package from spack.package import * class Middle(Package): homepage = "http://www.example.com" url = "http://www.example.com/root-1.0.tar.gz" version("1.0", sha256="abcde") depends_on("changing") """ package_py = packages_dir / "middle" / "package.py" package_py.parent.mkdir(parents=True) package_py.write_text(middle_pkg_str) changing_template = """ from spack_repo.builtin_mock.build_systems.generic import Package from spack.package import * class Changing(Package): homepage = "http://www.example.com" url = "http://www.example.com/changing-1.0.tar.gz" {% if not delete_version %} version("1.0", sha256="abcde") {% endif %} version("0.9", sha256="abcde") {% if not delete_variant %} variant("fee", default=True, description="nope") {% endif %} variant("foo", default=True, description="nope") {% if add_variant %} variant("fum", default=True, description="nope") variant("fum2", default=True, description="nope") {% endif %} """ with spack.repo.use_repositories(root, override=False) as repos: class _ChangingPackage: default_context = [ ("delete_version", True), ("delete_variant", False), ("add_variant", False), ] def __init__(self): cache_dir = tmp_path_factory.mktemp("cache") self.repo_cache = spack.util.file_cache.FileCache(str(cache_dir)) self.repo = spack.repo.Repo(root, cache=self.repo_cache) def change(self, changes=None): changes = changes or {} context = dict(self.default_context) context.update(changes) # Remove the repo object and delete Python modules repos.remove(self.repo) # TODO: this mocks a change in the recipe that should happen in a # TODO: different process space. Leaving this comment as a hint # TODO: in case tests using this fixture start failing. for module in [x for x in sys.modules if x.startswith("spack_repo.changing")]: del sys.modules[module] # Change the recipe t = spack.vendor.jinja2.Template(changing_template) changing_pkg_str = t.render(**context) package_py = packages_dir / "changing" / "package.py" package_py.parent.mkdir(parents=True, exist_ok=True) package_py.write_text(changing_pkg_str) # Re-add the repository self.repo = spack.repo.Repo(root, cache=self.repo_cache) repos.put_first(self.repo) _changing_pkg = _ChangingPackage() _changing_pkg.change( {"delete_version": False, "delete_variant": False, "add_variant": False} ) yield _changing_pkg @pytest.fixture() def clang12_with_flags(compiler_factory): c = compiler_factory(spec="llvm@12.2.0+clang os=redhat6") c["extra_attributes"]["flags"] = {"cflags": "-O3", "cxxflags": "-O3"} return c @pytest.fixture() def gcc11_with_flags(compiler_factory): c = compiler_factory(spec="gcc@11.1.0 languages:=c,c++,fortran os=redhat6") c["extra_attributes"]["flags"] = {"cflags": "-O0 -g", "cxxflags": "-O0 -g", "fflags": "-O0 -g"} return c def weights_from_result(result: Result, *, name: str) -> Dict[str, int]: weights = {} for x in result.criteria: if x.name == name and x.kind == spack.solver.asp.OptimizationKind.CONCRETE: weights["reused"] = x.value elif x.name == name and x.kind == spack.solver.asp.OptimizationKind.BUILD: weights["built"] = x.value return weights # This must use the mutable_config fixture because the test # adjusting_default_target_based_on_compiler uses the current_host fixture, # which changes the config. @pytest.mark.usefixtures("mutable_config", "mock_packages") class TestConcretize: def test_concretize(self, spec): check_concretize(spec) def test_concretize_mention_build_dep(self): spec = check_concretize("cmake-client ^cmake@=3.21.3") # Check parent's perspective of child to_dependencies = spec.edges_to_dependencies(name="cmake") assert len(to_dependencies) == 1 assert to_dependencies[0].depflag == dt.BUILD # Check child's perspective of parent cmake = spec["cmake"] from_dependents = cmake.edges_from_dependents(name="cmake-client") assert len(from_dependents) == 1 assert from_dependents[0].depflag == dt.BUILD def test_concretize_preferred_version(self): spec = check_concretize("python") assert spec.version == ver("=2.7.11") spec = check_concretize("python@3.5.1") assert spec.version == ver("=3.5.1") def test_concretize_with_restricted_virtual(self): check_concretize("mpileaks ^mpich2") concrete = check_concretize("mpileaks ^mpich2@1.1") assert concrete["mpich2"].satisfies("mpich2@1.1") concrete = check_concretize("mpileaks ^mpich2@1.2") assert concrete["mpich2"].satisfies("mpich2@1.2") concrete = check_concretize("mpileaks ^mpich2@:1.5") assert concrete["mpich2"].satisfies("mpich2@:1.5") concrete = check_concretize("mpileaks ^mpich2@:1.3") assert concrete["mpich2"].satisfies("mpich2@:1.3") concrete = check_concretize("mpileaks ^mpich2@:1.2") assert concrete["mpich2"].satisfies("mpich2@:1.2") concrete = check_concretize("mpileaks ^mpich2@:1.1") assert concrete["mpich2"].satisfies("mpich2@:1.1") concrete = check_concretize("mpileaks ^mpich2@1.1:") assert concrete["mpich2"].satisfies("mpich2@1.1:") concrete = check_concretize("mpileaks ^mpich2@1.5:") assert concrete["mpich2"].satisfies("mpich2@1.5:") concrete = check_concretize("mpileaks ^mpich2@1.3.1:1.4") assert concrete["mpich2"].satisfies("mpich2@1.3.1:1.4") def test_concretize_with_provides_when(self): """Make sure insufficient versions of MPI are not in providers list when we ask for some advanced version. """ repo = spack.repo.PATH assert not any(s.intersects("mpich2@:1.0") for s in repo.providers_for("mpi@2.1")) assert not any(s.intersects("mpich2@:1.1") for s in repo.providers_for("mpi@2.2")) assert not any(s.intersects("mpich@:1") for s in repo.providers_for("mpi@2")) assert not any(s.intersects("mpich@:1") for s in repo.providers_for("mpi@3")) assert not any(s.intersects("mpich2") for s in repo.providers_for("mpi@3")) def test_provides_handles_multiple_providers_of_same_version(self): """ """ providers = spack.repo.PATH.providers_for("mpi@3.0") # Note that providers are repo-specific, so we don't misinterpret # providers, but vdeps are not namespace-specific, so we can # associate vdeps across repos. assert Spec("builtin_mock.multi-provider-mpi@1.10.3") in providers assert Spec("builtin_mock.multi-provider-mpi@1.10.2") in providers assert Spec("builtin_mock.multi-provider-mpi@1.10.1") in providers assert Spec("builtin_mock.multi-provider-mpi@1.10.0") in providers assert Spec("builtin_mock.multi-provider-mpi@1.8.8") in providers def test_different_compilers_get_different_flags( self, mutable_config, clang12_with_flags, gcc11_with_flags ): """Tests that nodes get the flags of the associated compiler.""" mutable_config.set( "packages", { "llvm": {"externals": [clang12_with_flags]}, "gcc": {"externals": [gcc11_with_flags]}, }, ) t = spack.vendor.archspec.cpu.host().family client = spack.concretize.concretize_one( Spec( f"cmake-client platform=test os=redhat6 target={t} %gcc@11.1.0" f" ^cmake platform=test os=redhat6 target={t} %clang@12.2.0" ) ) cmake = client["cmake"] assert set(client.compiler_flags["cflags"]) == {"-O0", "-g"} assert set(cmake.compiler_flags["cflags"]) == {"-O3"} assert set(client.compiler_flags["fflags"]) == {"-O0", "-g"} assert not set(cmake.compiler_flags["fflags"]) @pytest.mark.regression("9908") def test_spec_flags_maintain_order(self, mutable_config, gcc11_with_flags): """Tests that Spack assembles flags in a consistent way (i.e. with the same ordering), for successive concretizations. """ mutable_config.set("packages", {"gcc": {"externals": [gcc11_with_flags]}}) spec_str = "libelf os=redhat6 %gcc@11.1.0" for _ in range(3): s = spack.concretize.concretize_one(spec_str) assert all( s.compiler_flags[x] == ["-O0", "-g"] for x in ("cflags", "cxxflags", "fflags") ) @pytest.mark.parametrize( "spec_str,expected,not_expected", [ # Simple flag propagation from the root ("hypre cflags=='-g' ^openblas", ["hypre cflags='-g'", "^openblas cflags='-g'"], []), ( "hypre cflags='-g' ^openblas", ["hypre cflags='-g'", "^openblas"], ["^openblas cflags='-g'"], ), # Setting a flag overrides propagation ( "hypre cflags=='-g' ^openblas cflags='-O3'", ["hypre cflags='-g'", "^openblas cflags='-O3'"], ["^openblas cflags='-g'"], ), # Propagation doesn't go across build dependencies ( "cmake-client cflags=='-O2 -g'", ["cmake-client cflags=='-O2 -g'", "^cmake"], ["cmake cflags=='-O2 -g'"], ), ], ) def test_compiler_flag_propagation(self, spec_str, expected, not_expected): root = spack.concretize.concretize_one(spec_str) for constraint in expected: assert root.satisfies(constraint) for constraint in not_expected: assert not root.satisfies(constraint) def test_mixing_compilers_only_affects_subdag(self): """Tests that, when we mix compilers, the one with lower penalty is used for nodes where the compiler is not forced. """ spec = spack.concretize.concretize_one("dt-diamond%clang ^dt-diamond-bottom%gcc") # This is intended to traverse the "root" unification set, and check compilers # on the nodes in the set for x in spec.traverse(deptype=("link", "run")): if "c" not in x or not x.name.startswith("dt-diamond"): continue expected_gcc = x.name != "dt-diamond" assert bool(x.dependencies(name="llvm", deptype="build")) is not expected_gcc, x.tree() assert bool(x.dependencies(name="gcc", deptype="build")) is expected_gcc assert x.satisfies("%clang") is not expected_gcc assert x.satisfies("%gcc") is expected_gcc def test_disable_mixing_prevents_mixing(self): with spack.config.override("concretizer", {"compiler_mixing": False}): with pytest.raises(spack.error.UnsatisfiableSpecError): spack.concretize.concretize_one("dt-diamond%clang ^dt-diamond-bottom%gcc") def test_disable_mixing_is_per_language(self): with spack.config.override("concretizer", {"compiler_mixing": False}): spack.concretize.concretize_one("openblas %c=llvm %fortran=gcc") def test_disable_mixing_override_by_package(self): with spack.config.override("concretizer", {"compiler_mixing": ["dt-diamond-bottom"]}): root = spack.concretize.concretize_one("dt-diamond%clang ^dt-diamond-bottom%gcc") assert root.satisfies("%clang") assert root["dt-diamond-bottom"].satisfies("%gcc") assert root["dt-diamond-left"].satisfies("%clang") with pytest.raises(spack.error.UnsatisfiableSpecError): spack.concretize.concretize_one("dt-diamond%clang ^dt-diamond-left%gcc") def test_disable_mixing_reuse(self, fake_db_install): # Install a spec left = spack.concretize.concretize_one("dt-diamond-left %gcc") fake_db_install(left) assert left.satisfies("%c=gcc") lefthash = left.dag_hash()[:7] # Check if mixing works when it's allowed spack.concretize.concretize_one(f"dt-diamond%clang ^/{lefthash}") # Now try to use it with compiler mixing disabled with spack.config.override("concretizer", {"compiler_mixing": False}): with pytest.raises(spack.error.UnsatisfiableSpecError): spack.concretize.concretize_one(f"dt-diamond%clang ^/{lefthash}") # Should be able to reuse if the compilers match spack.concretize.concretize_one(f"dt-diamond%gcc ^/{lefthash}") def test_disable_mixing_reuse_and_built(self, fake_db_install): r"""In this case we have x |\ y z Where y is a link dependency and z is a build dependency. We install y with a compiler c1, and we make sure we cannot ask for `x%c2 ^z%c1 ^/y This looks similar to `test_disable_mixing_reuse`. But the compiler nodes are handled differently in this case: this is the only test that explicitly exercises compiler unmixing rule #2. """ dep1 = spack.concretize.concretize_one("libdwarf %gcc") fake_db_install(dep1) assert dep1.satisfies("%c=gcc") dep1hash = dep1.dag_hash()[:7] spack.concretize.concretize_one(f"mixing-parent%clang ^cmake%gcc ^/{dep1hash}") with spack.config.override("concretizer", {"compiler_mixing": False}): with pytest.raises(spack.error.UnsatisfiableSpecError, match="mixing is disabled"): spack.concretize.concretize_one(f"mixing-parent%clang ^cmake%gcc ^/{dep1hash}") def test_disable_mixing_allow_compiler_link(self): """Check if we can use a compiler when mixing is disabled, and still depend on a separate compiler package (in the latter case not using it as a compiler but rather for some utility it provides). """ with spack.config.override("concretizer", {"compiler_mixing": False}): x = spack.concretize.concretize_one("llvm-client%gcc") assert x.satisfies("%cxx=gcc") assert x.satisfies("%c=gcc") assert "llvm" in x def test_compiler_run_dep_link_dep_not_forced(self, temporary_store): """When a compiler is used as a pure build dependency, its transitive run-reachable deps are unified in the build environment, but their pure link-type dependencies must NOT be forced onto the package. Scenario: compiler-with-deps has a run+link dep on binutils-for-test, which has a pure link dep on zlib. A package that depends on zlib and uses compiler-with-deps as its C compiler should be free to pick its own zlib (here: zlib@1.2.8) independently of the toolchain's zlib (zlib@1.2.11). Without the fix the imposed hash from binutils-for-test forces the toolchain version onto the package, causing a conflict. """ # Pre-install the compiler with its transitive deps binutils-for-test and zlib@1.2.11 compiler = spack.concretize.concretize_one("compiler-with-deps ^zlib@1.2.11") assert compiler["zlib"].satisfies("@1.2.11") PackageInstaller([compiler.package], fake=True, explicit=True).install() # Concretize a package that depends on a different zlib from its compiler's toolchain. pkg = spack.concretize.concretize_one( "pkg-with-zlib-dep %c=compiler-with-deps ^zlib@1.2.8" ) assert pkg["zlib"].satisfies("@1.2.8") def test_disable_mixing_env( self, mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mutable_config ): spack_yaml = tmp_path / ev.manifest_name spack_yaml.write_text( """\ spack: specs: - dt-diamond%gcc - dt-diamond%clang concretizer: compiler_mixing: false unify: when_possible """ ) with ev.Environment(tmp_path) as e: e.concretize() for root in e.roots(): if root.satisfies("%gcc"): assert root["dt-diamond-left"].satisfies("%gcc") assert root["dt-diamond-bottom"].satisfies("%gcc") else: assert root["dt-diamond-left"].satisfies("%llvm") assert root["dt-diamond-bottom"].satisfies("%llvm") def test_compiler_inherited_upwards(self): spec = spack.concretize.concretize_one("dt-diamond ^dt-diamond-bottom%clang") for x in spec.traverse(deptype=("link", "run")): if "c" not in x: continue assert x.satisfies("%clang") def test_architecture_deep_inheritance(self, mock_targets, compiler_factory): """Make sure that indirect dependencies receive architecture information from the root even when partial architecture information is provided by an intermediate dependency. """ cnl_compiler = compiler_factory( spec="gcc@4.5.0 os=CNL languages:=c,c++,fortran target=nocona" ) with spack.config.override("packages", {"gcc": {"externals": [cnl_compiler]}}): spec_str = "mpileaks os=CNL target=nocona %gcc@4.5.0 ^dyninst os=CNL ^callpath os=CNL" spec = spack.concretize.concretize_one(spec_str) for s in spec.traverse(root=False, deptype=("link", "run")): if s.external: continue assert s.architecture.target == spec.architecture.target def test_compiler_flags_from_user_are_grouped(self): spec = Spec('pkg-a cflags="-O -foo-flag foo-val" platform=test %gcc') spec = spack.concretize.concretize_one(spec) cflags = spec.compiler_flags["cflags"] assert any(x == "-foo-flag foo-val" for x in cflags) def concretize_multi_provider(self): s = Spec("mpileaks ^multi-provider-mpi@3.0") s = spack.concretize.concretize_one(s) assert s["mpi"].version == ver("1.10.3") def test_concretize_dependent_with_singlevalued_variant_type(self): s = Spec("singlevalue-variant-dependent-type") s = spack.concretize.concretize_one(s) @pytest.mark.parametrize("spec,version", [("dealii", "develop"), ("xsdk", "0.4.0")]) def concretize_difficult_packages(self, a, b): """Test a couple of large packages that are often broken due to current limitations in the concretizer""" s = Spec(a + "@" + b) s = spack.concretize.concretize_one(s) assert s[a].version == ver(b) def test_concretize_two_virtuals(self): """Test a package with multiple virtual dependencies.""" spack.concretize.concretize_one("hypre") def test_concretize_two_virtuals_with_one_bound(self, mutable_mock_repo): """Test a package with multiple virtual dependencies and one preset.""" spack.concretize.concretize_one("hypre ^openblas") def test_concretize_two_virtuals_with_two_bound(self): """Test a package with multiple virtual deps and two of them preset.""" spack.concretize.concretize_one("hypre ^netlib-lapack") def test_concretize_two_virtuals_with_dual_provider(self): """Test a package with multiple virtual dependencies and force a provider that provides both. """ spack.concretize.concretize_one("hypre ^openblas-with-lapack") @pytest.mark.parametrize("max_dupes_default", [1, 2, 3]) def test_concretize_two_virtuals_with_dual_provider_and_a_conflict( self, max_dupes_default, mutable_config ): """Test a package with multiple virtual dependencies and force a provider that provides both, and another conflicting package that provides one. """ mutable_config.set("concretizer:duplicates:max_dupes:default", max_dupes_default) s = Spec("hypre ^openblas-with-lapack ^netlib-lapack") with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one(s) @pytest.mark.parametrize( "spec_str,expected_propagation", [ # Propagates past a node that doesn't have the variant ("hypre~~shared ^openblas", [("hypre", "~shared"), ("openblas", "~shared")]), # Propagates from root node to all nodes ( "ascent~~shared +adios2", [("ascent", "~shared"), ("adios2", "~shared"), ("bzip2", "~shared")], ), # Propagate from a node that is not the root node ( "ascent +adios2 ^adios2~~shared", [("ascent", "+shared"), ("adios2", "~shared"), ("bzip2", "~shared")], ), ], ) def test_concretize_propagate_disabled_variant(self, spec_str, expected_propagation): """Tests various patterns of boolean variant propagation""" spec = spack.concretize.concretize_one(spec_str) for key, expected_satisfies in expected_propagation: spec[key].satisfies(expected_satisfies) def test_concretize_propagate_variant_not_dependencies(self): """Test that when propagating a variant it is not propagated to dependencies that do not have that variant""" spec = Spec("quantum-espresso~~invino") spec = spack.concretize.concretize_one(spec) for dep in spec.traverse(root=False): assert "invino" not in dep.variants.keys() def test_concretize_propagate_variant_exclude_dependency_fail(self): """Tests that a propagating variant cannot be allowed to be excluded by any of the source package's dependencies""" spec = Spec("hypre ~~shared ^openblas +shared") with pytest.raises(spack.error.UnsatisfiableSpecError): spec = spack.concretize.concretize_one(spec) def test_concretize_propagate_same_variant_from_direct_dep_fail(self): """Test that when propagating a variant from the source package and a direct dependency also propagates the same variant with a different value. Raises error""" spec = Spec("ascent +adios2 ++shared ^adios2 ~~shared") with pytest.raises(spack.error.UnsatisfiableSpecError): spec = spack.concretize.concretize_one(spec) def test_concretize_propagate_same_variant_in_dependency_fail(self): """Test that when propagating a variant from the source package, none of it's dependencies can propagate that variant with a different value. Raises error.""" spec = Spec("ascent +adios2 ++shared ^bzip2 ~~shared") with pytest.raises(spack.error.UnsatisfiableSpecError): spec = spack.concretize.concretize_one(spec) def test_concretize_propagate_same_variant_virtual_dependency_fail(self): """Test that when propagating a variant from the source package and a direct dependency (that is a virtual pkg) also propagates the same variant with a different value. Raises error""" spec = Spec("hypre ++shared ^openblas ~~shared") with pytest.raises(spack.error.UnsatisfiableSpecError): spec = spack.concretize.concretize_one(spec) def test_concretize_propagate_same_variant_multiple_sources_diamond_dep_fail(self): """Test that fails when propagating the same variant with different values from multiple sources that share a dependency""" spec = Spec("parent-foo-bar ^dependency-foo-bar++bar ^direct-dep-foo-bar~~bar") with pytest.raises(spack.error.UnsatisfiableSpecError): spec = spack.concretize.concretize_one(spec) def test_concretize_propagate_specified_variant(self): """Test that only the specified variant is propagated to the dependencies""" spec = Spec("parent-foo-bar ~~foo") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("^dependency-foo-bar~foo") assert spec.satisfies("^second-dependency-foo-bar-fee~foo") assert spec.satisfies("^direct-dep-foo-bar~foo") assert not spec.satisfies("^dependency-foo-bar+bar") assert not spec.satisfies("^second-dependency-foo-bar-fee+bar") assert not spec.satisfies("^direct-dep-foo-bar+bar") def test_concretize_propagate_one_variant(self): """Test that you can specify to propagate one variant and not all""" spec = Spec("parent-foo-bar ++bar ~foo") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("~foo") and not spec.satisfies("^dependency-foo-bar~foo") assert spec.satisfies("+bar") and spec.satisfies("^dependency-foo-bar+bar") def test_concretize_propagate_through_first_level_deps(self): """Test that boolean valued variants can be propagated past first level dependencies even if the first level dependency does have the variant""" spec = Spec("parent-foo-bar-fee ++fee") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("+fee") and not spec.satisfies("dependency-foo-bar+fee") assert spec.satisfies("^second-dependency-foo-bar-fee+fee") def test_concretize_propagate_multiple_variants(self): """Test that multiple boolean valued variants can be propagated from the same source package""" spec = Spec("parent-foo-bar-fee ~~foo ++bar") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("~foo") and spec.satisfies("+bar") assert spec.satisfies("^dependency-foo-bar ~foo +bar") assert spec.satisfies("^second-dependency-foo-bar-fee ~foo +bar") def test_concretize_propagate_multiple_variants_mulitple_sources(self): """Test the propagates multiple different variants for multiple sources in a diamond dependency""" spec = Spec("parent-foo-bar ^dependency-foo-bar++bar ^direct-dep-foo-bar~~foo") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("^second-dependency-foo-bar-fee+bar") assert spec.satisfies("^second-dependency-foo-bar-fee~foo") assert not spec.satisfies("^dependency-foo-bar~foo") assert not spec.satisfies("^direct-dep-foo-bar+bar") def test_concretize_propagate_single_valued_variant(self): """Test propagation for single valued variants""" spec = Spec("multivalue-variant libs==static") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("libs=static") assert spec.satisfies("^pkg-a libs=static") def test_concretize_propagate_multivalue_variant(self): """Test that multivalue variants are propagating the specified value(s) to their dependencies. The dependencies should not have the default value""" spec = Spec("multivalue-variant foo==baz,fee") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("^pkg-a foo=baz,fee") assert spec.satisfies("^pkg-b foo=baz,fee") assert not spec.satisfies("^pkg-a foo=bar") assert not spec.satisfies("^pkg-b foo=bar") def test_concretize_propagate_multiple_multivalue_variant(self): """Tests propagating the same mulitvalued variant from different sources allows the dependents to accept all propagated values""" spec = Spec("multivalue-variant foo==bar ^pkg-a foo==baz") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("multivalue-variant foo=bar") assert spec.satisfies("^pkg-a foo=bar,baz") assert spec.satisfies("^pkg-b foo=bar,baz") def test_concretize_propagate_variant_not_in_source(self): """Test that variant is still propagated even if the source pkg doesn't have the variant""" spec = Spec("callpath++debug") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("^mpich+debug") assert not spec.satisfies("callpath+debug") assert not spec.satisfies("^dyninst+debug") def test_concretize_propagate_variant_multiple_deps_not_in_source(self): """Test that a variant can be propagated to multiple dependencies when the variant is not in the source package""" spec = Spec("netlib-lapack++shared") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("^openblas+shared") assert spec.satisfies("^perl+shared") assert not spec.satisfies("netlib-lapack+shared") def test_concretize_propagate_variant_second_level_dep_not_in_source(self): """Test that a variant can be propagated past first level dependencies when the variant is not in the source package or any of the first level dependencies""" spec = Spec("parent-foo-bar ++fee") spec = spack.concretize.concretize_one(spec) assert spec.satisfies("^second-dependency-foo-bar-fee +fee") assert not spec.satisfies("parent-foo-bar +fee") def test_no_matching_compiler_specs(self): s = Spec("pkg-a %gcc@0.0.0") with pytest.raises(spack.solver.asp.InvalidVersionError): spack.concretize.concretize_one(s) def test_no_compilers_for_arch(self): s = Spec("pkg-a arch=linux-rhel0-x86_64") with pytest.raises(spack.error.SpackError): s = spack.concretize.concretize_one(s) def test_virtual_is_fully_expanded_for_callpath(self): # force dependence on fake "zmpi" by asking for MPI 10.0 spec = Spec("callpath ^mpi@10.0") assert len(spec.dependencies(name="mpi")) == 1 assert "fake" not in spec spec = spack.concretize.concretize_one(spec) assert len(spec.dependencies(name="zmpi")) == 1 assert all(not d.dependencies(name="mpi") for d in spec.traverse()) assert all(x in spec for x in ("zmpi", "mpi")) edges_to_zmpi = spec.edges_to_dependencies(name="zmpi") assert len(edges_to_zmpi) == 1 assert "fake" in edges_to_zmpi[0].spec def test_virtual_is_fully_expanded_for_mpileaks(self): spec = Spec("mpileaks ^mpi@10.0") assert len(spec.dependencies(name="mpi")) == 1 assert "fake" not in spec spec = spack.concretize.concretize_one(spec) assert len(spec.dependencies(name="zmpi")) == 1 assert len(spec.dependencies(name="callpath")) == 1 callpath = spec.dependencies(name="callpath")[0] assert len(callpath.dependencies(name="zmpi")) == 1 zmpi = callpath.dependencies(name="zmpi")[0] assert len(zmpi.dependencies(name="fake")) == 1 assert all(not d.dependencies(name="mpi") for d in spec.traverse()) assert all(x in spec for x in ("zmpi", "mpi")) @pytest.mark.parametrize( "spec_str,expected,not_expected", [ # clang (llvm~flang) only provides C, and C++ compilers, while gcc has also fortran # # If we ask mpileaks%clang, then %gcc must be used for fortran, and since # %gcc is preferred to clang in config, it will be used for most nodes ( "mpileaks %clang", {"mpileaks": "%clang", "libdwarf": "%gcc", "libelf": "%gcc"}, {"libdwarf": "%clang", "libelf": "%clang"}, ), ( "mpileaks %clang@:15.0.0", {"mpileaks": "%clang", "libdwarf": "%gcc", "libelf": "%gcc"}, {"libdwarf": "%clang", "libelf": "%clang"}, ), ( "mpileaks %gcc", {"mpileaks": "%gcc", "libdwarf": "%gcc", "libelf": "%gcc"}, {"mpileaks": "%clang", "libdwarf": "%clang", "libelf": "%clang"}, ), ( "mpileaks %gcc@10.2.1", {"mpileaks": "%gcc", "libdwarf": "%gcc", "libelf": "%gcc"}, {"mpileaks": "%clang", "libdwarf": "%clang", "libelf": "%clang"}, ), # dyninst doesn't require fortran, so %clang is propagated ( "dyninst %clang", {"dyninst": "%clang", "libdwarf": "%clang", "libelf": "%clang"}, {"libdwarf": "%gcc", "libelf": "%gcc"}, ), ], ) def test_compiler_inheritance(self, spec_str, expected, not_expected): """Spack tries to propagate compilers as much as possible, but prefers using a single toolchain on a node, rather than mixing them. """ spec = spack.concretize.concretize_one(spec_str) for name, constraint in expected.items(): assert spec[name].satisfies(constraint) for name, constraint in not_expected.items(): assert not spec[name].satisfies(constraint) def test_external_package(self): """Tests that an external is preferred, if present, and that it does not have dependencies. """ spec = spack.concretize.concretize_one("externaltool") assert spec.external_path == os.path.sep + os.path.join("path", "to", "external_tool") assert not spec.dependencies() def test_nobuild_package(self): """Test that a non-buildable package raise an error if no specs in packages.yaml are compatible with the request. """ spec = Spec("externaltool%clang") with pytest.raises(spack.error.SpecError): spec = spack.concretize.concretize_one(spec) def test_external_and_virtual(self, mutable_config): mutable_config.set("packages:stuff", {"buildable": False}) spec = spack.concretize.concretize_one("externaltest") assert spec["externaltool"].external_path == os.path.sep + os.path.join( "path", "to", "external_tool" ) # "stuff" is a virtual provided by externalvirtual assert spec["stuff"].external_path == os.path.sep + os.path.join( "path", "to", "external_virtual_clang" ) def test_compiler_child(self): s = Spec("mpileaks target=x86_64 %clang ^dyninst%gcc") s = spack.concretize.concretize_one(s) assert s["mpileaks"].satisfies("%clang") assert s["dyninst"].satisfies("%gcc") def test_conflicts_in_spec(self, conflict_spec): s = Spec(conflict_spec) with pytest.raises(spack.error.SpackError): s = spack.concretize.concretize_one(s) def test_conflicts_show_cores(self, conflict_spec, monkeypatch): s = Spec(conflict_spec) with pytest.raises(spack.error.SpackError) as e: s = spack.concretize.concretize_one(s) assert "conflict" in e.value.message def test_conflict_in_all_directives_true(self): s = Spec("when-directives-true") with pytest.raises(spack.error.SpackError): s = spack.concretize.concretize_one(s) @pytest.mark.parametrize("spec_str", ["unsat-provider@1.0+foo"]) def test_no_conflict_in_external_specs(self, spec_str): # Modify the configuration to have the spec with conflict # registered as an external ext = Spec(spec_str) data = {"externals": [{"spec": spec_str, "prefix": "/fake/path"}]} spack.config.set("packages::{0}".format(ext.name), data) ext = spack.concretize.concretize_one(ext) # failure raises exception def test_regression_issue_4492(self): # Constructing a spec which has no dependencies, but is otherwise # concrete is kind of difficult. What we will do is to concretize # a spec, and then modify it to have no dependency and reset the # cache values. s = Spec("mpileaks") s = spack.concretize.concretize_one(s) # Check that now the Spec is concrete, store the hash assert s.concrete # Remove the dependencies and reset caches s.clear_dependencies() s._concrete = False assert not s.concrete @pytest.mark.regression("7239") def test_regression_issue_7239(self): # Constructing a SpecBuildInterface from another SpecBuildInterface # results in an inconsistent MRO # Normal Spec s = Spec("mpileaks") s = spack.concretize.concretize_one(s) assert spack.llnl.util.lang.ObjectWrapper not in s.__class__.__mro__ # Spec wrapped in a build interface build_interface = s["mpileaks"] assert spack.llnl.util.lang.ObjectWrapper in build_interface.__class__.__mro__ # Mimics asking the build interface from a build interface build_interface = s["mpileaks"]["mpileaks"] assert spack.llnl.util.lang.ObjectWrapper in build_interface.__class__.__mro__ @pytest.mark.regression("7705") def test_regression_issue_7705(self): # spec.package.provides(name) doesn't account for conditional # constraints in the concretized spec s = Spec("simple-inheritance~openblas") s = spack.concretize.concretize_one(s) assert not s.package.provides("lapack") @pytest.mark.regression("7941") def test_regression_issue_7941(self): # The string representation of a spec containing # an explicit multi-valued variant and a dependency # might be parsed differently than the originating spec s = Spec("pkg-a foobar=bar ^pkg-b") t = Spec(str(s)) s = spack.concretize.concretize_one(s) t = spack.concretize.concretize_one(t) assert s.dag_hash() == t.dag_hash() @pytest.mark.parametrize( "abstract_specs", [ # Establish a baseline - concretize a single spec ("mpileaks",), # When concretized together with older version of callpath # and dyninst it uses those older versions ("mpileaks", "callpath@0.9", "dyninst@8.1.1"), # Handle recursive syntax within specs ("mpileaks", "callpath@0.9 ^dyninst@8.1.1", "dyninst"), # Test specs that have overlapping dependencies but are not # one a dependency of the other ("mpileaks", "direct-mpich"), ], ) def test_simultaneous_concretization_of_specs(self, abstract_specs): abstract_specs = [Spec(x) for x in abstract_specs] concrete_specs = spack.concretize._concretize_specs_together(abstract_specs) # Check there's only one configuration of each package in the DAG names = set( dep.name for spec in concrete_specs for dep in spec.traverse(deptype=("link", "run")) ) for name in names: name_specs = set(spec[name] for spec in concrete_specs if name in spec) assert len(name_specs) == 1 # Check that there's at least one Spec that satisfies the # initial abstract request for aspec in abstract_specs: assert any(cspec.satisfies(aspec) for cspec in concrete_specs) # Make sure the concrete spec are top-level specs with no dependents for spec in concrete_specs: assert not spec.dependents() @pytest.mark.parametrize("spec", ["noversion", "noversion-bundle"]) def test_noversion_pkg(self, spec): """Test concretization failures for no-version packages.""" with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one(spec) @pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.parametrize( "spec,compiler_spec,best_achievable", [ ( "mpileaks%gcc@=4.4.7 ^dyninst@=10.2.1 target=x86_64:", "gcc@4.4.7 languages=c,c++,fortran", "core2", ), ("mpileaks target=x86_64: %gcc@=4.8", "gcc@4.8 languages=c,c++,fortran", "haswell"), ( "mpileaks target=x86_64: %gcc@=5.3.0", "gcc@5.3.0 languages=c,c++,fortran", "broadwell", ), ], ) @pytest.mark.regression("13361", "20537") @pytest.mark.usefixtures("mock_targets") def test_adjusting_default_target_based_on_compiler( self, spec, compiler_spec, best_achievable, current_host, compiler_factory, mutable_config ): best_achievable = spack.vendor.archspec.cpu.TARGETS[best_achievable] expected = best_achievable if best_achievable < current_host else current_host mutable_config.set( "packages", {"gcc": {"externals": [compiler_factory(spec=f"{compiler_spec}")]}} ) s = spack.concretize.concretize_one(spec) assert str(s.architecture.target) == str(expected) @pytest.mark.parametrize( "constraint,expected", [("%gcc@10.2", "@=10.2.1"), ("%gcc@10.2:", "@=10.2.1")] ) def test_compiler_version_matches_any_entry_in_packages_yaml(self, constraint, expected): # The behavior here has changed since #8735 / #14730. Now %gcc@10.2 is an abstract # compiler spec, and it should first find a matching compiler gcc@=10.2.1 s = spack.concretize.concretize_one(f"mpileaks {constraint}") gcc_deps = s.dependencies(name="gcc", deptype="build") assert len(gcc_deps) == 1 assert gcc_deps[0].satisfies(expected) def test_concretize_anonymous(self): with pytest.raises(spack.error.SpackError): s = Spec("+variant") s = spack.concretize.concretize_one(s) @pytest.mark.parametrize("spec_str", ["mpileaks ^%gcc", "mpileaks ^cflags=-g"]) def test_concretize_anonymous_dep(self, spec_str): with pytest.raises(spack.error.SpackError): s = Spec(spec_str) s = spack.concretize.concretize_one(s) @pytest.mark.parametrize( "spec_str,expected_str", [ # Unconstrained versions select default compiler (gcc@10.2.1) ("bowtie@1.4.0", "%gcc@10.2.1"), # Version with conflicts and no valid gcc select another compiler ("bowtie@1.3.0", "%clang@15.0.0"), # If a higher gcc is available, with a worse os, still prefer that, # assuming the two operating systems are compatible ("bowtie@1.2.2 %gcc", "%gcc@11.1.0"), ], ) def test_compiler_conflicts_in_package_py( self, spec_str, expected_str, gcc11_with_flags, mutable_config ): mutable_config.set( "concretizer:os_compatible", {"debian6": ["redhat6"], "redhat6": ["debian6"]} ) with spack.config.override("packages", {"gcc": {"externals": [gcc11_with_flags]}}): s = spack.concretize.concretize_one(spec_str) assert s.satisfies(expected_str) @pytest.mark.parametrize( "spec_str,expected,unexpected", [ ("conditional-variant-pkg@1.0", ["two_whens"], ["version_based", "variant_based"]), ("conditional-variant-pkg@2.0", ["version_based", "variant_based"], ["two_whens"]), ( "conditional-variant-pkg@2.0~version_based", ["version_based"], ["variant_based", "two_whens"], ), ( "conditional-variant-pkg@2.0+version_based+variant_based", ["version_based", "variant_based", "two_whens"], [], ), ], ) def test_conditional_variants(self, spec_str, expected, unexpected): s = spack.concretize.concretize_one(spec_str) for var in expected: assert s.satisfies("%s=*" % var) for var in unexpected: assert not s.satisfies("%s=*" % var) @pytest.mark.parametrize( "bad_spec", [ "@1.0~version_based", "@1.0+version_based", "@2.0~version_based+variant_based", "@2.0+version_based~variant_based+two_whens", ], ) def test_conditional_variants_fail(self, bad_spec): with pytest.raises( (spack.error.UnsatisfiableSpecError, spack.spec.InvalidVariantForSpecError) ): _ = spack.concretize.concretize_one("conditional-variant-pkg" + bad_spec) @pytest.mark.parametrize( "spec_str,expected,unexpected", [ ("py-extension3 ^python@3.5.1", [], ["py-extension1"]), ("py-extension3 ^python@2.7.11", ["py-extension1"], []), ("py-extension3@1.0 ^python@2.7.11", ["patchelf@0.9"], []), ("py-extension3@1.1 ^python@2.7.11", ["patchelf@0.9"], []), ("py-extension3@1.0 ^python@3.5.1", ["patchelf@0.10"], []), ], ) def test_conditional_dependencies(self, spec_str, expected, unexpected, fuzz_dep_order): """Tests that conditional dependencies are correctly attached. The original concretizer can be sensitive to the iteration order over the dependencies of a package, so we use a fuzzer function to test concretization with dependencies iterated forwards and backwards. """ fuzz_dep_order("py-extension3") # test forwards and backwards s = spack.concretize.concretize_one(spec_str) for dep in expected: msg = '"{0}" is not in "{1}" and was expected' assert dep in s, msg.format(dep, spec_str) for dep in unexpected: msg = '"{0}" is in "{1}" but was unexpected' assert dep not in s, msg.format(dep, spec_str) @pytest.mark.parametrize( "spec_str,patched_deps", [ ("patch-several-dependencies", [("libelf", 1), ("fake", 2)]), ("patch-several-dependencies@1.0", [("libelf", 1), ("fake", 2), ("libdwarf", 1)]), ( "patch-several-dependencies@1.0 ^libdwarf@20111030", [("libelf", 1), ("fake", 2), ("libdwarf", 2)], ), ("patch-several-dependencies ^libelf@0.8.10", [("libelf", 2), ("fake", 2)]), ("patch-several-dependencies +foo", [("libelf", 2), ("fake", 2)]), ], ) def test_patching_dependencies(self, spec_str, patched_deps): s = spack.concretize.concretize_one(spec_str) for dep, num_patches in patched_deps: assert s[dep].satisfies("patches=*") assert len(s[dep].variants["patches"].value) == num_patches @pytest.mark.regression("267,303,1781,2310,2632,3628") @pytest.mark.parametrize( "spec_str, expected", [ # Need to understand that this configuration is possible # only if we use the +mpi variant, which is not the default ("fftw ^mpich", ["+mpi"]), # This spec imposes two orthogonal constraints on a dependency, # one of which is conditional. The original concretizer fail since # when it applies the first constraint, it sets the unknown variants # of the dependency to their default values ("quantum-espresso", ["^fftw@1.0+mpi"]), # This triggers a conditional dependency on ^fftw@1.0 ("quantum-espresso", ["^openblas"]), # This constructs a constraint for a dependency og the type # @x.y:x.z where the lower bound is unconditional, the upper bound # is conditional to having a variant set ("quantum-espresso", ["^libelf@0.8.12"]), ("quantum-espresso~veritas", ["^libelf@0.8.13"]), ], ) def test_working_around_conflicting_defaults(self, spec_str, expected): s = spack.concretize.concretize_one(spec_str) assert s.concrete for constraint in expected: assert s.satisfies(constraint) @pytest.mark.regression("5651") def test_package_with_constraint_not_met_by_external(self): """Check that if we have an external package A at version X.Y in packages.yaml, but our spec doesn't allow X.Y as a version, then a new version of A is built that meets the requirements. """ packages_yaml = {"libelf": {"externals": [{"spec": "libelf@0.8.13", "prefix": "/usr"}]}} spack.config.set("packages", packages_yaml) # quantum-espresso+veritas requires libelf@:0.8.12 s = spack.concretize.concretize_one("quantum-espresso+veritas") assert s.satisfies("^libelf@0.8.12") assert not s["libelf"].external @pytest.mark.regression("9744") def test_cumulative_version_ranges_with_different_length(self): s = spack.concretize.concretize_one("cumulative-vrange-root") assert s.concrete assert s.satisfies("^cumulative-vrange-bottom@2.2") @pytest.mark.regression("9937") def test_dependency_conditional_on_another_dependency_state(self): root_str = "variant-on-dependency-condition-root" dep_str = "variant-on-dependency-condition-a" spec_str = "{0} ^{1}".format(root_str, dep_str) s = spack.concretize.concretize_one(spec_str) assert s.concrete assert s.satisfies("^variant-on-dependency-condition-b") s = spack.concretize.concretize_one(spec_str + "+x") assert s.concrete assert s.satisfies("^variant-on-dependency-condition-b") s = spack.concretize.concretize_one(spec_str + "~x") assert s.concrete assert not s.satisfies("^variant-on-dependency-condition-b") def test_external_that_would_require_a_virtual_dependency(self): s = spack.concretize.concretize_one("requires-virtual") assert s.external assert "stuff" not in s def test_transitive_conditional_virtual_dependency(self, mutable_config): """Test that an external is used as provider if the virtual is non-buildable""" mutable_config.set("packages:stuff", {"buildable": False}) s = spack.concretize.concretize_one("transitive-conditional-virtual-dependency") # Test that the default +stuff~mpi is maintained, and the right provider is selected assert s.satisfies("^conditional-virtual-dependency +stuff~mpi") assert s.satisfies("^[virtuals=stuff] externalvirtual") @pytest.mark.regression("20040") def test_conditional_provides_or_depends_on(self): # Check that we can concretize correctly a spec that can either # provide a virtual or depend on it based on the value of a variant s = spack.concretize.concretize_one("v1-consumer ^conditional-provider +disable-v1") assert "v1-provider" in s assert s["v1"].name == "v1-provider" assert s["v2"].name == "conditional-provider" @pytest.mark.regression("20079") @pytest.mark.parametrize( "spec_str,tests_arg,with_dep,without_dep", [ # Check that True is treated correctly and attaches test deps # to all nodes in the DAG ("pkg-a", True, ["pkg-a"], []), ("pkg-a foobar=bar", True, ["pkg-a", "pkg-b"], []), # Check that a list of names activates the dependency only for # packages in that list ("pkg-a foobar=bar", ["pkg-a"], ["pkg-a"], ["pkg-b"]), ("pkg-a foobar=bar", ["pkg-b"], ["pkg-b"], ["pkg-a"]), # Check that False disregard test dependencies ("pkg-a foobar=bar", False, [], ["pkg-a", "pkg-b"]), ], ) def test_activating_test_dependencies(self, spec_str, tests_arg, with_dep, without_dep): s = spack.concretize.concretize_one(spec_str, tests=tests_arg) for pkg_name in with_dep: msg = "Cannot find test dependency in package '{0}'" node = s[pkg_name] assert node.dependencies(deptype="test"), msg.format(pkg_name) for pkg_name in without_dep: msg = "Test dependency in package '{0}' is unexpected" node = s[pkg_name] assert not node.dependencies(deptype="test"), msg.format(pkg_name) @pytest.mark.regression("19981") def test_target_ranges_in_conflicts(self): with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one("impossible-concretization") def test_target_compatibility(self): with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one( Spec("libdwarf target=x86_64 ^libelf target=x86_64_v2") ) @pytest.mark.regression("20040") def test_variant_not_default(self): s = spack.concretize.concretize_one("ecp-viz-sdk") # Check default variant value for the package assert "+dep" in s["conditional-constrained-dependencies"] # Check that non-default variant values are forced on the dependency d = s["dep-with-variants"] assert "+foo+bar+baz" in d def test_all_patches_applied(self): uuidpatch = ( "a60a42b73e03f207433c5579de207c6ed61d58e4d12dd3b5142eb525728d89ea" if sys.platform != "win32" else "d0df7988457ec999c148a4a2af25ce831bfaad13954ba18a4446374cb0aef55e" ) localpatch = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" spec = Spec("conditionally-patch-dependency+jasper") spec = spack.concretize.concretize_one(spec) assert (uuidpatch, localpatch) == spec["libelf"].variants["patches"].value def test_dont_select_version_that_brings_more_variants_in(self): s = spack.concretize.concretize_one("dep-with-variants-if-develop-root") assert s["dep-with-variants-if-develop"].satisfies("@1.0") @pytest.mark.regression("20244,20736") @pytest.mark.parametrize( "spec_str,is_external,expected", [ # These are all externals, and 0_8 is a version not in package.py ("externaltool@1.0", True, "@1.0"), ("externaltool@0.9", True, "@0.9"), ("externaltool@0_8", True, "@0_8"), # This external package is buildable, has a custom version # in packages.yaml that is greater than the ones in package.py # and specifies a variant ("external-buildable-with-variant +baz", True, "@1.1.special +baz"), ("external-buildable-with-variant ~baz", False, "@1.0 ~baz"), ("external-buildable-with-variant@1.0: ~baz", False, "@1.0 ~baz"), # This uses an external version that meets the condition for # having an additional dependency, but the dependency shouldn't # appear in the answer set ("external-buildable-with-variant@0.9 +baz", True, "@0.9"), # This package has an external version declared that would be # the least preferred if Spack had to build it ("old-external", True, "@1.0.0"), ], ) def test_external_package_versions(self, spec_str, is_external, expected): s = spack.concretize.concretize_one(spec_str) assert s.external == is_external assert s.satisfies(expected) @pytest.mark.parametrize("dev_first", [True, False]) @pytest.mark.parametrize( "spec", ["dev-build-test-install", "dev-build-test-dependent ^dev-build-test-install"] ) @pytest.mark.parametrize("mock_db", [True, False]) def test_reuse_does_not_overwrite_dev_specs( self, dev_first, spec, mock_db, tmp_path: pathlib.Path, temporary_store, monkeypatch ): """Test that reuse does not mix dev specs with non-dev specs. Tests for either order (dev specs are not reused for non-dev, and non-dev specs are not reused for dev specs) Tests for a spec in which the root is developed and a spec in which a dep is developed. Tests for both reuse from database and reuse from buildcache""" # dev and non-dev specs that are otherwise identical spec = Spec(spec) dev_spec = spec.copy() dev_spec["dev-build-test-install"].constrain(f"dev_path={tmp_path}") # run the test in both orders first_spec = dev_spec if dev_first else spec second_spec = spec if dev_first else dev_spec # concretize and setup spack to reuse in the appropriate manner first_spec = spack.concretize.concretize_one(first_spec) def mock_fn(*args, **kwargs): return [first_spec] if mock_db: temporary_store.db.add(first_spec) else: monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", mock_fn) # concretize and ensure we did not reuse with spack.config.override("concretizer:reuse", True): second_spec = spack.concretize.concretize_one(second_spec) assert first_spec.dag_hash() != second_spec.dag_hash() @pytest.mark.regression("20292") @pytest.mark.parametrize( "context", [ {"add_variant": True, "delete_variant": False}, {"add_variant": False, "delete_variant": True}, {"add_variant": True, "delete_variant": True}, ], ) def test_reuse_installed_packages_when_package_def_changes( self, context, mutable_database, repo_with_changing_recipe ): # test applies only with reuse turned off in concretizer spack.config.set("concretizer:reuse", False) # Install a spec root = spack.concretize.concretize_one("root") dependency = root["changing"].copy() PackageInstaller([root.package], fake=True, explicit=True).install() # Modify package.py repo_with_changing_recipe.change(context) # Try to concretize with the spec installed previously new_root_with_reuse = spack.concretize.concretize_one( Spec("root ^/{0}".format(dependency.dag_hash())) ) new_root_without_reuse = spack.concretize.concretize_one("root") # validate that the graphs are the same with reuse, but not without assert ht.build_hash(root) == ht.build_hash(new_root_with_reuse) assert ht.build_hash(root) != ht.build_hash(new_root_without_reuse) # DAG hash should be the same with reuse since only the dependency changed assert root.dag_hash() == new_root_with_reuse.dag_hash() # Structure and package hash will be different without reuse assert root.dag_hash() != new_root_without_reuse.dag_hash() @pytest.mark.regression("43663") def test_no_reuse_when_variant_condition_does_not_hold(self, mutable_database, mock_packages): spack.config.set("concretizer:reuse", True) # Install a spec for which the `version_based` variant condition does not hold old = spack.concretize.concretize_one("conditional-variant-pkg @1") PackageInstaller([old.package], fake=True, explicit=True).install() # Then explicitly require a spec with `+version_based`, which shouldn't reuse previous spec new1 = spack.concretize.concretize_one("conditional-variant-pkg +version_based") assert new1.satisfies("@2 +version_based") new2 = spack.concretize.concretize_one("conditional-variant-pkg +two_whens") assert new2.satisfies("@2 +two_whens +version_based") def test_reuse_with_flags(self, mutable_database, mutable_config): spack.config.set("concretizer:reuse", True) spec = spack.concretize.concretize_one("pkg-a cflags=-g cxxflags=-g") PackageInstaller([spec.package], fake=True, explicit=True).install() testspec = spack.concretize.concretize_one("pkg-a cflags=-g") assert testspec == spec, testspec.tree() @pytest.mark.regression("20784") def test_concretization_of_test_dependencies(self): # With clingo we emit dependency_conditions regardless of the type # of the dependency. We need to ensure that there's at least one # dependency type declared to infer that the dependency holds. s = spack.concretize.concretize_one("test-dep-with-imposed-conditions") assert "c" not in s @pytest.mark.parametrize( "spec_str", ["wrong-variant-in-conflicts", "wrong-variant-in-depends-on"] ) def test_error_message_for_inconsistent_variants(self, spec_str): s = Spec(spec_str) with pytest.raises(vt.UnknownVariantError): s = spack.concretize.concretize_one(s) @pytest.mark.regression("22533") @pytest.mark.parametrize( "spec_str,variant_name,expected_values", [ # Test the default value 'auto' ("mvapich2", "file_systems", ("auto",)), # Test setting a single value from the disjoint set ("mvapich2 file_systems=lustre", "file_systems", ("lustre",)), # Test setting multiple values from the disjoint set ("mvapich2 file_systems=lustre,gpfs", "file_systems", ("lustre", "gpfs")), ], ) def test_mv_variants_disjoint_sets_from_spec(self, spec_str, variant_name, expected_values): s = spack.concretize.concretize_one(spec_str) assert set(expected_values) == set(s.variants[variant_name].value) @pytest.mark.regression("22533") def test_mv_variants_disjoint_sets_from_packages_yaml(self): external_mvapich2 = { "mvapich2": { "buildable": False, "externals": [{"spec": "mvapich2@2.3.1 file_systems=nfs,ufs", "prefix": "/usr"}], } } spack.config.set("packages", external_mvapich2) s = spack.concretize.concretize_one("mvapich2") assert set(s.variants["file_systems"].value) == set(["ufs", "nfs"]) @pytest.mark.regression("22596") def test_external_with_non_default_variant_as_dependency(self): # This package depends on another that is registered as an external # with 'buildable: true' and a variant with a non-default value set s = spack.concretize.concretize_one("trigger-external-non-default-variant") assert "~foo" in s["external-non-default-variant"] assert "~bar" in s["external-non-default-variant"] assert s["external-non-default-variant"].external @pytest.mark.regression("22718") @pytest.mark.parametrize( "spec_str,expected_compiler", [("mpileaks", "%gcc@10.2.1"), ("mpileaks ^mpich%clang@15.0.0", "%clang@15.0.0")], ) def test_compiler_is_unique(self, spec_str, expected_compiler): s = spack.concretize.concretize_one(spec_str) for node in s.traverse(): if not node.satisfies("^ c"): continue assert node.satisfies(expected_compiler) @pytest.mark.parametrize( "spec_str,expected_dict", [ # Check the defaults from the package (libs=shared) ("multivalue-variant", {"libs=shared": True, "libs=static": False}), # Check that libs=static doesn't extend the default ("multivalue-variant libs=static", {"libs=shared": False, "libs=static": True}), ], ) def test_multivalued_variants_from_cli(self, spec_str, expected_dict): s = spack.concretize.concretize_one(spec_str) for constraint, value in expected_dict.items(): assert s.satisfies(constraint) == value @pytest.mark.regression("22351") @pytest.mark.parametrize( "spec_str,expected", [ # Version 1.1.0 is deprecated and should not be selected, unless we # explicitly asked for that ("deprecated-versions", "deprecated-versions@1.0.0"), ("deprecated-versions@=1.1.0", "deprecated-versions@1.1.0"), ], ) def test_deprecated_versions_not_selected(self, spec_str, expected): with spack.config.override("config:deprecated", True): s = spack.concretize.concretize_one(spec_str) s.satisfies(expected) @pytest.mark.regression("24196") def test_version_badness_more_important_than_default_mv_variants(self): # If a dependency had an old version that for some reason pulls in # a transitive dependency with a multi-valued variant, that old # version was preferred because of the order of our optimization # criteria. s = spack.concretize.concretize_one("root") assert s["gmt"].satisfies("@2.0") @pytest.mark.regression("24205") def test_provider_must_meet_requirements(self): # A package can be a provider of a virtual only if the underlying # requirements are met. s = Spec("unsat-virtual-dependency") with pytest.raises((RuntimeError, spack.error.UnsatisfiableSpecError)): s = spack.concretize.concretize_one(s) @pytest.mark.regression("23951") def test_newer_dependency_adds_a_transitive_virtual(self): # Ensure that a package doesn't concretize any of its transitive # dependencies to an old version because newer versions pull in # a new virtual dependency. The possible concretizations here are: # # root@1.0 <- middle@1.0 <- leaf@2.0 <- blas # root@1.0 <- middle@1.0 <- leaf@1.0 # # and "blas" is pulled in only by newer versions of "leaf" s = spack.concretize.concretize_one("root-adds-virtual") assert s["leaf-adds-virtual"].satisfies("@2.0") assert "blas" in s @pytest.mark.regression("26718") def test_versions_in_virtual_dependencies(self): # Ensure that a package that needs a given version of a virtual # package doesn't end up using a later implementation s = spack.concretize.concretize_one("hpcviewer@2019.02") assert s["java"].satisfies("virtual-with-versions@1.8.0") @pytest.mark.regression("26866") def test_non_default_provider_of_multiple_virtuals(self): s = spack.concretize.concretize_one("many-virtual-consumer ^low-priority-provider") assert s["mpi"].name == "low-priority-provider" assert s["lapack"].name == "low-priority-provider" for virtual_pkg in ("mpi", "lapack"): for pkg in spack.repo.PATH.providers_for(virtual_pkg): if pkg.name == "low-priority-provider": continue assert pkg not in s @pytest.mark.regression("27237") @pytest.mark.parametrize( "spec_str,expect_installed", [("mpich", True), ("mpich+debug", False), ("mpich~debug", True)], ) def test_concrete_specs_are_not_modified_on_reuse( self, mutable_database, spec_str, expect_installed ): # Test the internal consistency of solve + DAG reconstruction # when reused specs are added to the mix. This prevents things # like additional constraints being added to concrete specs in # the answer set produced by clingo. with spack.config.override("concretizer:reuse", True): s = spack.concretize.concretize_one(spec_str) assert s.installed is expect_installed assert s.satisfies(spec_str) @pytest.mark.regression("26721,19736") def test_sticky_variant_in_package(self): # Here we test that a sticky variant cannot be changed from its default value # by the ASP solver if not set explicitly. The package used in the test needs # to have +allow-gcc set to be concretized with %gcc and clingo is not allowed # to change the default ~allow-gcc with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one("sticky-variant %gcc") s = spack.concretize.concretize_one("sticky-variant+allow-gcc %gcc") assert s.satisfies("%gcc") and s.satisfies("+allow-gcc") s = spack.concretize.concretize_one("sticky-variant %clang") assert s.satisfies("%clang") and s.satisfies("~allow-gcc") @pytest.mark.regression("42172") @pytest.mark.parametrize( "spec,allow_gcc", [ ("sticky-variant@1.0+allow-gcc", True), ("sticky-variant@1.0~allow-gcc", False), # FIXME (externals as concrete) ("sticky-variant@1.0", False), ], ) def test_sticky_variant_in_external(self, spec, allow_gcc): # setup external for sticky-variant+allow-gcc config = {"externals": [{"spec": spec, "prefix": "/fake/path"}], "buildable": False} spack.config.set("packages:sticky-variant", config) maybe = spack.llnl.util.lang.nullcontext if allow_gcc else pytest.raises with maybe(spack.error.SpackError): s = spack.concretize.concretize_one("sticky-variant-dependent%gcc") if allow_gcc: assert s.satisfies("%gcc") assert s["sticky-variant"].satisfies("+allow-gcc") assert s["sticky-variant"].external def test_do_not_invent_new_concrete_versions_unless_necessary(self): # ensure we select a known satisfying version rather than creating # a new '2.7' version. assert ver("=2.7.11") == spack.concretize.concretize_one("python@2.7").version # Here there is no known satisfying version - use the one on the spec. assert ver("=2.7.21") == spack.concretize.concretize_one("python@=2.7.21").version @pytest.mark.parametrize( "spec_str,valid", [ ("conditional-values-in-variant@1.62.0 cxxstd=17", False), ("conditional-values-in-variant@1.62.0 cxxstd=2a", False), ("conditional-values-in-variant@1.72.0 cxxstd=2a", False), # Ensure disjoint set of values work too ("conditional-values-in-variant@1.72.0 staging=flexpath", False), # Ensure conditional values set False fail too ("conditional-values-in-variant foo=bar", False), ("conditional-values-in-variant foo=foo", True), ], ) def test_conditional_values_in_variants(self, spec_str, valid): s = Spec(spec_str) raises = pytest.raises((RuntimeError, spack.error.UnsatisfiableSpecError)) with spack.llnl.util.lang.nullcontext() if valid else raises: s = spack.concretize.concretize_one(s) def test_conditional_values_in_conditional_variant(self): """Test that conditional variants play well with conditional possible values""" s = spack.concretize.concretize_one("conditional-values-in-variant@1.50.0") assert "cxxstd" not in s.variants s = spack.concretize.concretize_one("conditional-values-in-variant@1.60.0") assert "cxxstd" in s.variants def test_target_granularity(self): # The test architecture uses core2 as the default target. Check that when # we configure Spack for "generic" granularity we concretize for x86_64 default_target = spack.platforms.test.Test.default generic_target = spack.vendor.archspec.cpu.TARGETS[default_target].generic.name s = Spec("python") assert spack.concretize.concretize_one(s).satisfies("target=%s" % default_target) with spack.config.override("concretizer:targets", {"granularity": "generic"}): assert spack.concretize.concretize_one(s).satisfies("target=%s" % generic_target) def test_host_compatible_concretization(self): # Check that after setting "host_compatible" to false we cannot concretize. # Here we use "k10" to set a target non-compatible with the current host # to avoid a lot of boilerplate when mocking the test platform. The issue # is that the defaults for the test platform are very old, so there's no # compiler supporting e.g. icelake etc. s = Spec("python target=k10") assert spack.concretize.concretize_one(s) with spack.config.override("concretizer:targets", {"host_compatible": True}): with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one(s) def test_add_microarchitectures_on_explicit_request(self): # Check that if we consider only "generic" targets, we can still solve for # specific microarchitectures on explicit requests with spack.config.override("concretizer:targets", {"granularity": "generic"}): s = spack.concretize.concretize_one("python target=k10") assert s.satisfies("target=k10") @pytest.mark.regression("29201") def test_delete_version_and_reuse(self, mutable_database, repo_with_changing_recipe): """Test that we can reuse installed specs with versions not declared in package.py """ root = spack.concretize.concretize_one("root") PackageInstaller([root.package], fake=True, explicit=True).install() repo_with_changing_recipe.change({"delete_version": True}) with spack.config.override("concretizer:reuse", True): new_root = spack.concretize.concretize_one("root") assert root.dag_hash() == new_root.dag_hash() @pytest.mark.regression("29201") def test_installed_version_is_selected_only_for_reuse( self, mutable_database, repo_with_changing_recipe ): """Test that a version coming from an installed spec is a possible version only for reuse """ # Install a dependency that cannot be reused with "root" # because of a conflict in a variant, then delete its version dependency = spack.concretize.concretize_one("changing@1.0~foo") PackageInstaller([dependency.package], fake=True, explicit=True).install() repo_with_changing_recipe.change({"delete_version": True}) with spack.config.override("concretizer:reuse", True): new_root = spack.concretize.concretize_one("root") assert not new_root["changing"].satisfies("@1.0") @pytest.mark.regression("28259") def test_reuse_with_unknown_namespace_dont_raise( self, temporary_store, mock_custom_repository ): with spack.repo.use_repositories(mock_custom_repository, override=False): s = spack.concretize.concretize_one("pkg-c") assert s.namespace != "builtin_mock" PackageInstaller([s.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): s = spack.concretize.concretize_one("pkg-c") assert s.namespace == "builtin_mock" @pytest.mark.regression("45538") def test_reuse_from_other_namespace_no_raise( self, temporary_store, monkeypatch, repo_builder: RepoBuilder ): repo_builder.add_package("zlib") builtin = spack.concretize.concretize_one("zlib") PackageInstaller([builtin.package], fake=True, explicit=True).install() with spack.repo.use_repositories(repo_builder.root, override=False): with spack.config.override("concretizer:reuse", True): zlib = spack.concretize.concretize_one(f"{repo_builder.namespace}.zlib") assert zlib.namespace == repo_builder.namespace @pytest.mark.regression("28259") def test_reuse_with_unknown_package_dont_raise( self, temporary_store, monkeypatch, repo_builder: RepoBuilder ): repo_builder.add_package("pkg-c") with spack.repo.use_repositories(repo_builder.root, override=False): s = spack.concretize.concretize_one("pkg-c") assert s.namespace == repo_builder.namespace PackageInstaller([s.package], fake=True, explicit=True).install() del sys.modules[f"spack_repo.{repo_builder.namespace}.packages.pkg_c"] repo_builder.remove("pkg-c") with spack.repo.use_repositories(repo_builder.root, override=False) as repos: repos.repos[0]._pkg_checker.invalidate() with spack.config.override("concretizer:reuse", True): s = spack.concretize.concretize_one("pkg-c") assert s.namespace == "builtin_mock" @pytest.mark.parametrize( "specs,checks", [ (["libelf", "libelf@0.8.10"], {"libelf": 1}), (["libdwarf%gcc", "libelf%clang"], {"libdwarf": 1, "libelf": 1}), (["libdwarf%gcc", "libdwarf%clang"], {"libdwarf": 2, "libelf": 1}), (["libdwarf^libelf@0.8.12", "libdwarf^libelf@0.8.13"], {"libdwarf": 2, "libelf": 2}), (["hdf5", "zmpi"], {"zmpi": 1, "fake": 1}), (["hdf5", "mpich"], {"mpich": 1}), (["hdf5^zmpi", "mpich"], {"mpi": 2, "mpich": 1, "zmpi": 1, "fake": 1}), (["mpi", "zmpi"], {"mpi": 1, "mpich": 0, "zmpi": 1, "fake": 1}), (["mpi", "mpich"], {"mpi": 1, "mpich": 1, "zmpi": 0}), ], ) def test_best_effort_coconcretize(self, specs, checks): specs = [Spec(s) for s in specs] solver = spack.solver.asp.Solver() solver.reuse = False concrete_specs = set() for result in solver.solve_in_rounds(specs): for s in result.specs: concrete_specs.update(s.traverse()) for matching_spec, expected_count in checks.items(): matches = [x for x in concrete_specs if x.satisfies(matching_spec)] assert len(matches) == expected_count @pytest.mark.parametrize( "specs,expected_spec,occurrences", [ # The algorithm is greedy, and it might decide to solve the "best" # spec early in which case reuse is suboptimal. In this case the most # recent version of libdwarf is selected and concretized to libelf@0.8.13 ( [ "libdwarf@20111030^libelf@0.8.10", "libdwarf@20130207^libelf@0.8.12", "libdwarf@20130729", ], "libelf@0.8.12", 1, ), # Check we reuse the best libelf in the environment ( [ "libdwarf@20130729^libelf@0.8.10", "libdwarf@20130207^libelf@0.8.12", "libdwarf@20111030", ], "libelf@0.8.12", 2, ), (["libdwarf@20130729", "libdwarf@20130207", "libdwarf@20111030"], "libelf@0.8.13", 3), # We need to solve in 2 rounds and we expect mpich to be preferred to zmpi (["hdf5+mpi", "zmpi", "mpich"], "mpich", 2), ], ) def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occurrences): """Test package preferences during coconcretization.""" specs = [Spec(s) for s in specs] solver = spack.solver.asp.Solver() solver.reuse = False concrete_specs = {} for result in solver.solve_in_rounds(specs): concrete_specs.update(result.specs_by_input) counter = 0 for spec in concrete_specs.values(): if expected_spec in spec: counter += 1 assert counter == occurrences, concrete_specs def test_solve_in_rounds_all_unsolved(self, monkeypatch, mock_packages): specs = [Spec(x) for x in ["libdwarf%gcc", "libdwarf%clang"]] solver = spack.solver.asp.Solver() solver.reuse = False simulate_unsolved_property = list((x, None) for x in specs) monkeypatch.setattr(spack.solver.asp.Result, "unsolved_specs", simulate_unsolved_property) monkeypatch.setattr(spack.solver.asp.Result, "specs", list()) with pytest.raises(spack.solver.asp.OutputDoesNotSatisfyInputError): list(solver.solve_in_rounds(specs)) def test_coconcretize_reuse_and_virtuals(self): reusable_specs = [] for s in ["mpileaks ^mpich", "zmpi"]: reusable_specs.extend(spack.concretize.concretize_one(s).traverse(root=True)) root_specs = [Spec("mpileaks"), Spec("zmpi")] with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, root_specs, reuse=reusable_specs) for spec in result.specs: assert "zmpi" in spec @pytest.mark.regression("30864") def test_misleading_error_message_on_version(self, mutable_database): # For this bug to be triggered we need a reusable dependency # that is not optimal in terms of optimization scores. # We pick an old version of "b" reusable_specs = [spack.concretize.concretize_one("non-existing-conditional-dep@1.0")] root_spec = Spec("non-existing-conditional-dep@2.0") with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() with pytest.raises(spack.solver.asp.UnsatisfiableSpecError, match="Cannot satisfy"): solver.driver.solve(setup, [root_spec], reuse=reusable_specs) @pytest.mark.regression("31148") def test_version_weight_and_provenance(self, mutable_config): """Test package preferences during concretization.""" reusable_specs = [ spack.concretize.concretize_one(spec_str) for spec_str in ("pkg-b@0.9", "pkg-b@1.0") ] root_spec = Spec("pkg-a foobar=bar") packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], exclude=[], ).selected_specs() with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve( setup, [root_spec], reuse=reusable_specs + external_specs ) # Version badness should be > 0 only for reused specs. For instance, for pkg-b # the version provenance is: # # pkg_fact("pkg-b", version_declared("1.0", 0)). # pkg_fact("pkg-b", version_origin("1.0", "installed")). # pkg_fact("pkg-b", version_origin("1.0", "package_py")). # pkg_fact("pkg-b", version_declared("0.9", 1)). # pkg_fact("pkg-b", version_origin("0.9", "installed")). # pkg_fact("pkg-b", version_origin("0.9", "package_py")). weights = weights_from_result(result, name="version badness (non roots)") assert weights["reused"] == 3 and weights["built"] == 0 result_spec = result.specs[0] assert result_spec.satisfies("^pkg-b@1.0") assert result_spec["pkg-b"].dag_hash() == reusable_specs[1].dag_hash() @pytest.mark.regression("51112") def test_variant_penalty(self, mutable_config): """Test package preferences during concretization.""" packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], exclude=[], ).selected_specs() # The variant definition is similar to # # % Variant cxxstd in package trilinos # pkg_fact("trilinos",variant_definition("cxxstd",195)). # variant_type(195,"single"). # pkg_fact("trilinos",variant_default_value_from_package_py(195,"14")). # pkg_fact("trilinos",variant_penalty(195,"14",1)). # pkg_fact("trilinos",variant_penalty(195,"17",2)). # pkg_fact("trilinos",variant_penalty(195,"20",3)). # pkg_fact("trilinos",variant_possible_value(195,"14")). # pkg_fact("trilinos",variant_possible_value(195,"17")). # pkg_fact("trilinos",variant_possible_value(195,"20")). solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() # Ensure that since the default value of 14 cannot be taken, we select "17" result, _, _ = solver.driver.solve(setup, [Spec("trilinos")], reuse=external_specs) weights = weights_from_result(result, name="variant penalty (roots)") assert weights["reused"] == 0 and weights["built"] == 2 trilinos = result.specs[0] assert trilinos.satisfies("cxxstd=17") # If we disable "17", then "20" is next, and the penalty is higher result, _, _ = solver.driver.solve( setup, [Spec("trilinos+disable17")], reuse=external_specs ) weights = weights_from_result(result, name="variant penalty (roots)") assert weights["reused"] == 0 and weights["built"] == 3 trilinos = result.specs[0] assert trilinos.satisfies("cxxstd=20") # Test a disjoint set of values to ensure declared package order is respected result, _, _ = solver.driver.solve(setup, [Spec("mvapich2")], reuse=external_specs) weights = weights_from_result(result, name="variant penalty (roots)") assert weights["reused"] == 0 and weights["built"] == 0 mvapich2 = result.specs[0] assert mvapich2.satisfies("file_systems=auto") result, _, _ = solver.driver.solve(setup, [Spec("mvapich2+noauto")], reuse=external_specs) weights = weights_from_result(result, name="variant penalty (roots)") assert weights["reused"] == 0 and weights["built"] == 2 mvapich2 = result.specs[0] assert mvapich2.satisfies("file_systems=lustre") @pytest.mark.regression("51267") @pytest.mark.parametrize( "packages_config,expected", [ # Two preferences on different virtuals ( """ packages: c: prefer: - clang mpi: prefer: - mpich2 """, [ 'provider_weight_from_config("mpi","mpich2",0).', 'provider_weight_from_config("c","clang",0).', ], ), # A requirement and a preference on the same virtual ( """ packages: c: require: - gcc prefer: - clang """, [ 'provider_weight_from_config("c","gcc",0).', 'provider_weight_from_config("c","clang",1).', ], ), ( """ packages: c: require: - clang prefer: - gcc """, [ 'provider_weight_from_config("c","gcc",1).', 'provider_weight_from_config("c","clang",0).', ], ), # Multiple requirements with priorities ( """ packages: all: providers: mpi: [low-priority-mpi] mpi: require: - any_of: [mpich2, zmpi] prefer: - mpich """, [ 'provider_weight_from_config("mpi","mpich2",0).', 'provider_weight_from_config("mpi","zmpi",1).', 'provider_weight_from_config("mpi","mpich",2).', 'provider_weight_from_config("mpi","low-priority-mpi",3).', ], ), # Configuration with conflicts ( """ packages: all: providers: mpi: [mpich, low-priority-mpi] mpi: require: - mpich2 conflict: - mpich """, [ 'provider_weight_from_config("mpi","mpich2",0).', 'provider_weight_from_config("mpi","low-priority-mpi",1).', ], ), ( """ packages: all: providers: mpi: [mpich, low-priority-mpi] mpi: require: - mpich2 conflict: - mpich@1 """, [ 'provider_weight_from_config("mpi","mpich2",0).', 'provider_weight_from_config("mpi","mpich",1).', 'provider_weight_from_config("mpi","low-priority-mpi",2).', ], ), ], ) def test_requirements_and_weights(self, packages_config, expected, mutable_config): """Checks that requirements and strong preferences on virtual packages influence the weights for providers, even if "package preferences" are not set consistently. """ packages_yaml = syaml.load_config(packages_config) mutable_config.set("packages", packages_yaml["packages"]) setup = spack.solver.asp.SpackSolverSetup() asp_problem = setup.setup([Spec("mpileaks")], reuse=[], allow_deprecated=False).asp_problem assert all(x in asp_problem for x in expected) def test_reuse_succeeds_with_config_compatible_os(self): root_spec = Spec("pkg-b") s = spack.concretize.concretize_one(root_spec) other_os = s.copy() mock_os = "ubuntu2204" other_os.architecture = spack.spec.ArchSpec( "test-{os}-{target}".format(os=mock_os, target=str(s.architecture.target)) ) reusable_specs = [other_os] overrides = {"concretizer": {"reuse": True, "os_compatible": {s.os: [mock_os]}}} custom_scope = spack.config.InternalConfigScope("concretize_override", overrides) with spack.config.override(custom_scope): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, [root_spec], reuse=reusable_specs) concrete_spec = result.specs[0] assert concrete_spec.satisfies("os={}".format(other_os.architecture.os)) def test_git_hash_assigned_version_is_preferred(self): hash = "a" * 40 s = Spec("develop-branch-version@%s=develop" % hash) c = spack.concretize.concretize_one(s) assert hash in str(c) @pytest.mark.parametrize("git_ref", ("a" * 40, "0.2.15", "main")) def test_git_ref_version_is_equivalent_to_specified_version(self, git_ref): s = Spec("develop-branch-version@git.%s=develop" % git_ref) c = spack.concretize.concretize_one(s) assert git_ref in str(c) assert s.satisfies("@develop") assert s.satisfies("@0.1:") @pytest.mark.parametrize("git_ref", ("a" * 40, "0.2.15", "fbranch")) def test_git_ref_version_succeeds_with_unknown_version(self, git_ref): # main is not defined in the package.py for this file s = Spec("develop-branch-version@git.%s=main" % git_ref) s = spack.concretize.concretize_one(s) assert s.satisfies("develop-branch-version@main") @pytest.mark.regression("31484") def test_installed_externals_are_reused( self, mutable_database, repo_with_changing_recipe, tmp_path: pathlib.Path ): """Tests that external specs that are in the DB can be reused, if they result in a better optimization score. """ external_conf = { "changing": { "buildable": False, "externals": [{"spec": "changing@1.0", "prefix": str(tmp_path)}], } } spack.config.set("packages", external_conf) # Install the external spec middle_pkg = spack.concretize.concretize_one("middle") PackageInstaller([middle_pkg.package], fake=True, explicit=True).install() assert middle_pkg["changing"].external changing_external = middle_pkg["changing"] # Modify the package.py file repo_with_changing_recipe.change({"delete_variant": True}) # Try to concretize the external without reuse and confirm the hash changed with spack.config.override("concretizer:reuse", False): root_no_reuse = spack.concretize.concretize_one("root") assert root_no_reuse["changing"].dag_hash() != changing_external.dag_hash() # ... while with reuse we have the same hash with spack.config.override("concretizer:reuse", True): root_with_reuse = spack.concretize.concretize_one("root") assert root_with_reuse["changing"].dag_hash() == changing_external.dag_hash() @pytest.mark.regression("31484") def test_user_can_select_externals_with_require( self, mutable_database, tmp_path: pathlib.Path ): """Test that users have means to select an external even in presence of reusable specs.""" external_conf: Dict[str, Any] = { "mpi": {"buildable": False}, "multi-provider-mpi": { "externals": [{"spec": "multi-provider-mpi@2.0.0", "prefix": str(tmp_path)}] }, } spack.config.set("packages", external_conf) # mpich and others are installed, so check that # fresh use the external, reuse does not with spack.config.override("concretizer:reuse", False): mpi_spec = spack.concretize.concretize_one("mpi") assert mpi_spec.name == "multi-provider-mpi" with spack.config.override("concretizer:reuse", True): mpi_spec = spack.concretize.concretize_one("mpi") assert mpi_spec.name != "multi-provider-mpi" external_conf["mpi"]["require"] = "multi-provider-mpi" spack.config.set("packages", external_conf) with spack.config.override("concretizer:reuse", True): mpi_spec = spack.concretize.concretize_one("mpi") assert mpi_spec.name == "multi-provider-mpi" @pytest.mark.regression("31484") def test_installed_specs_disregard_conflicts(self, mutable_database, monkeypatch): """Test that installed specs do not trigger conflicts. This covers for the rare case where a conflict is added on a package after a spec matching the conflict was installed. """ # Add a conflict to "mpich" that match an already installed "mpich~debug" pkg_cls = spack.repo.PATH.get_pkg_class("mpich") monkeypatch.setitem(pkg_cls.conflicts, Spec(), [(Spec("~debug"), None)]) # If we concretize with --fresh the conflict is taken into account with spack.config.override("concretizer:reuse", False): s = spack.concretize.concretize_one("mpich") assert s.satisfies("+debug") # If we concretize with --reuse it is not, since "mpich~debug" was already installed with spack.config.override("concretizer:reuse", True): s = spack.concretize.concretize_one("mpich") assert s.installed assert s.satisfies("~debug"), s @pytest.mark.regression("32471") def test_require_targets_are_allowed(self, mutable_config, mutable_database): """Test that users can set target constraints under the require attribute.""" # Configuration to be added to packages.yaml required_target = spack.vendor.archspec.cpu.TARGETS[ spack.platforms.test.Test.default ].family external_conf = {"all": {"require": f"target={required_target}"}} mutable_config.set("packages", external_conf) with spack.config.override("concretizer:reuse", False): spec = spack.concretize.concretize_one("mpich") for s in spec.traverse(deptype=("link", "run")): assert s.satisfies(f"target={required_target}") target = spack.platforms.test.Test.default def test_external_python_extension_find_dependency_from_config(self, mutable_config, tmp_path): """Tests that an external Python extension gets a dependency on Python.""" packages_yaml = f""" packages: py-extension1: buildable: false externals: - spec: py-extension1@2.0 prefix: {tmp_path / "py-extension1"} python: externals: - spec: python@3.8.13 prefix: {tmp_path / "python"} """ configuration = syaml.load_config(packages_yaml) mutable_config.set("packages", configuration["packages"]) py_extension = spack.concretize.concretize_one("py-extension1") assert py_extension.external assert py_extension["python"].external assert py_extension["python"].prefix == str(tmp_path / "python") @pytest.mark.regression("36190") @pytest.mark.parametrize( "specs", [ ["mpileaks^ callpath ^dyninst@8.1.1:8 ^mpich2@1.3:1"], ["multivalue-variant ^pkg-a@2:2"], ["v1-consumer ^conditional-provider@1:1 +disable-v1"], ], ) def test_result_specs_is_not_empty(self, mutable_config, specs): """Check that the implementation of "result.specs" is correct in cases where we know a concretization exists. """ specs = [Spec(s) for s in specs] packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], exclude=[], ).selected_specs() solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, specs, reuse=external_specs) assert result.specs @pytest.mark.regression("38664") def test_unsolved_specs_raises_error(self, monkeypatch, mock_packages): """Check that the solver raises an exception when input specs are not satisfied. """ specs = [Spec("zlib")] solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() simulate_unsolved_property = list((x, None) for x in specs) monkeypatch.setattr(spack.solver.asp.Result, "unsolved_specs", simulate_unsolved_property) with pytest.raises( spack.solver.asp.InternalConcretizerError, match="the solver completed but produced specs", ): solver.driver.solve(setup, specs, reuse=[]) @pytest.mark.regression("43141") @pytest.mark.parametrize( "spec_str,expected_match", [ # A package does not exist ("pkg-a ^foo", "since 'foo' does not exist"), # Request a compiler for a package that doesn't need it ("pkg-c %gcc", "cannot depend on gcc"), ], ) def test_errors_on_statically_checked_preconditions(self, spec_str, expected_match): """Tests that the solver can report a case where the compiler cannot be set""" with pytest.raises(spack.error.UnsatisfiableSpecError, match=expected_match): spack.concretize.concretize_one(spec_str) @pytest.mark.regression("36339") @pytest.mark.parametrize( "compiler_str,expected", [ ("gcc@:9", "@=9.4.0"), ("gcc@:10", "@=10.2.1"), ("gcc@10", "@=10.2.1"), ("gcc@10:", "@=10.2.1"), ], ) def test_compiler_match_constraints_when_selected(self, compiler_str, expected): """Test that, when multiple compilers with the same name are in the configuration we ensure that the selected one matches all the required constraints. """ s = spack.concretize.concretize_one(f"pkg-a %{compiler_str}") assert s["gcc"].satisfies(expected) @pytest.mark.parametrize("spec_str", ["mpileaks", "mpileaks ^mpich"]) def test_virtuals_are_annotated_on_edges(self, spec_str): """Tests that information on virtuals is annotated on DAG edges""" spec = spack.concretize.concretize_one(spec_str) mpi_provider = spec["mpi"].name edges = spec.edges_to_dependencies(name=mpi_provider) assert len(edges) == 1 and edges[0].virtuals == ("mpi",) edges = spec.edges_to_dependencies(name="callpath") assert len(edges) == 1 and edges[0].virtuals == () @pytest.mark.parametrize("transitive", [True, False]) def test_explicit_splices( self, mutable_config, database_mutable_config, mock_packages, transitive, capfd ): mpich_spec = database_mutable_config.query("mpich")[0] splice_info = { "target": "mpi", "replacement": f"/{mpich_spec.dag_hash()}", "transitive": transitive, } spack.config.CONFIG.set("concretizer", {"splice": {"explicit": [splice_info]}}) spec = spack.concretize.concretize_one("hdf5 ^zmpi") assert spec.satisfies(f"^mpich@{mpich_spec.version}") assert spec.build_spec.dependencies(name="zmpi", deptype="link") assert spec["mpi"].build_spec.satisfies(mpich_spec) assert not spec.build_spec.satisfies(f"^mpich/{mpich_spec.dag_hash()}") assert not spec.dependencies(name="zmpi", deptype="link") captured = capfd.readouterr() assert "Warning: explicit splice configuration has caused" in captured.err assert "hdf5 ^zmpi" in captured.err assert str(spec) in captured.err def test_explicit_splice_fails_nonexistent(mutable_config, mock_packages, mock_store): splice_info = {"target": "mpi", "replacement": "mpich/doesnotexist"} spack.config.CONFIG.set("concretizer", {"splice": {"explicit": [splice_info]}}) with pytest.raises(spack.spec.InvalidHashError): _ = spack.concretize.concretize_one("hdf5^zmpi") def test_explicit_splice_fails_no_hash(mutable_config, mock_packages, mock_store): splice_info = {"target": "mpi", "replacement": "mpich"} spack.config.CONFIG.set("concretizer", {"splice": {"explicit": [splice_info]}}) with pytest.raises(spack.solver.asp.InvalidSpliceError, match="must be specified by hash"): _ = spack.concretize.concretize_one("hdf5^zmpi") def test_explicit_splice_non_match_nonexistent_succeeds( mutable_config, mock_packages, mock_store ): """When we have a nonexistent splice configured but are not using it, don't fail.""" splice_info = {"target": "will_not_match", "replacement": "nonexistent/doesnotexist"} spack.config.CONFIG.set("concretizer", {"splice": {"explicit": [splice_info]}}) spec = spack.concretize.concretize_one("zlib") # the main test is that it does not raise assert not spec.spliced @pytest.mark.db @pytest.mark.parametrize( "spec_str,mpi_name", [("mpileaks", "mpich"), ("mpileaks ^mpich2", "mpich2"), ("mpileaks ^zmpi", "zmpi")], ) def test_virtuals_are_reconstructed_on_reuse(self, spec_str, mpi_name, mutable_database): """Tests that when we reuse a spec, virtual on edges are reconstructed correctly""" with spack.config.override("concretizer:reuse", True): spec = spack.concretize.concretize_one(spec_str) assert spec.installed mpi_edges = spec.edges_to_dependencies(mpi_name) assert len(mpi_edges) == 1 assert "mpi" in mpi_edges[0].virtuals def test_dont_define_new_version_from_input_if_checksum_required(self, working_env): os.environ["SPACK_CONCRETIZER_REQUIRE_CHECKSUM"] = "yes" with pytest.raises(spack.error.UnsatisfiableSpecError): # normally spack concretizes to @=3.0 if it's not defined in package.py, except # when checksums are required spack.concretize.concretize_one("pkg-a@=3.0") @pytest.mark.regression("39570") @pytest.mark.db def test_reuse_python_from_cli_and_extension_from_db(self, mutable_database): """Tests that reusing python with and explicit request on the command line, when the spec also reuses a python extension from the DB, doesn't fail. """ s = spack.concretize.concretize_one("py-extension1") python_hash = s["python"].dag_hash() PackageInstaller([s.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): with_reuse = spack.concretize.concretize_one(f"py-extension2 ^/{python_hash}") with spack.config.override("concretizer:reuse", False): without_reuse = spack.concretize.concretize_one("py-extension2") assert with_reuse.dag_hash() == without_reuse.dag_hash() @pytest.mark.regression("35536") @pytest.mark.parametrize( "spec_str,expected_namespaces", [ # Single node with fully qualified namespace ("builtin_mock.gmake", {"gmake": "builtin_mock"}), # Dependency with fully qualified namespace ("hdf5 ^builtin_mock.gmake", {"gmake": "builtin_mock", "hdf5": "duplicates_test"}), ("hdf5 ^gmake", {"gmake": "duplicates_test", "hdf5": "duplicates_test"}), ], ) def test_select_lower_priority_package_from_repository_stack( self, spec_str, expected_namespaces ): """Tests that a user can explicitly select a lower priority, fully qualified dependency from cli. """ # 'builtin_mock" and "duplicates_test" share a 'gmake' package additional_repo = os.path.join( spack.paths.test_repos_path, "spack_repo", "duplicates_test" ) with spack.repo.use_repositories(additional_repo, override=False): s = spack.concretize.concretize_one(spec_str) for name, namespace in expected_namespaces.items(): assert s[name].concrete assert s[name].namespace == namespace def test_reuse_specs_from_non_available_compilers(self, mutable_config, mutable_database): """Tests that we can reuse specs with compilers that are not configured locally.""" # All the specs in the mutable DB have been compiled with %gcc@10.2.1 mpileaks = [s for s in mutable_database.query_local() if s.name == "mpileaks"] # Remove gcc@10.2.1 remover = spack.compilers.config.CompilerRemover(mutable_config) remover.mark_compilers(match="gcc@=10.2.1") remover.flush() mutable_config.set("concretizer:reuse", True) # mpileaks is in the database, it will be reused with gcc@=10.2.1 root = spack.concretize.concretize_one("mpileaks") assert root.satisfies("%gcc@10.2.1") assert any(root.dag_hash() == x.dag_hash() for x in mpileaks) # fftw is not in the database, therefore it will be compiled with gcc@=9.4.0 root = spack.concretize.concretize_one("fftw~mpi") assert root.satisfies("%gcc@9.4.0") @pytest.mark.regression("43406") def test_externals_with_platform_explicitly_set(self, tmp_path: pathlib.Path): """Tests that users can specify platform=xxx in an external spec""" external_conf = { "mpich": { "buildable": False, "externals": [{"spec": "mpich@=2.0.0 platform=test", "prefix": str(tmp_path)}], } } spack.config.set("packages", external_conf) s = spack.concretize.concretize_one("mpich") assert s.external @pytest.mark.regression("43267") def test_spec_with_build_dep_from_json(self, tmp_path: pathlib.Path): """Tests that we can correctly concretize a spec, when we express its dependency as a concrete spec to be read from JSON. The bug was triggered by missing virtuals on edges that were trimmed from pure build dependencies. """ build_dep = spack.concretize.concretize_one("dttop") json_file = tmp_path / "build.json" json_file.write_text(build_dep.to_json()) s = spack.concretize.concretize_one(f"dtuse ^{str(json_file)}") assert s["dttop"].dag_hash() == build_dep.dag_hash() @pytest.mark.regression("44040") def test_exclude_specs_from_reuse(self, monkeypatch): r"""Tests that we can exclude a spec from reuse when concretizing, and that the spec is not added back to the solve as a dependency of another reusable spec. The expected spec is: o callpath@1.0 |\ o | mpich@3.0.4 |\ \ | |\ \ | | | o dyninst@8.2 | |_|/| |/| |/| | |/|/| | | | |\ | | | | o libdwarf@20130729 | |_|_|/| |/| |_|/| | |/| |/| | | |/|/ | | | o libelf@0.8.13 | |_|/| |/| |/| | |/|/ | o | gcc-runtime@10.5.0 |/| | | |/ o | glibc@2.31 / o gcc@10.5.0 """ # Prepare a mock mirror that returns an old version of dyninst request_str = "callpath ^mpich" reused = spack.concretize.concretize_one(f"{request_str} ^dyninst@8.1.1") monkeypatch.setattr(spack.solver.reuse, "_specs_from_mirror", lambda: [reused]) # Exclude dyninst from reuse, so we expect that the old version is not taken into account with spack.config.override( "concretizer:reuse", {"from": [{"type": "buildcache", "exclude": ["dyninst"]}, {"type": "external"}]}, ): result = spack.concretize.concretize_one(request_str) assert result.dag_hash() != reused.dag_hash() assert result["mpich"].dag_hash() == reused["mpich"].dag_hash() assert result["dyninst"].dag_hash() != reused["dyninst"].dag_hash() assert result["dyninst"].satisfies("@=8.2") for dep in result["dyninst"].traverse(root=False): assert dep.dag_hash() == reused[dep.name].dag_hash() @pytest.mark.regression("44091") @pytest.mark.parametrize( "included_externals", [ ["deprecated-versions"], # Try the empty list, to ensure that in that case everything will be included # since filtering should happen only when the list is non-empty [], ], ) def test_include_specs_from_externals_and_libcs( self, included_externals, mutable_config, tmp_path: pathlib.Path ): """Tests that when we include specs from externals, we always include libcs.""" mutable_config.set( "packages", { "deprecated-versions": { "externals": [{"spec": "deprecated-versions@1.1.0", "prefix": str(tmp_path)}] } }, ) request_str = "deprecated-client" # When using the external the version is selected even if deprecated with spack.config.override( "concretizer:reuse", {"from": [{"type": "external", "include": included_externals}]} ): result = spack.concretize.concretize_one(request_str) assert result["deprecated-versions"].satisfies("@1.1.0") # When excluding it, we pick the non-deprecated version with spack.config.override( "concretizer:reuse", {"from": [{"type": "external", "exclude": ["deprecated-versions"]}]}, ): result = spack.concretize.concretize_one(request_str) assert result["deprecated-versions"].satisfies("@1.0.0") @pytest.mark.regression("44085") def test_can_reuse_concrete_externals_for_dependents(self, mutable_config): """Test that external specs that are in the DB can be reused. This means they are preferred to concretizing another external from packages.yaml """ packages_yaml = { "externaltool": {"externals": [{"spec": "externaltool@0.9", "prefix": "/fake/path"}]} } mutable_config.set("packages", packages_yaml) # Concretize with v0.9 to get a suboptimal spec, since we have gcc@10 available external_spec = spack.concretize.concretize_one("externaltool@0.9") assert external_spec.external root_specs = [Spec("sombrero")] with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, root_specs, reuse=[external_spec]) assert len(result.specs) == 1 sombrero = result.specs[0] assert sombrero["externaltool"].dag_hash() == external_spec.dag_hash() def test_cannot_reuse_host_incompatible_libc(self): """Test whether reuse concretization correctly fails to reuse a spec with a host incompatible libc.""" if not spack.solver.core.using_libc_compatibility(): pytest.skip("This test requires libc nodes") # We install b@1 ^glibc@2.30, and b@0 ^glibc@2.28. The former is not host compatible, the # latter is. fst = spack.concretize.concretize_one("pkg-b@1") fst._mark_concrete(False) fst.dependencies("glibc")[0].versions = VersionList(["=2.30"]) fst._mark_concrete(True) snd = spack.concretize.concretize_one("pkg-b@0") # The spec b@1 ^glibc@2.30 is "more optimal" than b@0 ^glibc@2.28, but due to glibc # incompatibility, it should not be reused. solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, [Spec("pkg-b")], reuse=[fst, snd]) assert len(result.specs) == 1 assert result.specs[0] == snd @pytest.mark.regression("45321") @pytest.mark.parametrize( "corrupted_str", [ "cmake@3.4.3 foo=bar", # cmake has no variant "foo" "mvdefaults@1.0 foo=a,d", # variant "foo" has no value "d" "cmake %gcc", # spec has no version ], ) def test_corrupted_external_does_not_halt_concretization(self, corrupted_str, mutable_config): """Tests that having a wrong variant in an external spec doesn't stop concretization""" corrupted_spec = Spec(corrupted_str) packages_yaml = { f"{corrupted_spec.name}": { "externals": [{"spec": corrupted_str, "prefix": "/dev/null"}] } } mutable_config.set("packages", packages_yaml) # Assert we don't raise due to the corrupted external entry above s = spack.concretize.concretize_one("pkg-a") assert s.concrete @pytest.mark.regression("44828") @pytest.mark.not_on_windows("Tests use linux paths") def test_correct_external_is_selected_from_packages_yaml(self, mutable_config): """Tests that when filtering external specs, the correct external is selected to reconstruct the prefix, and other external attributes. """ packages_yaml = { "mpileaks": { "externals": [ {"spec": "mpileaks@2.3 +opt", "prefix": "/tmp/prefix1"}, {"spec": "mpileaks@2.3 ~opt", "prefix": "/tmp/prefix2"}, ] } } concretizer_yaml = { "reuse": {"roots": True, "from": [{"type": "external", "exclude": ["+opt"]}]} } mutable_config.set("packages", packages_yaml) mutable_config.set("concretizer", concretizer_yaml) s = spack.concretize.concretize_one("mpileaks") # Check that we got the properties from the right external assert s.external assert s.satisfies("~opt") assert s.prefix == "/tmp/prefix2" def test_git_based_version_must_exist_to_use_ref(self): # gmake should fail, only has sha256 with pytest.raises(spack.error.UnsatisfiableSpecError) as e: spack.concretize.concretize_one(f"gmake commit={'a' * 40}") assert "Cannot use commit variant with" in e.value.message @pytest.fixture() def duplicates_test_repository(): repository_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "duplicates_test") with spack.repo.use_repositories(repository_path) as mock_repo: yield mock_repo @pytest.mark.usefixtures("mutable_config", "duplicates_test_repository") class TestConcretizeSeparately: """Collects test on separate concretization""" @pytest.mark.parametrize("strategy", ["minimal", "full"]) def test_two_gmake(self, strategy): """Tests that we can concretize a spec with nodes using the same build dependency pinned at different versions. o hdf5@1.0 |\ o | pinned-gmake@1.0 o | gmake@3.0 / o gmake@4.1 """ spack.config.CONFIG.set("concretizer:duplicates:strategy", strategy) s = spack.concretize.concretize_one("hdf5") # Check that hdf5 depends on gmake@=4.1 hdf5_gmake = s["hdf5"].dependencies(name="gmake", deptype="build") assert len(hdf5_gmake) == 1 and hdf5_gmake[0].satisfies("@=4.1") # Check that pinned-gmake depends on gmake@=3.0 pinned_gmake = s["pinned-gmake"].dependencies(name="gmake", deptype="build") assert len(pinned_gmake) == 1 and pinned_gmake[0].satisfies("@=3.0") @pytest.mark.parametrize("strategy", ["minimal", "full"]) def test_two_setuptools(self, strategy): """Tests that we can concretize separate build dependencies, when we are dealing with extensions. o py-shapely@1.25.0 |\ | |\ | o | py-setuptools@60 |/ / | o py-numpy@1.25.0 |/| | |\ | o | py-setuptools@59 |/ / o | python@3.11.2 o | gmake@3.0 / o gmake@4.1 """ spack.config.CONFIG.set("concretizer:duplicates:strategy", strategy) s = spack.concretize.concretize_one("py-shapely") # Requirements on py-shapely setuptools = s["py-shapely"].dependencies(name="py-setuptools", deptype="build") assert len(setuptools) == 1 and setuptools[0].satisfies("@=60") # Requirements on py-numpy setuptools = s["py-numpy"].dependencies(name="py-setuptools", deptype="build") assert len(setuptools) == 1 and setuptools[0].satisfies("@=59") gmake = s["py-numpy"].dependencies(name="gmake", deptype="build") assert len(gmake) == 1 and gmake[0].satisfies("@=4.1") # Requirements on python gmake = s["python"].dependencies(name="gmake", deptype="build") assert len(gmake) == 1 and gmake[0].satisfies("@=3.0") def test_solution_without_cycles(self): """Tests that when we concretize a spec with cycles, a fallback kicks in to recompute a solution without cycles. """ s = spack.concretize.concretize_one("cycle-a") assert s["cycle-a"].satisfies("+cycle") assert s["cycle-b"].satisfies("~cycle") s = spack.concretize.concretize_one("cycle-b") assert s["cycle-a"].satisfies("~cycle") assert s["cycle-b"].satisfies("+cycle") @pytest.mark.parametrize("strategy", ["minimal", "full"]) def test_pure_build_virtual_dependency(self, strategy): """Tests that we can concretize a pure build virtual dependency, and ensures that pure build virtual dependencies are accounted in the list of possible virtual dependencies. virtual-build@1.0 | [type=build, virtual=pkgconfig] pkg-config@1.0 """ spack.config.CONFIG.set("concretizer:duplicates:strategy", strategy) s = spack.concretize.concretize_one("virtual-build") assert s["pkgconfig"].name == "pkg-config" @pytest.mark.regression("40595") def test_no_multiple_solutions_with_different_edges_same_nodes(self): r"""Tests that the root node, which has a dependency on py-setuptools without constraint, doesn't randomly pick one of the two setuptools (@=59, @=60) needed by its dependency. o py-floating@1.25.0/3baitsp |\ | |\ | | |\ | o | | py-shapely@1.25.0/4hep6my |/| | | | |\| | | | |/ | |/| | | o py-setuptools@60/cwhbthc | |/ |/| | o py-numpy@1.25.0/5q5fx4d |/| | |\ | o | py-setuptools@59/jvsa7sd |/ / o | python@3.11.2/pdmjekv o | gmake@3.0/jv7k2bl / o gmake@4.1/uo6ot3d """ spec_str = "py-floating" root = spack.concretize.concretize_one(spec_str) assert root["py-shapely"].satisfies("^py-setuptools@=60") assert root["py-numpy"].satisfies("^py-setuptools@=59") edges = root.edges_to_dependencies("py-setuptools") assert len(edges) == 1 assert edges[0].spec.satisfies("@=60") def test_build_environment_is_unified(self): """A pure build dep that is marked build-tool can creates its own unification set. This test ensures that its sibling build dependencies are unified with it, together with their runtime dependencies. It ensures the same package cannot appear multiple times in a single build environment, for example when it's both a direct build dep, as well as pulled in as a transitive runtime dep of a sibling build dep.""" spack.config.CONFIG.set("concretizer:duplicates", {"max_dupes": {"unify-build-deps-c": 2}}) # Fails because unify-build-deps-c version @1 and @2 are needed in the build environment with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one("unify-build-deps-a@1.0") # Succeeds because unify-build-deps-c version @2 is not needed in the build environment spack.concretize.concretize_one("unify-build-deps-a@2.0") # Lastly, a sanity check that max_dupes is a requirement for this to work. spack.config.CONFIG.set("concretizer:duplicates", {"max_dupes": {"unify-build-deps-c": 1}}) with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one("unify-build-deps-a@2.0") @pytest.mark.regression("43647") def test_specifying_different_versions_build_deps(self): """Tests that we can concretize a spec with nodes using the same build dependency pinned at different versions, when the constraint is specified in the root spec. o hdf5@1.0 |\ o | pinned-gmake@1.0 o | gmake@3.0 / o gmake@4.1 """ hdf5_str = "hdf5@1.0 ^gmake@4.1" pinned_str = "pinned-gmake@1.0 ^gmake@3.0" input_specs = [Spec(hdf5_str), Spec(pinned_str)] solver = spack.solver.asp.Solver() result = solver.solve(input_specs) assert any(x.satisfies(hdf5_str) for x in result.specs) assert any(x.satisfies(pinned_str) for x in result.specs) @pytest.mark.regression("44289") def test_all_extensions_depend_on_same_extendee(self): """Tests that we don't reuse dependencies that bring in a different extendee""" setuptools = spack.concretize.concretize_one("py-setuptools ^python@3.10") solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve( setup, [Spec("py-floating ^python@3.11")], reuse=list(setuptools.traverse()) ) assert len(result.specs) == 1 floating = result.specs[0] assert all(setuptools.dag_hash() != x.dag_hash() for x in floating.traverse()) pythons = [x for x in floating.traverse() if x.name == "python"] assert len(pythons) == 1 and pythons[0].satisfies("@3.11") @pytest.mark.parametrize( "v_str,v_opts,checksummed", [ ("1.2.3", {"sha256": f"{1:064x}"}, True), # it's not about the version being "infinite", # but whether it has a digest ("develop", {"sha256": f"{1:064x}"}, True), # other hash types ("1.2.3", {"checksum": f"{1:064x}"}, True), ("1.2.3", {"md5": f"{1:032x}"}, True), ("1.2.3", {"sha1": f"{1:040x}"}, True), ("1.2.3", {"sha224": f"{1:056x}"}, True), ("1.2.3", {"sha384": f"{1:096x}"}, True), ("1.2.3", {"sha512": f"{1:0128x}"}, True), # no digest key ("1.2.3", {"bogus": f"{1:064x}"}, False), # git version with full commit sha ("1.2.3", {"commit": f"{1:040x}"}, True), (f"{1:040x}=1.2.3", {}, True), # git version with short commit sha ("1.2.3", {"commit": f"{1:07x}"}, False), (f"{1:07x}=1.2.3", {}, False), # git tag is a moving target ("1.2.3", {"tag": "v1.2.3"}, False), ("1.2.3", {"tag": "v1.2.3", "commit": f"{1:07x}"}, False), # git branch is a moving target ("1.2.3", {"branch": "releases/1.2"}, False), # git ref is a moving target ("git.branch=1.2.3", {}, False), ], ) def test_drop_moving_targets(v_str, v_opts, checksummed): v = Version(v_str) assert spack.solver.asp._is_checksummed_version((v, v_opts)) == checksummed class TestConcreteSpecsByHash: """Tests the container of concrete specs""" @pytest.mark.parametrize( "input_specs", [["pkg-a"], ["pkg-a foobar=bar", "pkg-b"], ["pkg-a foobar=baz", "pkg-b"]] ) def test_adding_specs(self, input_specs, default_mock_concretization): """Tests that concrete specs in the container are equivalent, but stored as different objects in memory. """ container = spack.solver.asp.ConcreteSpecsByHash() input_specs = [spack.concretize.concretize_one(s) for s in input_specs] for s in input_specs: container.add(s) for root in input_specs: for node in root.traverse(root=True): assert node == container[node.dag_hash()] assert node.dag_hash() in container assert node is not container[node.dag_hash()] @pytest.fixture() def edges_test_repository(): repository_path = os.path.join(spack.paths.test_repos_path, "spack_repo", "edges_test") with spack.repo.use_repositories(repository_path) as mock_repo: yield mock_repo @pytest.mark.usefixtures("mutable_config", "edges_test_repository") class TestConcretizeEdges: """Collects tests on edge properties""" @pytest.mark.parametrize( "spec_str,expected_satisfies,expected_not_satisfies", [ ("conditional-edge", ["^zlib@2.0"], ["^zlib-api"]), ("conditional-edge~foo", ["^zlib@2.0"], ["^zlib-api"]), ( "conditional-edge+foo", ["^zlib@1.0", "^zlib-api", "^[virtuals=zlib-api] zlib"], ["^[virtuals=mpi] zlib"], ), ], ) def test_condition_triggered_by_edge_property( self, spec_str, expected_satisfies, expected_not_satisfies ): """Tests that we can enforce constraints based on edge attributes""" s = spack.concretize.concretize_one(spec_str) for expected in expected_satisfies: assert s.satisfies(expected), str(expected) for not_expected in expected_not_satisfies: assert not s.satisfies(not_expected), str(not_expected) def test_virtuals_provided_together_but_only_one_required_in_dag(self): """Tests that we can use a provider that provides more than one virtual together, and is providing only one, iff the others are not needed in the DAG. o blas-only-client | [virtual=blas] o openblas (provides blas and lapack together) """ s = spack.concretize.concretize_one("blas-only-client ^openblas") assert s.satisfies("^[virtuals=blas] openblas") assert not s.satisfies("^[virtuals=blas,lapack] openblas") def test_reusable_externals_match(mock_packages, tmp_path: pathlib.Path): spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0") spec.external_path = str(tmp_path) spec.external_modules = ["mpich/4.1"] spec._mark_concrete() assert spack.solver.reuse._is_reusable( spec, { "mpich": { "externals": [ {"spec": "mpich@4.1", "prefix": str(tmp_path), "modules": ["mpich/4.1"]} ] } }, local=False, ) def test_reusable_externals_match_virtual(mock_packages, tmp_path: pathlib.Path): spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0") spec.external_path = str(tmp_path) spec.external_modules = ["mpich/4.1"] spec._mark_concrete() assert spack.solver.reuse._is_reusable( spec, { "mpi": { "externals": [ {"spec": "mpich@4.1", "prefix": str(tmp_path), "modules": ["mpich/4.1"]} ] } }, local=False, ) def test_reusable_externals_different_prefix(mock_packages, tmp_path: pathlib.Path): spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0") spec.external_path = "/other/path" spec.external_modules = ["mpich/4.1"] spec._mark_concrete() assert not spack.solver.reuse._is_reusable( spec, { "mpich": { "externals": [ {"spec": "mpich@4.1", "prefix": str(tmp_path), "modules": ["mpich/4.1"]} ] } }, local=False, ) @pytest.mark.parametrize("modules", [None, ["mpich/4.1", "libfabric/1.19"]]) def test_reusable_externals_different_modules(mock_packages, tmp_path: pathlib.Path, modules): spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0") spec.external_path = str(tmp_path) spec.external_modules = modules spec._mark_concrete() assert not spack.solver.reuse._is_reusable( spec, { "mpich": { "externals": [ {"spec": "mpich@4.1", "prefix": str(tmp_path), "modules": ["mpich/4.1"]} ] } }, local=False, ) def test_reusable_externals_different_spec(mock_packages, tmp_path: pathlib.Path): spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0") spec.external_path = str(tmp_path) spec._mark_concrete() assert not spack.solver.reuse._is_reusable( spec, {"mpich": {"externals": [{"spec": "mpich@4.1 +debug", "prefix": str(tmp_path)}]}}, local=False, ) def test_concretization_version_order(): versions = [ (Version("develop"), {}), (Version("1.0"), {}), (Version("2.0"), {"deprecated": True}), (Version("1.1"), {}), (Version("1.1alpha1"), {}), (Version("0.9"), {"preferred": True}), ] result = [ v for v, _ in sorted( versions, key=spack.package_base.concretization_version_order, reverse=True ) ] assert result == [ Version("0.9"), # preferred Version("2.0"), # deprecation is accounted for separately Version("1.1"), # latest non-deprecated final version Version("1.0"), # latest non-deprecated final version Version("1.1alpha1"), # prereleases Version("develop"), # likely development version ] @pytest.mark.parametrize( "roots,reuse_yaml,expected,not_expected,expected_length", [ ( ["mpileaks"], {"roots": True, "include": ["^mpich"]}, ["^mpich"], ["^mpich2", "^zmpi"], # Reused from store + externals 2 + 15, ), ( ["mpileaks"], {"roots": True, "include": ["externaltest"]}, ["externaltest"], ["^mpich", "^mpich2", "^zmpi"], # Reused from store + externals 1 + 15, ), ], ) @pytest.mark.usefixtures("mutable_database", "mock_store") @pytest.mark.not_on_windows("Expected length is different on Windows") def test_filtering_reused_specs( roots, reuse_yaml, expected, not_expected, expected_length, mutable_config ): """Tests that we can select which specs are to be reused, using constraints as filters""" # Assume all specs have a runtime dependency mutable_config.set("concretizer:reuse", reuse_yaml) packages_with_externals = spack.solver.runtimes.external_config_with_implicit_externals( mutable_config ) completion_mode = mutable_config.get("concretizer:externals:completion") selector = spack.solver.asp.ReusableSpecsSelector( configuration=mutable_config, external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, ) specs = selector.reusable_specs(roots) assert len(specs) == expected_length for constraint in expected: assert all(x.satisfies(constraint) for x in specs if not x.external) for constraint in not_expected: assert all(not x.satisfies(constraint) for x in specs if not x.external) @pytest.mark.usefixtures("mutable_database", "mock_store") @pytest.mark.parametrize( "reuse_yaml,expected_length", [ ( {"from": [{"type": "local"}]}, # Local store + externals 19 + 15, ), ( {"from": [{"type": "buildcache"}]}, # Local store + externals 0 + 15, ), ], ) @pytest.mark.not_on_windows("Expected length is different on Windows") def test_selecting_reused_sources(reuse_yaml, expected_length, mutable_config): """Tests that we can turn on/off sources of reusable specs""" # Assume all specs have a runtime dependency mutable_config.set("concretizer:reuse", reuse_yaml) packages_with_externals = spack.solver.runtimes.external_config_with_implicit_externals( mutable_config ) completion_mode = mutable_config.get("concretizer:externals:completion") selector = spack.solver.asp.ReusableSpecsSelector( configuration=mutable_config, external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, ) specs = selector.reusable_specs(["mpileaks"]) assert len(specs) == expected_length # Compiler wrapper is not reused, as it might have changed from previous installations assert not [x for x in specs if x.name == "compiler-wrapper"] @pytest.mark.parametrize( "specs,include,exclude,expected", [ # "foo" discarded by include rules (everything compiled with GCC) (["cmake@3.27.9 %gcc", "foo %clang"], ["%gcc"], [], ["cmake@3.27.9 %gcc"]), # "cmake" discarded by exclude rules (everything compiled with GCC but cmake) (["cmake@3.27.9 %gcc", "foo %gcc"], ["%gcc"], ["cmake"], ["foo %gcc"]), ], ) def test_spec_filters(specs, include, exclude, expected): specs = [Spec(x) for x in specs] expected = [Spec(x) for x in expected] f = spack.spec_filter.SpecFilter( factory=lambda: specs, is_usable=lambda x: True, include=include, exclude=exclude ) assert f.selected_specs() == expected @pytest.mark.regression("38484") def test_git_ref_version_can_be_reused(install_mockery): first_spec = spack.concretize.concretize_one( spack.spec.Spec("git-ref-package@git.2.1.5=2.1.5~opt") ) PackageInstaller([first_spec.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): # reproducer of the issue is that spack will solve when there is a change to the base spec second_spec = spack.concretize.concretize_one( spack.spec.Spec("git-ref-package@git.2.1.5=2.1.5+opt") ) assert second_spec.dag_hash() != first_spec.dag_hash() # we also want to confirm that reuse actually works so leave variant off to # let solver reuse third_spec = spack.spec.Spec("git-ref-package@git.2.1.5=2.1.5") assert first_spec.satisfies(third_spec) third_spec = spack.concretize.concretize_one(third_spec) assert third_spec.dag_hash() == first_spec.dag_hash() @pytest.mark.parametrize("standard_version", ["2.0.0", "2.1.5", "2.1.6"]) def test_reuse_prefers_standard_over_git_versions(standard_version, install_mockery): """ order matters in this test. typically reuse would pick the highest versioned installed match but we want to prefer the standard version over git ref based versions so install git ref last and ensure it is not picked up by reuse """ standard_spec = spack.concretize.concretize_one( spack.spec.Spec(f"git-ref-package@{standard_version}") ) PackageInstaller([standard_spec.package], fake=True, explicit=True).install() git_spec = spack.concretize.concretize_one("git-ref-package@git.2.1.5=2.1.5") PackageInstaller([git_spec.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): test_spec = spack.concretize.concretize_one("git-ref-package@2") assert git_spec.dag_hash() != test_spec.dag_hash() assert standard_spec.dag_hash() == test_spec.dag_hash() @pytest.mark.parametrize("unify", [True, "when_possible", False]) def test_spec_unification(unify, mutable_config, mock_packages): spack.config.set("concretizer:unify", unify) a = "pkg-a" a_restricted = "pkg-a^pkg-b foo=baz" b = "pkg-b foo=none" unrestricted = spack.cmd.parse_specs([a, b], concretize=True) a_concrete_unrestricted = [s for s in unrestricted if s.name == "pkg-a"][0] b_concrete_unrestricted = [s for s in unrestricted if s.name == "pkg-b"][0] assert (a_concrete_unrestricted["pkg-b"] == b_concrete_unrestricted) == (unify is not False) maybe_fails = pytest.raises if unify is True else spack.llnl.util.lang.nullcontext with maybe_fails(spack.solver.asp.UnsatisfiableSpecError): _ = spack.cmd.parse_specs([a_restricted, b], concretize=True) @pytest.mark.not_on_windows("parallelism unsupported on Windows") @pytest.mark.enable_parallelism def test_parallel_concretization(mutable_config, mock_packages): """Test whether parallel unify-false style concretization works.""" specs = [(Spec("pkg-a"), None), (Spec("pkg-b"), None)] result = spack.concretize.concretize_separately(specs) assert {s.name for s, _ in result} == {"pkg-a", "pkg-b"} @pytest.mark.usefixtures("mutable_config", "mock_packages") @pytest.mark.parametrize( "spec_str, error_type", [ (f"git-ref-package@main commit={'a' * 40}", None), (f"git-ref-package@main commit={'a' * 39}", AssertionError), (f"git-ref-package@2.1.6 commit={'a' * 40}", spack.error.UnsatisfiableSpecError), (f"git-ref-package@git.2.1.6=2.1.6 commit={'a' * 40}", None), (f"git-ref-package@git.{'a' * 40}=2.1.6 commit={'a' * 40}", None), ], ) def test_spec_containing_commit_variant(spec_str, error_type): spec = spack.spec.Spec(spec_str) if error_type is None: spack.concretize.concretize_one(spec) else: with pytest.raises(error_type): spack.concretize.concretize_one(spec) @pytest.mark.usefixtures("mutable_config", "mock_packages") @pytest.mark.parametrize( "spec_str", [ f"git-test-commit@git.main commit={'a' * 40}", f"git-test-commit@git.v1.0 commit={'a' * 40}", "git-test-commit@{sha} commit={sha}", "git-test-commit@{sha} commit=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", ], ) def test_spec_with_commit_interacts_with_lookup(mock_git_version_info, monkeypatch, spec_str): # This test will be short lived. Technically we could do further checks with a Lookup # but skipping impl since we are going to deprecate repo_path, filename, commits = mock_git_version_info file_url = pathlib.Path(repo_path).as_uri() monkeypatch.setattr(spack.package_base.PackageBase, "git", file_url, raising=False) spec = spack.spec.Spec(spec_str.format(sha=commits[-1])) spack.concretize.concretize_one(spec) @pytest.mark.usefixtures("mutable_config", "mock_packages") @pytest.mark.parametrize("version_str", [f"git.{'a' * 40}=main", "git.2.1.5=main"]) def test_relationship_git_versions_and_commit_variant(version_str): """ Confirm that GitVersions auto assign and populates the commit variant correctly """ # This should be a short lived test and can be deleted when we remove GitVersions spec = spack.spec.Spec(f"git-ref-package@{version_str}") spec = spack.concretize.concretize_one(spec) if spec.version.commit_sha: assert spec.version.commit_sha == spec.variants["commit"].value else: assert "commit" not in spec.variants @pytest.mark.usefixtures("install_mockery") def test_abstract_commit_spec_reuse(): commit = "abcd" * 10 spec_str_1 = f"git-ref-package@develop commit={commit}" spec_str_2 = f"git-ref-package commit={commit}" spec1 = spack.concretize.concretize_one(spec_str_1) PackageInstaller([spec1.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): spec2 = spack.concretize.concretize_one(spec_str_2) assert spec2.dag_hash() == spec1.dag_hash() @pytest.mark.usefixtures("install_mockery") @pytest.mark.parametrize( "installed_commit, incoming_commit, reusable", [("a" * 40, "b" * 40, False), (None, "b" * 40, False), ("a" * 40, None, True)], ) def test_commit_variant_can_be_reused(installed_commit, incoming_commit, reusable): # install a non-default variant to test if reuse picks it if installed_commit: spec_str_1 = f"git-ref-package@develop commit={installed_commit} ~opt" else: spec_str_1 = "git-ref-package@develop ~opt" if incoming_commit: spec_str_2 = f"git-ref-package@develop commit={incoming_commit}" else: spec_str_2 = "git-ref-package@develop" spec1 = spack.concretize.concretize_one(spack.spec.Spec(spec_str_1)) PackageInstaller([spec1.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): spec2 = spack.spec.Spec(spec_str_2) spec2 = spack.concretize.concretize_one(spec2) assert (spec1.dag_hash() == spec2.dag_hash()) == reusable @pytest.mark.regression("42679") @pytest.mark.parametrize("compiler_str", ["gcc@=9.4.0", "gcc@=9.4.0-foo"]) def test_selecting_compiler_with_suffix(mutable_config, mock_packages, compiler_str): """Tests that we can select compilers whose versions differ only for a suffix.""" packages_yaml = syaml.load_config( """ packages: gcc: externals: - spec: "gcc@9.4.0-foo languages='c,c++'" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ """ ) mutable_config.set("packages", packages_yaml["packages"]) s = spack.concretize.concretize_one(f"libelf %{compiler_str}") assert s["c"].satisfies(compiler_str) def test_duplicate_compiler_in_externals(mutable_config, mock_packages): """Tests that having duplicate compilers in packages.yaml do not raise and error.""" packages_yaml = syaml.load_config( """ packages: gcc: externals: - spec: "gcc@9.4.0 languages='c,c++'" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ - spec: "gcc@9.4.0 languages='c,c++'" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ """ ) mutable_config.set("packages", packages_yaml["packages"]) s = spack.concretize.concretize_one("libelf %gcc@9.4") assert s["c"].satisfies("gcc@9.4.0") @pytest.mark.parametrize( "spec_str,expected", [ ("gcc@14 %gcc@9.4.0", ["gcc@14", "%c,cxx=gcc@9.4.0", "^gcc-runtime@9.4.0"]), # If we don't specify a compiler, we should get the default compiler which is gcc ("gcc@14", ["gcc@14", "%c,cxx=gcc@10", "^gcc-runtime@10"]), ], ) def test_compiler_can_depend_on_themselves_to_build( spec_str, expected, default_mock_concretization ): """Tests that a compiler can depend on "itself" to bootstrap.""" s = default_mock_concretization(spec_str) assert not s.external for c in expected: assert s.satisfies(c) def test_compiler_attribute_is_tolerated_in_externals( mutable_config, mock_packages, tmp_path: pathlib.Path ): """Tests that we don't error out if an external specifies a compiler in the old way, provided that a suitable external compiler exists. """ packages_yaml = syaml.load_config( f""" packages: cmake: externals: - spec: "cmake@3.27.4 %gcc@10" prefix: {tmp_path} buildable: false """ ) mutable_config.set("packages", packages_yaml["packages"]) s = spack.concretize.concretize_one("cmake") assert s.external and s.external_path == str(tmp_path) def test_compiler_can_be_built_with_other_compilers(config, mock_packages): """Tests that a compiler can be built also with another compiler.""" s = spack.concretize.concretize_one("llvm@18 +clang %gcc") assert s.satisfies("llvm@18") c_compiler = s.dependencies(virtuals=("c",)) assert len(c_compiler) == 1 and c_compiler[0].satisfies("gcc@10") @pytest.mark.parametrize( "spec_str,expected", [ # Only one compiler is in the DAG, so pick the external associated with it ("dyninst %clang", "clang"), ("dyninst %gcc", "gcc"), # Both compilers are in the DAG, so pick the best external according to other criteria ("dyninst %clang ^libdwarf%gcc", "clang"), ("dyninst %gcc ^libdwarf%clang", "clang"), ], ) def test_compiler_match_for_externals_is_taken_into_account( spec_str, expected, mutable_config, mock_packages, tmp_path: pathlib.Path ): """Tests that compiler annotation for externals are somehow taken into account for a match""" packages_yaml = syaml.load_config( f""" packages: libelf: externals: - spec: "libelf@0.8.12 %gcc@10" prefix: {tmp_path / "gcc"} - spec: "libelf@0.8.13 %clang" prefix: {tmp_path / "clang"} """ ) mutable_config.set("packages", packages_yaml["packages"]) s = spack.concretize.concretize_one(spec_str) libelf = s["libelf"] assert libelf.external and libelf.external_path == str(tmp_path / expected) @pytest.mark.parametrize( "spec_str,expected", [ # Only one compiler is in the DAG, so pick the external associated with it ("dyninst %gcc@10", "libelf-gcc10"), ("dyninst %gcc@9", "libelf-gcc9"), # Both compilers are in the DAG, so pick the best external according to other criteria ("dyninst %gcc@10 ^libdwarf%gcc@9", "libelf-gcc9"), ], ) def test_compiler_match_for_externals_with_versions( spec_str, expected, mutable_config, mock_packages, tmp_path: pathlib.Path ): """Tests that version constraints are taken into account for compiler annotations on externals """ packages_yaml = syaml.load_config( f""" packages: libelf: buildable: false externals: - spec: "libelf@0.8.12 %gcc@10" prefix: {tmp_path / "libelf-gcc10"} - spec: "libelf@0.8.13 %gcc@9.4.0" prefix: {tmp_path / "libelf-gcc9"} """ ) mutable_config.set("packages", packages_yaml["packages"]) s = spack.concretize.concretize_one(spec_str) libelf = s["libelf"] assert libelf.external and libelf.external_path == str(tmp_path / expected) def test_specifying_compilers_with_virtuals_syntax(default_mock_concretization): """Tests that we can pin compilers to nodes using the %[virtuals=...] syntax""" # clang will be used for both C and C++, since they are provided together mpich = default_mock_concretization("mpich %[virtuals=fortran] gcc %clang") assert mpich["fortran"].satisfies("gcc") assert mpich["c"].satisfies("llvm") assert mpich["cxx"].satisfies("llvm") # gcc is the default compiler mpileaks = default_mock_concretization( "mpileaks ^libdwarf %gcc ^mpich %[virtuals=fortran] gcc %clang" ) assert mpileaks["c"].satisfies("gcc") libdwarf = mpileaks["libdwarf"] assert libdwarf["c"].satisfies("gcc") assert libdwarf["c"].satisfies("gcc") mpich = mpileaks["mpi"] assert mpich["fortran"].satisfies("gcc") assert mpich["c"].satisfies("llvm") assert mpich["cxx"].satisfies("llvm") @pytest.mark.regression("49847") @pytest.mark.xfail(sys.platform == "win32", reason="issues with install mockery") def test_reuse_when_input_specifies_build_dep(install_mockery): """Test that we can reuse a spec when specifying build dependencies in the input""" pkgb_old = spack.concretize.concretize_one(spack.spec.Spec("pkg-b@0.9 %gcc@9")) PackageInstaller([pkgb_old.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): result = spack.concretize.concretize_one("pkg-b %gcc") assert pkgb_old.dag_hash() == result.dag_hash() result = spack.concretize.concretize_one("pkg-a ^pkg-b %gcc@9") assert pkgb_old.dag_hash() == result["pkg-b"].dag_hash() assert result.satisfies("%gcc@9") result = spack.concretize.concretize_one("pkg-a %gcc@10 ^pkg-b %gcc@9") assert pkgb_old.dag_hash() == result["pkg-b"].dag_hash() @pytest.mark.regression("49847") def test_reuse_when_requiring_build_dep(install_mockery, mutable_config): """Test that we can reuse a spec when specifying build dependencies in requirements""" mutable_config.set("packages:all:require", "%gcc") pkgb_old = spack.concretize.concretize_one(spack.spec.Spec("pkg-b@0.9")) PackageInstaller([pkgb_old.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): result = spack.concretize.concretize_one("pkg-b") assert pkgb_old.dag_hash() == result.dag_hash(), result.tree() @pytest.mark.regression("50167") def test_input_analysis_and_conditional_requirements(default_mock_concretization): """Tests that input analysis doesn't account for conditional requirement to discard possible dependencies. If the requirement is conditional, and impossible to achieve on the current platform, the valid search space is still the complement of the condition that activates the requirement. """ libceed = default_mock_concretization("libceed") assert libceed["libxsmm"].satisfies("@main") assert libceed["libxsmm"].satisfies("platform=test") @pytest.mark.parametrize( "compiler_str,expected,not_expected", [ # Compilers are matched to some other external, so the compiler that picked is concrete ("gcc@10", ["%gcc", "%gcc@10"], ["%clang", "%gcc@9"]), ("gcc@9.4.0", ["%gcc", "%gcc@9"], ["%clang", "%gcc@10"]), ("clang", ["%clang", "%llvm+clang"], ["%gcc", "%gcc@9", "%gcc@10"]), ], ) @pytest.mark.regression("49841") def test_installing_external_with_compilers_directly( compiler_str, expected, not_expected, mutable_config, mock_packages, tmp_path: pathlib.Path ): """Tests that version constraints are taken into account for compiler annotations on externals """ spec_str = f"libelf@0.8.12 %{compiler_str}" packages_yaml = syaml.load_config( f""" packages: libelf: buildable: false externals: - spec: {spec_str} prefix: {tmp_path / "libelf"} """ ) mutable_config.set("packages", packages_yaml["packages"]) s = spack.concretize.concretize_one(spec_str) assert s.external assert all(s.satisfies(c) for c in expected) assert all(not s.satisfies(c) for c in not_expected) @pytest.mark.regression("49841") def test_using_externals_with_compilers(mutable_config, mock_packages, tmp_path: pathlib.Path): """Tests that version constraints are taken into account for compiler annotations on externals, even imposed as transitive deps. """ packages_yaml = syaml.load_config( f""" packages: libelf: buildable: false externals: - spec: libelf@0.8.12 %gcc@10 prefix: {tmp_path / "libelf"} """ ) mutable_config.set("packages", packages_yaml["packages"]) with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one("dyninst%gcc@10.2.1 ^libelf@0.8.12 %gcc@:9") s = spack.concretize.concretize_one("dyninst%gcc@10.2.1 ^libelf@0.8.12 %gcc@10:") libelf = s["libelf"] assert libelf.external and libelf.satisfies("%gcc") @pytest.mark.regression("50161") def test_installed_compiler_and_better_external(install_mockery, mutable_config): """Tests that we always prefer a higher-priority external compiler, when we have a lower-priority compiler installed, and we try to concretize a spec without specifying the compiler dependency. """ pkg_b = spack.concretize.concretize_one(spack.spec.Spec("pkg-b %clang")) PackageInstaller([pkg_b.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", False): pkg_a = spack.concretize.concretize_one("pkg-a") assert pkg_a["c"].satisfies("gcc@10"), pkg_a.tree() assert pkg_a["pkg-b"]["c"].satisfies("gcc@10") with spack.config.override("concretizer:reuse", False): mpileaks = spack.concretize.concretize_one("mpileaks") assert mpileaks.satisfies("%gcc@10") @pytest.mark.regression("50006") def test_concrete_multi_valued_variants_in_externals( mutable_config, mock_packages, tmp_path: pathlib.Path ): """Tests that concrete multivalued variants in externals cannot be extended with additional values when concretizing. """ packages_yaml = syaml.load_config( f""" packages: gcc: buildable: false externals: - spec: gcc@12.1.0 languages:='c,c++' prefix: {tmp_path / "gcc-12"} extra_attributes: compilers: c: {tmp_path / "gcc-12"}/bin/gcc cxx: {tmp_path / "gcc-12"}/bin/g++ - spec: gcc@14.1.0 languages:=fortran prefix: {tmp_path / "gcc-14"} extra_attributes: compilers: fortran: {tmp_path / "gcc-14"}/bin/gfortran """ ) mutable_config.set("packages", packages_yaml["packages"]) with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one("pkg-b %gcc@14") s = spack.concretize.concretize_one("pkg-b %gcc") assert s["c"].satisfies("gcc@12.1.0"), s.tree() assert s["c"].external assert s["c"].satisfies("languages=c,c++") and not s["c"].satisfies("languages=fortran") def test_concrete_multi_valued_in_input_specs(default_mock_concretization): """Tests that we can use := to specify exactly multivalued variants in input specs.""" s = default_mock_concretization("gcc languages:=fortran") assert not s.external and s["c"].external assert s.satisfies("languages:=fortran") assert not s.satisfies("languages=c") and not s.satisfies("languages=c++") def test_concrete_multi_valued_variants_in_requirements(mutable_config, mock_packages): """Tests that concrete multivalued variants can be imposed by requirements.""" packages_yaml = syaml.load_config( """ packages: pkg-a: require: - libs:=static """ ) mutable_config.set("packages", packages_yaml["packages"]) with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one("pkg-a libs=shared") spack.concretize.concretize_one("pkg-a libs=shared,static") s = spack.concretize.concretize_one("pkg-a") assert s.satisfies("libs:=static") assert not s.satisfies("libs=shared") def test_concrete_multi_valued_variants_in_depends_on(default_mock_concretization): """Tests the use of := in depends_on directives""" with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): default_mock_concretization("gmt-concrete-mv-dependency ^mvdefaults foo:=c") default_mock_concretization("gmt-concrete-mv-dependency ^mvdefaults foo:=a,c") default_mock_concretization("gmt-concrete-mv-dependency ^mvdefaults foo:=b,c") s = default_mock_concretization("gmt-concrete-mv-dependency") assert s.satisfies("^mvdefaults foo:=a,b"), s.tree() assert not s.satisfies("^mvdefaults foo=c") def test_concrete_multi_valued_variants_when_args(default_mock_concretization): """Tests the use of := in conflicts and when= arguments""" # Check conflicts("foo:=a,b", when="@0.9") with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): default_mock_concretization("mvdefaults@0.9 foo:=a,b") for c in ("foo:=a", "foo:=a,b,c", "foo:=a,c", "foo:=b,c"): s = default_mock_concretization(f"mvdefaults@0.9 {c}") assert s.satisfies(c) # Check depends_on("pkg-b", when="foo:=b,c") s = default_mock_concretization("mvdefaults foo:=b,c") assert s.satisfies("^pkg-b") for c in ("foo:=a", "foo:=a,b,c", "foo:=a,b", "foo:=a,c"): s = default_mock_concretization(f"mvdefaults {c}") assert not s.satisfies("^pkg-b") @pytest.mark.usefixtures("mock_packages") @pytest.mark.parametrize( "constraint_in_yaml,unsat_request,sat_request", [ # Arch parts pytest.param( "target=x86_64", "target=core2", "target=x86_64", marks=pytest.mark.skipif( platform.machine() != "x86_64", reason="only valid for x86_64" ), ), pytest.param( "target=core2", "target=x86_64", "target=core2", marks=pytest.mark.skipif( platform.machine() != "x86_64", reason="only valid for x86_64" ), ), ("os=debian6", "os=redhat6", "os=debian6"), ("platform=test", "platform=linux", "platform=test"), # Variants ("~lld", "+lld", "~lld"), ("+lld", "~lld", "+lld"), ], ) def test_spec_parts_on_fresh_compilers( constraint_in_yaml, unsat_request, sat_request, mutable_config, tmp_path: pathlib.Path ): """Tests that spec parts like targets and variants in `% target= ` are associated with `package` for `%` just as they would be for `^`, when we concretize without reusing. """ packages_yaml = syaml.load_config( f""" packages: llvm:: buildable: false externals: - spec: "llvm@20 +clang {constraint_in_yaml}" prefix: {tmp_path / "llvm-20"} """ ) mutable_config.set("packages", packages_yaml["packages"]) # Check the abstract spec is formed correctly abstract_spec = Spec(f"pkg-a %llvm@20 +clang {unsat_request}") assert abstract_spec["llvm"].satisfies(f"@20 +clang {unsat_request}") # Check that we can't concretize the spec, since llvm is not buildable with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one(abstract_spec) # Check we can instead concretize if we use the correct constraint s = spack.concretize.concretize_one(f"pkg-a %llvm@20 +clang {sat_request}") assert s["c"].external and s["c"].satisfies(f"@20 +clang {sat_request}") @pytest.mark.usefixtures("mock_packages", "mutable_database") @pytest.mark.parametrize( "constraint_in_yaml,unsat_request,sat_request", [ # Arch parts pytest.param( "target=x86_64", "target=core2", "target=x86_64", marks=pytest.mark.skipif( platform.machine() != "x86_64", reason="only valid for x86_64" ), ), pytest.param( "target=core2", "target=x86_64", "target=core2", marks=pytest.mark.skipif( platform.machine() != "x86_64", reason="only valid for x86_64" ), ), ("os=debian6", "os=redhat6", "os=debian6"), ("platform=test", "platform=linux", "platform=test"), # Variants ("~lld", "+lld", "~lld"), ("+lld", "~lld", "+lld"), ], ) def test_spec_parts_on_reused_compilers( constraint_in_yaml, unsat_request, sat_request, mutable_config, tmp_path: pathlib.Path ): """Tests that requests of the form % are considered for reused specs, even though build dependency are not part of the ASP problem. """ packages_yaml = syaml.load_config( f""" packages: c: require: llvm cxx: require: llvm llvm:: buildable: false externals: - spec: "llvm+clang@20 {constraint_in_yaml}" prefix: {tmp_path / "llvm-20"} mpileaks: buildable: true """ ) mutable_config.set("packages", packages_yaml["packages"]) # Install the spec installed_spec = spack.concretize.concretize_one(f"mpileaks %llvm@20 {sat_request}") PackageInstaller([installed_spec.package], fake=True, explicit=True).install() # Make mpileaks not buildable mutable_config.set("packages:mpileaks:buildable", False) # Check we can't concretize with the unsat request... with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one(f"mpileaks %llvm@20 {unsat_request}") # ...but we can with the original constraint with spack.config.override("concretizer:reuse", True): s = spack.concretize.concretize_one(f"mpileaks %llvm@20 {sat_request}") assert s.dag_hash() == installed_spec.dag_hash() def test_use_compiler_by_hash(mock_packages, mutable_database, mutable_config): """Tests that we can reuse an installed compiler specifying its hash""" installed_spec = spack.concretize.concretize_one("gcc@14.0") PackageInstaller([installed_spec.package], fake=True, explicit=True).install() with spack.config.override("concretizer:reuse", True): s = spack.concretize.concretize_one(f"mpileaks %gcc/{installed_spec.dag_hash()}") assert s["c"].dag_hash() == installed_spec.dag_hash() @pytest.mark.parametrize( "spec_str,expected,not_expected", [ # Simple build requirement on gcc, as a provider for c ( "mpileaks %gcc", ["%[deptypes=build] gcc"], ["%[deptypes=link] gcc", "%[deptypes=run] gcc"], ), # Require mpich as a direct dependency of mpileaks ( "mpileaks %[deptypes=link] mpich", ["%[deptypes=build,link] mpich", "^callpath%[deptypes=build,link] mpich"], ["%[deptypes=run] mpich"], ), ( "mpileaks %[deptypes=link] mpich+debug", # non-default variant ["%[deptypes=build,link] mpich+debug"], ["% mpich~debug"], ), # Require mpich as a direct dependency of two nodes, with compatible constraints ( "mpileaks %mpich+debug ^callpath %mpich@3.0.3", # non-default variant [ "%[deptypes=build,link] mpich@3.0.3+debug", "^callpath %[deptypes=build,link] mpich@3.0.3+debug", ], ["%mpich~debug"], ), # Package that has a conditional link dependency on a compiler ("emacs +native", ["%[virtuals=c deptypes=build,link] gcc"], []), ("emacs +native %gcc", ["%[virtuals=c deptypes=build,link] gcc"], []), ("emacs +native %[virtuals=c] gcc", ["%[virtuals=c deptypes=build,link] gcc"], []), # Package that depends on llvm as a library and also needs C and C++ compilers ( "llvm-client", ["%[virtuals=c,cxx deptypes=build] gcc", "%[deptypes=build,link] llvm"], ["%c=llvm"], ), ( "llvm-client %c,cxx=gcc", ["%[virtuals=c,cxx deptypes=build] gcc", "%[deptypes=build,link] llvm"], ["%c=llvm"], ), ("llvm-client %c,cxx=llvm", ["%[virtuals=c,cxx deptypes=build,link] llvm"], ["%gcc"]), ], ) def test_specifying_direct_dependencies( spec_str, expected, not_expected, default_mock_concretization ): """Tests solving % in different scenarios, either for runtime or buildtime dependencies.""" concrete_spec = default_mock_concretization(spec_str) for c in expected: assert concrete_spec.satisfies(c) for c in not_expected: assert not concrete_spec.satisfies(c) @pytest.mark.parametrize( "spec_str,conditional_spec,expected", [ # Abstract spec is False, cause the set of possible solutions in the rhs is smaller ("mpich", "%[when=+debug] llvm", (False, True)), # Abstract spec is True, since we know the condition never applies ("mpich~debug", "%[when=+debug] llvm", (True, True)), # In this case we know the condition applies ("mpich+debug", "%[when=+debug] llvm", (False, False)), ("mpich+debug %llvm+clang", "%[when=+debug] llvm", (True, True)), ("mpich+debug", "%[when=+debug] gcc", (False, True)), # Conditional specs on the lhs ("mpich %[when=+debug] gcc", "mpich %gcc", (False, True)), ("mpich %[when=+debug] gcc", "mpich %llvm", (False, False)), ("mpich %[when=+debug] gcc", "mpich %[when=+debug] gcc", (True, True)), ("mpileaks ^[when=+opt] callpath@0.9", "mpileaks ^callpath@1.0", (False, True)), ("mpileaks ^[when=+opt] callpath@1.0", "mpileaks ^callpath@1.0", (False, True)), ("mpileaks ^[when=+opt] callpath@1.0", "mpileaks ^[when=+opt] callpath@1.0", (True, True)), # Conditional specs on both sides ( "mpileaks ^[when=+opt] callpath@1.0", "mpileaks ^[when=+opt+debug] callpath@1.0", (True, True), ), ( "mpileaks ^[when=+opt+debug] callpath@1.0", "mpileaks ^[when=+opt] callpath@1.0", (False, True), ), ( "mpileaks ^[when=+opt] callpath@1.0", "mpileaks ^[when=~debug] callpath@1.0", (False, True), ), # Different conditional specs associated with different nodes in the DAG, where one does # not apply since the condition is not met ( "mpileaks %[when='%mpi' virtuals=mpi] zmpi ^libelf %[when='%mpi' virtuals=mpi] mpich", "mpileaks %[virtuals=mpi] zmpi", (False, True), ), ( "mpileaks %[when='%mpi' virtuals=mpi] mpich ^libelf %[when='%mpi' virtuals=mpi] zmpi", "mpileaks %[virtuals=mpi] mpich", (False, True), ), ], ) def test_satisfies_conditional_spec( spec_str, conditional_spec, expected, default_mock_concretization ): """Tests satisfies semantic when testing an abstract spec and its concretized counterpart with a conditional spec. """ abstract_spec = Spec(spec_str) concrete_spec = default_mock_concretization(spec_str) expected_abstract, expected_concrete = expected assert abstract_spec.satisfies(conditional_spec) is expected_abstract assert concrete_spec.satisfies(conditional_spec) is expected_concrete assert concrete_spec.satisfies(abstract_spec) @pytest.mark.not_on_windows("Tests use linux paths") @pytest.mark.regression("51001") def test_selecting_externals_with_compilers_as_root(mutable_config, mock_packages): """Tests that we can select externals that have a compiler in their spec, even when they are root. """ packages_yaml = syaml.load_config( """ packages: gcc:: externals: - spec: "gcc@9.4.0 languages='c,c++'" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ llvm:: buildable: false externals: - spec: "llvm@20 +clang" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ mpich: buildable: false externals: - spec: "mpich@3.4.3 %gcc" prefix: /path/mpich/gcc - spec: "mpich@3.4.3 %clang" prefix: /path/mpich/clang """ ) mutable_config.set("packages", packages_yaml["packages"]) # Select mpich as the root spec s = spack.concretize.concretize_one("mpich %clang") assert s.external assert s.prefix == "/path/mpich/clang" s = spack.concretize.concretize_one("mpich %gcc") assert s.external assert s.prefix == "/path/mpich/gcc" # Select mpich as a dependency s = spack.concretize.concretize_one("mpileaks ^mpi=mpich %clang") assert s["mpi"].external assert s["mpi"].prefix == "/path/mpich/clang" s = spack.concretize.concretize_one("mpileaks ^mpi=mpich %gcc") assert s["mpi"].external assert s["mpi"].prefix == "/path/mpich/gcc" @pytest.mark.not_on_windows("Tests use linux paths") @pytest.mark.regression("51001") @pytest.mark.parametrize( "external_compiler,spec_str", [("gcc@8", "mpich %gcc@8.4"), ("gcc@8.4.0", "mpich %gcc@8")] ) def test_selecting_externals_with_compilers_and_versions( external_compiler, spec_str, mutable_config, mock_packages ): """Tests different scenarios of having a compiler specified with a version constraint, either in the input spec or in the external spec. """ packages_yaml = syaml.load_config( f""" packages: gcc: externals: - spec: "gcc@8.4.0 languages='c,c++'" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ mpich: buildable: false externals: - spec: "mpich@3.4.3 %{external_compiler}" prefix: /path/mpich/gcc - spec: "mpich@3.4.3 %clang" prefix: /path/mpich/clang """ ) mutable_config.set("packages", packages_yaml["packages"]) s = spack.concretize.concretize_one(spec_str) assert s.external assert s.prefix == "/path/mpich/gcc" @pytest.mark.regression("51001") @pytest.mark.parametrize( "external_compiler,spec_str,error_match", [ # Compiler is underspecified ("gcc", "mpich %gcc", "there are multiple external specs"), ("gcc@9", "mpich %gcc", "there are multiple external specs"), # Compiler does not exist ("%oneapi", "mpich %gcc@8", "there is no"), ], ) def test_errors_when_specifying_externals_with_compilers( external_compiler, spec_str, error_match, mutable_config, mock_packages ): """Tests different errors that can occur in an external spec with a compiler specified.""" packages_yaml = syaml.load_config( f""" packages: mpich: buildable: false externals: - spec: "mpich@3.4.3 %{external_compiler}" prefix: /path/mpich/gcc - spec: "mpich@3.4.3 %clang" prefix: /path/mpich/clang """ ) mutable_config.set("packages", packages_yaml["packages"]) with pytest.raises(ExternalDependencyError, match=error_match): _ = spack.concretize.concretize_one(spec_str) @pytest.mark.regression("51146,51067") def test_caret_in_input_cannot_set_transitive_build_dependencies(default_mock_concretization): """Tests that a caret in the input spec does not set transitive build dependencies, and errors with an appropriate message. """ with pytest.raises(spack.solver.asp.UnsatisfiableSpecError, match="transitive 'link' or"): _ = default_mock_concretization("multivalue-variant ^gmake") @pytest.mark.regression("51167") @pytest.mark.require_provenance def test_commit_variant_enters_the_hash(mutable_config, mock_packages, monkeypatch): """Tests that an implicit commit variant, obtained from resolving the commit sha of a branch, enters the hash of the spec. """ first_call = True def _mock_resolve(spec) -> None: if first_call: spec.variants["commit"] = vt.SingleValuedVariant("commit", f"{'b' * 40}") return spec.variants["commit"] = vt.SingleValuedVariant("commit", f"{'a' * 40}") monkeypatch.setattr(spack.package_base.PackageBase, "_resolve_git_provenance", _mock_resolve) before = spack.concretize.concretize_one("git-ref-package@develop") first_call = False after = spack.concretize.concretize_one("git-ref-package@develop") assert before.package.needs_commit(before.version) assert before.satisfies(f"commit={'b' * 40}") assert after.satisfies(f"commit={'a' * 40}") assert before.dag_hash() != after.dag_hash() @pytest.mark.regression("51180") def test_reuse_with_mixed_compilers(mutable_config, mock_packages): """Tests that potentially reusing a spec with a mixed compiler set, will not interfere with a request on one of the languages for the same package. """ packages_yaml = syaml.load_config( """ packages: gcc: externals: - spec: "gcc@15.1 languages='c,c++,fortran'" prefix: /path1 extra_attributes: compilers: c: /path1/bin/gcc cxx: /path1/bin/g++ fortran: /path1/bin/gfortran llvm: externals: - spec: "llvm@20 +flang+clang" prefix: /path2 extra_attributes: compilers: c: /path2/bin/clang cxx: /path2/bin/clang++ fortran: /path2/bin/flang """ ) mutable_config.set("packages", packages_yaml["packages"]) s = spack.concretize.concretize_one("openblas %c=gcc %fortran=llvm") reusable_specs = list(s.traverse(root=True)) root_specs = [Spec("openblas %fortran=gcc")] with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, root_specs, reuse=reusable_specs) assert len(result.specs) == 1 r = result.specs[0] assert r.satisfies("openblas %fortran=gcc") assert r.dag_hash() != s.dag_hash() @pytest.mark.regression("51224") def test_when_possible_above_all(mutable_config, mock_packages): """Tests that the criterion to solve as many specs as possible is above all other criteria.""" specs = [Spec("pkg-a"), Spec("pkg-b")] solver = spack.solver.asp.Solver() for result in solver.solve_in_rounds(specs): criteria = sorted(result.criteria, reverse=True) assert criteria[0].name == "number of input specs not concretized" def test_concretization_cache_roundtrip( mock_packages, use_concretization_cache, monkeypatch, mutable_config ): """Tests whether we can write the results of a clingo solve to the cache and load the same spec request from the cache to produce identical specs""" assert spack.config.get("concretizer:concretization_cache:enable") # run one standard concretization to populate the cache and the setup method # memoization h = spack.concretize.concretize_one("hdf5") # ASP output should be stable, concretizing the same spec # should have the same problem output # assert that we're not storing any new cache entries def _ensure_no_store(self, problem: str, result, statistics, test=False): # always throw, we never want to reach this code path assert False, "Concretization cache hit expected" # Assert that we're actually hitting the cache cache_fetch = spack.solver.asp.ConcretizationCache.fetch def _ensure_cache_hits(self, problem: str): result, statistics = cache_fetch(self, problem) assert result, "Expected successful concretization cache hit" assert statistics, "Expected statistics to be non null on cache hit" return result, statistics monkeypatch.setattr(spack.solver.asp.ConcretizationCache, "store", _ensure_no_store) monkeypatch.setattr(spack.solver.asp.ConcretizationCache, "fetch", _ensure_cache_hits) # ensure subsequent concretizations of the same spec produce the same spec # object for _ in range(5): hdf5 = spack.concretize.concretize_one("hdf5") assert h.to_json(pretty=True) == hdf5.to_json(pretty=True) assert h == hdf5 def test_concretization_cache_roundtrip_result(use_concretization_cache): """Ensure the concretization cache doesn't change Solver Result objects.""" specs = [Spec("hdf5")] solver = spack.solver.asp.Solver() result1 = solver.solve(specs) result2 = solver.solve(specs) assert result1 == result2 def test_concretization_cache_count_cleanup(use_concretization_cache, mutable_config): """Tests to ensure we are cleaning the cache when we should be respective to the number of entries allowed in the cache""" conc_cache_dir = use_concretization_cache spack.config.set("concretizer:concretization_cache:entry_limit", 1000) def names(): return set( x.name for x in conc_cache_dir.iterdir() if (not x.is_dir() and not x.name.startswith(".")) ) assert len(names()) == 0 for i in range(1000): name = spack.util.hash.b32_hash(f"mock_cache_file_{i}") mock_cache_file = conc_cache_dir / name mock_cache_file.touch() before = names() assert len(before) == 1000 # cleanup should be run after the 1,001st execution spack.concretize.concretize_one("hdf5") # ensure that half the elements were removed and that one more was created after = names() assert len(after) == 501 assert len(after - before) == 1 # one additional hash added by 1001st concretization def test_concretization_cache_uncompressed_entry(use_concretization_cache, monkeypatch): def _store(self, problem, result, statistics): cache_path = self._cache_path_from_problem(problem) with self.write_transaction(cache_path) as exists: if exists: return try: with open(cache_path, "x", encoding="utf-8") as cache_entry: cache_dict = {"results": result.to_dict(), "statistics": statistics} cache_entry.write(json.dumps(cache_dict)) except FileExistsError: pass monkeypatch.setattr(spack.solver.asp.ConcretizationCache, "store", _store) # Store the results in plaintext spack.concretize.concretize_one("zlib") # Ensure fetch can handle the plaintext cache entry spack.concretize.concretize_one("zlib") @pytest.mark.parametrize( "asp_file", [ "concretize.lp", "heuristic.lp", "display.lp", "direct_dependency.lp", "when_possible.lp", "libc_compatibility.lp", "os_compatibility.lp", "splices.lp", ], ) def test_concretization_cache_asp_canonicalization(asp_file): path = os.path.join(os.path.dirname(spack.solver.asp.__file__), asp_file) with open(path, "r", encoding="utf-8") as f: original = [line.strip() for line in f.readlines()] stripped = spack.solver.asp.strip_asp_problem(original) diff = list(difflib.unified_diff(original, stripped)) assert all( [ line == "-" or line.startswith("-%") for line in diff if line.startswith("-") and not line.startswith("---") ] ) @pytest.mark.parametrize( "node_completion,expected,not_expected", [ ("architecture_only", ["+clang", "~flang", "platform=test"], ["lld=*"]), ( "default_variants", ["+clang", "~flang", "+lld", "platform=test"], ["~clang", "+flang", "~lld"], ), ], ) def test_external_node_completion_from_config( node_completion, expected, not_expected, mutable_config, mock_packages ): """Tests the different options for external node completion in the configuration file.""" mutable_config.set("concretizer:externals:completion", node_completion) s = spack.concretize.concretize_one("llvm") assert s.external assert all(s.satisfies(c) for c in expected) assert all(not s.satisfies(c) for c in not_expected) @pytest.mark.parametrize( "spec_str,packages_yaml,expected", [ ( "mpileaks", """ packages: mpileaks: externals: - spec: "mpileaks@2.3~debug+opt" prefix: /user/path dependencies: - id: callpath_id deptypes: link - id: mpich_id deptypes: - "build" - "link" virtuals: "mpi" callpath: externals: - spec: "callpath@1.0" prefix: /user/path id: callpath_id dependencies: - id: mpich_id deptypes: - "build" - "link" virtuals: "mpi" mpich: externals: - spec: "mpich@3.0.4" prefix: /user/path id: mpich_id """, [ "%mpi=mpich@3.0.4", "^callpath %mpi=mpich@3.0.4", "%[deptypes=link] callpath", "%[deptypes=build,link] mpich", ], ), # Same, but using `spec:` instead of `id:` for dependencies ( "mpileaks", """ packages: mpileaks: externals: - spec: "mpileaks@2.3~debug+opt" prefix: /user/path dependencies: - spec: callpath deptypes: link - spec: mpich virtuals: "mpi" callpath: externals: - spec: "callpath@1.0" prefix: /user/path dependencies: - spec: mpich virtuals: "mpi" mpich: externals: - spec: "mpich@3.0.4" prefix: /user/path """, [ "%mpi=mpich@3.0.4", "^callpath %mpi=mpich@3.0.4", "%[deptypes=link] callpath", "%[deptypes=build,link] mpich", ], ), ], ) def test_external_specs_with_dependencies( spec_str, packages_yaml, expected, mutable_config, mock_packages ): """Tests that we can reconstruct external specs with dependencies.""" configuration = syaml.load_config(packages_yaml) mutable_config.set("packages", configuration["packages"]) s = spack.concretize.concretize_one(spec_str) assert all(node.external for node in s.traverse()) assert all(s.satisfies(c) for c in expected) @pytest.mark.parametrize( "default_target,expected", [ # Specific target requested ("x86_64_v3", ["callpath target=x86_64_v3", "^mpich target=x86_64_v3"]), # With ranges, be conservative by default (":x86_64_v3", ["callpath target=x86_64", "^mpich target=x86_64"]), ("x86_64:x86_64_v3", ["callpath target=x86_64", "^mpich target=x86_64"]), ("x86_64:", ["callpath target=x86_64", "^mpich target=x86_64"]), ], ) @pytest.mark.skipif( spack.vendor.archspec.cpu.host().family != "x86_64", reason="test data for x86_64" ) def test_target_requirements(default_target, expected, mutable_config, mock_packages): """Tests different scenarios where targets might be constrained by configuration and are not specified in external specs """ configuration = syaml.load_config( f""" packages: all: require: - "target={default_target}" callpath: buildable: false externals: - spec: "callpath@1.0" prefix: /user/path id: callpath_id dependencies: - id: mpich_id deptypes: - "build" - "link" virtuals: "mpi" mpich: externals: - spec: "mpich@3.0.4" prefix: /user/path id: mpich_id """ ) mutable_config.set("packages", configuration["packages"]) s = spack.concretize.concretize_one("callpath") assert s.external assert all(s.satisfies(x) for x in expected), s.tree() @pytest.mark.parametrize( "spec_str,inline,yaml", [ ( "cmake-client", """ packages: cmake-client: externals: - spec: cmake-client@1.0 %cmake prefix: /mock cmake: externals: - spec: cmake@3.23.0 prefix: /mock """, """ packages: cmake-client: externals: - spec: cmake-client@1.0 prefix: /mock dependencies: - spec: cmake cmake: externals: - spec: cmake@3.23.0 prefix: /mock """, ), ( "mpileaks", """ packages: mpileaks: externals: - spec: "mpileaks@2.3~debug+opt %mpi=mpich %[deptypes=link] callpath" prefix: /user/path callpath: externals: - spec: "callpath@1.0 %mpi=mpich" prefix: /user/path mpich: externals: - spec: "mpich@3.0.4" prefix: /user/path """, """ packages: mpileaks: externals: - spec: "mpileaks@2.3~debug+opt" prefix: /user/path dependencies: - spec: callpath deptypes: link - spec: mpich virtuals: "mpi" callpath: externals: - spec: "callpath@1.0" prefix: /user/path dependencies: - spec: mpich virtuals: "mpi" mpich: externals: - spec: "mpich@3.0.4" prefix: /user/path """, ), ], ) def test_external_inline_equivalent_to_yaml(spec_str, inline, yaml, mutable_config, mock_packages): """Tests that the inline syntax for external specs is equivalent to the YAML syntax.""" configuration = syaml.load_config(inline) mutable_config.set("packages", configuration["packages"]) inline_spec = spack.concretize.concretize_one(spec_str) configuration = syaml.load_config(yaml) mutable_config.set("packages", configuration["packages"]) yaml_spec = spack.concretize.concretize_one(spec_str) assert inline_spec == yaml_spec @pytest.mark.regression("51556") def test_reusing_gcc_same_version_different_libcs(monkeypatch, mutable_config, mock_packages): """Tests that Spack can solve for specs when it reuses 2 GCCs at the same version, but injecting different libcs. """ packages_yaml = syaml.load_config( """ packages: gcc: externals: - spec: "gcc@12.3.0 languages='c,c++,fortran' os=debian6" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ fortran: /path/bin/gfortran - spec: "gcc@12.3.0 languages='c,c++,fortran' os=redhat6" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ fortran: /path/bin/gfortran """ ) mutable_config.set("packages", packages_yaml["packages"]) mutable_config.set("concretizer:reuse", True) def _mock_libc(self): if self.spec.satisfies("os=debian6"): return spack.spec.Spec("glibc@=2.31", external_path="/rocky9/path") return spack.spec.Spec("glibc@=2.28", external_path="/rocky8/path") monkeypatch.setattr( spack.compilers.libraries.CompilerPropertyDetector, "default_libc", _mock_libc ) # This should not raise mpileaks = spack.concretize.concretize_one("mpileaks %c=gcc@12") assert mpileaks.satisfies("%c=gcc@12") def test_concrete_specs_skip_prechecks(mock_packages): """Test that concrete specs are not checked for unknown versions and dependencies.""" specs = [spack.spec.Spec("zlib"), spack.spec.Spec("deprecated-versions@=1.1.0")] with pytest.raises(spack.solver.asp.DeprecatedVersionError): spack.solver.asp.SpackSolverSetup().setup(specs) with spack.config.override("config:deprecated", True): concrete_spec = spack.concretize.concretize_one(specs[1]) # Try again with the same version but a concrete spec specs[1] = concrete_spec spack.solver.asp.SpackSolverSetup().setup(specs) @pytest.mark.regression("51683") def test_activating_variant_for_conditional_language_dependency(default_mock_concretization): """Tests that a dependency on a conditional language can be concretized, and that the solver turn on the correct variant to enable the language dependency """ # To trigger the bug, we need at least another node needing fortran, in this case mpich s = default_mock_concretization("mpileaks %fortran=gcc %mpi=mpich") assert s.satisfies("+fortran") # Try just asking for fortran, without the provider s = default_mock_concretization("mpileaks %fortran %mpi=mpich") assert s.satisfies("+fortran") def test_when_condition_with_direct_dependency_on_virtual_provider(default_mock_concretization): """If a when condition contains a direct dependency on a provider of a virtual, it should only trigger if the provider is used for that current package, and not if the provider happens to be a dependency, without its virtual being depended on.""" s = default_mock_concretization("direct-dep-virtuals-one") assert s.satisfies("%netlib-blas") assert s["direct-dep-virtuals-two"].satisfies("%blas=netlib-blas") def test_conflict_with_direct_dependency_on_virtual_provider(default_mock_concretization): """Test that conflicts on virtual providers as direct dependencies work""" s = default_mock_concretization("conflict-virtual") assert s.satisfies("%blas=netlib-blas") with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): default_mock_concretization("conflict-virtual +conflict_direct") with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): default_mock_concretization("conflict-virtual +conflict_transitive") def test_imposed_spec_dependency_duplication(mock_packages: spack.repo.Repo): """Tests that imposed dependencies triggered by identical conditions are grouped together, and that imposed dependencies that differ on a deptype are not grouped together.""" # The trigger-and-effect-deps pkg has 4 conditions, 2 triggers, and 4 effects in total: # +x -> depends on pkg-a with deptype link # +x -> depends on pkg-b with deptype link # +y -> depends on pkg-a with deptype run # +y -> depends on pkg-b with deptype run pkg = mock_packages.get_pkg_class("trigger-and-effect-deps") setup = spack.solver.asp.SpackSolverSetup() setup.gen = spack.solver.asp.ProblemInstanceBuilder() setup.package_dependencies_rules(pkg) setup.trigger_rules() setup.effect_rules() asp = setup.gen.asp_problem # There should be 4 conditions total assert len([line for line in asp if re.search(r"condition\(\d+\)", line)]) == 4 # There should be 2 triggers total assert len([line for line in asp if re.search(r"trigger_id\(\d+\)", line)]) == 2 # There should be 4 effects total assert len([line for line in asp if re.search(r"effect_id\(\d+\)", line)]) == 4 @pytest.mark.regression("51842") @pytest.mark.parametrize( "spec_str,expected", [ ("variant-function-validator", "generator=make %adios2~bzip2"), ("variant-function-validator generator=make", "generator=make %adios2~bzip2"), ("variant-function-validator generator=ninja", "generator=ninja %adios2+bzip2"), ("variant-function-validator generator=other", "generator=other %adios2+bzip2"), ], ) def test_penalties_for_variant_defined_by_function( default_mock_concretization, spec_str, expected ): """Tests that we have penalties for variants defined by functions, and that variant values are consistent with defaults and optimization rules. """ s = default_mock_concretization(spec_str) assert s.satisfies(expected) def test_default_values_used_if_subset_required_by_dependent(mock_packages): """If a dependent requires *at least* a subset of default values of a multi-valued variant of a dependency, that should not influence concretization; the default values should be used.""" # multivalue-variant-multi-defaults-dependent requires myvariant=bar without baz. a = spack.concretize.concretize_one("multivalue-variant-multi-defaults-dependent") # we still end up using baz, and we don't drop it to avoid an extra dependency. assert a.satisfies("%multivalue-variant-multi-defaults myvariant=bar,baz") def test_virtual_gets_multiple_dupes(mock_packages, config): """Tests that virtual packages always get multiple dupes, according to what we have in the configuration files. """ specs = [spack.spec.Spec("pkg-with-c-link-dep")] possible_graph = spack.solver.input_analysis.NoStaticAnalysis( configuration=spack.config.CONFIG, repo=spack.repo.PATH ) counter = spack.solver.input_analysis.MinimalDuplicatesCounter( specs, tests=False, possible_graph=possible_graph ) gen = spack.solver.asp.ProblemInstanceBuilder() counter.possible_packages_facts(gen, spack.solver.core.fn) asp = gen.asp_problem # "c" is a compiler language virtual and must allow multiple nodes, not be capped at 1 selected_lines = [line for line in asp if line.startswith('max_dupes("c"')] assert len(selected_lines) == 1 max_dupes_c = selected_lines[0] assert 'max_dupes("c",2).' == max_dupes_c, f"should have max_dupes=2, but got: {max_dupes_c}" def test_compiler_selection_when_external_has_variant_penalty(mutable_config, mock_packages): """Tests that a compiler that should be preferred is not swapped with a less preferred compiler because of penalties on variants. """ packages_yaml = syaml.load_config( """ packages: gcc:: externals: - spec: "gcc@15.2.0 languages='c,c++' ~binutils" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ llvm:: buildable: false externals: - spec: "llvm@20 +clang" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ """ ) mutable_config.set("packages", packages_yaml["packages"]) concrete = spack.concretize.concretize_one("libdwarf") # GCC is the preferred provider, but has a penalty on its variants assert concrete.satisfies("%gcc@15.2.0 ~binutils"), concrete.tree() # LLVM is the second provider choice, with no penalty on variants assert not concrete.satisfies("%llvm@20 +clang") def test_mpi_selection_when_external_has_variant_penalty(mutable_config, mock_packages): """Tests that conflicting with a default provider doesn't cause a variant values to be flipped to avoid the variant dependency. """ packages_yaml = syaml.load_config( """ packages: all: variants: +mpi mpich: buildable: false """ ) mutable_config.set("packages", packages_yaml["packages"]) concrete = spack.concretize.concretize_one("transitive-conditional-virtual-dependency") # GCC is the preferred provider, but has a penalty on its variants assert concrete.satisfies("%conditional-virtual-dependency+mpi"), concrete.tree() # LLVM is the second provider choice, with no penalty on variants assert concrete.satisfies("^mpi=zmpi") def test_preferring_different_compilers_for_different_languages(mutable_config, mock_packages): """Tests that in a case where we prefer different compilers for different languages, steering towards using a unique toolchain is lower priority with respect to flipping variants to turn off a language, or selecting a non-default provider. """ packages_yaml = syaml.load_config( """ packages: all: providers: c:: [llvm, gcc] cxx:: [llvm, gcc] fortran:: [gcc] c: prefer: - llvm cxx: prefer: - llvm fortran: prefer: - gcc mpileaks: variants: +fortran """ ) mutable_config.set("packages", packages_yaml["packages"]) mpileaks = spack.concretize.concretize_one("mpileaks") assert mpileaks.satisfies("%c,cxx=llvm %fortran=gcc"), mpileaks.tree() assert mpileaks.satisfies("%mpi=mpich") assert mpileaks["mpich"].satisfies("%c,cxx=llvm %fortran=gcc") ================================================ FILE: lib/spack/spack/test/concretization/errors.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Regression tests for concretizer error messages. Every test asserts two properties: 1. The correct exception type is raised. 2. The message contains every "actionable part" -- a string from the user's input (spec token, config key, package name) that helps identify what to change. """ import pathlib from io import StringIO from typing import List import pytest import spack.concretize import spack.config import spack.error import spack.main import spack.solver.asp import spack.spec version_error_messages = [ "Cannot satisfy", " required because quantum-espresso depends on fftw@:1.0", " required because quantum-espresso ^fftw@1.1: requested explicitly", " required because quantum-espresso ^fftw@1.1: requested explicitly", ] external_error_messages = [ "Cannot build quantum-espresso, since it is configured `buildable:false` and " "no externals satisfy the request" ] variant_error_messages = [ "'fftw' requires conflicting variant values '~mpi' and '+mpi'", " required because quantum-espresso depends on fftw+mpi when +invino", " required because quantum-espresso+invino ^fftw~mpi requested explicitly", " required because quantum-espresso+invino ^fftw~mpi requested explicitly", ] external_config = { "packages:quantum-espresso": { "buildable": False, "externals": [{"spec": "quantum-espresso@1.0~veritas", "prefix": "/path/to/qe"}], } } @pytest.mark.parametrize( "error_messages,config_set,spec", [ (version_error_messages, {}, "quantum-espresso^fftw@1.1:"), (external_error_messages, external_config, "quantum-espresso+veritas"), (variant_error_messages, {}, "quantum-espresso+invino^fftw~mpi"), ], ) def test_error_messages(error_messages, config_set, spec, mock_packages, mutable_config): for path, conf in config_set.items(): spack.config.set(path, conf) with pytest.raises(spack.solver.asp.UnsatisfiableSpecError) as e: _ = spack.concretize.concretize_one(spec) for em in error_messages: assert em in str(e.value), str(e.value) @pytest.mark.parametrize( "spec", ["deprecated-versions@1.1.0", "deprecated-client ^deprecated-versions@1.1.0"] ) def test_deprecated_version_error(spec, mock_packages, mutable_config): with pytest.raises(spack.solver.asp.DeprecatedVersionError, match="deprecated-versions@1.1.0"): _ = spack.concretize.concretize_one(spec) spack.config.set("config:deprecated", True) spack.concretize.concretize_one(spec) @pytest.mark.parametrize( "spec", ["deprecated-versions@99.9", "deprecated-client ^deprecated-versions@99.9"] ) def test_nonexistent_version_error(spec, mock_packages, mutable_config): with pytest.raises(spack.solver.asp.InvalidVersionError, match="deprecated-versions@99.9"): _ = spack.concretize.concretize_one(spec) def test_internal_error_handling_formatting(tmp_path: pathlib.Path): log = StringIO() input_to_output = [ (spack.spec.Spec("foo+x"), spack.spec.Spec("foo@=1.0~x")), (spack.spec.Spec("bar+y"), spack.spec.Spec("x@=1.0~y")), (spack.spec.Spec("baz+z"), None), ] spack.main._handle_solver_bug( spack.solver.asp.OutputDoesNotSatisfyInputError(input_to_output), root=tmp_path, out=log ) output = log.getvalue() assert "the following specs were not solved:\n - baz+z\n" in output assert ( "the following specs were concretized, but do not satisfy the input:\n" " - input: foo+x\n" " output: foo@=1.0~x\n" " - input: bar+y\n" " output: x@=1.0~y" ) in output files = {f.name: str(f) for f in tmp_path.glob("spack-asp-*/*.json")} assert {"input-1.json", "input-2.json", "output-1.json", "output-2.json"} == set(files.keys()) assert spack.spec.Spec.from_specfile(files["input-1.json"]) == spack.spec.Spec("foo+x") assert spack.spec.Spec.from_specfile(files["input-2.json"]) == spack.spec.Spec("bar+y") assert spack.spec.Spec.from_specfile(files["output-1.json"]) == spack.spec.Spec("foo@=1.0~x") assert spack.spec.Spec.from_specfile(files["output-2.json"]) == spack.spec.Spec("x@=1.0~y") def assert_actionable_error(exc_info, *required_part: str) -> None: """Verify that the error message contains every required part, which is usually a string that the user can recognize in their own input. """ msg = str(exc_info.value) missing = [h for h in required_part if h not in msg] assert not missing, f"Error message is missing parts {missing!r}\nFull message:\n{msg}" @pytest.mark.parametrize( "input_spec,expected_parts", [ # fftw is constrained to ~mpi by the explicit request, but quantum-espresso # requires fftw+mpi when +invino. Both values cannot coexist. pytest.param( "quantum-espresso+invino^fftw~mpi", ["fftw", "mpi"], id="variant_value_conflict" ), # The user requests a variant that does not exist on the package. pytest.param( "quantum-espresso+nonexistent", ["quantum-espresso", "nonexistent", "No such variant"], id="variant_undefined", ), # quantum-espresso has only version 1.0; @:0.1 cannot be satisfied. pytest.param( "quantum-espresso@:0.1", ["quantum-espresso@:0.1", "No version exists"], id="version_constraint_unsatisfied", ), # hypre propagates ~~shared to its deps, but openblas is explicitly +shared. pytest.param( "hypre ~~shared ^openblas +shared", ["shared", "hypre", "'openblas' requires conflicting variant values"], id="propagation_excluded", ), # dependency-foo-bar (++bar) and direct-dep-foo-bar (~~bar) both propagate # variant "bar" with different values to their shared transitive dependency. pytest.param( "parent-foo-bar ^dependency-foo-bar++bar ^direct-dep-foo-bar~~bar", ["cannot both propagate variant 'bar'"], id="propagation_conflict_to_dep", ), # gmake is a build dependency of a transitive dep, not directly reachable # via link/run from multivalue-variant. pytest.param( "multivalue-variant ^gmake", ["gmake is not a direct 'build' or"], id="literal_not_in_dag", ), # mvapich2 file_systems uses auto_or_any_combination_of, but "auto" and "lustre" # come from disjoint sets and cannot be combined. pytest.param( "mvapich2 file_systems=auto,lustre", ["mvapich2", "file_systems", "the value 'auto' is mutually exclusive"], id="variant_disjoint_sets", ), ], ) def test_input_spec_driven_errors( input_spec: str, expected_parts: List[str], mock_packages, mutable_config ) -> None: """Tests errors caused by a token in the CLI input spec. The message must name both the affected package and the specific token (variant, version, flag, dep) the user supplied. """ with pytest.raises(spack.error.SpackError) as exc_info: spack.concretize.concretize_one(input_spec) assert_actionable_error(exc_info, *expected_parts) @pytest.mark.parametrize( "packages_config,input_spec,expected_parts", [ # quantum-espresso is set buildable:false; the available external does not # satisfy +veritas, so no valid spec can be found. pytest.param( { "packages:quantum-espresso": { "buildable": False, "externals": [ {"spec": "quantum-espresso@1.0~veritas", "prefix": "/path/to/qe"} ], } }, "quantum-espresso+veritas", ["quantum-espresso", "it is configured `buildable:false`"], id="buildable_false", ), # The user provided a packages.yaml `require:` with a message field. The error must surface # the custom message so the user knows the policy and the package name so they can find # the config section. pytest.param( { "packages:libelf": { "require": [{"spec": "%clang", "message": "must be compiled with clang"}] } }, "libelf%gcc", ["libelf", "must be compiled with clang"], id="requirement_unsatisfied_custom_message", ), # Generic message must still name the package so the user knows which entry to look at pytest.param( {"packages:libelf": {"require": ["%clang"]}}, "libelf%gcc", ["libelf"], id="requirement_unsatisfied_generic", ), ], ) def test_config_driven_errors( packages_config, input_spec: str, expected_parts: List[str], mock_packages, mutable_config ) -> None: """Tests errors caused by user configuration, e,g, a setting in packages.yaml. The message must identify the package and the config value to fix. """ for path, conf in packages_config.items(): spack.config.set(path, conf) with pytest.raises(spack.error.SpackError) as exc_info: spack.concretize.concretize_one(input_spec) assert_actionable_error(exc_info, *expected_parts) @pytest.mark.parametrize( "input_spec,expected_handles", [ # conflict-parent@0.9 has conflicts("^conflict~foo", when="@0.9"). When the user requests # `^conflict~foo` the conflict fires. The auto-generated message includes the package name # and the when-spec version, giving the user two places to look. pytest.param( "conflict-parent@0.9 ^conflict~foo", ["conflict-parent", "'^conflict~foo' conflicts with '@0.9'"], id="conflicts_directive", ), # requires-clang has `requires("%clang", msg="can only be compiled with Clang")`. When # compiled with %gcc the requirement is unsatisfied and the custom message is shown pytest.param("requires-clang %gcc", ["requires-clang", "Clang"], id="requires_directive"), ], ) def test_package_py_driven_errors( input_spec: str, expected_handles: List[str], mock_packages, mutable_config ) -> None: """Tests errors involving directives in package.py recipes. The error message must name the package whose directive caused the failure. """ with pytest.raises(spack.error.SpackError) as exc_info: spack.concretize.concretize_one(input_spec) assert_actionable_error(exc_info, *expected_handles) ================================================ FILE: lib/spack/spack/test/concretization/flag_mixing.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ These tests include the following package DAGs: Firstly, w, x, y where w and x apply cflags to y. w |\ x | |/ y Secondly, v, y which where v does not apply cflags to y - this is for testing mixing with compiler flag propagation in the absence of compiler flags applied by dependents. v | y Finally, a diamond dag to check that the topological order is resolved into a total order: t |\ u x |/ y """ import pathlib import pytest import spack.concretize import spack.config import spack.environment as ev import spack.paths import spack.repo import spack.spec import spack.util.spack_yaml as syaml @pytest.fixture def test_repo(mutable_config, monkeypatch, mock_stage): repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "flags_test" with spack.repo.use_repositories(str(repo_dir)) as mock_packages_repo: yield mock_packages_repo def update_concretize_scope(conf_str, section): conf = syaml.load_config(conf_str) spack.config.set(section, conf[section], scope="concretize") def test_mix_spec_and_requirements(concretize_scope, test_repo): conf_str = """\ packages: y: require: cflags="-c" """ update_concretize_scope(conf_str, "packages") s1 = spack.concretize.concretize_one('y cflags="-a"') assert s1.satisfies('cflags="-a -c"') def test_mix_spec_and_dependent(concretize_scope, test_repo): s1 = spack.concretize.concretize_one('x ^y cflags="-a"') assert s1["y"].satisfies('cflags="-a -d1"') def _compiler_cfg_one_entry_with_cflags(cflags): return f"""\ packages: gcc: externals: - spec: gcc@12.100.100 languages:=c,c++ prefix: /fake extra_attributes: compilers: c: /fake/bin/gcc cxx: /fake/bin/g++ flags: cflags: {cflags} """ def test_mix_spec_and_compiler_cfg(concretize_scope, test_repo): conf_str = _compiler_cfg_one_entry_with_cflags("-Wall") update_concretize_scope(conf_str, "packages") s1 = spack.concretize.concretize_one('y cflags="-O2" %gcc@12.100.100') assert s1.satisfies('cflags="-Wall -O2"') def test_pkg_flags_from_compiler_and_none(concretize_scope, mock_packages): packages_yaml = f""" {_compiler_cfg_one_entry_with_cflags("-Wall")} llvm: externals: - spec: llvm+clang@19.1.0 prefix: /fake extra_attributes: compilers: c: /fake/bin/clang cxx: /fake/bin/clang++ """ update_concretize_scope(packages_yaml, "packages") s1 = spack.spec.Spec("cmake%gcc@12.100.100") s2 = spack.spec.Spec("cmake-client^cmake%clang@19.1.0") concrete = dict(spack.concretize.concretize_together([(s1, None), (s2, None)])) assert concrete[s1].compiler_flags["cflags"] == ["-Wall"] assert concrete[s2]["cmake"].compiler_flags["cflags"] == [] @pytest.mark.parametrize( "cmd_flags,req_flags,cmp_flags,dflags,expected_order", [ ("-a -b", "-c", None, False, "-c -a -b"), ("-x7 -x4", "-x5 -x6", None, False, "-x5 -x6 -x7 -x4"), ("-x7 -x4", "-x5 -x6", "-x3 -x8", False, "-x3 -x8 -x5 -x6 -x7 -x4"), ("-x7 -x4", "-x5 -x6", "-x3 -x8", True, "-x3 -x8 -d1 -d2 -x5 -x6 -x7 -x4"), ("-x7 -x4", None, "-x3 -x8", False, "-x3 -x8 -x7 -x4"), ("-x7 -x4", None, "-x3 -x8", True, "-x3 -x8 -d1 -d2 -x7 -x4"), # The remaining test cover cases of intersection ("-a -b", "-a -c", None, False, "-c -a -b"), ("-a -b", None, "-a -c", False, "-c -a -b"), ("-a -b", "-a -c", "-a -d", False, "-d -c -a -b"), ("-a -d2 -d1", "-d2 -c", "-d1 -b", True, "-b -c -a -d2 -d1"), ("-a", "-d0 -d2 -c", "-d1 -b", True, "-b -d1 -d0 -d2 -c -a"), ], ) def test_flag_order_and_grouping( concretize_scope, test_repo, cmd_flags, req_flags, cmp_flags, dflags, expected_order ): """Check consistent flag ordering and grouping on a package "y" with flags introduced from a variety of sources. The ordering rules are explained in ``asp.SpecBuilder.reorder_flags``. """ conf_str = """ packages: """ if cmp_flags: conf_str = _compiler_cfg_one_entry_with_cflags(cmp_flags) if req_flags: conf_str = f"""\ {conf_str} y: require: cflags="{req_flags}" """ update_concretize_scope(conf_str, "packages") compiler_spec = "" if cmp_flags: compiler_spec = "%gcc@12.100.100" cmd_flags_str = f'cflags="{cmd_flags}"' if cmd_flags else "" if dflags: spec_str = f"x+activatemultiflag {compiler_spec} ^y {cmd_flags_str}" expected_dflags = "-d1 -d2" else: spec_str = f"y {cmd_flags_str} {compiler_spec}" expected_dflags = None root_spec = spack.concretize.concretize_one(spec_str) spec = root_spec["y"] satisfy_flags = " ".join(x for x in [cmd_flags, req_flags, cmp_flags, expected_dflags] if x) assert spec.satisfies(f'cflags="{satisfy_flags}"') assert spec.compiler_flags["cflags"] == expected_order.split() def test_two_dependents_flag_mixing(concretize_scope, test_repo): root_spec1 = spack.concretize.concretize_one("w~moveflaglater") spec1 = root_spec1["y"] assert spec1.compiler_flags["cflags"] == "-d0 -d1 -d2".split() root_spec2 = spack.concretize.concretize_one("w+moveflaglater") spec2 = root_spec2["y"] assert spec2.compiler_flags["cflags"] == "-d3 -d1 -d2".split() def test_propagate_and_compiler_cfg(concretize_scope, test_repo): conf_str = _compiler_cfg_one_entry_with_cflags("-f2") update_concretize_scope(conf_str, "packages") root_spec = spack.concretize.concretize_one("v cflags=='-f1' %gcc@12.100.100") assert root_spec["y"].satisfies("cflags='-f1 -f2'") def test_propagate_and_pkg_dep(concretize_scope, test_repo): root_spec1 = spack.concretize.concretize_one("x ~activatemultiflag cflags=='-f1'") assert root_spec1["y"].satisfies("cflags='-f1 -d1'") def test_propagate_and_require(concretize_scope, test_repo): conf_str = """\ packages: y: require: cflags="-f2" """ update_concretize_scope(conf_str, "packages") root_spec1 = spack.concretize.concretize_one("v cflags=='-f1'") assert root_spec1["y"].satisfies("cflags='-f1 -f2'") # Next, check that a requirement does not "undo" a request for # propagation from the command-line spec conf_str = """\ packages: v: require: cflags="-f1" """ update_concretize_scope(conf_str, "packages") root_spec2 = spack.concretize.concretize_one("v cflags=='-f1'") assert root_spec2["y"].satisfies("cflags='-f1'") # Note: requirements cannot enforce propagation: any attempt to do # so will generate a concretization error; this likely relates to # the note about #37180 in concretize.lp def test_dev_mix_flags(tmp_path: pathlib.Path, concretize_scope, mutable_mock_env_path, test_repo): src_dir = tmp_path / "x-src" env_content = f"""\ spack: specs: - y cflags=='-fsanitize=address' %gcc@12.100.100 develop: y: spec: y cflags=='-fsanitize=address' path: {src_dir} """ conf_str = _compiler_cfg_one_entry_with_cflags("-f1") update_concretize_scope(conf_str, "packages") manifest_file = tmp_path / ev.manifest_name manifest_file.write_text(env_content) e = ev.create("test", manifest_file) with e: e.concretize() e.write() (result,) = list(j for i, j in e.concretized_specs() if j.name == "y") assert result["y"].satisfies("cflags='-fsanitize=address -f1'") def test_diamond_dep_flag_mixing(concretize_scope, test_repo): """A diamond where each dependent applies flags to the bottom dependency. The goal is to ensure that the flag ordering is (a) topological and (b) repeatable for elements not subject to this partial ordering (i.e. the flags for the left and right nodes of the diamond always appear in the same order). `Spec.traverse` is responsible for handling both of these needs. """ root_spec1 = spack.concretize.concretize_one("t") spec1 = root_spec1["y"] assert spec1.satisfies('cflags="-c1 -c2 -d1 -d2 -e1 -e2"') assert spec1.compiler_flags["cflags"] == "-c1 -c2 -e1 -e2 -d1 -d2".split() def test_flag_injection_different_compilers(mock_packages, mutable_config): """Tests that flag propagation is not activated on nodes with a compiler that is different from the propagation source. """ s = spack.concretize.concretize_one('mpileaks cflags=="-O2" %gcc ^callpath %llvm') assert s.satisfies('cflags="-O2"') and s["c"].name == "gcc" assert not s["callpath"].satisfies('cflags="-O2"') and s["callpath"]["c"].name == "llvm" @pytest.mark.regression("51209") @pytest.mark.parametrize( "spec_str,expected,not_expected", [ # gcc using flags compiled with another gcc not using flags ("gcc@14 cflags='-O3'", ["gcc@14 cflags='-O3'", "%gcc@10"], ["%gcc cflags='-O3'"]), # Parent and child, imposing different flags on gmake ( "7zip-dependent %gmake cflags='-O2' ^7zip %gmake cflags='-g'", ["%gmake cflags='-O2'", "^7zip %gmake cflags='-g'"], ["%gmake cflags='-g'"], ), ], ) def test_flags_and_duplicate_nodes(spec_str, expected, not_expected, default_mock_concretization): """Tests that we can concretize a spec with flags on a node that is present with duplicates in the DAG. For instance, a compiler built with a previous version of itself. """ s = default_mock_concretization(spec_str) assert all(s.satisfies(x) for x in expected) assert all(not s.satisfies(x) for x in not_expected) ================================================ FILE: lib/spack/spack/test/concretization/preferences.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import stat import pytest import spack.concretize import spack.config import spack.package_prefs import spack.repo import spack.util.module_cmd import spack.util.spack_yaml as syaml from spack.error import ConfigError from spack.spec import Spec from spack.version import Version @pytest.fixture() def configure_permissions(): conf = syaml.load_config( """\ all: permissions: read: group write: group group: all mpich: permissions: read: user write: user mpileaks: permissions: write: user group: mpileaks callpath: permissions: write: world """ ) spack.config.set("packages", conf, scope="concretize") yield def concretize(abstract_spec): return spack.concretize.concretize_one(abstract_spec) def update_packages(pkgname, section, value): """Update config and reread package list""" conf = {pkgname: {section: value}} spack.config.set("packages", conf, scope="concretize") def assert_variant_values(spec, **variants): concrete = concretize(spec) for variant, value in variants.items(): assert concrete.variants[variant].value == value @pytest.mark.usefixtures("concretize_scope", "mock_packages") class TestConcretizePreferences: @pytest.mark.parametrize( "package_name,variant_value,expected_results", [ ( "mpileaks", "~debug~opt+shared+static", {"debug": False, "opt": False, "shared": True, "static": True}, ), # Check that using a list of variants instead of a single string works ( "mpileaks", ["~debug", "~opt", "+shared", "+static"], {"debug": False, "opt": False, "shared": True, "static": True}, ), # Use different values for the variants and check them again ( "mpileaks", ["+debug", "+opt", "~shared", "-static"], {"debug": True, "opt": True, "shared": False, "static": False}, ), # Check a multivalued variant with multiple values set ( "multivalue-variant", ["foo=bar,baz", "fee=bar"], {"foo": ("bar", "baz"), "fee": "bar"}, ), ("singlevalue-variant", ["fum=why"], {"fum": "why"}), ], ) def test_preferred_variants(self, package_name, variant_value, expected_results): """Test preferred variants are applied correctly""" update_packages(package_name, "variants", variant_value) assert_variant_values(package_name, **expected_results) @pytest.mark.regression("50921") @pytest.mark.parametrize("config_type", ("require", "prefer")) def test_preferred_commit_variant(self, config_type): """Tests that we can use auto-variants in requirements and preferences.""" commit_value = "b" * 40 name = "git-ref-package" value = f"commit={commit_value}" update_packages(name, config_type, [value]) assert_variant_values(name, **{"commit": commit_value}) def test_preferred_variants_from_wildcard(self): """ Test that 'foo=*' concretizes to any value """ update_packages("multivalue-variant", "variants", "foo=bar") assert_variant_values("multivalue-variant foo=*", foo=("bar",)) def test_preferred_target(self, mutable_mock_repo): """Test preferred targets are applied correctly""" spec = concretize("mpich") default = str(spec.target) preferred = str(spec.target.family) update_packages("all", "target", [preferred]) spec = concretize("mpich") assert str(spec.target) == preferred spec = concretize("mpileaks") assert str(spec["mpileaks"].target) == preferred assert str(spec["mpi"].target) == preferred update_packages("all", "target", [default]) spec = concretize("mpileaks") assert str(spec["mpileaks"].target) == default assert str(spec["mpi"].target) == default def test_preferred_versions(self): """Test preferred package versions are applied correctly""" update_packages("mpileaks", "version", ["2.3"]) spec = concretize("mpileaks") assert spec.version == Version("2.3") update_packages("mpileaks", "version", ["2.2"]) spec = concretize("mpileaks") assert spec.version == Version("2.2") def test_preferred_versions_mixed_version_types(self): update_packages("mixedversions", "version", ["=2.0"]) spec = concretize("mixedversions") assert spec.version == Version("2.0") def test_preferred_providers(self): """Test preferred providers of virtual packages are applied correctly """ update_packages("all", "providers", {"mpi": ["mpich"]}) spec = concretize("mpileaks") assert "mpich" in spec update_packages("all", "providers", {"mpi": ["zmpi"]}) spec = concretize("mpileaks") assert "zmpi" in spec @pytest.mark.parametrize( "update,expected", [ ( {"url": "http://www.somewhereelse.com/mpileaks-1.0.tar.gz"}, "http://www.somewhereelse.com/mpileaks-2.3.tar.gz", ), ({}, "http://www.spack.llnl.gov/mpileaks-2.3.tar.gz"), ], ) def test_config_set_pkg_property_url(self, update, expected, mock_packages_repo): """Test setting an existing attribute in the package class""" update_packages("mpileaks", "package_attributes", update) with spack.repo.use_repositories(mock_packages_repo): spec = concretize("mpileaks") assert spec.package.fetcher.url == expected def test_config_set_pkg_property_new(self, mock_packages_repo): """Test that you can set arbitrary attributes on the Package class""" conf = syaml.load_config( """\ mpileaks: package_attributes: v1: 1 v2: true v3: yesterday v4: "true" v5: x: 1 y: 2 v6: - 1 - 2 """ ) spack.config.set("packages", conf, scope="concretize") with spack.repo.use_repositories(mock_packages_repo): spec = concretize("mpileaks") assert spec.package.v1 == 1 assert spec.package.v2 is True assert spec.package.v3 == "yesterday" assert spec.package.v4 == "true" assert dict(spec.package.v5) == {"x": 1, "y": 2} assert list(spec.package.v6) == [1, 2] update_packages("mpileaks", "package_attributes", {}) with spack.repo.use_repositories(mock_packages_repo): spec = concretize("mpileaks") with pytest.raises(AttributeError): spec.package.v1 def test_preferred(self): """ "Test packages with some version marked as preferred=True""" spec = spack.concretize.concretize_one("python") assert spec.version == Version("2.7.11") # now add packages.yaml with versions other than preferred # ensure that once config is in place, non-preferred version is used update_packages("python", "version", ["3.5.0"]) spec = spack.concretize.concretize_one("python") assert spec.version == Version("3.5.0") def test_preferred_undefined_raises(self): """Preference should not specify an undefined version""" update_packages("python", "version", ["3.5.0.1"]) spec = Spec("python") with pytest.raises(ConfigError): spack.concretize.concretize_one(spec) def test_preferred_truncated(self): """Versions without "=" are treated as version ranges: if there is a satisfying version defined in the package.py, we should use that (don't define a new version). """ update_packages("python", "version", ["3.5"]) spec = spack.concretize.concretize_one("python") assert spec.satisfies("@3.5.1") def test_develop(self): """Test concretization with develop-like versions""" spec = spack.concretize.concretize_one("develop-test") assert spec.version == Version("0.2.15") spec = spack.concretize.concretize_one("develop-test2") assert spec.version == Version("0.2.15") # now add packages.yaml with develop-like versions # ensure that once config is in place, develop-like version is used update_packages("develop-test", "version", ["develop"]) spec = spack.concretize.concretize_one("develop-test") assert spec.version == Version("develop") update_packages("develop-test2", "version", ["0.2.15.develop"]) spec = spack.concretize.concretize_one("develop-test2") assert spec.version == Version("0.2.15.develop") def test_external_mpi(self): # make sure this doesn't give us an external first. spec = spack.concretize.concretize_one("mpi") assert not spec.external and spec.package.provides("mpi") # load config conf = syaml.load_config( """\ all: providers: mpi: [mpich] mpich: buildable: false externals: - spec: mpich@3.0.4 prefix: /dummy/path """ ) spack.config.set("packages", conf, scope="concretize") # ensure that once config is in place, external is used spec = spack.concretize.concretize_one("mpi") assert spec["mpich"].external_path == os.path.sep + os.path.join("dummy", "path") def test_external_module(self, monkeypatch): """Test that packages can find externals specified by module The specific code for parsing the module is tested elsewhere. This just tests that the preference is accounted for""" # make sure this doesn't give us an external first. def mock_module(cmd, module): return "prepend-path PATH /dummy/path" monkeypatch.setattr(spack.util.module_cmd, "module", mock_module) spec = spack.concretize.concretize_one("mpi") assert not spec.external and spec.package.provides("mpi") # load config conf = syaml.load_config( """\ all: providers: mpi: [mpich] mpi: buildable: false externals: - spec: mpich@3.0.4 modules: [dummy] """ ) spack.config.set("packages", conf, scope="concretize") # ensure that once config is in place, external is used spec = spack.concretize.concretize_one("mpi") assert spec["mpich"].external_path == os.path.sep + os.path.join("dummy", "path") def test_buildable_false(self): conf = syaml.load_config( """\ libelf: buildable: false """ ) spack.config.set("packages", conf, scope="concretize") spec = Spec("libelf") assert not spack.package_prefs.is_spec_buildable(spec) spec = Spec("mpich") assert spack.package_prefs.is_spec_buildable(spec) def test_buildable_false_virtual(self): conf = syaml.load_config( """\ mpi: buildable: false """ ) spack.config.set("packages", conf, scope="concretize") spec = Spec("libelf") assert spack.package_prefs.is_spec_buildable(spec) spec = Spec("mpich") assert not spack.package_prefs.is_spec_buildable(spec) def test_buildable_false_all(self): conf = syaml.load_config( """\ all: buildable: false """ ) spack.config.set("packages", conf, scope="concretize") spec = Spec("libelf") assert not spack.package_prefs.is_spec_buildable(spec) spec = Spec("mpich") assert not spack.package_prefs.is_spec_buildable(spec) def test_buildable_false_all_true_package(self): conf = syaml.load_config( """\ all: buildable: false libelf: buildable: true """ ) spack.config.set("packages", conf, scope="concretize") spec = Spec("libelf") assert spack.package_prefs.is_spec_buildable(spec) spec = Spec("mpich") assert not spack.package_prefs.is_spec_buildable(spec) def test_buildable_false_all_true_virtual(self): conf = syaml.load_config( """\ all: buildable: false mpi: buildable: true """ ) spack.config.set("packages", conf, scope="concretize") spec = Spec("libelf") assert not spack.package_prefs.is_spec_buildable(spec) spec = Spec("mpich") assert spack.package_prefs.is_spec_buildable(spec) def test_buildable_false_virtual_true_pacakge(self): conf = syaml.load_config( """\ mpi: buildable: false mpich: buildable: true """ ) spack.config.set("packages", conf, scope="concretize") spec = Spec("zmpi") assert not spack.package_prefs.is_spec_buildable(spec) spec = Spec("mpich") assert spack.package_prefs.is_spec_buildable(spec) def test_config_permissions_from_all(self, configure_permissions): # Although these aren't strictly about concretization, they are # configured in the same file and therefore convenient to test here. # Make sure we can configure readable and writable # Test inheriting from 'all' spec = Spec("zmpi") perms = spack.package_prefs.get_package_permissions(spec) assert perms == stat.S_IRWXU | stat.S_IRWXG dir_perms = spack.package_prefs.get_package_dir_permissions(spec) assert dir_perms == stat.S_IRWXU | stat.S_IRWXG | stat.S_ISGID group = spack.package_prefs.get_package_group(spec) assert group == "all" def test_config_permissions_from_package(self, configure_permissions): # Test overriding 'all' spec = Spec("mpich") perms = spack.package_prefs.get_package_permissions(spec) assert perms == stat.S_IRWXU dir_perms = spack.package_prefs.get_package_dir_permissions(spec) assert dir_perms == stat.S_IRWXU group = spack.package_prefs.get_package_group(spec) assert group == "all" def test_config_permissions_differ_read_write(self, configure_permissions): # Test overriding group from 'all' and different readable/writable spec = Spec("mpileaks") perms = spack.package_prefs.get_package_permissions(spec) assert perms == stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP dir_perms = spack.package_prefs.get_package_dir_permissions(spec) expected = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_ISGID assert dir_perms == expected group = spack.package_prefs.get_package_group(spec) assert group == "mpileaks" def test_config_perms_fail_write_gt_read(self, configure_permissions): # Test failure for writable more permissive than readable spec = Spec("callpath") with pytest.raises(ConfigError): spack.package_prefs.get_package_permissions(spec) @pytest.mark.regression("20040") def test_variant_not_flipped_to_pull_externals(self): """Test that a package doesn't prefer pulling in an external to using the default value of a variant. """ s = spack.concretize.concretize_one("vdefault-or-external-root") assert "~external" in s["vdefault-or-external"] assert "externaltool" not in s @pytest.mark.regression("25585") def test_dependencies_cant_make_version_parent_score_better(self): """Test that a package can't select a worse version for a dependent because doing so it can pull-in a dependency that makes the overall version score even or better and maybe has a better score in some lower priority criteria. """ s = spack.concretize.concretize_one("version-test-root") assert s.satisfies("^version-test-pkg@2.4.6") assert "version-test-dependency-preferred" not in s @pytest.mark.regression("26598") def test_multivalued_variants_are_lower_priority_than_providers(self): """Test that the rule to maximize the number of values for multivalued variants is considered at lower priority than selecting the default provider for virtual dependencies. This ensures that we don't e.g. select openmpi over mpich even if we specified mpich as the default mpi provider, just because openmpi supports more fabrics by default. """ with spack.config.override( "packages:all", {"providers": {"somevirtual": ["some-virtual-preferred"]}} ): s = spack.concretize.concretize_one("somevirtual") assert s.name == "some-virtual-preferred" @pytest.mark.regression("26721,19736") def test_sticky_variant_accounts_for_packages_yaml(self): with spack.config.override("packages:sticky-variant", {"variants": "+allow-gcc"}): s = spack.concretize.concretize_one("sticky-variant %gcc") assert s.satisfies("%gcc") and s.satisfies("+allow-gcc") @pytest.mark.regression("41134") def test_default_preference_variant_different_type_does_not_error(self): """Tests that a different type for an existing variant in the 'all:' section of packages.yaml doesn't fail with an error. """ with spack.config.override("packages:all", {"variants": "+foo"}): s = spack.concretize.concretize_one("pkg-a") assert s.satisfies("foo=bar") def test_version_preference_cannot_generate_buildable_versions(self): """Tests that a version preference not mentioned in package.py cannot be used in a built spec. """ mpileaks_external = syaml.load_config( """ mpileaks: # Version 0.9 is not mentioned in package.py version: ["0.9"] buildable: true externals: - spec: mpileaks@0.9 +debug prefix: /path """ ) with spack.config.override("packages", mpileaks_external): # Asking for mpileaks+debug results in the external being chosen mpileaks = spack.concretize.concretize_one("mpileaks+debug") assert mpileaks.external and mpileaks.satisfies("@0.9 +debug") # Asking for ~debug results in the highest known version being chosen mpileaks = spack.concretize.concretize_one("mpileaks~debug") assert not mpileaks.external and mpileaks.satisfies("@2.3 ~debug") ================================================ FILE: lib/spack/spack/test/concretization/requirements.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.concretize import spack.config import spack.error import spack.installer import spack.package_base import spack.paths import spack.platforms import spack.repo import spack.solver.asp import spack.spec import spack.store import spack.util.spack_yaml as syaml import spack.version from spack.installer import PackageInstaller from spack.solver.asp import InternalConcretizerError, UnsatisfiableSpecError from spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml from spack.solver.runtimes import external_config_with_implicit_externals from spack.spec import Spec from spack.util.url import path_to_file_url def update_packages_config(conf_str): conf = syaml.load_config(conf_str) spack.config.set("packages", conf["packages"], scope="concretize") @pytest.fixture def test_repo(mutable_config, monkeypatch, mock_stage): repo_dir = pathlib.Path(spack.paths.test_repos_path) / "spack_repo" / "requirements_test" with spack.repo.use_repositories(str(repo_dir)) as mock_packages_repo: yield mock_packages_repo def test_one_package_multiple_reqs(concretize_scope, test_repo): conf_str = """\ packages: y: require: - "@2.4" - "~shared" """ update_packages_config(conf_str) y_spec = spack.concretize.concretize_one("y") assert y_spec.satisfies("@2.4~shared") def test_requirement_isnt_optional(concretize_scope, test_repo): """If a user spec requests something that directly conflicts with a requirement, make sure we get an error. """ conf_str = """\ packages: x: require: "@1.0" """ update_packages_config(conf_str) with pytest.raises(UnsatisfiableSpecError): spack.concretize.concretize_one("x@1.1") def test_require_undefined_version(concretize_scope, test_repo): """If a requirement specifies a numbered version that isn't in the associated package.py and isn't part of a Git hash equivalence (hash=number), then Spack should raise an error (it is assumed this is a typo, and raising the error here avoids a likely error when Spack attempts to fetch the version). """ conf_str = """\ packages: x: require: "@1.2" """ update_packages_config(conf_str) with pytest.raises(spack.error.ConfigError): spack.concretize.concretize_one("x") def test_require_truncated(concretize_scope, test_repo): """A requirement specifies a version range, with satisfying versions defined in the package.py. Make sure we choose one of the defined versions (vs. allowing the requirement to define a new version). """ conf_str = """\ packages: x: require: "@1" """ update_packages_config(conf_str) xspec = spack.concretize.concretize_one("x") assert xspec.satisfies("@1.1") def test_git_user_supplied_reference_satisfaction( concretize_scope, test_repo, mock_git_version_info, monkeypatch ): repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False ) hash_eq_ver = Spec(f"v@{commits[0]}=2.2") hash_eq_ver_copy = Spec(f"v@{commits[0]}=2.2") just_hash = Spec(f"v@{commits[0]}") just_ver = Spec("v@=2.2") hash_eq_other_ver = Spec(f"v@{commits[0]}=2.3") assert not hash_eq_ver == just_hash assert not hash_eq_ver.satisfies(just_hash) assert not hash_eq_ver.intersects(just_hash) # Git versions and literal versions are distinct versions, like # pkg@10.1.0 and pkg@10.1.0-suffix are distinct versions. assert not hash_eq_ver.satisfies(just_ver) assert not just_ver.satisfies(hash_eq_ver) assert not hash_eq_ver.intersects(just_ver) assert hash_eq_ver != just_ver assert just_ver != hash_eq_ver assert not hash_eq_ver == just_ver assert not just_ver == hash_eq_ver # When a different version is associated, they're not equal assert not hash_eq_ver.satisfies(hash_eq_other_ver) assert not hash_eq_other_ver.satisfies(hash_eq_ver) assert not hash_eq_ver.intersects(hash_eq_other_ver) assert not hash_eq_other_ver.intersects(hash_eq_ver) assert hash_eq_ver != hash_eq_other_ver assert hash_eq_other_ver != hash_eq_ver assert not hash_eq_ver == hash_eq_other_ver assert not hash_eq_other_ver == hash_eq_ver # These should be equal assert hash_eq_ver == hash_eq_ver_copy assert not hash_eq_ver != hash_eq_ver_copy assert hash_eq_ver.satisfies(hash_eq_ver_copy) assert hash_eq_ver_copy.satisfies(hash_eq_ver) assert hash_eq_ver.intersects(hash_eq_ver_copy) assert hash_eq_ver_copy.intersects(hash_eq_ver) def test_requirement_adds_new_version( concretize_scope, test_repo, mock_git_version_info, monkeypatch ): repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False ) a_commit_hash = commits[0] conf_str = """\ packages: v: require: "@{0}=2.2" """.format(a_commit_hash) update_packages_config(conf_str) s1 = spack.concretize.concretize_one("v") assert s1.satisfies("@2.2") # Make sure the git commit info is retained assert isinstance(s1.version, spack.version.GitVersion) assert s1.version.ref == a_commit_hash def test_requirement_adds_version_satisfies( concretize_scope, test_repo, mock_git_version_info, monkeypatch ): """Make sure that new versions added by requirements are factored into conditions. In this case create a new version that satisfies a depends_on condition and make sure it is triggered (i.e. the dependency is added). """ repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False ) # Sanity check: early version of T does not include U s0 = spack.concretize.concretize_one("t@2.0") assert "u" not in s0 conf_str = """\ packages: t: require: "@{0}=2.2" """.format(commits[0]) update_packages_config(conf_str) s1 = spack.concretize.concretize_one("t") assert "u" in s1 assert s1.satisfies("@2.2") @pytest.mark.parametrize("require_checksum", (True, False)) def test_requirement_adds_git_hash_version( require_checksum, concretize_scope, test_repo, mock_git_version_info, monkeypatch ): # A full commit sha is a checksummed version, so this test should pass in both cases if require_checksum: monkeypatch.setenv("SPACK_CONCRETIZER_REQUIRE_CHECKSUM", "yes") repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False ) a_commit_hash = commits[0] conf_str = f"""\ packages: v: require: "@{a_commit_hash}" """ update_packages_config(conf_str) s1 = spack.concretize.concretize_one("v") assert isinstance(s1.version, spack.version.GitVersion) assert s1.satisfies(f"v@{a_commit_hash}") def test_requirement_adds_multiple_new_versions( concretize_scope, test_repo, mock_git_version_info, monkeypatch ): repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False ) conf_str = f"""\ packages: v: require: - one_of: ["@{commits[0]}=2.2", "@{commits[1]}=2.3"] """ update_packages_config(conf_str) assert spack.concretize.concretize_one("v").satisfies(f"@{commits[0]}=2.2") assert spack.concretize.concretize_one("v@2.3").satisfies(f"v@{commits[1]}=2.3") # TODO: this belongs in the concretize_preferences test module but uses # fixtures defined only here def test_preference_adds_new_version( concretize_scope, test_repo, mock_git_version_info, monkeypatch ): """Normally a preference cannot define a new version, but that constraint is ignored if the version is a Git hash-based version. """ repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", path_to_file_url(repo_path), raising=False ) conf_str = f"""\ packages: v: version: ["{commits[0]}=2.2", "{commits[1]}=2.3"] """ update_packages_config(conf_str) assert spack.concretize.concretize_one("v").satisfies(f"@{commits[0]}=2.2") assert spack.concretize.concretize_one("v@2.3").satisfies(f"@{commits[1]}=2.3") # When installing by hash, a lookup is triggered, so it's not mapped to =2.3. s3 = spack.concretize.concretize_one(f"v@{commits[1]}") assert s3.satisfies(f"v@{commits[1]}") assert not s3.satisfies("@2.3") def test_external_adds_new_version_that_is_preferred(concretize_scope, test_repo): """Test that we can use a version, not declared in package recipe, as the preferred version if that version appears in an external spec. """ conf_str = """\ packages: y: version: ["2.7"] externals: - spec: y@2.7 # Not defined in y prefix: /fake/nonexistent/path/ buildable: false """ update_packages_config(conf_str) spec = spack.concretize.concretize_one("x") assert spec["y"].satisfies("@2.7") assert spack.version.Version("2.7") not in spec["y"].package.versions def test_requirement_is_successfully_applied(concretize_scope, test_repo): """If a simple requirement can be satisfied, make sure the concretization succeeds and the requirement spec is applied. """ s1 = spack.concretize.concretize_one("x") # Without any requirements/preferences, the later version is preferred assert s1.satisfies("@1.1") conf_str = """\ packages: x: require: "@1.0" """ update_packages_config(conf_str) s2 = spack.concretize.concretize_one("x") # The requirement forces choosing the earlier version assert s2.satisfies("@1.0") def test_require_hash(mock_fetch, install_mockery, concretize_scope, test_repo): """Apply a requirement to use a specific hash. Install multiple hashes to ensure non-default concretization""" s1 = spack.concretize.concretize_one("x@1.1") s2 = spack.concretize.concretize_one("x@1.0") builder = spack.installer.PackageInstaller([s1.package, s2.package], fake=True) builder.install() conf_str = f"""\ packages: x: require: x/{s2.dag_hash()} """ update_packages_config(conf_str) test_spec = spack.concretize.concretize_one("x") assert test_spec == s2 def test_multiple_packages_requirements_are_respected(concretize_scope, test_repo): """Apply requirements to two packages; make sure the concretization succeeds and both requirements are respected. """ conf_str = """\ packages: x: require: "@1.0" y: require: "@2.4" """ update_packages_config(conf_str) spec = spack.concretize.concretize_one("x") assert spec["x"].satisfies("@1.0") assert spec["y"].satisfies("@2.4") def test_oneof(concretize_scope, test_repo): """'one_of' allows forcing the concretizer to satisfy one of the specs in the group (but not all have to be satisfied). """ conf_str = """\ packages: y: require: - one_of: ["@2.4", "~shared"] """ update_packages_config(conf_str) spec = spack.concretize.concretize_one("x") # The concretizer only has to satisfy one of @2.4/~shared, and @2.4 # comes first so it is prioritized assert spec["y"].satisfies("@2.4+shared") def test_one_package_multiple_oneof_groups(concretize_scope, test_repo): """One package has two 'one_of' groups; check that both are applied. """ conf_str = """\ packages: y: require: - one_of: ["@2.4%gcc", "@2.5%clang"] - one_of: ["@2.5~shared", "@2.4+shared"] """ update_packages_config(conf_str) s1 = spack.concretize.concretize_one("y@2.5") assert s1.satisfies("~shared%clang") s2 = spack.concretize.concretize_one("y@2.4") assert s2.satisfies("+shared%gcc") @pytest.mark.regression("34241") def test_require_cflags(concretize_scope, mock_packages): """Ensures that flags can be required from configuration.""" conf_str = """\ packages: mpich2: require: cflags="-g" mpi: require: mpich cflags="-O1" """ update_packages_config(conf_str) mpich2 = spack.concretize.concretize_one("mpich2") assert mpich2.satisfies("cflags=-g") mpileaks = spack.concretize.concretize_one("mpileaks") assert mpileaks["mpi"].satisfies("mpich cflags=-O1") mpi = spack.concretize.concretize_one("mpi") assert mpi.satisfies("mpich cflags=-O1") def test_requirements_for_package_that_is_not_needed(concretize_scope, test_repo): """Specify requirements for specs that are not concretized or a dependency of a concretized spec (in other words, none of the requirements are used for the requested spec). """ # Note that the exact contents aren't important since this isn't # intended to be used, but the important thing is that a number of # packages have requirements applied conf_str = """\ packages: x: require: "@1.0" y: require: - one_of: ["@2.4%gcc", "@2.5%clang"] - one_of: ["@2.5~shared", "@2.4+shared"] """ update_packages_config(conf_str) s1 = spack.concretize.concretize_one("v") assert s1.satisfies("@2.1") def test_oneof_ordering(concretize_scope, test_repo): """Ensure that earlier elements of 'one_of' have higher priority. This priority should override default priority (e.g. choosing later versions). """ conf_str = """\ packages: y: require: - one_of: ["@2.4", "@2.5"] """ update_packages_config(conf_str) s1 = spack.concretize.concretize_one("y") assert s1.satisfies("@2.4") s2 = spack.concretize.concretize_one("y@2.5") assert s2.satisfies("@2.5") def test_reuse_oneof(concretize_scope, test_repo, tmp_path: pathlib.Path, mock_fetch): conf_str = """\ packages: y: require: - one_of: ["@2.5", "~shared"] """ store_dir = tmp_path / "store" with spack.store.use_store(str(store_dir)): s1 = spack.concretize.concretize_one("y@2.5~shared") PackageInstaller([s1.package], fake=True, explicit=True).install() update_packages_config(conf_str) with spack.config.override("concretizer:reuse", True): s2 = spack.concretize.concretize_one("y") assert not s2.satisfies("@2.5~shared") @pytest.mark.parametrize( "allow_deprecated,expected,not_expected", [(True, ["@=2.3", "%gcc"], []), (False, ["%gcc"], ["@=2.3"])], ) def test_requirements_and_deprecated_versions( allow_deprecated, expected, not_expected, concretize_scope, test_repo ): """Tests the expected behavior of requirements and deprecated versions. If deprecated versions are not allowed, concretization should just pick the other requirement. If deprecated versions are allowed, both requirements are honored. """ # 2.3 is a deprecated versions. Ensure that any_of picks both constraints, # since they are possible conf_str = """\ packages: y: require: - any_of: ["@=2.3", "%gcc"] """ update_packages_config(conf_str) with spack.config.override("config:deprecated", allow_deprecated): s1 = spack.concretize.concretize_one("y") for constrain in expected: assert s1.satisfies(constrain) for constrain in not_expected: assert not s1.satisfies(constrain) @pytest.mark.parametrize("spec_str,requirement_str", [("x", "%gcc"), ("x", "%clang")]) def test_default_requirements_with_all(spec_str, requirement_str, concretize_scope, test_repo): """Test that default requirements are applied to all packages.""" conf_str = f"""\ packages: all: require: "{requirement_str}" """ update_packages_config(conf_str) spec = spack.concretize.concretize_one(spec_str) assert "c" in spec for s in spec.traverse(): if "c" in s and s.name not in ("gcc", "llvm"): assert s.satisfies(requirement_str) @pytest.mark.parametrize( "requirements,expectations", [ (("%gcc", "%clang"), ("%gcc", "%clang")), (("~shared%gcc", "@1.0"), ("~shared%gcc", "@1.0+shared")), ], ) def test_default_and_package_specific_requirements( concretize_scope, requirements, expectations, test_repo ): """Test that specific package requirements override default package requirements.""" generic_req, specific_req = requirements generic_exp, specific_exp = expectations conf_str = f"""\ packages: all: require: "{generic_req}" x: require: "{specific_req}" """ update_packages_config(conf_str) spec = spack.concretize.concretize_one("x") assert spec.satisfies(specific_exp) assert spec["y"].satisfies(generic_exp) @pytest.mark.parametrize("mpi_requirement", ["mpich", "mpich2", "zmpi"]) def test_requirements_on_virtual(mpi_requirement, concretize_scope, mock_packages): conf_str = f"""\ packages: mpi: require: "{mpi_requirement}" """ update_packages_config(conf_str) spec = spack.concretize.concretize_one("callpath") assert "mpi" in spec assert mpi_requirement in spec @pytest.mark.parametrize( "mpi_requirement,specific_requirement", [("mpich", "@3.0.3"), ("mpich2", "%clang"), ("zmpi", "%gcc")], ) def test_requirements_on_virtual_and_on_package( mpi_requirement, specific_requirement, concretize_scope, mock_packages ): conf_str = f"""\ packages: mpi: require: "{mpi_requirement}" {mpi_requirement}: require: "{specific_requirement}" """ update_packages_config(conf_str) spec = spack.concretize.concretize_one("callpath") assert "mpi" in spec assert mpi_requirement in spec assert spec["mpi"].satisfies(specific_requirement) def test_incompatible_virtual_requirements_raise(concretize_scope, mock_packages): conf_str = """\ packages: mpi: require: "mpich" """ update_packages_config(conf_str) spec = Spec("callpath^zmpi") # TODO (multiple nodes): recover a better error message later with pytest.raises((UnsatisfiableSpecError, InternalConcretizerError)): spack.concretize.concretize_one(spec) def test_non_existing_variants_under_all(concretize_scope, mock_packages): conf_str = """\ packages: all: require: - any_of: ["~foo", "@:"] """ update_packages_config(conf_str) spec = spack.concretize.concretize_one("callpath^zmpi") assert "~foo" not in spec @pytest.mark.parametrize( "packages_yaml,spec_str,expected_satisfies", [ # In the tests below we set the compiler preference to "gcc" to be explicit on the # fact that "clang" is not the preferred compiler. That helps making more robust the # tests that verify enforcing "%clang" as a requirement. ( """\ packages: all: compiler: ["gcc", "clang"] libelf: require: - one_of: ["%clang"] when: "@0.8.13" """, "libelf", [("@0.8.13%clang", True), ("%gcc", False)], ), ( """\ packages: all: compiler: ["gcc", "clang"] libelf: require: - one_of: ["%clang"] when: "@0.8.13" """, "libelf@0.8.12", [("%clang", False), ("%gcc", True)], ), ( """\ packages: all: compiler: ["gcc", "clang"] libelf: require: - spec: "%clang" when: "@0.8.13" """, "libelf@0.8.12", [("%clang", False), ("%gcc", True)], ), ( """\ packages: all: compiler: ["gcc", "clang"] libelf: require: - spec: "@0.8.13" when: "%clang" """, "libelf@0.8.13%gcc", [("%clang", False), ("%gcc", True), ("@0.8.13", True)], ), ], ) def test_conditional_requirements_from_packages_yaml( packages_yaml, spec_str, expected_satisfies, concretize_scope, mock_packages ): """Test that conditional requirements are required when the condition is met, and optional when the condition is not met. """ update_packages_config(packages_yaml) spec = spack.concretize.concretize_one(spec_str) for match_str, expected in expected_satisfies: assert spec.satisfies(match_str) is expected @pytest.mark.parametrize( "packages_yaml,spec_str,expected_message", [ ( """\ packages: mpileaks: require: - one_of: ["~debug"] message: "debug is not allowed" """, "mpileaks+debug", "debug is not allowed", ), ( """\ packages: libelf: require: - one_of: ["%clang"] message: "can only be compiled with clang" """, "libelf%gcc", "can only be compiled with clang", ), ( """\ packages: libelf: require: - one_of: ["%clang"] when: platform=test message: "can only be compiled with clang on the test platform" """, "libelf%gcc", "can only be compiled with clang on ", ), ( """\ packages: libelf: require: - spec: "%clang" when: platform=test message: "can only be compiled with clang on the test platform" """, "libelf%gcc", "can only be compiled with clang on ", ), ( """\ packages: libelf: require: - one_of: ["%clang", "%intel"] when: platform=test message: "can only be compiled with clang or intel on the test platform" """, "libelf%gcc", "can only be compiled with clang or intel", ), ], ) def test_requirements_fail_with_custom_message( packages_yaml, spec_str, expected_message, concretize_scope, mock_packages ): """Test that specs failing due to requirements not being satisfiable fail with a custom error message. """ update_packages_config(packages_yaml) with pytest.raises(spack.error.SpackError, match=expected_message): spack.concretize.concretize_one(spec_str) def test_skip_requirement_when_default_requirement_condition_cannot_be_met( concretize_scope, mock_packages ): """Tests that we can express a requirement condition under 'all' also in cases where the corresponding condition spec mentions variants or versions that don't exist in the package. For those packages the requirement rule is not emitted, since it can be determined to be always false. """ packages_yaml = """ packages: all: require: - one_of: ["%clang"] when: "+shared" """ update_packages_config(packages_yaml) s = spack.concretize.concretize_one("mpileaks") assert s.satisfies("+shared %clang") # Sanity checks that 'callpath' doesn't have the shared variant, but that didn't # cause failures during concretization. assert "shared" not in s["callpath"].variants def test_requires_directive(mock_packages, config): # This package requires either clang or gcc s = spack.concretize.concretize_one("requires-clang-or-gcc") assert s.satisfies("%gcc") s = spack.concretize.concretize_one("requires-clang-or-gcc %gcc") assert s.satisfies("%gcc") s = spack.concretize.concretize_one("requires-clang-or-gcc %clang") # Test both the real package (llvm) and its alias (clang) assert s.satisfies("%llvm") and s.satisfies("%clang") # This package can only be compiled with clang s = spack.concretize.concretize_one("requires-clang") assert s.satisfies("%llvm") s = spack.concretize.concretize_one("requires-clang %clang") assert s.satisfies("%llvm") with pytest.raises(spack.error.SpackError, match="can only be compiled with Clang"): spack.concretize.concretize_one("requires-clang %gcc") @pytest.mark.parametrize( "packages_yaml", [ # Simple string """ packages: all: require: "+shared" """, # List of strings """ packages: all: require: - "+shared" """, # Objects with attributes """ packages: all: require: - spec: "+shared" """, """ packages: all: require: - one_of: ["+shared"] """, ], ) def test_default_requirements_semantic(packages_yaml, concretize_scope, mock_packages): """Tests that requirements under 'all:' are by default applied only if the variant/property required exists, but are strict otherwise. For example: packages: all: require: "+shared" should enforce the value of "+shared" when a Boolean variant named "shared" exists. This is not overridable from the command line, so with the configuration above: > spack spec zlib~shared is unsatisfiable. """ update_packages_config(packages_yaml) # Regular zlib concretize to+shared s = spack.concretize.concretize_one("zlib") assert s.satisfies("+shared") # If we specify the variant we can concretize only the one matching the constraint s = spack.concretize.concretize_one("zlib+shared") assert s.satisfies("+shared") with pytest.raises(UnsatisfiableSpecError): spack.concretize.concretize_one("zlib~shared") # A spec without the shared variant still concretize s = spack.concretize.concretize_one("pkg-a") assert not s.satisfies("pkg-a+shared") assert not s.satisfies("pkg-a~shared") @pytest.mark.parametrize( "packages_yaml,spec_str,expected,not_expected", [ # The package has a 'libs' mv variant defaulting to 'libs=shared' ( """ packages: all: require: "+libs" """, "multivalue-variant", ["libs=shared"], ["libs=static", "+libs"], ), ( """ packages: all: require: "libs=foo" """, "multivalue-variant", ["libs=shared"], ["libs=static", "libs=foo"], ), ( # (TODO): revisit this case when we'll have exact value semantic for mv variants """ packages: all: require: "libs=static" """, "multivalue-variant", ["libs=static", "libs=shared"], [], ), ( # Constraint apply as a whole, so having a non-existing variant # invalidate the entire constraint """ packages: all: require: "libs=static+feefoo" """, "multivalue-variant", ["libs=shared"], ["libs=static"], ), ], ) def test_default_requirements_semantic_with_mv_variants( packages_yaml, spec_str, expected, not_expected, concretize_scope, mock_packages ): """Tests that requirements under 'all:' are behaving correctly under cases that could stem from MV variants. """ update_packages_config(packages_yaml) s = spack.concretize.concretize_one(spec_str) for constraint in expected: assert s.satisfies(constraint), constraint for constraint in not_expected: assert not s.satisfies(constraint), constraint @pytest.mark.regression("42084") def test_requiring_package_on_multiple_virtuals(concretize_scope, mock_packages): update_packages_config( """ packages: all: providers: scalapack: [netlib-scalapack] blas: require: intel-parallel-studio lapack: require: intel-parallel-studio scalapack: require: intel-parallel-studio """ ) s = spack.concretize.concretize_one("dla-future") assert s["blas"].name == "intel-parallel-studio" assert s["lapack"].name == "intel-parallel-studio" assert s["scalapack"].name == "intel-parallel-studio" @pytest.mark.parametrize( "packages_yaml,spec_str,expected,not_expected", [ ( """ packages: all: prefer: - "%clang" """, "multivalue-variant", ["%[virtuals=c] llvm"], ["%gcc"], ), ( """ packages: all: prefer: - "%clang" """, "multivalue-variant %gcc", ["%[virtuals=c] gcc"], ["%llvm"], ), # Test parsing objects instead of strings ( """ packages: all: prefer: - spec: "%clang" """, "multivalue-variant", ["%[virtuals=c] llvm"], ["%gcc"], ), # Test using preferences on virtuals ( """ packages: all: providers: mpi: [mpich] mpi: prefer: - zmpi """, "mpileaks", ["^[virtuals=mpi] zmpi"], ["^[virtuals=mpi] mpich"], ), ( """ packages: all: providers: mpi: [mpich] mpi: prefer: - zmpi """, "mpileaks ^[virtuals=mpi] mpich", ["^[virtuals=mpi] mpich"], ["^[virtuals=mpi] zmpi"], ), # Tests that strong preferences can be overridden by requirements ( """ packages: all: providers: mpi: [zmpi] mpi: require: - mpich prefer: - zmpi """, "mpileaks", ["^[virtuals=mpi] mpich"], ["^[virtuals=mpi] zmpi"], ), ], ) def test_strong_preferences_packages_yaml( packages_yaml, spec_str, expected, not_expected, concretize_scope, mock_packages ): """Tests that strong preferences are taken into account for compilers.""" update_packages_config(packages_yaml) s = spack.concretize.concretize_one(spec_str) for constraint in expected: assert s.satisfies(constraint) for constraint in not_expected: assert not s.satisfies(constraint) @pytest.mark.parametrize( "packages_yaml,spec_str", [ ( """ packages: all: conflict: - "%clang" """, "multivalue-variant %clang", ), # Use an object instead of a string in configuration ( """ packages: all: conflict: - spec: "%clang" message: "cannot use clang" """, "multivalue-variant %clang", ), ( """ packages: multivalue-variant: conflict: - spec: "%clang" when: "@2" message: "cannot use clang with version 2" """, "multivalue-variant@=2.3 %clang", ), # Test using conflict on virtual ( """ packages: mpi: conflict: - mpich """, "mpileaks ^[virtuals=mpi] mpich", ), ], ) def test_conflict_packages_yaml(packages_yaml, spec_str, concretize_scope, mock_packages): """Tests conflicts that are specified from configuration files.""" update_packages_config(packages_yaml) with pytest.raises(UnsatisfiableSpecError): spack.concretize.concretize_one(spec_str) @pytest.mark.parametrize( "spec_str,expected,not_expected", [ ( "forward-multi-value+cuda cuda_arch=10 ^dependency-mv~cuda", ["cuda_arch=10", "^dependency-mv~cuda"], ["cuda_arch=11", "^dependency-mv cuda_arch=10", "^dependency-mv cuda_arch=11"], ), ( "forward-multi-value+cuda cuda_arch=10 ^dependency-mv+cuda", ["cuda_arch=10", "^dependency-mv cuda_arch=10"], ["cuda_arch=11", "^dependency-mv cuda_arch=11"], ), ( "forward-multi-value+cuda cuda_arch=11 ^dependency-mv+cuda", ["cuda_arch=11", "^dependency-mv cuda_arch=11"], ["cuda_arch=10", "^dependency-mv cuda_arch=10"], ), ( "forward-multi-value+cuda cuda_arch=10,11 ^dependency-mv+cuda", ["cuda_arch=10,11", "^dependency-mv cuda_arch=10,11"], [], ), ], ) def test_forward_multi_valued_variant_using_requires( spec_str, expected, not_expected, config, mock_packages ): """Tests that a package can forward multivalue variants to dependencies, using `requires` directives of the form: for _val in ("shared", "static"): requires(f"^some-virtual-mv libs={_val}", when=f"libs={_val}^some-virtual-mv") """ s = spack.concretize.concretize_one(spec_str) for constraint in expected: assert s.satisfies(constraint) for constraint in not_expected: assert not s.satisfies(constraint) def test_strong_preferences_higher_priority_than_reuse(concretize_scope, mock_packages): """Tests that strong preferences have a higher priority than reusing specs.""" reused_spec = spack.concretize.concretize_one("adios2~bzip2") reuse_nodes = list(reused_spec.traverse()) root_specs = [Spec("ascent+adios2")] # Check that without further configuration adios2 is reused with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, root_specs, reuse=reuse_nodes) ascent = result.specs[0] assert ascent["adios2"].dag_hash() == reused_spec.dag_hash(), ascent # If we stick a preference, adios2 is not reused update_packages_config( """ packages: adios2: prefer: - "+bzip2" """ ) with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, root_specs, reuse=reuse_nodes) ascent = result.specs[0] assert ascent["adios2"].dag_hash() != reused_spec.dag_hash() assert ascent["adios2"].satisfies("+bzip2") # A preference is still preference, so we can override from input with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve( setup, [Spec("ascent+adios2^adios2~bzip2")], reuse=reuse_nodes ) ascent = result.specs[0] assert ascent["adios2"].dag_hash() == reused_spec.dag_hash(), ascent @pytest.mark.parametrize( "packages_yaml,err_match", [ ( """ packages: mpi: require: - "+bzip2" """, "expected a named spec", ), ( """ packages: mpi: require: - one_of: ["+bzip2", openmpi] """, "expected a named spec", ), ( """ packages: mpi: require: - "^mpich" """, "Did you mean", ), ], ) def test_anonymous_spec_cannot_be_used_in_virtual_requirements( packages_yaml, err_match, concretize_scope, mock_packages ): """Tests that using anonymous specs in requirements for virtual packages raises an appropriate error message. """ update_packages_config(packages_yaml) with pytest.raises(spack.error.SpackError, match=err_match): spack.concretize.concretize_one("mpileaks") def test_virtual_requirement_respects_any_of(concretize_scope, mock_packages): """Tests that "any of" requirements can be used with virtuals""" conf_str = """\ packages: mpi: require: - any_of: ["mpich2", "mpich"] """ update_packages_config(conf_str) s = spack.concretize.concretize_one("mpileaks") assert s.satisfies("^[virtuals=mpi] mpich2") s = spack.concretize.concretize_one("mpileaks ^mpich2") assert s.satisfies("^[virtuals=mpi] mpich2") s = spack.concretize.concretize_one("mpileaks ^mpich") assert s.satisfies("^[virtuals=mpi] mpich") with pytest.raises(spack.error.SpackError): spack.concretize.concretize_one("mpileaks ^[virtuals=mpi] zmpi") @pytest.mark.parametrize( "packages_yaml,expected_reuse,expected_contraints", [ ( """ packages: all: require: - "%gcc" """, True, # To minimize installed specs we reuse pkg-b compiler, since the requirement allows it ["%gcc@9"], ), ( """ packages: all: require: - "%gcc@10" """, False, ["%gcc@10"], ), ( """ packages: all: require: - "%gcc@9" """, True, ["%gcc@9"], ), ], ) @pytest.mark.regression("49847") def test_requirements_on_compilers_and_reuse( concretize_scope, mock_packages, mutable_config, packages_yaml, expected_reuse, expected_contraints, ): """Tests that we can require compilers with `%` in configuration files, and still get reuse of specs (even though reused specs have no build dependency in the ASP encoding). """ input_spec = "pkg-a" reused_spec = spack.concretize.concretize_one("pkg-b@0.9 %gcc@9") reused_nodes = list(reused_spec.traverse()) update_packages_config(packages_yaml) root_specs = [Spec(input_spec)] packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], exclude=[], ).selected_specs() with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve(setup, root_specs, reuse=reused_nodes + external_specs) pkga = result.specs[0] is_pkgb_reused = pkga["pkg-b"].dag_hash() == reused_spec.dag_hash() assert is_pkgb_reused == expected_reuse for c in expected_contraints: assert pkga.satisfies(c) @pytest.mark.parametrize( "abstract,req_is_noop", [ ("hdf5+mpi", False), ("hdf5~mpi", True), ("conditional-languages+c", False), ("conditional-languages+cxx", False), ("conditional-languages+fortran", False), ("conditional-languages~c~cxx~fortran", True), ], ) def test_requirements_conditional_deps( abstract, req_is_noop, mutable_config, mock_packages, config_two_gccs ): required_spec = ( "%[when='^c' virtuals=c]gcc@10.3.1 " "%[when='^cxx' virtuals=cxx]gcc@10.3.1 " "%[when='^fortran' virtuals=fortran]gcc@10.3.1 " "^[when='^mpi' virtuals=mpi]zmpi" ) abstract = spack.spec.Spec(abstract) no_requirements = spack.concretize.concretize_one(abstract) spack.config.CONFIG.set(f"packages:{abstract.name}", {"require": required_spec}) requirements = spack.concretize.concretize_one(abstract) assert requirements.satisfies(required_spec) assert (requirements == no_requirements) == req_is_noop # show the reqs change concretization @pytest.mark.regression("50898") def test_preferring_compilers_can_be_overridden(mutable_config, mock_packages): """Tests that we can override preferences for languages, without triggering an error.""" mutable_config.set("packages:c", {"prefer": ["llvm"]}) s = spack.spec.Spec("pkg-a %gcc ^pkg-b %llvm") concrete = spack.concretize.concretize_one(s) assert concrete.satisfies("%c=gcc") assert concrete["pkg-b"].satisfies("%c=llvm") @pytest.mark.regression("50955") def test_multiple_externals_and_requirement( concretize_scope, mock_packages, tmp_path: pathlib.Path ): """Tests that we can concretize a required virtual, when we have multiple externals specs for it, differing only by the compiler. """ packages_yaml = f""" packages: c: require: gcc mpi: require: mpich mpich: buildable: false externals: - spec: "mpich@4.3.0 %gcc@10" prefix: {tmp_path / "gcc"} - spec: "mpich@4.3.0 %clang" prefix: {tmp_path / "clang"} """ update_packages_config(packages_yaml) s = spack.spec.Spec("mpileaks") concrete = spack.concretize.concretize_one(s) assert concrete.satisfies("%gcc") assert concrete["mpi"].satisfies("mpich@4.3.0") assert concrete["mpi"].prefix == str(tmp_path / "gcc") @pytest.mark.regression("51262") @pytest.mark.parametrize( "input_constraint", [ # Override the compiler preference with a different version of gcc "%c=gcc@10", # Same, but without specifying the virtual "%gcc@10", # Override the mpi preference with a different version of mpich "%mpi=mpich@3 ~debug", # Override the mpi preference with a different provider "%mpi=mpich2", ], ) def test_overriding_preference_with_provider_details( input_constraint, concretize_scope, mock_packages, tmp_path: pathlib.Path ): """Tests that if we have a preference with provider details, such as a version range, or a variant, we can override it from the command line, while we can't do the same when we have a requirement. """ # A preference can be overridden packages_yaml = """ packages: c: prefer: - gcc@9 mpi: prefer: - mpich@3 +debug """ update_packages_config(packages_yaml) concrete = spack.concretize.concretize_one(f"mpileaks {input_constraint}") assert concrete.satisfies(input_constraint) # A requirement cannot packages_yaml = """ packages: c: require: - gcc@9 mpi: require: - mpich@3 +debug """ update_packages_config(packages_yaml) with pytest.raises(UnsatisfiableSpecError): spack.concretize.concretize_one(f"mpileaks {input_constraint}") @pytest.mark.parametrize( "initial_preference,current_preference", [ # Different provider ("llvm", "gcc"), ("gcc", "llvm"), # Different version of the same provider ("gcc@9", "gcc@10"), ("gcc@10", "gcc@9"), # Different configuration of the same provider ("llvm+lld", "llvm~lld"), ("llvm~lld", "llvm+lld"), ], ) @pytest.mark.parametrize("constraint_kind", ["require", "prefer"]) def test_language_preferences_and_reuse( initial_preference, current_preference, constraint_kind, concretize_scope, mutable_config, mock_packages, ): """Tests that language preferences are respected when reusing specs.""" # Install mpileaks with a non-default variant to avoid "accidental" reuse packages_yaml = f""" packages: c: {constraint_kind}: - {initial_preference} cxx: {constraint_kind}: - {initial_preference} llvm: externals: - spec: "llvm@15.0.0 +clang~flang ~lld" prefix: /path1 extra_attributes: compilers: c: /path1/bin/clang cxx: /path1/bin/clang++ """ update_packages_config(packages_yaml) initial_mpileaks = spack.concretize.concretize_one("mpileaks+debug") reused_nodes = list(initial_mpileaks.traverse()) packages_with_externals = external_config_with_implicit_externals(mutable_config) completion_mode = mutable_config.get("concretizer:externals:completion") external_specs = spec_filter_from_packages_yaml( external_parser=create_external_parser(packages_with_externals, completion_mode), packages_with_externals=packages_with_externals, include=[], exclude=[], ).selected_specs() # Ask for just "mpileaks" and check the spec is reused with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve( setup, [Spec("mpileaks")], reuse=reused_nodes + external_specs ) reused_mpileaks = result.specs[0] assert reused_mpileaks.dag_hash() == initial_mpileaks.dag_hash() # Change the language preferences and verify reuse is not happening packages_yaml = f""" packages: c: {constraint_kind}: - {current_preference} cxx: {constraint_kind}: - {current_preference} llvm: externals: - spec: "llvm@15.0.0 +clang~flang ~lld" prefix: /path1 extra_attributes: compilers: c: /path1/bin/clang cxx: /path1/bin/clang++ """ update_packages_config(packages_yaml) with spack.config.override("concretizer:reuse", True): solver = spack.solver.asp.Solver() setup = spack.solver.asp.SpackSolverSetup() result, _, _ = solver.driver.solve( setup, [Spec("mpileaks")], reuse=reused_nodes + external_specs ) mpileaks = result.specs[0] assert initial_mpileaks.dag_hash() != mpileaks.dag_hash() for node in mpileaks.traverse(): assert node.satisfies(f"%[when=%c]c={current_preference}") assert node.satisfies(f"%[when=%cxx]cxx={current_preference}") def test_external_spec_completion_with_targets_required( concretize_scope, mock_packages, tmp_path: pathlib.Path ): """Tests that we can concretize a spec needing externals, when we require a specific target, without extra configuration. """ current_platform = spack.platforms.host() packages_yaml = f""" packages: all: require: - target={current_platform.default} mpich: buildable: false externals: - spec: "mpich@4.3.0" prefix: {tmp_path / "mpich"} """ update_packages_config(packages_yaml) s = spack.spec.Spec("mpileaks") concrete = spack.concretize.concretize_one(s) assert concrete.satisfies(f"target={current_platform.default}") def test_penalties_for_language_preferences(concretize_scope, mock_packages): """Tests the default behavior when we use more than one compiler package in a DAG, under different scenarios. """ # This test uses gcc compilers providing c,cxx and fortran, and clang providing only c and cxx dependency_names = ["mpi", "callpath", "libdwarf", "libelf"] # If we don't express requirements, Spack tries to use a single compiler package if possible s = spack.concretize.concretize_one("mpileaks %c=gcc@10") assert s.satisfies("%c=gcc@10") assert all(s[name].satisfies("%c=gcc@10") for name in dependency_names) # Same with clang, if nothing else requires fortran s = spack.concretize.concretize_one("mpileaks %c=clang ^mpi=mpich2") assert s.satisfies("%c=clang") assert all(s[name].satisfies("%c=clang") for name in dependency_names) # If something brings in fortran that node is compiled entirely with gcc, # because currently we prefer to use a single toolchain for any node s = spack.concretize.concretize_one("mpileaks %c=clang ^mpi=mpich") assert s.satisfies("%c=clang") assert s["mpich"].satisfies("%c,cxx,fortran=gcc@10") # If we prefer compilers in configuration, that has a higher priority update_packages_config( """ packages: c: prefer: [gcc] cxx: prefer: [gcc] fortran: prefer: [gcc] """ ) s = spack.concretize.concretize_one("mpileaks %c=clang ^mpi=mpich2") assert s.satisfies("%c=clang") assert all(s[name].satisfies("%c=gcc@10") for name in dependency_names) # Mixed compilers in the preferences update_packages_config( """ packages: c: prefer: [llvm] cxx: prefer: [llvm] fortran: prefer: [gcc] """ ) s = spack.concretize.concretize_one("mpileaks %c=gcc ^mpi=mpich") assert s.satisfies("%c=gcc@10") assert all(s[name].satisfies("%c=clang") for name in dependency_names) assert s["mpi"].satisfies("%c,cxx=clang %fortran=gcc@10") ================================================ FILE: lib/spack/spack/test/concretization/splicing.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test ABI-based splicing of dependencies""" from typing import List import pytest import spack.concretize import spack.config import spack.deptypes as dt from spack.installer import PackageInstaller from spack.solver.asp import SolverError, UnsatisfiableSpecError def _make_specs_non_buildable(specs: List[str]): output_config = {} for spec in specs: output_config[spec] = {"buildable": False} return output_config @pytest.fixture def install_specs(mutable_database, mock_packages, mutable_config, install_mockery): """Returns a function that concretizes and installs a list of abstract specs""" mutable_config.set("concretizer:reuse", True) def _impl(*specs_str): concrete_specs = [spack.concretize.concretize_one(s) for s in specs_str] PackageInstaller([s.package for s in concrete_specs], fake=True, explicit=True).install() return concrete_specs return _impl def _enable_splicing(): spack.config.set("concretizer:splice", {"automatic": True}) @pytest.mark.parametrize("spec_str", ["splice-z", "splice-h@1"]) def test_spec_reuse(spec_str, install_specs, mutable_config): """Tests reuse of splice-z, without splicing, as a root and as a dependency of splice-h""" splice_z = install_specs("splice-z@1.0.0+compat")[0] mutable_config.set("packages", _make_specs_non_buildable(["splice-z"])) concrete = spack.concretize.concretize_one(spec_str) assert concrete["splice-z"].satisfies(splice_z) @pytest.mark.regression("48578") def test_splice_installed_hash(install_specs, mutable_config): """Tests splicing the dependency of an installed spec, for another installed spec""" splice_t, splice_h = install_specs( "splice-t@1 ^splice-h@1.0.0+compat ^splice-z@1.0.0", "splice-h@1.0.2+compat ^splice-z@1.0.0", ) packages_config = _make_specs_non_buildable(["splice-t", "splice-h"]) mutable_config.set("packages", packages_config) goal_spec = "splice-t@1 ^splice-h@1.0.2+compat ^splice-z@1.0.0" with pytest.raises(UnsatisfiableSpecError): spack.concretize.concretize_one(goal_spec) _enable_splicing() concrete = spack.concretize.concretize_one(goal_spec) # splice-t has a dependency that is changing, thus its hash should be different assert concrete.dag_hash() != splice_t.dag_hash() assert concrete.build_spec.satisfies(splice_t) assert not concrete.satisfies(splice_t) # splice-h is reused, so the hash should stay the same assert concrete["splice-h"].satisfies(splice_h) assert concrete["splice-h"].build_spec.satisfies(splice_h) assert concrete["splice-h"].dag_hash() == splice_h.dag_hash() def test_splice_build_splice_node(install_specs, mutable_config): """Tests splicing the dependency of an installed spec, for a spec that is yet to be built""" splice_t = install_specs("splice-t@1 ^splice-h@1.0.0+compat ^splice-z@1.0.0+compat")[0] mutable_config.set("packages", _make_specs_non_buildable(["splice-t"])) goal_spec = "splice-t@1 ^splice-h@1.0.2+compat ^splice-z@1.0.0+compat" with pytest.raises(UnsatisfiableSpecError): spack.concretize.concretize_one(goal_spec) _enable_splicing() concrete = spack.concretize.concretize_one(goal_spec) # splice-t has a dependency that is changing, thus its hash should be different assert concrete.dag_hash() != splice_t.dag_hash() assert concrete.build_spec.satisfies(splice_t) assert not concrete.satisfies(splice_t) # splice-h should be different assert concrete["splice-h"].dag_hash() != splice_t["splice-h"].dag_hash() assert concrete["splice-h"].build_spec.dag_hash() == concrete["splice-h"].dag_hash() def test_double_splice(install_specs, mutable_config): """Tests splicing two dependencies of an installed spec, for other installed specs""" splice_t, splice_h, splice_z = install_specs( "splice-t@1 ^splice-h@1.0.0+compat ^splice-z@1.0.0+compat", "splice-h@1.0.2+compat ^splice-z@1.0.1+compat", "splice-z@1.0.2+compat", ) mutable_config.set("packages", _make_specs_non_buildable(["splice-t", "splice-h", "splice-z"])) goal_spec = "splice-t@1 ^splice-h@1.0.2+compat ^splice-z@1.0.2+compat" with pytest.raises(UnsatisfiableSpecError): spack.concretize.concretize_one(goal_spec) _enable_splicing() concrete = spack.concretize.concretize_one(goal_spec) # splice-t and splice-h have a dependency that is changing, thus its hash should be different assert concrete.dag_hash() != splice_t.dag_hash() assert concrete.build_spec.satisfies(splice_t) assert not concrete.satisfies(splice_t) assert concrete["splice-h"].dag_hash() != splice_h.dag_hash() assert concrete["splice-h"].build_spec.satisfies(splice_h) assert not concrete["splice-h"].satisfies(splice_h) # splice-z is reused, so the hash should stay the same assert concrete["splice-z"].dag_hash() == splice_z.dag_hash() @pytest.mark.parametrize( "original_spec,goal_spec", [ # `virtual-abi-1` can be spliced for `virtual-abi-multi abi=one` and vice-versa ( "depends-on-virtual-with-abi ^virtual-abi-1", "depends-on-virtual-with-abi ^virtual-abi-multi abi=one", ), ( "depends-on-virtual-with-abi ^virtual-abi-multi abi=one", "depends-on-virtual-with-abi ^virtual-abi-1", ), # `virtual-abi-2` can be spliced for `virtual-abi-multi abi=two` and vice-versa ( "depends-on-virtual-with-abi ^virtual-abi-2", "depends-on-virtual-with-abi ^virtual-abi-multi abi=two", ), ( "depends-on-virtual-with-abi ^virtual-abi-multi abi=two", "depends-on-virtual-with-abi ^virtual-abi-2", ), ], ) def test_virtual_multi_splices_in(original_spec, goal_spec, install_specs, mutable_config): """Tests that we can splice a virtual dependency with a different, but compatible, provider.""" original = install_specs(original_spec)[0] mutable_config.set("packages", _make_specs_non_buildable(["depends-on-virtual-with-abi"])) with pytest.raises(UnsatisfiableSpecError): spack.concretize.concretize_one(goal_spec) _enable_splicing() spliced = spack.concretize.concretize_one(goal_spec) assert spliced.dag_hash() != original.dag_hash() assert spliced.build_spec.dag_hash() == original.dag_hash() assert spliced["virtual-with-abi"].name != spliced.build_spec["virtual-with-abi"].name @pytest.mark.parametrize( "original_spec,goal_spec", [ # can_splice("manyvariants@1.0.0", when="@1.0.1", match_variants="*") ( "depends-on-manyvariants ^manyvariants@1.0.0+a+b c=v1 d=v2", "depends-on-manyvariants ^manyvariants@1.0.1+a+b c=v1 d=v2", ), ( "depends-on-manyvariants ^manyvariants@1.0.0~a~b c=v3 d=v3", "depends-on-manyvariants ^manyvariants@1.0.1~a~b c=v3 d=v3", ), # can_splice("manyvariants@2.0.0+a~b", when="@2.0.1~a+b", match_variants=["c", "d"]) ( "depends-on-manyvariants@2.0 ^manyvariants@2.0.0+a~b c=v3 d=v2", "depends-on-manyvariants@2.0 ^manyvariants@2.0.1~a+b c=v3 d=v2", ), # can_splice("manyvariants@2.0.0 c=v1 d=v1", when="@2.0.1+a+b") ( "depends-on-manyvariants@2.0 ^manyvariants@2.0.0~a~b c=v1 d=v1", "depends-on-manyvariants@2.0 ^manyvariants@2.0.1+a+b c=v3 d=v3", ), ], ) def test_manyvariant_matching_variant_splice( original_spec, goal_spec, install_specs, mutable_config ): """Tests splicing with different kind of matching on variants""" original = install_specs(original_spec)[0] mutable_config.set("packages", {"depends-on-manyvariants": {"buildable": False}}) with pytest.raises((UnsatisfiableSpecError, SolverError)): spack.concretize.concretize_one(goal_spec) _enable_splicing() spliced = spack.concretize.concretize_one(goal_spec) assert spliced.dag_hash() != original.dag_hash() assert spliced.build_spec.dag_hash() == original.dag_hash() # The spliced 'manyvariants' is yet to be built assert spliced["manyvariants"].dag_hash() != original["manyvariants"].dag_hash() assert spliced["manyvariants"].build_spec.dag_hash() == spliced["manyvariants"].dag_hash() def test_external_splice_same_name(install_specs, mutable_config): """Tests that externals can be spliced for non-external specs""" original_splice_h, original_splice_t = install_specs( "splice-h@1.0.0 ^splice-z@1.0.0+compat", "splice-t@1.0 ^splice-h@1.0.1 ^splice-z@1.0.1+compat", ) mutable_config.set("packages", _make_specs_non_buildable(["splice-t", "splice-h"])) mutable_config.set( "packages", { "splice-z": { "externals": [{"spec": "splice-z@1.0.2+compat", "prefix": "/usr"}], "buildable": False, } }, ) _enable_splicing() concrete_splice_h = spack.concretize.concretize_one("splice-h@1.0.0 ^splice-z@1.0.2") concrete_splice_t = spack.concretize.concretize_one( "splice-t@1.0 ^splice-h@1.0.1 ^splice-z@1.0.2" ) assert concrete_splice_h.dag_hash() != original_splice_h.dag_hash() assert concrete_splice_h.build_spec.dag_hash() == original_splice_h.dag_hash() assert concrete_splice_h["splice-z"].external assert concrete_splice_t.dag_hash() != original_splice_t.dag_hash() assert concrete_splice_t.build_spec.dag_hash() == original_splice_t.dag_hash() assert concrete_splice_t["splice-z"].external assert concrete_splice_t["splice-z"].dag_hash() == concrete_splice_h["splice-z"].dag_hash() def test_spliced_build_deps_only_in_build_spec(install_specs): """Tests that build specs are not reported in the spliced spec""" install_specs("splice-t@1.0 ^splice-h@1.0.1 ^splice-z@1.0.0") _enable_splicing() spliced = spack.concretize.concretize_one("splice-t@1.0 ^splice-h@1.0.2 ^splice-z@1.0.0") build_spec = spliced.build_spec # Spec has been spliced assert build_spec.dag_hash() != spliced.dag_hash() # Build spec has spliced build dependencies assert build_spec.dependencies("splice-h", dt.BUILD) assert build_spec.dependencies("splice-z", dt.BUILD) # Spliced build dependencies are removed assert len(spliced.dependencies(None, dt.BUILD)) == 0 def test_spliced_transitive_dependency(install_specs, mutable_config): """Tests that build specs are not reported, even for spliced transitive dependencies""" install_specs("splice-depends-on-t@1.0 ^splice-h@1.0.1") mutable_config.set("packages", _make_specs_non_buildable(["splice-depends-on-t"])) _enable_splicing() spliced = spack.concretize.concretize_one("splice-depends-on-t^splice-h@1.0.2") # Spec has been spliced assert spliced.build_spec.dag_hash() != spliced.dag_hash() assert spliced["splice-t"].build_spec.dag_hash() != spliced["splice-t"].dag_hash() # Spliced build dependencies are removed assert len(spliced.dependencies(None, dt.BUILD)) == 0 assert len(spliced["splice-t"].dependencies(None, dt.BUILD)) == 0 ================================================ FILE: lib/spack/spack/test/config.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import io import os import pathlib import sys import tempfile import textwrap from datetime import date import pytest import spack import spack.config import spack.directory_layout import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs import spack.package_base import spack.paths import spack.platforms import spack.repo import spack.schema.compilers import spack.schema.config import spack.schema.env import spack.schema.include import spack.schema.mirrors import spack.schema.repos import spack.spec import spack.store import spack.util.executable import spack.util.git import spack.util.path as spack_path import spack.util.spack_yaml as syaml from spack.enums import ConfigScopePriority from spack.llnl.util.filesystem import getuid, join_path, touch from spack.util.spack_yaml import DictWithLineInfo # sample config data config_low = { "config": { "install_tree": {"root": "install_tree_path"}, "build_stage": ["path1", "path2", "path3"], } } config_override_all = {"config:": {"install_tree:": {"root": "override_all"}}} config_override_key = {"config": {"install_tree:": {"root": "override_key"}}} config_merge_list = {"config": {"build_stage": ["patha", "pathb"]}} config_override_list = {"config": {"build_stage:": ["pathd", "pathe"]}} config_merge_dict = {"config": {"aliases": {"ls": "find", "dev": "develop"}}} config_override_dict = {"config": {"aliases:": {"be": "build-env", "deps": "dependencies"}}} @pytest.fixture() def env_yaml(tmp_path: pathlib.Path): """Return a sample env.yaml for test purposes""" env_yaml = str(tmp_path / "env.yaml") with open(env_yaml, "w", encoding="utf-8") as f: f.write( """\ spack: config: verify_ssl: False dirty: False packages: all: compiler: [ 'gcc@4.5.3' ] repos: z: /x/y/z """ ) return env_yaml def check_compiler_config(comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get("compilers") compiler_list = ["cc", "cxx", "f77", "fc"] flag_list = ["cflags", "cxxflags", "fflags", "cppflags", "ldflags", "ldlibs"] param_list = ["modules", "paths", "spec", "operating_system"] for compiler in config: conf = compiler["compiler"] if conf["spec"] in compiler_names: comp = next( (c["compiler"] for c in comps if c["compiler"]["spec"] == conf["spec"]), None ) if not comp: raise ValueError("Bad config spec") for p in param_list: assert conf[p] == comp[p] for f in flag_list: expected = comp.get("flags", {}).get(f, None) actual = conf.get("flags", {}).get(f, None) assert expected == actual for c in compiler_list: expected = comp["paths"][c] actual = conf["paths"][c] assert expected == actual # # Some sample compiler config data and tests. # a_comps = { "compilers": [ { "compiler": { "paths": {"cc": "/gcc473", "cxx": "/g++473", "f77": None, "fc": None}, "modules": None, "spec": "gcc@4.7.3", "operating_system": "CNL10", } }, { "compiler": { "paths": {"cc": "/gcc450", "cxx": "/g++450", "f77": "gfortran", "fc": "gfortran"}, "modules": None, "spec": "gcc@4.5.0", "operating_system": "CNL10", } }, { "compiler": { "paths": {"cc": "/gcc422", "cxx": "/g++422", "f77": "gfortran", "fc": "gfortran"}, "flags": {"cppflags": "-O0 -fpic", "fflags": "-f77"}, "modules": None, "spec": "gcc@4.2.2", "operating_system": "CNL10", } }, { "compiler": { "paths": { "cc": "", "cxx": "", "f77": "", "fc": "", }, "modules": None, "spec": "clang@3.3", "operating_system": "CNL10", } }, ] } b_comps = { "compilers": [ { "compiler": { "paths": {"cc": "/icc100", "cxx": "/icp100", "f77": None, "fc": None}, "modules": None, "spec": "icc@10.0", "operating_system": "CNL10", } }, { "compiler": { "paths": {"cc": "/icc111", "cxx": "/icp111", "f77": "ifort", "fc": "ifort"}, "modules": None, "spec": "icc@11.1", "operating_system": "CNL10", } }, { "compiler": { "paths": {"cc": "/icc123", "cxx": "/icp123", "f77": "ifort", "fc": "ifort"}, "flags": {"cppflags": "-O3", "fflags": "-f77rtl"}, "modules": None, "spec": "icc@12.3", "operating_system": "CNL10", } }, { "compiler": { "paths": { "cc": "", "cxx": "", "f77": "", "fc": "", }, "modules": None, "spec": "clang@3.3", "operating_system": "CNL10", } }, ] } @pytest.fixture() def compiler_specs(): """Returns a couple of compiler specs needed for the tests""" a = [ac["compiler"]["spec"] for ac in a_comps["compilers"]] b = [bc["compiler"]["spec"] for bc in b_comps["compilers"]] CompilerSpecs = collections.namedtuple("CompilerSpecs", ["a", "b"]) return CompilerSpecs(a=a, b=b) def test_write_key_in_memory(mock_low_high_config, compiler_specs): # Write b_comps "on top of" a_comps. spack.config.set("compilers", a_comps["compilers"], scope="low") spack.config.set("compilers", b_comps["compilers"], scope="high") # Make sure the config looks how we expect. check_compiler_config(a_comps["compilers"], *compiler_specs.a) check_compiler_config(b_comps["compilers"], *compiler_specs.b) def test_write_key_to_disk(mock_low_high_config, compiler_specs): # Write b_comps "on top of" a_comps. spack.config.set("compilers", a_comps["compilers"], scope="low") spack.config.set("compilers", b_comps["compilers"], scope="high") # Clear caches so we're forced to read from disk. spack.config.CONFIG.clear_caches() # Same check again, to ensure consistency. check_compiler_config(a_comps["compilers"], *compiler_specs.a) check_compiler_config(b_comps["compilers"], *compiler_specs.b) def test_write_to_same_priority_file(mock_low_high_config, compiler_specs): # Write b_comps in the same file as a_comps. spack.config.set("compilers", a_comps["compilers"], scope="low") spack.config.set("compilers", b_comps["compilers"], scope="low") # Clear caches so we're forced to read from disk. spack.config.CONFIG.clear_caches() # Same check again, to ensure consistency. check_compiler_config(a_comps["compilers"], *compiler_specs.a) check_compiler_config(b_comps["compilers"], *compiler_specs.b) # # Sample repo data and tests # repos_low = {"repos": {"low": "/some/path"}} repos_high = {"repos": {"high": "/some/other/path"}} # Test setting config values via path in filename def test_add_config_path(mutable_config): # Try setting a new install tree root path = "config:install_tree:root:/path/to/config.yaml" spack.config.add(path) set_value = spack.config.get("config")["install_tree"]["root"] assert set_value == "/path/to/config.yaml" # Now a package:all setting path = "packages:all:target:[x86_64]" spack.config.add(path) targets = spack.config.get("packages")["all"]["target"] assert "x86_64" in targets # Try quotes to escape brackets path = ( "config:install_tree:projections:cmake:" "'{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" ) spack.config.add(path) set_value = spack.config.get("config")["install_tree"]["projections"]["cmake"] assert set_value == "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}" path = 'modules:default:tcl:all:environment:set:"{name}_ROOT":"{prefix}"' spack.config.add(path) set_value = spack.config.get("modules")["default"]["tcl"]["all"]["environment"]["set"] assert r"{name}_ROOT" in set_value assert set_value[r"{name}_ROOT"] == r"{prefix}" assert spack.config.get('modules:default:tcl:all:environment:set:"{name}_ROOT"') == r"{prefix}" # NOTE: # The config path: "config:install_tree:root:" is unique in that it can accept multiple # schemas (such as a dropped "root" component) which is atypical and may lead to passing tests # when the behavior is in reality incorrect. # the config path below is such that no subkey accepts a string as a valid entry in our schema # try quotes to escape colons path = "config:build_stage:'C:\\path\\to\\config.yaml'" spack.config.add(path) set_value = spack.config.get("config")["build_stage"] assert "C:\\path\\to\\config.yaml" in set_value @pytest.mark.regression("17543,23259") def test_add_config_path_with_enumerated_type(mutable_config): spack.config.add("config:flags:keep_werror:all") assert spack.config.get("config")["flags"]["keep_werror"] == "all" spack.config.add("config:flags:keep_werror:specific") assert spack.config.get("config")["flags"]["keep_werror"] == "specific" with pytest.raises(spack.error.ConfigError): spack.config.add("config:flags:keep_werror:foo") def test_add_config_filename(mock_low_high_config, tmp_path: pathlib.Path): config_yaml = tmp_path / "config-filename.yaml" config_yaml.touch() with config_yaml.open("w") as f: syaml.dump_config(config_low, f) spack.config.add_from_file(str(config_yaml), scope="low") assert "build_stage" in spack.config.get("config") build_stages = spack.config.get("config")["build_stage"] for stage in config_low["config"]["build_stage"]: assert stage in build_stages # repos def test_write_list_in_memory(mock_low_high_config): spack.config.set("repos", repos_low["repos"], scope="low") spack.config.set("repos", repos_high["repos"], scope="high") config = spack.config.get("repos") assert config == {**repos_high["repos"], **repos_low["repos"]} class MockEnv: def __init__(self, path): self.path = path def test_substitute_config_variables(mock_low_high_config, monkeypatch, tmp_path: pathlib.Path): # Test $spack substitution at the start (valid on all platforms) assert os.path.join(spack.paths.prefix, "foo", "bar", "baz") == spack_path.canonicalize_path( "$spack/foo/bar/baz/" ) assert os.path.join(spack.paths.prefix, "foo", "bar", "baz") == spack_path.canonicalize_path( "${spack}/foo/bar/baz/" ) # Test $spack substitution in the middle. This only makes sense when using posix paths. if sys.platform != "win32": prefix = spack.paths.prefix.lstrip(os.sep) base = str(tmp_path) assert os.path.join(base, "foo", "bar", "baz", prefix) == spack_path.canonicalize_path( os.path.join(base, "foo", "bar", "baz", "$spack") ) assert os.path.join( base, "foo", "bar", "baz", prefix, "foo", "bar", "baz" ) == spack_path.canonicalize_path( os.path.join(base, "foo", "bar", "baz", "$spack", "foo", "bar", "baz") ) assert os.path.join(base, "foo", "bar", "baz", prefix) == spack_path.canonicalize_path( os.path.join(base, "foo", "bar", "baz", "${spack}") ) assert os.path.join( base, "foo", "bar", "baz", prefix, "foo", "bar", "baz" ) == spack_path.canonicalize_path( os.path.join(base, "foo", "bar", "baz", "${spack}", "foo", "bar", "baz") ) assert os.path.join( base, "foo", "bar", "baz", prefix, "foo", "bar", "baz" ) != spack_path.canonicalize_path( os.path.join(base, "foo", "bar", "baz", "${spack", "foo", "bar", "baz") ) # $env replacement is a no-op when no environment is active assert spack_path.canonicalize_path( os.path.join(str(tmp_path), "foo", "bar", "baz", "$env") ) == os.path.join(str(tmp_path), "foo", "bar", "baz", "$env") # Fake an active environment and $env is replaced properly fake_env_path = str(tmp_path / "quux" / "quuux") monkeypatch.setattr(ev, "active_environment", lambda: MockEnv(fake_env_path)) assert spack_path.canonicalize_path("$env/foo/bar/baz") == os.path.join( fake_env_path, os.path.join("foo", "bar", "baz") ) # relative paths without source information are relative to cwd assert spack_path.canonicalize_path(os.path.join("foo", "bar", "baz")) == os.path.abspath( os.path.join("foo", "bar", "baz") ) # relative paths with source information are relative to the file spack.config.set( "modules:default", {"roots": {"lmod": os.path.join("foo", "bar", "baz")}}, scope="low" ) spack.config.CONFIG.clear_caches() path = spack.config.get("modules:default:roots:lmod") assert spack_path.canonicalize_path(path) == os.path.normpath( os.path.join(mock_low_high_config.scopes["low"].path, os.path.join("foo", "bar", "baz")) ) # test architecture information is in replacements assert spack_path.canonicalize_path( os.path.join("foo", "$platform", "bar") ) == os.path.abspath(os.path.join("foo", "test", "bar")) host_target = spack.platforms.host().default_target() host_target_family = str(host_target.family) assert spack_path.canonicalize_path( os.path.join("foo", "$target_family", "bar") ) == os.path.abspath(os.path.join("foo", host_target_family, "bar")) packages_merge_low = {"packages": {"foo": {"variants": ["+v1"]}, "bar": {"variants": ["+v2"]}}} packages_merge_high = { "packages": { "foo": {"version": ["a"]}, "bar": {"version": ["b"], "variants": ["+v3"]}, "baz": {"version": ["c"]}, } } @pytest.mark.regression("7924") def test_merge_with_defaults(mock_low_high_config, write_config_file): """This ensures that specified preferences merge with defaults as expected. Originally all defaults were initialized with the exact same object, which led to aliasing problems. Therefore the test configs used here leave 'version' blank for multiple packages in 'packages_merge_low'. """ write_config_file("packages", packages_merge_low, "low") write_config_file("packages", packages_merge_high, "high") cfg = spack.config.get("packages") assert cfg["foo"]["version"] == ["a"] assert cfg["bar"]["version"] == ["b"] assert cfg["baz"]["version"] == ["c"] def test_substitute_user(mock_low_high_config, tmp_path: pathlib.Path): user = spack_path.get_user() base = str(tmp_path) assert os.path.join(base, "foo", "bar", user, "baz") == spack_path.canonicalize_path( os.path.join(base, "foo", "bar", "$user", "baz") ) def test_substitute_user_cache(mock_low_high_config): user_cache_path = spack.paths.user_cache_path assert os.path.join(user_cache_path, "baz") == spack_path.canonicalize_path( os.path.join("$user_cache_path", "baz") ) def test_substitute_tempdir(mock_low_high_config): tempdir = tempfile.gettempdir() assert tempdir == spack_path.canonicalize_path("$tempdir") assert os.path.join(tempdir, "foo", "bar", "baz") == spack_path.canonicalize_path( os.path.join("$tempdir", "foo", "bar", "baz") ) def test_substitute_date(mock_low_high_config): test_path = os.path.join("hello", "world", "on", "$date") new_path = spack_path.canonicalize_path(test_path) assert "$date" in test_path assert date.today().strftime("%Y-%m-%d") in new_path def test_substitute_spack_version(): version = spack.spack_version_info assert spack_path.canonicalize_path( "spack$spack_short_version/test" ) == spack_path.canonicalize_path(f"spack{version[0]}.{version[1]}/test") PAD_STRING = spack_path.SPACK_PATH_PADDING_CHARS MAX_PATH_LEN = spack_path.get_system_path_max() MAX_PADDED_LEN = MAX_PATH_LEN - spack_path.SPACK_MAX_INSTALL_PATH_LENGTH reps = [PAD_STRING for _ in range((MAX_PADDED_LEN // len(PAD_STRING) + 1) + 2)] full_padded_string = os.path.join(os.sep + "path", os.sep.join(reps))[:MAX_PADDED_LEN] @pytest.mark.parametrize( "config_settings_fn,expected_fn", [ (lambda p: [], lambda p: [None, None, None]), ( lambda p: [["config:install_tree:root", os.path.join(str(p), "path")]], lambda p: [os.path.join(str(p), "path"), None, None], ), ( lambda p: [["config:install_tree:projections", {"all": "{name}"}]], lambda p: [None, None, {"all": "{name}"}], ), ], ) def test_parse_install_tree(config_settings_fn, expected_fn, mutable_config, tmp_path): config_settings = config_settings_fn(tmp_path) expected = expected_fn(tmp_path) expected_root = expected[0] or mutable_config.get("config:install_tree:root") expected_unpadded_root = expected[1] or expected_root expected_proj = expected[2] or spack.directory_layout.default_projections # config settings is a list of 2-element lists, [path, value] # where path is a config path and value is the value to set at that path # these can be "splatted" in as the arguments to config.set for config_setting in config_settings: mutable_config.set(*config_setting) config_dict = mutable_config.get("config") root, unpadded_root, projections = spack.store.parse_install_tree(config_dict) assert root == expected_root assert unpadded_root == expected_unpadded_root assert projections == expected_proj def test_change_or_add(mutable_config, mock_packages): spack.config.add("packages:a:version:['1.0']", scope="user") spack.config.add("packages:b:version:['1.1']", scope="system") class ChangeTest: def __init__(self, pkg_name, new_version): self.pkg_name = pkg_name self.new_version = new_version def find_fn(self, section): return self.pkg_name in section def change_fn(self, section): pkg_section = section.get(self.pkg_name, {}) pkg_section["version"] = self.new_version section[self.pkg_name] = pkg_section change1 = ChangeTest("b", ["1.2"]) spack.config.change_or_add("packages", change1.find_fn, change1.change_fn) assert "b" not in mutable_config.get("packages", scope="user") assert mutable_config.get("packages")["b"]["version"] == ["1.2"] change2 = ChangeTest("c", ["1.0"]) spack.config.change_or_add("packages", change2.find_fn, change2.change_fn) assert "c" in mutable_config.get("packages", scope="user") @pytest.mark.not_on_windows("Padding unsupported on Windows") @pytest.mark.parametrize( "config_settings,expected", [ ( [ ["config:install_tree:root", os.sep + "path"], ["config:install_tree:padded_length", 11], ], [os.path.join(os.sep + "path", PAD_STRING[:5]), os.sep + "path", None], ), ( [["config:install_tree:root", "/path/$padding:11"]], [os.path.join(os.sep + "path", PAD_STRING[:5]), os.sep + "path", None], ), ([["config:install_tree:padded_length", False]], [None, None, None]), ( [ ["config:install_tree:padded_length", True], ["config:install_tree:root", os.sep + "path"], ], [full_padded_string, os.sep + "path", None], ), ], ) def test_parse_install_tree_padded(config_settings, expected, mutable_config): expected_root = expected[0] or mutable_config.get("config:install_tree:root") expected_unpadded_root = expected[1] or expected_root expected_proj = expected[2] or spack.directory_layout.default_projections # config settings is a list of 2-element lists, [path, value] # where path is a config path and value is the value to set at that path # these can be "splatted" in as the arguments to config.set for config_setting in config_settings: mutable_config.set(*config_setting) config_dict = mutable_config.get("config") root, unpadded_root, projections = spack.store.parse_install_tree(config_dict) assert root == expected_root assert unpadded_root == expected_unpadded_root assert projections == expected_proj def test_read_config(mock_low_high_config, write_config_file): write_config_file("config", config_low, "low") assert spack.config.get("config") == config_low["config"] def test_read_config_override_all(mock_low_high_config, write_config_file): write_config_file("config", config_low, "low") write_config_file("config", config_override_all, "high") assert spack.config.get("config") == {"install_tree": {"root": "override_all"}} def test_read_config_override_key(mock_low_high_config, write_config_file): write_config_file("config", config_low, "low") write_config_file("config", config_override_key, "high") assert spack.config.get("config") == { "install_tree": {"root": "override_key"}, "build_stage": ["path1", "path2", "path3"], } def test_read_config_merge_list(mock_low_high_config, write_config_file): write_config_file("config", config_low, "low") write_config_file("config", config_merge_list, "high") assert spack.config.get("config") == { "install_tree": {"root": "install_tree_path"}, "build_stage": ["patha", "pathb", "path1", "path2", "path3"], } def test_read_config_override_list(mock_low_high_config, write_config_file): write_config_file("config", config_low, "low") write_config_file("config", config_override_list, "high") assert spack.config.get("config") == { "install_tree": {"root": "install_tree_path"}, "build_stage": config_override_list["config"]["build_stage:"], } def test_internal_config_update(mock_low_high_config, write_config_file): write_config_file("config", config_low, "low") before = mock_low_high_config.get("config") assert before["install_tree"]["root"] == "install_tree_path" # add an internal configuration scope scope = spack.config.InternalConfigScope("command_line") assert "InternalConfigScope" in repr(scope) mock_low_high_config.push_scope(scope) command_config = mock_low_high_config.get("config", scope="command_line") command_config["install_tree"] = {"root": "foo/bar"} mock_low_high_config.set("config", command_config, scope="command_line") after = mock_low_high_config.get("config") assert after["install_tree"]["root"] == "foo/bar" def test_internal_config_filename(mock_low_high_config, write_config_file): write_config_file("config", config_low, "low") mock_low_high_config.push_scope(spack.config.InternalConfigScope("command_line")) with pytest.raises(NotImplementedError): mock_low_high_config.get_config_filename("command_line", "config") def test_mark_internal(): data = { "config": { "bool": False, "int": 6, "numbers": [1, 2, 3], "string": "foo", "dict": {"more_numbers": [1, 2, 3], "another_string": "foo", "another_int": 7}, } } marked = spack.config._mark_internal(data, "x") # marked version should be equal to the original assert data == marked def assert_marked(obj): if type(obj) is bool: return # can't subclass bool, so can't mark it assert hasattr(obj, "_start_mark") and obj._start_mark.name == "x" assert hasattr(obj, "_end_mark") and obj._end_mark.name == "x" # everything in the marked version should have marks checks = ( marked.keys(), marked.values(), marked["config"].keys(), marked["config"].values(), marked["config"]["numbers"], marked["config"]["dict"].keys(), marked["config"]["dict"].values(), marked["config"]["dict"]["more_numbers"], ) for seq in checks: for obj in seq: assert_marked(obj) def test_internal_config_from_data(): config = spack.config.create_from( spack.config.InternalConfigScope( "_builtin", {"config": {"verify_ssl": False, "build_jobs": 6}} ) ) assert config.get("config:verify_ssl", scope="_builtin") is False assert config.get("config:build_jobs", scope="_builtin") == 6 assert config.get("config:verify_ssl") is False assert config.get("config:build_jobs") == 6 # push one on top and see what happens. config.push_scope( spack.config.InternalConfigScope( "higher", {"config": {"checksum": True, "verify_ssl": True}} ) ) assert config.get("config:verify_ssl", scope="_builtin") is False assert config.get("config:build_jobs", scope="_builtin") == 6 assert config.get("config:verify_ssl", scope="higher") is True assert config.get("config:build_jobs", scope="higher") is None assert config.get("config:verify_ssl") is True assert config.get("config:build_jobs") == 6 assert config.get("config:checksum") is True assert config.get("config:checksum", scope="_builtin") is None assert config.get("config:checksum", scope="higher") is True def test_keys_are_ordered(configuration_dir): """Test that keys in Spack YAML files retain their order from the file.""" expected_order = ( "./bin", "./man", "./share/man", "./share/aclocal", "./lib/pkgconfig", "./lib64/pkgconfig", "./share/pkgconfig", "./", ) config_scope = spack.config.DirectoryConfigScope("modules", configuration_dir / "site") data = config_scope.get_section("modules") prefix_inspections = data["modules"]["prefix_inspections"] for actual, expected in zip(prefix_inspections, expected_order): assert actual == expected def test_config_format_error(mutable_config): """This is raised when we try to write a bad configuration.""" with pytest.raises(spack.config.ConfigFormatError): spack.config.set("compilers", {"bad": "data"}, scope="site") def get_config_error(filename, schema, yaml_string): """Parse a YAML string and return the resulting ConfigFormatError. Fail if there is no ConfigFormatError """ with open(filename, "w", encoding="utf-8") as f: f.write(yaml_string) # parse and return error, or fail. try: spack.config.read_config_file(filename, schema) except spack.config.ConfigFormatError as e: return e else: pytest.fail("ConfigFormatError was not raised!") def test_config_parse_dict_in_list(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): e = get_config_error( "repos.yaml", spack.schema.repos.schema, """\ repos: a: https://foobar.com/foo b: https://foobar.com/bar c: error: - abcdef d: https://foobar.com/baz """, ) assert "repos.yaml:2" in str(e) def test_config_parse_str_not_bool(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): e = get_config_error( "config.yaml", spack.schema.config.schema, """\ config: verify_ssl: False checksum: foobar dirty: True """, ) assert "config.yaml:3" in str(e) def test_config_parse_list_in_dict(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): e = get_config_error( "mirrors.yaml", spack.schema.mirrors.schema, """\ mirrors: foo: http://foobar.com/baz bar: http://barbaz.com/foo baz: http://bazfoo.com/bar travis: [1, 2, 3] """, ) assert "mirrors.yaml:5" in str(e) def test_bad_config_section(mock_low_high_config): """Test that getting or setting a bad section gives an error.""" with pytest.raises(spack.config.ConfigSectionError): spack.config.set("foobar", "foobar") with pytest.raises(spack.config.ConfigSectionError): spack.config.get("foobar") def test_nested_override(): """Ensure proper scope naming of nested overrides.""" base_name = spack.config._OVERRIDES_BASE_NAME def _check_scopes(num_expected, debug_values): scope_names = [ s.name for s in spack.config.CONFIG.scopes.values() if s.name.startswith(base_name) ] for i in range(num_expected): name = "{0}{1}".format(base_name, i) assert name in scope_names data = spack.config.CONFIG.get_config("config", name) assert data["debug"] == debug_values[i] # Check results from single and nested override with spack.config.override("config:debug", True): with spack.config.override("config:debug", False): _check_scopes(2, [True, False]) _check_scopes(1, [True]) def test_alternate_override(monkeypatch): """Ensure proper scope naming of override when conflict present.""" base_name = spack.config._OVERRIDES_BASE_NAME def _matching_scopes(regexpr): return [spack.config.InternalConfigScope("{0}1".format(base_name))] # Check that the alternate naming works monkeypatch.setattr(spack.config.CONFIG, "matching_scopes", _matching_scopes) with spack.config.override("config:debug", False): name = "{0}2".format(base_name) scope_names = [ s.name for s in spack.config.CONFIG.scopes.values() if s.name.startswith(base_name) ] assert name in scope_names data = spack.config.CONFIG.get_config("config", name) assert data["debug"] is False def test_immutable_scope(tmp_path: pathlib.Path): config_yaml = str(tmp_path / "config.yaml") with open(config_yaml, "w", encoding="utf-8") as f: f.write( """\ config: install_tree: root: dummy_tree_value """ ) scope = spack.config.DirectoryConfigScope("test", str(tmp_path), writable=False) data = scope.get_section("config") assert data is not None assert data["config"]["install_tree"] == {"root": "dummy_tree_value"} with pytest.raises(spack.error.ConfigError): scope._write_section("config") def test_single_file_scope(config, env_yaml): scope = spack.config.SingleFileScope( "env", env_yaml, spack.schema.env.schema, yaml_path=["spack"] ) with spack.config.override(scope): # from the single-file config assert spack.config.get("config:verify_ssl") is False assert spack.config.get("config:dirty") is False # from the lower config scopes assert spack.config.get("config:checksum") is True assert spack.config.get("config:checksum") is True assert spack.config.get("packages:externalmodule:buildable") is False assert spack.config.get("repos") == { "z": "/x/y/z", "builtin_mock": "$spack/var/spack/test_repos/spack_repo/builtin_mock", } def test_single_file_scope_section_override(tmp_path: pathlib.Path, config): """Check that individual config sections can be overridden in an environment config. The config here primarily differs in that the ``packages`` section is intended to override all other scopes (using the "::" syntax). """ env_yaml = str(tmp_path / "env.yaml") with open(env_yaml, "w", encoding="utf-8") as f: f.write( """\ spack: config: verify_ssl: False packages:: all: target: [ x86_64 ] repos: z: /x/y/z """ ) scope = spack.config.SingleFileScope( "env", env_yaml, spack.schema.env.schema, yaml_path=["spack"] ) with spack.config.override(scope): # from the single-file config assert spack.config.get("config:verify_ssl") is False assert spack.config.get("packages:all:target") == ["x86_64"] # from the lower config scopes assert spack.config.get("config:checksum") is True assert not spack.config.get("packages:externalmodule") assert spack.config.get("repos") == { "z": "/x/y/z", "builtin_mock": "$spack/var/spack/test_repos/spack_repo/builtin_mock", } def test_write_empty_single_file_scope(tmp_path: pathlib.Path): env_schema = spack.schema.env.schema config_file = tmp_path / "config.yaml" config_file.touch() scope = spack.config.SingleFileScope("test", str(config_file), env_schema, yaml_path=["spack"]) scope._write_section("config") # confirm we can write empty config assert not scope.get_section("config") def check_schema(name, file_contents): """Check a Spack YAML schema against some data""" f = io.StringIO(file_contents) data = syaml.load_config(f) spack.config.validate(data, name) def test_good_env_yaml(): check_schema( spack.schema.env.schema, """\ spack: config: verify_ssl: False dirty: False repos: - ~/my/repo/location mirrors: remote: /foo/bar/baz compilers: - compiler: spec: cce@2.1 operating_system: cnl modules: [] paths: cc: /path/to/cc cxx: /path/to/cxx fc: /path/to/fc f77: /path/to/f77 """, ) def test_bad_env_yaml(): with pytest.raises(spack.config.ConfigFormatError): check_schema( spack.schema.env.schema, """\ spack: foobar: verify_ssl: False dirty: False """, ) def test_bad_config_yaml(): with pytest.raises(spack.config.ConfigFormatError): check_schema( spack.schema.config.schema, """\ config: verify_ssl: False install_tree: root: extra_level: foo """, ) def test_bad_include_yaml(): with pytest.raises(spack.config.ConfigFormatError, match="is not of type"): check_schema( spack.schema.include.schema, """\ include: $HOME/include.yaml """, ) def test_bad_mirrors_yaml(): with pytest.raises(spack.config.ConfigFormatError): check_schema( spack.schema.mirrors.schema, """\ mirrors: local: True """, ) def test_bad_repos_yaml(): with pytest.raises(spack.config.ConfigFormatError): check_schema( spack.schema.repos.schema, """\ repos: True """, ) def test_bad_compilers_yaml(): with pytest.raises(spack.config.ConfigFormatError): check_schema( spack.schema.compilers.schema, """\ compilers: key_instead_of_list: 'value' """, ) with pytest.raises(spack.config.ConfigFormatError): check_schema( spack.schema.compilers.schema, """\ compilers: - shmompiler: environment: /bad/value """, ) with pytest.raises(spack.config.ConfigFormatError): check_schema( spack.schema.compilers.schema, """\ compilers: - compiler: fenfironfent: /bad/value """, ) def test_internal_config_section_override(mock_low_high_config, write_config_file): write_config_file("config", config_merge_list, "low") wanted_list = config_override_list["config"]["build_stage:"] mock_low_high_config.push_scope( spack.config.InternalConfigScope("high", {"config:": {"build_stage": wanted_list}}) ) assert mock_low_high_config.get("config:build_stage") == wanted_list def test_internal_config_dict_override(mock_low_high_config, write_config_file): write_config_file("config", config_merge_dict, "low") wanted_dict = config_override_dict["config"]["aliases:"] mock_low_high_config.push_scope(spack.config.InternalConfigScope("high", config_override_dict)) assert mock_low_high_config.get("config:aliases") == wanted_dict def test_internal_config_list_override(mock_low_high_config, write_config_file): write_config_file("config", config_merge_list, "low") wanted_list = config_override_list["config"]["build_stage:"] mock_low_high_config.push_scope(spack.config.InternalConfigScope("high", config_override_list)) assert mock_low_high_config.get("config:build_stage") == wanted_list def test_set_section_override(mock_low_high_config, write_config_file): write_config_file("config", config_merge_list, "low") wanted_list = config_override_list["config"]["build_stage:"] with spack.config.override("config::build_stage", wanted_list): assert mock_low_high_config.get("config:build_stage") == wanted_list assert config_merge_list["config"]["build_stage"] == mock_low_high_config.get( "config:build_stage" ) def test_set_list_override(mock_low_high_config, write_config_file): write_config_file("config", config_merge_list, "low") wanted_list = config_override_list["config"]["build_stage:"] with spack.config.override("config:build_stage:", wanted_list): assert wanted_list == mock_low_high_config.get("config:build_stage") assert config_merge_list["config"]["build_stage"] == mock_low_high_config.get( "config:build_stage" ) def test_set_dict_override(mock_low_high_config, write_config_file): write_config_file("config", config_merge_dict, "low") wanted_dict = config_override_dict["config"]["aliases:"] with spack.config.override("config:aliases:", wanted_dict): assert wanted_dict == mock_low_high_config.get("config:aliases") assert config_merge_dict["config"]["aliases"] == mock_low_high_config.get("config:aliases") def test_set_bad_path(config): with pytest.raises(ValueError): with spack.config.override(":bad:path", ""): pass def test_bad_path_double_override(config): with pytest.raises(syaml.SpackYAMLError, match="Meaningless second override"): with spack.config.override("bad::double:override::directive", ""): pass def test_license_dir_config(mutable_config, mock_packages, tmp_path): """Ensure license directory is customizable""" expected_dir = spack.paths.default_license_dir assert spack.config.get("config:license_dir") == expected_dir assert spack.package_base.PackageBase.global_license_dir == expected_dir assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == expected_dir abs_path = str(tmp_path / "foo" / "bar" / "baz") spack.config.set("config:license_dir", abs_path) assert spack.config.get("config:license_dir") == abs_path assert spack.package_base.PackageBase.global_license_dir == abs_path assert spack.repo.PATH.get_pkg_class("pkg-a").global_license_dir == abs_path @pytest.mark.regression("22547") def test_single_file_scope_cache_clearing(env_yaml): scope = spack.config.SingleFileScope( "env", env_yaml, spack.schema.env.schema, yaml_path=["spack"] ) # Check that we can retrieve data from the single file scope before = scope.get_section("config") assert before # Clear the cache of the Single file scope scope.clear() # Check that the section can be retrieved again and it's # the same as before after = scope.get_section("config") assert after assert before == after @pytest.mark.regression("22611") def test_internal_config_scope_cache_clearing(): """ An InternalConfigScope object is constructed from data that is already in memory, therefore it doesn't have any cache to clear. Here we ensure that calling the clear method is consistent with that.. """ data = {"config": {"build_jobs": 10}} internal_scope = spack.config.InternalConfigScope("internal", data) # Ensure that the initial object is properly set assert internal_scope.sections["config"] == data # Call the clear method internal_scope.clear() # Check that this didn't affect the scope object assert internal_scope.sections["config"] == data def test_system_config_path_is_overridable(working_env): p = "/some/path" os.environ["SPACK_SYSTEM_CONFIG_PATH"] = p assert spack.paths._get_system_config_path() == p def test_system_config_path_is_default_when_env_var_is_empty(working_env): os.environ["SPACK_SYSTEM_CONFIG_PATH"] = "" assert os.sep + os.path.join("etc", "spack") == spack.paths._get_system_config_path() def test_user_config_path_is_overridable(working_env): p = "/some/path" os.environ["SPACK_USER_CONFIG_PATH"] = p assert p == spack.paths._get_user_config_path() def test_user_config_path_is_default_when_env_var_is_empty(working_env): os.environ["SPACK_USER_CONFIG_PATH"] = "" assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_config_path() def test_default_install_tree(monkeypatch, default_config): s = spack.spec.Spec("nonexistent@x.y.z arch=foo-bar-baz") monkeypatch.setattr(s, "dag_hash", lambda length: "abc123") _, _, projections = spack.store.parse_install_tree(spack.config.get("config")) assert s.format(projections["all"]) == "foo-baz/nonexistent-x.y.z-abc123" @pytest.fixture def mock_include_scope(tmp_path): for subdir in ["defaults", "test1", "test2", "test3"]: path = tmp_path / subdir path.mkdir() include = tmp_path / "include.yaml" with include.open("w", encoding="utf-8") as f: f.write( textwrap.dedent( """\ include:: - name: "test1" path: "test1" when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' - name: "test2" path: "test2" - name: "test3" path: "test3" when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' """ ) ) yield tmp_path @pytest.fixture def include_config_factory(mock_include_scope): def make_config(): cfg = spack.config.Configuration() cfg.push_scope( spack.config.DirectoryConfigScope("defaults", str(mock_include_scope / "defaults")), priority=ConfigScopePriority.DEFAULTS, ) cfg.push_scope( spack.config.DirectoryConfigScope("tmp_path", str(mock_include_scope)), priority=ConfigScopePriority.CONFIG_FILES, ) return cfg yield make_config def test_modify_scope_precedence(working_env, include_config_factory, tmp_path): """Test how spack selects the scope to modify when commands write config.""" cfg = include_config_factory() # ensure highest precedence writable scope is selected by default assert cfg.highest_precedence_scope().name == "tmp_path" include_yaml = tmp_path / "include.yaml" subdir = tmp_path / "subdir" subdir2 = tmp_path / "subdir2" subdir.mkdir() subdir2.mkdir() with include_yaml.open("w", encoding="utf-8") as f: f.write( textwrap.dedent( """\ include:: - name: "subdir" path: "subdir" """ ) ) cfg.push_scope( spack.config.DirectoryConfigScope("override", str(tmp_path)), priority=ConfigScopePriority.CONFIG_FILES, ) # ensure override scope is selected when it is on top assert cfg.highest_precedence_scope().name == "override" cfg.remove_scope("override") with include_yaml.open("w", encoding="utf-8") as f: f.write( textwrap.dedent( """\ include:: - name: "subdir" path: "subdir" prefer_modify: true """ ) ) cfg.push_scope( spack.config.DirectoryConfigScope("override", str(tmp_path)), priority=ConfigScopePriority.CONFIG_FILES, ) # if the top scope prefers another, ensure it is selected assert cfg.highest_precedence_scope().name == "subdir" cfg.remove_scope("override") with include_yaml.open("w", encoding="utf-8") as f: f.write( textwrap.dedent( """\ include:: - name: "subdir" path: "subdir" - name: "subdir2" path: "subdir2" prefer_modify: true """ ) ) cfg.push_scope( spack.config.DirectoryConfigScope("override", str(tmp_path)), priority=ConfigScopePriority.CONFIG_FILES, ) # if there are multiple scopes and one is preferred, make sure it's that one assert cfg.highest_precedence_scope().name == "subdir2" def test_local_config_can_be_disabled(working_env, include_config_factory): """Ensure that SPACK_DISABLE_LOCAL_CONFIG disables configurations with `when:`.""" os.environ["SPACK_DISABLE_LOCAL_CONFIG"] = "true" cfg = include_config_factory() assert "defaults" in cfg.scopes assert "test1" not in cfg.scopes assert "test2" in cfg.scopes assert "test3" not in cfg.scopes os.environ["SPACK_DISABLE_LOCAL_CONFIG"] = "" cfg = include_config_factory() assert "defaults" in cfg.scopes assert "test1" not in cfg.scopes assert "test2" in cfg.scopes assert "test3" not in cfg.scopes del os.environ["SPACK_DISABLE_LOCAL_CONFIG"] cfg = include_config_factory() assert "defaults" in cfg.scopes assert "test1" in cfg.scopes assert "test2" in cfg.scopes assert "test3" in cfg.scopes def test_override_included_config(working_env, tmp_path, include_config_factory): override_scope = tmp_path / "override" override_scope.mkdir() include_yaml = override_scope / "include.yaml" subdir = override_scope / "subdir" subdir.mkdir() anotherdir = override_scope / "anotherdir" anotherdir.mkdir() with include_yaml.open("w", encoding="utf-8") as f: f.write( textwrap.dedent( """\ include:: - name: "subdir" path: "subdir" """ ) ) with (subdir / "include.yaml").open("w", encoding="utf-8") as f: f.write( textwrap.dedent( """\ include: - name: "anotherdir" path: "../anotherdir" """ ) ) # check the mock config is correct cfg = include_config_factory() assert "defaults" in cfg.scopes assert "tmp_path" in cfg.scopes assert "test1" in cfg.scopes assert "test2" in cfg.scopes assert "test3" in cfg.scopes active_names = [s.name for s in cfg.active_scopes] assert "defaults" in active_names assert "tmp_path" in active_names assert "test1" in active_names assert "test2" in active_names assert "test3" in active_names includes = str(cfg.get("include")) assert "subdir" not in includes assert "anotherdir" not in includes assert "test1" in includes assert "test2" in includes assert "test3" in includes # push a scope that overrides everything under it but includes a subdir. # its included subdir should be active, but scopes *not* included by the overriding # scope should not. cfg.push_scope( spack.config.DirectoryConfigScope("override", str(override_scope)), priority=ConfigScopePriority.CONFIG_FILES, ) assert "defaults" in cfg.scopes assert "tmp_path" in cfg.scopes assert "test1" in cfg.scopes assert "test2" in cfg.scopes assert "test3" in cfg.scopes assert "override" in cfg.scopes assert "subdir" in cfg.scopes assert "anotherdir" in cfg.scopes active_names = [s.name for s in cfg.active_scopes] assert "defaults" in active_names assert "tmp_path" in active_names assert "test1" not in active_names assert "test2" not in active_names assert "test3" not in active_names assert "override" in active_names assert "subdir" in active_names assert "anotherdir" not in active_names includes = str(cfg.get("include")) assert "subdir" in includes assert "anotherdir" not in includes assert "test1" not in includes assert "test2" not in includes assert "test3" not in includes # remove the override and ensure everything is back to normal cfg.remove_scope("override") assert "defaults" in cfg.scopes assert "tmp_path" in cfg.scopes assert "test1" in cfg.scopes assert "test2" in cfg.scopes assert "test3" in cfg.scopes active_names = [s.name for s in cfg.active_scopes] assert "defaults" in active_names assert "tmp_path" in active_names assert "test1" in active_names assert "test2" in active_names assert "test3" in active_names includes = str(cfg.get("include")) assert "subdir" not in includes assert "anotherdir" not in includes assert "test1" in includes assert "test2" in includes assert "test3" in includes def test_user_cache_path_is_overridable(working_env): p = "/some/path" os.environ["SPACK_USER_CACHE_PATH"] = p assert spack.paths._get_user_cache_path() == p def test_user_cache_path_is_default_when_env_var_is_empty(working_env): os.environ["SPACK_USER_CACHE_PATH"] = "" assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_cache_path() def test_config_file_dir_failure(tmp_path: pathlib.Path, mutable_empty_config): with pytest.raises(spack.config.ConfigFileError, match="not a file"): spack.config.read_config_file(str(tmp_path)) @pytest.mark.not_on_windows("chmod not supported on Windows") @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_config_file_read_perms_failure(tmp_path: pathlib.Path, mutable_empty_config): """Test reading a configuration file without permissions to ensure ConfigFileError is raised.""" filename = join_path(str(tmp_path), "test.yaml") touch(filename) os.chmod(filename, 0o200) with pytest.raises(spack.config.ConfigFileError, match="not readable"): spack.config.read_config_file(filename) def test_config_file_read_invalid_yaml(tmp_path: pathlib.Path, mutable_empty_config): """Test reading a configuration file with invalid (unparsable) YAML raises a ConfigFileError.""" filename = join_path(str(tmp_path), "test.yaml") with open(filename, "w", encoding="utf-8") as f: f.write("spack:\nview") with pytest.raises(spack.config.ConfigFileError, match="parsing YAML"): spack.config.read_config_file(filename) @pytest.mark.parametrize( "path,it_should_work,expected_parsed", [ ("x:y:z", True, ["x:", "y:", "z"]), ("x+::y:z", True, ["x+::", "y:", "z"]), ('x:y:"{z}"', True, ["x:", "y:", '"{z}"']), ('x:"y"+:z', True, ["x:", '"y"+:', "z"]), ('x:"y"trail:z', False, None), ("x:y:[1.0]", True, ["x:", "y:", "[1.0]"]), ("x:y:['1.0']", True, ["x:", "y:", "['1.0']"]), ("x:{y}:z", True, ["x:", "{y}:", "z"]), ("x:'{y}':z", True, ["x:", "'{y}':", "z"]), ("x:{y}", True, ["x:", "{y}"]), ], ) def test_config_path_dsl(path, it_should_work, expected_parsed): if it_should_work: assert spack.config.ConfigPath._validate(path) == expected_parsed else: with pytest.raises(ValueError): spack.config.ConfigPath._validate(path) @pytest.mark.regression("48254") def test_env_activation_preserves_command_line_scope(mutable_mock_env_path): """Check that the "command_line" scope remains the highest priority scope, when we activate, or deactivate, environments. """ expected_cl_scope = spack.config.CONFIG.highest() assert expected_cl_scope.name == "command_line" # Creating an environment pushes a new scope ev.create("test") with ev.read("test"): assert spack.config.CONFIG.highest() == expected_cl_scope # No active environment pops the scope with ev.no_active_environment(): assert spack.config.CONFIG.highest() == expected_cl_scope assert spack.config.CONFIG.highest() == expected_cl_scope # Switch the environment to another one ev.create("test-2") with ev.read("test-2"): assert spack.config.CONFIG.highest() == expected_cl_scope assert spack.config.CONFIG.highest() == expected_cl_scope assert spack.config.CONFIG.highest() == expected_cl_scope @pytest.mark.regression("48414") @pytest.mark.regression("49188") def test_env_activation_preserves_config_scopes(mutable_mock_env_path): """Check that the priority of scopes is respected when merging configuration files.""" custom_scope = spack.config.InternalConfigScope("custom_scope") spack.config.CONFIG.push_scope(custom_scope, priority=ConfigScopePriority.CUSTOM) expected_scopes_without_env = ["custom_scope", "command_line"] expected_scopes_with_first_env = ["env:test", "custom_scope", "command_line"] expected_scopes_with_second_env = ["env:test-2", "custom_scope", "command_line"] def highest_priority_scopes(config, *, nscopes): return list(config.scopes)[-nscopes:] assert highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env # Creating an environment pushes a new scope ev.create("test") with ev.read("test"): assert ( highest_priority_scopes(spack.config.CONFIG, nscopes=3) == expected_scopes_with_first_env ) # No active environment pops the scope with ev.no_active_environment(): assert ( highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env ) assert ( highest_priority_scopes(spack.config.CONFIG, nscopes=3) == expected_scopes_with_first_env ) # Switch the environment to another one ev.create("test-2") with ev.read("test-2"): assert ( highest_priority_scopes(spack.config.CONFIG, nscopes=3) == expected_scopes_with_second_env ) assert ( highest_priority_scopes(spack.config.CONFIG, nscopes=3) == expected_scopes_with_first_env ) assert highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env @pytest.mark.regression("51059") def test_config_include_similar_name(tmp_path: pathlib.Path): config_a = tmp_path / "a" / "config" config_b = tmp_path / "b" / "config" os.makedirs(config_a) with open(config_a / "config.yaml", "w", encoding="utf-8") as fd: syaml.dump_config({"config": {"install_tree": {"root": str(tmp_path)}}}, fd) os.makedirs(config_b) with open(config_b / "config.yaml", "w", encoding="utf-8") as fd: syaml.dump_config({"config": {"install_tree": {"padded_length": 64}}}, fd) with open(tmp_path / "include.yaml", "w", encoding="utf-8") as fd: syaml.dump_config({"include": [str(config_a), str(config_b)]}, fd) config = spack.config.create_from(spack.config.DirectoryConfigScope("test", str(tmp_path))) # Ensure all of the scopes are found assert len(config.matching_scopes("^test$")) == 1 assert len(config.matching_scopes("^test:a/config$")) == 1 assert len(config.matching_scopes("^test:b/config$")) == 1 def test_deepcopy_as_builtin(env_yaml): cfg = spack.config.create_from( spack.config.SingleFileScope("env", env_yaml, spack.schema.env.schema, yaml_path=["spack"]) ) config_copy = cfg.deepcopy_as_builtin("config") assert config_copy == cfg.get_config("config") assert type(config_copy) is DictWithLineInfo assert type(config_copy["verify_ssl"]) is bool packages_copy = cfg.deepcopy_as_builtin("packages") assert type(packages_copy) is DictWithLineInfo assert type(packages_copy["all"]) is DictWithLineInfo assert type(packages_copy["all"]["compiler"]) is list assert type(packages_copy["all"]["compiler"][0]) is str def test_included_optional_include_scopes(): with pytest.raises(NotImplementedError): spack.config.OptionalInclude({}).scopes(spack.config.ConfigScope("fail")) def test_included_path_string( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, capfd ): path = tmp_path / "local" / "config.yaml" path.parent.mkdir() include = spack.config.included_path(path) assert isinstance(include, spack.config.IncludePath) assert include.path == str(path) assert not include.optional assert include.evaluate_condition() parent_scope = mock_low_high_config.scopes["low"] # Trigger failure when required path does not exist with pytest.raises(ValueError, match="does not exist"): include.scopes(parent_scope) # First successful pass builds the scope path.touch() scopes = include.scopes(parent_scope) assert scopes and len(scopes) == 1 assert isinstance(scopes[0], spack.config.SingleFileScope) # Second pass uses the scopes previously built assert include._scopes is not None scopes = include.scopes(parent_scope) captured = capfd.readouterr()[1] assert "Using existing scopes" in captured def test_included_path_string_no_parent_path( tmp_path: pathlib.Path, config, ensure_debug, monkeypatch ): """Use a relative include path and no parent scope path so destination will be rooted in the current working directory (usually SPACK_ROOT).""" entry = {"path": "config.yaml", "optional": True} include = spack.config.included_path(entry) parent_scope = spack.config.InternalConfigScope("parent-scope") included_scopes = include.scopes(parent_scope) # ensure scope is returned even if there is no parent path assert len(included_scopes) == 1 # ensure scope for include is singlefile as it ends in .yaml assert isinstance(included_scopes[0], spack.config.SingleFileScope) destination = include.destination curr_dir = os.getcwd() assert curr_dir == os.path.commonprefix([curr_dir, destination]) # type: ignore[list-item] def test_included_path_substitution(): # check a straight path substitution entry = {"path": "$user_cache_path/path/to/config.yaml"} include = spack.config.included_path(entry) assert spack.paths.user_cache_path in include.path # check path through an environment variable path = "/path/to/project/packages.yaml" os.environ["SPACK_TEST_PATH_SUB"] = path entry = {"name": "vartest", "path": "$SPACK_TEST_PATH_SUB"} include = spack.config.included_path(entry) assert path in include.path def test_included_path_conditional_bad_when( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, capfd ): path = tmp_path / "local" path.mkdir() entry = {"path": str(path), "when": 'platform == "nosuchplatform"', "optional": True} include = spack.config.included_path(entry) assert isinstance(include, spack.config.IncludePath) assert include.path == entry["path"] assert include.when == entry["when"] assert include.optional assert not include.evaluate_condition() scopes = include.scopes(mock_low_high_config.scopes["low"]) captured = capfd.readouterr()[1] assert "condition is not satisfied" in captured assert not scopes def test_included_path_conditional_success(tmp_path: pathlib.Path, mock_low_high_config): path = tmp_path / "local" path.mkdir() entry = {"path": str(path), "when": 'platform == "test"', "optional": True} include = spack.config.included_path(entry) assert isinstance(include, spack.config.IncludePath) assert include.path == entry["path"] assert include.when == entry["when"] assert include.optional assert include.evaluate_condition() scopes = include.scopes(mock_low_high_config.scopes["low"]) assert scopes and len(scopes) == 1 assert isinstance(scopes[0], spack.config.DirectoryConfigScope) def test_included_path_git_missing_args(): # must have one or more of: branch, tag and commit so fail if missing any entry = {"git": "https://example.com/windows/configs.git", "paths": ["config.yaml"]} with pytest.raises(spack.error.ConfigError, match="specify one or more"): spack.config.included_path(entry) # must have one or more paths entry["tag"] = "v1.0" entry["paths"] = [] with pytest.raises(spack.error.ConfigError, match="must include one or more"): spack.config.included_path(entry) def test_included_path_git_unsat( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, capfd ): paths = ["config.yaml", "packages.yaml"] entry = { "git": "https://example.com/windows/configs.git", "tag": "v1.0", "paths": paths, "when": 'platform == "nosuchplatform"', } include = spack.config.included_path(entry) assert isinstance(include, spack.config.GitIncludePaths) assert include.git == entry["git"] assert include.tag == entry["tag"] assert include.paths == entry["paths"] assert include.when == entry["when"] assert not include.optional and not include.evaluate_condition() scopes = include.scopes(mock_low_high_config.scopes["low"]) captured = capfd.readouterr()[1] assert "condition is not satisfied" in captured assert not scopes def test_included_path_git_substitutions(): # check path substitutions for the git url *and* paths paths = ["./$platform/config.yaml", "$platform/packages.yaml"] entry = { "git": "https://example.com/$platform/configs.git", "branch": "develop", "name": "site", "paths": paths, "when": 'platform == "test"', } include = spack.config.included_path(entry) assert isinstance(include, spack.config.GitIncludePaths) assert not include.optional and include.evaluate_condition() assert "test" in include.git, "Expected the git url to contain the platform" for path in include.paths: assert "test" in path, "Expected the included git path to contain the platform" # check environment substitution for the git url url = "https://example.com/path/to/configs.git" os.environ["SPACK_TEST_URL_SUB"] = url entry["git"] = "$SPACK_TEST_URL_SUB" include = spack.config.included_path(entry) assert include.git == url, "Expected git url environment var substitution" @pytest.mark.parametrize( "key,value", [("branch", "main"), ("commit", "abcdef123456"), ("tag", "v1.0")] ) def test_included_path_git( tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd ): """Check git includes for branch, commit, and tag using relative paths. Note the mock config fixture does NOT create the scope path so a temporary directory will be used for caching the files. """ # Specifying two relative paths, one explicit, one implicit paths = ["./config.yaml", "packages.yaml"] entry = { "git": "https://example.com/windows/configs.git", key: value, "name": "site", "paths": paths, "when": 'platform == "test"', } include = spack.config.included_path(entry) assert isinstance(include, spack.config.GitIncludePaths) assert not include.optional and include.evaluate_condition() # set up minimal git and repository operations class MockIncludeGit(spack.util.executable.Executable): def __init__(self, required: bool): pass def __call__(self, *args, **kwargs) -> str: # type: ignore action = args[0] if action == "config": return "origin" return "" monkeypatch.setattr(spack.util.git, "git", MockIncludeGit) def _init_repo(*args, **kwargs): # Make sure the directory exists, where assuming called from within # the working directory. fs.mkdirp(fs.join_path(os.getcwd(), ".git")) def _checkout(*args, **kwargs): # Make sure the files exist at the clone destination, where assuming # called from within the working directory. with fs.working_dir(os.getcwd()): for p in paths: fs.touch(p) monkeypatch.setattr(spack.util.git, "init_git_repo", _init_repo) monkeypatch.setattr(spack.util.git, f"pull_checkout_{key}", _checkout) # First successful pass builds the scope parent_scope = mock_low_high_config.scopes["low"] scopes = include.scopes(parent_scope) assert len(scopes) == len(paths) base_paths = [os.path.basename(p) for p in paths] for scope in scopes: assert isinstance(scope, spack.config.SingleFileScope) assert os.path.basename(scope.path) in base_paths # type: ignore[union-attr] assert scope.name.split(":")[1] in base_paths # Second pass uses the scopes previously built. # Only need to do this for one of the parameters. if key == "branch": assert include._scopes is not None scopes = include.scopes(parent_scope) captured = capfd.readouterr()[1] assert "Using existing scopes" in captured # A direct clone now returns already cloned destination and debug message. # Again only need to run this test once. if key == "tag": assert include._clone(parent_scope) == include.destination captured = capfd.readouterr()[1] assert "already cloned" in captured @pytest.mark.parametrize("path", ["./config.yaml", "/path/to/my/special/package.yaml"]) def test_included_path_local_no_dest(path): """Confirm that local paths have no cache destination.""" entry = {"path": path} include = spack.config.included_path(entry) destination = include.base_directory(entry["path"]) assert not destination, f"Expected local include ({include}) to NOT have a cache destination" def test_included_path_url_temp_dest(mock_low_high_config): """Check that remote (raw) path under different scopes end up with temporary cache destinations.""" entry = { "path": "https://github.com/path/to/raw/config/config.yaml", "sha256": "26e871804a92cd07bb3d611b31b4156ae93d35b6a6d6e0ef3a67871fcb1d258b", } include = spack.config.included_path(entry) parent_scope = mock_low_high_config.scopes["low"] parent_scope.path = "" pre = f"Expected temporary cache destination for raw include path ({include}) for " for scope in [None, parent_scope]: rest = "parent scope with no path" if scope else "no parent scope" destination = include.base_directory(entry["path"], parent_scope=scope) dest_dir = str(pathlib.Path(destination).parent) temp_dir = tempfile.gettempdir() assert dest_dir == temp_dir, pre + rest def test_included_path_git_temp_dest(mock_low_high_config): """Check a remote (relative) path with different parent scope options that result in a temporary cache destination.""" entry = { "git": "https://example.com/linux/configs.git", "branch": "develop", "paths": ["config.yaml"], } include = spack.config.included_path(entry) parent_scope = mock_low_high_config.scopes["low"] parent_scope.path = "" pre = f"Expected temporary cache destination for git include path ({include}) for " for scope in [None, parent_scope]: rest = "parent scope with no path" if scope else "no parent scope" destination = include.base_directory(entry["git"], parent_scope=scope) dest_dir = str(pathlib.Path(destination).parent) temp_dir = tempfile.gettempdir() assert dest_dir == temp_dir, pre + rest def test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch): monkeypatch.setattr(spack.paths, "user_cache_path", str(tmp_path)) paths = ["concretizer.yaml"] entry = { "git": "https://example.com/linux/configs.git", "branch": "develop", "paths": paths, "when": 'platform == "test"', } include = spack.config.included_path(entry) parent_scope = mock_low_high_config.scopes["low"] # fail to initialize the repository def _failing_init(*args, **kwargs): raise spack.util.executable.ProcessError("mock init repo failure") monkeypatch.setattr(spack.util.git, "init_git_repo", _failing_init) with pytest.raises(spack.error.ConfigError, match="Unable to initialize"): include.scopes(parent_scope) # fail in git config (so use default remote) *and* git checkout def _init_repo(*args, **kwargs): fs.mkdirp(fs.join_path(include.destination, ".git")) class MockIncludeGit(spack.util.executable.Executable): def __init__(self, required: bool): pass def __call__(self, *args, **kwargs) -> str: # type: ignore raise spack.util.executable.ProcessError("mock git failure") monkeypatch.setattr(spack.util.git, "init_git_repo", _init_repo) monkeypatch.setattr(spack.util.git, "git", MockIncludeGit) with pytest.raises(spack.error.ConfigError, match="Unable to check out"): include.scopes(parent_scope) # set up invalid option failure include.branch = "" # type: ignore[union-attr] with pytest.raises(spack.error.ConfigError, match="Missing or unsupported options"): include.scopes(parent_scope) def test_missing_include_scope_list(mock_missing_dir_include_scopes): """Tests that an included scope with a non existent file/directory is still listed as a scope under spack.config.CONFIG.scopes""" assert "sub_base" in list(spack.config.CONFIG.scopes), ( "Missing Optional Scope Missing from Config Scopes" ) def test_missing_include_scope_writable_list(mock_missing_dir_include_scopes): """Tests that missing include scopes are included in writeable config lists""" assert [x for x in spack.config.CONFIG.writable_scopes if x.name == "sub_base"] def test_missing_include_scope_not_readable_list(mock_missing_dir_include_scopes): """Tests that missing include scopes are not included in existing config lists""" existing_scopes = [x for x in spack.config.CONFIG.existing_scopes if x.name != "sub_base"] assert len(existing_scopes) == 1 assert existing_scopes[0].name != "sub_base" def test_missing_include_scope_default_created_as_dir_scope(mock_missing_dir_include_scopes): """Tests that an optional include with no existing file/directory and no yaml extension is created as a directoryscope object""" missing_inc_scope = spack.config.CONFIG.scopes["sub_base"] assert isinstance(missing_inc_scope, spack.config.DirectoryConfigScope) def test_missing_include_scope_yaml_ext_is_file_scope(mock_missing_file_include_scopes): """Tests that an optional include scope with no existing file/directory and a yaml extension is created as a file scope""" missing_inc_scope = spack.config.CONFIG.scopes["sub_base"] assert isinstance(missing_inc_scope, spack.config.SingleFileScope) def test_missing_include_scope_writeable_not_readable(mock_missing_dir_include_scopes): """Tests that an included scope with a non existent file/directory can be written to (and created)""" assert spack.config.CONFIG.scopes["sub_base"].writable, ( "Missing Optional Scope should be writable" ) assert not spack.config.CONFIG.scopes["sub_base"].exists, ( "Missing Optional Scope should not exist" ) def test_missing_include_scope_empty_read(mock_missing_dir_include_scopes): """Tests that an included scope with a non existent file/directory returns an empty dict on read and has "exists" set to false""" assert spack.config.CONFIG.get("config", scope="sub_base") == {}, ( "Missing optional include scope does not return an empty value." ) assert not spack.config.CONFIG.scopes["sub_base"].exists, ( "Missing optional include should not be created on read" ) def test_missing_include_scope_file_empty_read(mock_missing_file_include_scopes): """Tests that an include scope with a non existent file returns an empty dict and has exists set to false""" assert spack.config.CONFIG.get("config", scope="sub_base") == {}, ( "Missing optional include scope does not return an empty value." ) assert not spack.config.CONFIG.scopes["sub_base"].exists, ( "Missing optional include should not be created on read" ) def test_missing_include_scope_write_directory(mock_missing_dir_include_scopes): """Tests that an include scope with a non existent directory creates said directory and the appropriate section file on write""" install_tree = syaml.syaml_dict({"install_tree": {"root": "$spack/tmp/spack"}}) spack.config.CONFIG.set("config", install_tree, scope="sub_base") assert os.path.exists(spack.config.CONFIG.scopes["sub_base"].path) install_root = spack.config.CONFIG.get("config:install_tree:root", scope="sub_base") assert install_root == "$spack/tmp/spack" def test_missing_include_scope_write_file(mock_missing_file_include_scopes): """Tests that an include scope with a non existent file creates said file with the appropriate section entry""" install_tree = syaml.syaml_dict({"install_tree": {"root": "$spack/tmp/spack"}}) spack.config.CONFIG.set("config", install_tree, scope="sub_base") assert os.path.exists(spack.config.CONFIG.scopes["sub_base"].path) install_root = spack.config.CONFIG.get("config:install_tree:root", scope="sub_base") assert install_root == "$spack/tmp/spack" def test_config_scope_empty_write(tmp_path: pathlib.Path): """Confirm skipping attempt to write non-existent scope section.""" config_scope = spack.config.DirectoryConfigScope("test", str(tmp_path)) assert config_scope.get_section("include") is None def test_include_bad_parent_scope(tmp_path: pathlib.Path): """Test parent scope validation.""" path = tmp_path / "config.yaml" path.touch() entry = {"path": str(path)} include = spack.config.included_path(entry) # Confirm require a ConfigScope parent with pytest.raises(AssertionError, match="configuration scope"): _ = include.scopes("_builtin") # type: ignore # Confirm require a named parent scope for name in ["", " "]: parent_scope = spack.config.InternalConfigScope(name, spack.config.CONFIG_DEFAULTS) with pytest.raises(AssertionError, match="must have a name"): _ = include.scopes(parent_scope) def test_config_invalid_scope(mock_low_high_config): err = "Must be one of \\['low', 'high'\\]" # noqa: W605 with pytest.raises(ValueError, match=err): spack.config.CONFIG.get_config_filename("noscope", "nosection") ================================================ FILE: lib/spack/spack/test/config_values.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.concretize import spack.store @pytest.mark.parametrize("hash_length", [1, 2, 3, 4, 5, 9]) @pytest.mark.usefixtures("mock_packages") def test_set_install_hash_length(hash_length, mutable_config, tmp_path: pathlib.Path): mutable_config.set("config:install_hash_length", hash_length) with spack.store.use_store(str(tmp_path)): spec = spack.concretize.concretize_one("libelf") prefix = spec.prefix hash_str = prefix.rsplit("-")[-1] assert len(hash_str) == hash_length @pytest.mark.usefixtures("mock_packages") def test_set_install_hash_length_upper_case(mutable_config, tmp_path: pathlib.Path): mutable_config.set("config:install_hash_length", 5) with spack.store.use_store( str(tmp_path), extra_data={"projections": {"all": "{name}-{HASH}"}} ): spec = spack.concretize.concretize_one("libelf") prefix = spec.prefix hash_str = prefix.rsplit("-")[-1] assert len(hash_str) == 5 ================================================ FILE: lib/spack/spack/test/conftest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import base64 import collections import datetime import email.message import errno import functools import inspect import io import itertools import json import os import re import shutil import stat import sys import tempfile import textwrap import xml.etree.ElementTree from pathlib import Path from typing import Callable, List, Optional, Tuple import pytest import spack.vendor.archspec.cpu import spack.vendor.archspec.cpu.microarchitecture import spack.vendor.archspec.cpu.schema import spack.binary_distribution import spack.bootstrap import spack.caches import spack.compilers.config import spack.compilers.libraries import spack.concretize import spack.config import spack.directives_meta import spack.environment as ev import spack.error import spack.extensions import spack.hash_types import spack.llnl.util.lang import spack.llnl.util.lock import spack.llnl.util.tty as tty import spack.llnl.util.tty.color import spack.modules.common import spack.package_base import spack.paths import spack.platforms import spack.repo import spack.solver.asp import spack.solver.reuse import spack.spec import spack.stage import spack.store import spack.subprocess_context import spack.tengine import spack.util.executable import spack.util.file_cache import spack.util.git import spack.util.gpg import spack.util.naming import spack.util.parallel import spack.util.spack_yaml as syaml import spack.util.url as url_util import spack.util.web import spack.version from spack.enums import ConfigScopePriority from spack.fetch_strategy import URLFetchStrategy from spack.installer import PackageInstaller from spack.llnl.util.filesystem import ( copy, copy_tree, join_path, mkdirp, remove_linked_tree, working_dir, ) from spack.main import SpackCommand from spack.util.pattern import Bunch from spack.util.remote_file_cache import raw_github_gitlab_url mirror_cmd = SpackCommand("mirror") def _recursive_chmod(path: Path, mode: int): """Recursively change permissions of a directory and all its contents.""" path.chmod(mode) for root, dirs, files in os.walk(path): for file in files: os.chmod(os.path.join(root, file), mode) for dir in dirs: os.chmod(os.path.join(root, dir), mode) @pytest.fixture(autouse=True) def clear_sys_modules(): """Clear package repos from sys.modules before each test.""" for key in list(sys.modules.keys()): if key.startswith("spack_repo.") or key == "spack_repo": del sys.modules[key] yield @pytest.fixture(autouse=True) def check_config_fixture(request): if "config" in request.fixturenames and "mutable_config" in request.fixturenames: raise RuntimeError("'config' and 'mutable_config' are both requested") def ensure_configuration_fixture_run_before(request): """Ensure that fixture mutating the configuration run before the one where the function is called. """ if "config" in request.fixturenames: request.getfixturevalue("config") if "mutable_config" in request.fixturenames: request.getfixturevalue("mutable_config") @pytest.fixture(scope="session") def git(): """Fixture for tests that use git.""" try: return spack.util.git.git(required=True) except spack.util.executable.CommandNotFoundError: pytest.skip("requires git to be installed") # # Return list of shas for latest two git commits in local spack repo # @pytest.fixture(scope="session") def last_two_git_commits(git): spack_git_path = spack.paths.prefix with working_dir(spack_git_path): git_log_out = git("log", "-n", "2", output=str, error=os.devnull) regex = re.compile(r"^commit\s([^\s]+$)", re.MULTILINE) yield regex.findall(git_log_out) def write_file(filename, contents): with open(filename, "w", encoding="utf-8") as f: f.write(contents) commit_counter = 0 @pytest.fixture def override_git_repos_cache_path(tmp_path: Path): saved = spack.paths.user_repos_cache_path tmp_git_path = tmp_path / "git-repo-cache-path-for-tests" tmp_git_path.mkdir() spack.paths.user_repos_cache_path = str(tmp_git_path) yield spack.paths.user_repos_cache_path = saved @pytest.fixture def mock_git_version_info(git, tmp_path: Path, override_git_repos_cache_path): """Create a mock git repo with known structure The structure of commits in this repo is as follows:: | o fourth 1.x commit (1.2) | o third 1.x commit | | o | fourth main commit (v2.0) o | third main commit | | | o second 1.x commit (v1.1) | o first 1.x commit | / |/ o second commit (v1.0) o first commit The repo consists of a single file, in which the GitVersion.std_version representation of each commit is expressed as a string. Important attributes of the repo for test coverage are: multiple branches, version tags on multiple branches, and version order is not equal to time order or topological order. """ repo_dir = tmp_path / "git_version_info_repo" repo_dir.mkdir() repo_path = str(repo_dir) filename = "file.txt" def commit(message): global commit_counter git( "commit", "--no-gpg-sign", "--date", "2020-01-%02d 12:0:00 +0300" % commit_counter, "-am", message, ) commit_counter += 1 with working_dir(repo_path): git("init") git("config", "user.name", "Spack") git("config", "user.email", "spack@spack.io") git("checkout", "-b", "main") commits = [] def latest_commit(): return git("rev-list", "-n1", "HEAD", output=str, error=str).strip() # Add two commits on main branch # A commit without a previous version counts as "0" write_file(filename, "[0]") git("add", filename) commit("first commit") commits.append(latest_commit()) # Tag second commit as v1.0 write_file(filename, "[1, 0]") commit("second commit") commits.append(latest_commit()) git("tag", "v1.0") # Add two commits and a tag on 1.x branch git("checkout", "-b", "1.x") write_file(filename, "[1, 0, 'git', 1]") commit("first 1.x commit") commits.append(latest_commit()) write_file(filename, "[1, 1]") commit("second 1.x commit") commits.append(latest_commit()) git("tag", "v1.1") # Add two commits and a tag on main branch git("checkout", "main") write_file(filename, "[1, 0, 'git', 1]") commit("third main commit") commits.append(latest_commit()) write_file(filename, "[2, 0]") commit("fourth main commit") commits.append(latest_commit()) git("tag", "v2.0") # Add two more commits on 1.x branch to ensure we aren't cheating by using time git("checkout", "1.x") write_file(filename, "[1, 1, 'git', 1]") commit("third 1.x commit") commits.append(latest_commit()) write_file(filename, "[1, 2]") commit("fourth 1.x commit") commits.append(latest_commit()) git("tag", "1.2") # test robust parsing to different syntax, no v # The commits are ordered with the last commit first in the list commits = list(reversed(commits)) # Return the git directory to install, the filename used, and the commits yield repo_path, filename, commits @pytest.fixture def mock_git_package_changes(git, tmp_path: Path, override_git_repos_cache_path, monkeypatch): """Create a mock git repo with known structure of package edits The structure of commits in this repo is as follows:: o diff-test: add v2.1.7 and v2.1.8 (invalid duplicated checksum) | o diff-test: add v2.1.6 (from a git ref) | o diff-test: add v2.1.5 (from source tarball) | o diff-test: new package (testing multiple added versions) The repo consists of a single package.py file for DiffTest. Important attributes of the repo for test coverage are: multiple package versions are added with some coming from a tarball and some from git refs. """ filename = "diff_test/package.py" repo_path, _ = spack.repo.create_repo(str(tmp_path), namespace="myrepo") cache_dir = tmp_path / "cache" cache_dir.mkdir() repo_cache = spack.util.file_cache.FileCache(str(cache_dir)) repo = spack.repo.Repo(repo_path, cache=repo_cache) def commit(message): global commit_counter git( "commit", "--no-gpg-sign", "--date", "2020-01-%02d 12:0:00 +0300" % commit_counter, "-am", message, ) commit_counter += 1 with working_dir(repo.packages_path): git("init") git("config", "user.name", "Spack") git("config", "user.email", "spack@spack.io") commits = [] def latest_commit(): return git("rev-list", "-n1", "HEAD", output=str, error=str).strip() os.makedirs(os.path.dirname(filename)) # add diff-test as a new package to the repository shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-0.txt", filename) git("add", filename) commit("diff-test: new package") commits.append(latest_commit()) # add v2.1.5 to diff-test shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-1.txt", filename) git("add", filename) commit("diff-test: add v2.1.5") commits.append(latest_commit()) # add v2.1.6 to diff-test shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-2.txt", filename) git("add", filename) commit("diff-test: add v2.1.6") commits.append(latest_commit()) # add v2.1.7 and v2.1.8 to diff-test shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-3.txt", filename) git("add", filename) commit("diff-test: add v2.1.7 and v2.1.8") commits.append(latest_commit()) # The commits are ordered with the last commit first in the list commits = list(reversed(commits)) # Return the git directory to install, the filename used, and the commits yield repo, filename, commits @pytest.fixture(autouse=True) def clear_recorded_monkeypatches(): yield spack.subprocess_context.MONKEYPATCHES.clear() @pytest.fixture(scope="session", autouse=True) def record_monkeypatch_setattr(): import _pytest saved_setattr = _pytest.monkeypatch.MonkeyPatch.setattr def record_setattr(cls, target, name, value, *args, **kwargs): spack.subprocess_context.MONKEYPATCHES.append((target, name)) saved_setattr(cls, target, name, value, *args, **kwargs) _pytest.monkeypatch.MonkeyPatch.setattr = record_setattr try: yield finally: _pytest.monkeypatch.MonkeyPatch.setattr = saved_setattr def _can_access(path, perms): return False @pytest.fixture def no_path_access(monkeypatch): monkeypatch.setattr(os, "access", _can_access) # # Disable any active Spack environment BEFORE all tests # @pytest.fixture(scope="session", autouse=True) def clean_user_environment(): spack_env_value = os.environ.pop(ev.spack_env_var, None) with ev.no_active_environment(): yield if spack_env_value: os.environ[ev.spack_env_var] = spack_env_value # # Make sure global state of active env does not leak between tests. # @pytest.fixture(scope="function", autouse=True) def clean_test_environment(): yield ev.deactivate() def _host(): """Mock archspec host so there is no inconsistency on the Windows platform This function cannot be local as it needs to be pickleable""" return spack.vendor.archspec.cpu.Microarchitecture("x86_64", [], "generic", [], {}, 0) @pytest.fixture(scope="function") def archspec_host_is_spack_test_host(monkeypatch): monkeypatch.setattr(spack.vendor.archspec.cpu, "host", _host) # Hooks to add command line options or set other custom behaviors. # They must be placed here to be found by pytest. See: # # https://docs.pytest.org/en/latest/writing_plugins.html # def pytest_addoption(parser): group = parser.getgroup("Spack specific command line options") group.addoption( "--fast", action="store_true", default=False, help='runs only "fast" unit tests, instead of the whole suite', ) def pytest_collection_modifyitems(config, items): if not config.getoption("--fast"): # --fast not given, run all the tests return slow_tests = ["db", "network", "maybeslow"] skip_as_slow = pytest.mark.skip(reason="skipped slow test [--fast command line option given]") for item in items: if any(x in item.keywords for x in slow_tests): item.add_marker(skip_as_slow) @pytest.fixture(scope="function") def use_concretization_cache(mock_packages, mutable_config, tmp_path: Path): """Enables the use of the concretization cache""" conc_cache_dir = tmp_path / "concretization" conc_cache_dir.mkdir() # ensure we have an isolated concretization cache while using fixture with spack.config.override( "concretizer:concretization_cache", {"enable": True, "url": str(conc_cache_dir)} ): yield conc_cache_dir # # These fixtures are applied to all tests # @pytest.fixture(scope="function", autouse=True) def no_chdir(): """Ensure that no test changes Spack's working directory. This prevents Spack tests (and therefore Spack commands) from changing the working directory and causing other tests to fail mysteriously. Tests should use ``working_dir`` or ``py.path``'s ``.as_cwd()`` instead of ``os.chdir`` to avoid failing this check. We assert that the working directory hasn't changed, unless the original wd somehow ceased to exist. """ original_wd = os.getcwd() yield if os.path.isdir(original_wd): assert os.getcwd() == original_wd def onerror(func, path, error_info): # Python on Windows is unable to remove paths without # write (IWUSR) permissions (such as those generated by Git on Windows) # This method changes file permissions to allow removal by Python os.chmod(path, stat.S_IWUSR) func(path) @pytest.fixture(scope="function", autouse=True) def mock_stage(tmp_path_factory: pytest.TempPathFactory, monkeypatch, request): """Establish the temporary build_stage for the mock archive.""" # The approach with this autouse fixture is to set the stage root # instead of using spack.config.override() to avoid configuration # conflicts with dozens of tests that rely on other configuration # fixtures, such as config. if "nomockstage" in request.keywords: # Tests can opt-out with @pytest.mark.nomockstage yield None return # Set the build stage to the requested path new_stage = tmp_path_factory.mktemp("mock-stage") # Ensure the source directory exists within the new stage path source_path = new_stage / spack.stage._source_path_subdir source_path.mkdir(parents=True, exist_ok=True) monkeypatch.setattr(spack.stage, "_stage_root", str(new_stage)) yield str(new_stage) # Clean up the test stage directory if new_stage.is_dir(): shutil.rmtree(new_stage, onerror=onerror) @pytest.fixture(scope="session") def mock_stage_for_database(tmp_path_factory: pytest.TempPathFactory, monkeypatch_session): """A session-scoped analog of mock_stage, so that the mock_store fixture uses its own stage vs. the global stage root for spack. """ new_stage = tmp_path_factory.mktemp("mock-stage") source_path = new_stage / spack.stage._source_path_subdir source_path.mkdir(parents=True, exist_ok=True) monkeypatch_session.setattr(spack.stage, "_stage_root", str(new_stage)) yield str(new_stage) # Clean up the test stage directory if new_stage.is_dir(): shutil.rmtree(new_stage, onerror=onerror) @pytest.fixture(scope="session") def ignore_stage_files(): """Session-scoped helper for check_for_leftover_stage_files. Used to track which leftover files in the stage have been seen. """ # to start with, ignore the .lock file at the stage root. return set([".lock", spack.stage._source_path_subdir, "build_cache"]) def remove_whatever_it_is(path): """Type-agnostic remove.""" if os.path.isfile(path): os.remove(path) elif os.path.islink(path): remove_linked_tree(path) else: shutil.rmtree(path, onerror=onerror) @pytest.fixture def working_env(): saved_env = os.environ.copy() yield # os.environ = saved_env doesn't work # it causes module_parsing::test_module_function to fail # when it's run after any test using this fixutre os.environ.clear() os.environ.update(saved_env) @pytest.fixture(scope="function", autouse=True) def check_for_leftover_stage_files(request, mock_stage, ignore_stage_files): """ Ensure that each (mock_stage) test leaves a clean stage when done. Tests that are expected to dirty the stage can disable the check by adding:: @pytest.mark.disable_clean_stage_check and the associated stage files will be removed. """ yield if mock_stage is None: # When tests opt out with @pytest.mark.nomockstage, do not check for left-over files return files_in_stage = set() try: stage_files = os.listdir(mock_stage) files_in_stage = set(stage_files) - ignore_stage_files except OSError as err: if err.errno == errno.ENOENT or err.errno == errno.EINVAL: pass else: raise if "disable_clean_stage_check" in request.keywords: # clean up after tests that are expected to be dirty for f in files_in_stage: path = os.path.join(mock_stage, f) remove_whatever_it_is(path) else: ignore_stage_files |= files_in_stage assert not files_in_stage class MockCache: def store(self, copy_cmd, relative_dest): pass def fetcher(self, target_path, digest, **kwargs): return MockCacheFetcher() class MockCacheFetcher: def fetch(self): raise spack.error.FetchError("Mock cache always fails for tests") def __str__(self): return "[mock fetch cache]" @pytest.fixture(autouse=True) def mock_fetch_cache(monkeypatch): """Substitutes spack.paths.FETCH_CACHE with a mock object that does nothing and raises on fetch. """ monkeypatch.setattr(spack.caches, "FETCH_CACHE", MockCache()) @pytest.fixture() def mock_binary_index(monkeypatch, tmp_path_factory: pytest.TempPathFactory): """Changes the directory for the binary index and creates binary index for every test. Clears its own index when it's done. """ tmpdir = tmp_path_factory.mktemp("mock_binary_index") index_path = tmpdir / "binary_index" mock_index = spack.binary_distribution.BinaryCacheIndex(str(index_path)) monkeypatch.setattr(spack.binary_distribution, "BINARY_INDEX", mock_index) yield @pytest.fixture(autouse=True) def _skip_if_missing_executables(request, monkeypatch): """Permits to mark tests with 'require_executables' and skip the tests if the executables passed as arguments are not found. """ marker = request.node.get_closest_marker("requires_executables") if marker: required_execs = marker.args missing_execs = [x for x in required_execs if spack.util.executable.which(x) is None] if missing_execs: msg = "could not find executables: {0}" pytest.skip(msg.format(", ".join(missing_execs))) # In case we require a compiler, clear the caches used to speed-up detection monkeypatch.setattr(spack.compilers.libraries.DefaultDynamicLinkerFilter, "_CACHE", {}) @pytest.fixture(scope="session") def test_platform(): return spack.platforms.Test() @pytest.fixture(autouse=True, scope="session") def _use_test_platform(test_platform): # This is the only context manager used at session scope (see note # below for more insight) since we want to use the test platform as # a default during tests. with spack.platforms.use_platform(test_platform): yield # # Note on context managers used by fixtures # # Because these context managers modify global state, they should really # ONLY be used persistently (i.e., around yield statements) in # function-scoped fixtures, OR in autouse session- or module-scoped # fixtures. # # If they're used in regular tests or in module-scoped fixtures that are # then injected as function arguments, weird things can happen, because # the original state won't be restored until *after* the fixture is # destroyed. This makes sense for an autouse fixture, where you know # everything in the module/session is going to need the modified # behavior, but modifying global state for one function in a way that # won't be restored until after the module or session is done essentially # leaves garbage behind for other tests. # # In general, we should module- or session-scope the *STATE* required for # these global objects, but we shouldn't module- or session-scope their # *USE*, or things can get really confusing. # # # Test-specific fixtures # @pytest.fixture(scope="session") def mock_packages_repo(): yield spack.repo.from_path(spack.paths.mock_packages_path) def _pkg_install_fn(pkg, spec, prefix): # sanity_check_prefix requires something in the install directory mkdirp(prefix.bin) @pytest.fixture def mock_pkg_install(monkeypatch): monkeypatch.setattr(spack.package_base.PackageBase, "install", _pkg_install_fn, raising=False) @pytest.fixture(scope="function") def fake_db_install(tmp_path): """This fakes "enough" of the installation process to make Spack think of a spec as being installed as far as the concretizer and parser are concerned. It does not run any build phase defined in the package, simply acting as though the installation had completed successfully. It allows doing things like ``spack.concretize.concretize_one(f"x ^/hash-of-y")`` after doing something like ``fake_db_install(y)`` """ with spack.store.use_store(str(tmp_path)) as the_store: def _install(a_spec): the_store.db.add(a_spec) yield _install @pytest.fixture(scope="function") def mock_packages(mock_packages_repo, mock_pkg_install, request): """Use the 'builtin_mock' repository instead of 'builtin'""" ensure_configuration_fixture_run_before(request) with spack.repo.use_repositories(mock_packages_repo) as mock_repo: yield mock_repo @pytest.fixture(scope="function") def mutable_mock_repo(mock_packages_repo, request): """Function-scoped mock packages, for tests that need to modify them.""" ensure_configuration_fixture_run_before(request) mock_repo = spack.repo.from_path(spack.paths.mock_packages_path) with spack.repo.use_repositories(mock_repo) as mock_packages_repo: yield mock_packages_repo class RepoBuilder: """Build a mock repository in a directory""" _counter = 0 def __init__(self, root_directory: str) -> None: RepoBuilder._counter += 1 namespace = f"test_namespace_{RepoBuilder._counter}" repo_root = os.path.join(root_directory, namespace) os.makedirs(repo_root, exist_ok=True) self.template_dirs = (os.path.join(spack.paths.share_path, "templates"),) self.root, self.namespace = spack.repo.create_repo(repo_root, namespace) self.build_system_name = f"test_build_system_{self.namespace}" self._add_build_system() def add_package( self, name: str, dependencies: Optional[List[Tuple[str, Optional[str], Optional[str]]]] = None, ) -> None: """Create a mock package in the repository, using a Jinja2 template. Args: name: name of the new package dependencies: list of ("dep_spec", "dep_type", "condition") tuples. Both "dep_type" and "condition" can default to ``None`` in which case ``spack.dependency.default_deptype`` and ``spack.spec.Spec()`` are used. """ dependencies = dependencies or [] context = { "cls_name": spack.util.naming.pkg_name_to_class_name(name), "dependencies": dependencies, } template = spack.tengine.make_environment_from_dirs(self.template_dirs).get_template( "mock-repository/package.pyt" ) package_py = self._recipe_filename(name) os.makedirs(os.path.dirname(package_py), exist_ok=True) with open(package_py, "w", encoding="utf-8") as f: f.write(template.render(context)) def remove(self, name: str) -> None: package_py = self._recipe_filename(name) shutil.rmtree(os.path.dirname(package_py)) def _add_build_system(self) -> None: """Add spack_repo..build_systems.test_build_system with build_system=test_build_system_.""" template = spack.tengine.make_environment_from_dirs(self.template_dirs).get_template( "mock-repository/build_system.pyt" ) text = template.render({"build_system_name": self.build_system_name}) build_system_py = os.path.join(self.root, "build_systems", "test_build_system.py") os.makedirs(os.path.dirname(build_system_py), exist_ok=True) with open(build_system_py, "w", encoding="utf-8") as f: f.write(text) def _recipe_filename(self, name: str) -> str: return os.path.join( self.root, "packages", spack.util.naming.pkg_name_to_pkg_dir(name, package_api=(2, 0)), "package.py", ) @pytest.fixture def repo_builder(tmp_path: Path): return RepoBuilder(str(tmp_path)) @pytest.fixture() def mock_custom_repository(tmp_path: Path, mutable_mock_repo): """Create a custom repository with a single package "c" and return its path.""" builder = RepoBuilder(str(tmp_path)) builder.add_package("pkg-c") return builder.root @pytest.fixture(scope="session") def linux_os(): """Returns a named tuple with attributes 'name' and 'version' representing the OS. """ platform = spack.platforms.host() name, version = "debian", "6" if platform.name == "linux": current_os = platform.default_operating_system() name, version = current_os.name, current_os.version LinuxOS = collections.namedtuple("LinuxOS", ["name", "version"]) return LinuxOS(name=name, version=version) @pytest.fixture def ensure_debug(monkeypatch): current_debug_level = tty.debug_level() tty.set_debug(1) yield tty.set_debug(current_debug_level) @pytest.fixture def default_config(): """Isolates the default configuration from the user configs. This ensures we can test the real default configuration without having tests fail when the user overrides the defaults that we test against.""" defaults_path = os.path.join(spack.paths.etc_path, "defaults") if sys.platform == "win32": defaults_path = os.path.join(defaults_path, "windows") with spack.config.use_configuration(defaults_path) as defaults_config: yield defaults_config @pytest.fixture(scope="session") def mock_uarch_json(tmp_path_factory: pytest.TempPathFactory): """Mock microarchitectures.json with test architecture descriptions.""" tmpdir = tmp_path_factory.mktemp("microarchitectures") uarch_json_source = ( Path(spack.paths.test_path) / "data" / "microarchitectures" / "microarchitectures.json" ) uarch_json_dest = tmpdir / "microarchitectures.json" shutil.copy2(uarch_json_source, uarch_json_dest) yield str(uarch_json_dest) @pytest.fixture(scope="session") def mock_uarch_configuration(mock_uarch_json): """Create mock dictionaries for the spack.vendor.archspec.cpu.""" def load_json(): with open(mock_uarch_json, encoding="utf-8") as f: return json.load(f) targets_json = load_json() targets = spack.vendor.archspec.cpu.microarchitecture._known_microarchitectures() yield targets_json, targets @pytest.fixture(scope="function") def mock_targets(mock_uarch_configuration, monkeypatch): """Use this fixture to enable mock uarch targets for testing.""" targets_json, targets = mock_uarch_configuration monkeypatch.setattr(spack.vendor.archspec.cpu.schema, "TARGETS_JSON", targets_json) monkeypatch.setattr(spack.vendor.archspec.cpu.microarchitecture, "TARGETS", targets) @pytest.fixture(scope="session") def configuration_dir(tmp_path_factory: pytest.TempPathFactory, linux_os): """Copies mock configuration files in a temporary directory. Returns the directory path. """ tmp_path = tmp_path_factory.mktemp("configurations") install_tree_root = tmp_path_factory.mktemp("opt") modules_root = tmp_path_factory.mktemp("share") tcl_root = modules_root / "modules" tcl_root.mkdir() lmod_root = modules_root / "lmod" lmod_root.mkdir() # /data/config has mock config yaml files in it # copy these to the site config. test_config = Path(spack.paths.test_path) / "data" / "config" shutil.copytree(test_config, tmp_path / "site") # Create temporary 'defaults', 'site' and 'user' folders (tmp_path / "user").mkdir() # Fill out config.yaml, compilers.yaml and modules.yaml templates. locks = sys.platform != "win32" config = tmp_path / "site" / "config.yaml" config_template = test_config / "config.yaml" config.write_text(config_template.read_text().format(install_tree_root, locks)) target = str(spack.vendor.archspec.cpu.host().family) compilers = tmp_path / "site" / "packages.yaml" compilers_template = test_config / "packages.yaml" compilers.write_text(compilers_template.read_text().format(linux_os=linux_os, target=target)) modules = tmp_path / "site" / "modules.yaml" modules_template = test_config / "modules.yaml" modules.write_text(modules_template.read_text().format(tcl_root, lmod_root)) for scope in ("spack", "user", "site", "system"): scope_path = tmp_path / scope scope_path.mkdir(exist_ok=True) include = tmp_path / "spack" / "include.yaml" # Need to use relative include paths here so it works for mutable_config fixture too with include.open("w", encoding="utf-8") as f: f.write( textwrap.dedent( """ include: # user configuration scope - name: "user" path_override_env_var: SPACK_USER_CONFIG_PATH path: ../user optional: true prefer_modify: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' # site configuration scope - name: "site" path: ../site optional: true # system configuration scope - name: "system" path_override_env_var: SPACK_SYSTEM_CONFIG_PATH path: ../system optional: true when: '"SPACK_DISABLE_LOCAL_CONFIG" not in env' """ ) ) yield tmp_path def _create_mock_configuration_scopes(configuration_dir): """Create the configuration scopes used in `config` and `mutable_config`.""" return [ ( ConfigScopePriority.DEFAULTS, spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS), ), ( ConfigScopePriority.CONFIG_FILES, spack.config.DirectoryConfigScope("spack", str(configuration_dir / "spack")), ), (ConfigScopePriority.COMMAND_LINE, spack.config.InternalConfigScope("command_line")), ] @pytest.fixture(scope="session") def mock_configuration_scopes(configuration_dir): """Create a persistent Configuration object from the configuration_dir.""" yield _create_mock_configuration_scopes(configuration_dir) @pytest.fixture(scope="function") def config(mock_configuration_scopes): """This fixture activates/deactivates the mock configuration.""" with spack.config.use_configuration(*mock_configuration_scopes) as config: yield config @pytest.fixture(scope="function") def mutable_config(tmp_path_factory: pytest.TempPathFactory, configuration_dir): """Like config, but tests can modify the configuration.""" mutable_dir = tmp_path_factory.mktemp("mutable_config") / "tmp" shutil.copytree(configuration_dir, mutable_dir) scopes = _create_mock_configuration_scopes(mutable_dir) with spack.config.use_configuration(*scopes) as cfg: yield cfg @pytest.fixture(scope="function") def mutable_empty_config(tmp_path_factory: pytest.TempPathFactory, configuration_dir): """Empty configuration that can be modified by the tests.""" mutable_dir = tmp_path_factory.mktemp("mutable_config") / "tmp" scopes = [ spack.config.DirectoryConfigScope(name, str(mutable_dir / name)) for name in ["site", "system", "user"] ] with spack.config.use_configuration(*scopes) as cfg: yield cfg # From https://github.com/pytest-dev/pytest/issues/363#issuecomment-1335631998 # Current suggested implementation from issue compatible with pytest >= 6.2 # this may be subject to change as new versions of Pytest are released # and update the suggested solution @pytest.fixture(scope="session") def monkeypatch_session(): with pytest.MonkeyPatch.context() as monkeypatch: yield monkeypatch @pytest.fixture(autouse=True) def mock_wsdk_externals(monkeypatch): """Skip check for required external packages on Windows during testing.""" monkeypatch.setattr(spack.bootstrap, "ensure_winsdk_external_or_raise", _return_none) @pytest.fixture(scope="function") def concretize_scope(mutable_config, tmp_path: Path): """Adds a scope for concretization preferences""" concretize_dir = tmp_path / "concretize" concretize_dir.mkdir() with spack.config.override( spack.config.DirectoryConfigScope("concretize", str(concretize_dir)) ): yield str(concretize_dir) spack.repo.PATH._provider_index = None @pytest.fixture def no_packages_yaml(mutable_config): """Creates a temporary configuration without compilers.yaml""" for local_config in mutable_config.scopes.values(): if not isinstance(local_config, spack.config.DirectoryConfigScope): continue compilers_yaml = local_config.get_section_filename("packages") if os.path.exists(compilers_yaml): os.remove(compilers_yaml) mutable_config.clear_caches() return mutable_config @pytest.fixture() def mock_low_high_config(tmp_path: Path): """Mocks two configuration scopes: 'low' and 'high'.""" scopes = [ spack.config.DirectoryConfigScope(name, str(tmp_path / name)) for name in ["low", "high"] ] with spack.config.use_configuration(*scopes) as config: yield config def create_config_scope(path: Path, name: str) -> spack.config.DirectoryConfigScope: """helper for creating config scopes with included file/directory scopes that do not have existing representation on the filesystem""" base_scope_dir = path / "base" config_data = syaml.syaml_dict( { "include": [ { "name": "sub_base", "path": str(path / name), "optional": True, "prefer_modify": True, } ] } ) base_scope_dir.mkdir() with open(str(base_scope_dir / "include.yaml"), "w+", encoding="utf-8") as f: syaml.dump_config(config_data, stream=f, default_flow_style=False) scope = spack.config.DirectoryConfigScope("base", str(base_scope_dir)) return scope @pytest.fixture() def mock_missing_dir_include_scopes(tmp_path: Path): """Mocks a config scope containing optional directory scope includes that do not have representation on the filesystem""" scope = create_config_scope(tmp_path, "sub") with spack.config.use_configuration(scope) as config: yield config @pytest.fixture def mock_missing_file_include_scopes(tmp_path: Path): """Mocks a config scope containing optional file scope includes that do not have representation on the filesystem""" scope = create_config_scope(tmp_path, "sub.yaml") with spack.config.use_configuration(scope) as config: yield config def _populate(mock_db): r"""Populate a mock database with packages. Here is what the mock DB looks like (explicit roots at top): o mpileaks o mpileaks' o mpileaks'' o externaltest o trivial-smoke-test |\ |\ |\ | | o callpath | o callpath' | o callpath'' o externaltool |/| |/| |/| | o | mpich o | mpich2 o | zmpi o externalvirtual | | o | fake | | | | |______________/ | .____________/ |/ o dyninst |\ | o libdwarf |/ o libelf """ def _install(spec): s = spack.concretize.concretize_one(spec) PackageInstaller([s.package], fake=True, explicit=True).install() _install("mpileaks ^mpich") _install("mpileaks ^mpich2") _install("mpileaks ^zmpi") _install("externaltest ^externalvirtual") _install("trivial-smoke-test") @pytest.fixture(scope="session") def _store_dir_and_cache(tmp_path_factory: pytest.TempPathFactory): """Returns the directory where to build the mock database and where to cache it. """ store = tmp_path_factory.mktemp("mock_store") cache = tmp_path_factory.mktemp("mock_store_cache") return store, cache @pytest.fixture(scope="session") def mock_store( tmp_path_factory: pytest.TempPathFactory, mock_packages_repo, mock_configuration_scopes, _store_dir_and_cache: Tuple[Path, Path], mock_stage_for_database, ): """Creates a read-only mock database with some packages installed note that the ref count for dyninst here will be 3, as it's recycled across each install. This does not actually activate the store for use by Spack -- see the ``database`` fixture for that. """ store_path, store_cache = _store_dir_and_cache _mock_wsdk_externals = spack.bootstrap.ensure_winsdk_external_or_raise # Make the DB filesystem read-only to ensure constructors don't modify anything in it. # We want Spack to be able to point to a DB on a read-only filesystem easily. _recursive_chmod(store_path, 0o555) # If the cache does not exist populate the store and create it if not os.path.exists(str(store_cache / ".spack-db")): with spack.config.use_configuration(*mock_configuration_scopes): with spack.store.use_store(str(store_path)) as store: with spack.repo.use_repositories(mock_packages_repo): # make the DB filesystem writable only while we populate it _recursive_chmod(store_path, 0o755) try: spack.bootstrap.ensure_winsdk_external_or_raise = _return_none _populate(store.db) finally: spack.bootstrap.ensure_winsdk_external_or_raise = _mock_wsdk_externals _recursive_chmod(store_path, 0o555) _recursive_chmod(store_cache, 0o755) copy_tree(str(store_path), str(store_cache)) _recursive_chmod(store_cache, 0o555) yield store_path @pytest.fixture(scope="function") def database(mock_store, mock_packages, config): """This activates the mock store, packages, AND config.""" with spack.store.use_store(str(mock_store)) as store: yield store.db # Force reading the database again between tests store.db.last_seen_verifier = "" @pytest.fixture(scope="function") def database_mutable_config(mock_store, mock_packages, mutable_config, monkeypatch): """This activates the mock store, packages, AND config.""" with spack.store.use_store(str(mock_store)) as store: yield store.db store.db.last_seen_verifier = "" @pytest.fixture(scope="function") def mutable_database(database_mutable_config, _store_dir_and_cache: Tuple[Path, Path]): """Writeable version of the fixture, restored to its initial state after each test. """ # Make the database writeable, as we are going to modify it store_path, store_cache = _store_dir_and_cache _recursive_chmod(store_path, 0o755) yield database_mutable_config # Restore the initial state by copying the content of the cache back into # the store and making the database read-only shutil.rmtree(store_path) copy_tree(str(store_cache), str(store_path)) _recursive_chmod(store_path, 0o555) @pytest.fixture() def dirs_with_libfiles(tmp_path_factory: pytest.TempPathFactory): lib_to_libfiles = { "libstdc++": ["libstdc++.so", "libstdc++.tbd"], "libgfortran": ["libgfortran.a", "libgfortran.dylib"], "libirc": ["libirc.a", "libirc.so"], } root = tmp_path_factory.mktemp("root") lib_to_dirs = {} i = 0 for lib, libfiles in lib_to_libfiles.items(): dirs = [] for libfile in libfiles: lib_dir = root / str(i) lib_dir.mkdir() (lib_dir / libfile).touch() dirs.append(str(lib_dir)) i += 1 lib_to_dirs[lib] = dirs all_dirs = list(itertools.chain.from_iterable(lib_to_dirs.values())) yield lib_to_dirs, all_dirs def _return_none(*args): return None @pytest.fixture(autouse=True) def disable_compiler_output_cache(monkeypatch): monkeypatch.setattr( spack.compilers.libraries, "COMPILER_CACHE", spack.compilers.libraries.CompilerCache() ) @pytest.fixture(scope="function") def install_mockery(temporary_store: spack.store.Store, mutable_config, mock_packages): """Hooks a fake install directory, DB, and stage directory into Spack.""" # We use a fake package, so temporarily disable checksumming with spack.config.override("config:checksum", False): yield # Wipe out any cached prefix failure locks (associated with the session-scoped mock archive) temporary_store.failure_tracker.clear_all() @pytest.fixture(scope="function") def temporary_mirror(mutable_config, tmp_path_factory): mirror_dir = tmp_path_factory.mktemp("mirror") mirror_cmd("add", "test-mirror-func", mirror_dir.as_uri()) yield str(mirror_dir) @pytest.fixture(scope="function") def temporary_store(tmp_path: Path, request): """Hooks a temporary empty store for the test function.""" ensure_configuration_fixture_run_before(request) temporary_store_path = tmp_path / "opt" with spack.store.use_store(str(temporary_store_path)) as s: yield s if temporary_store_path.exists(): shutil.rmtree(temporary_store_path) @pytest.fixture() def mock_fetch(mock_archive, monkeypatch): """Fake the URL for a package so it downloads from a file.""" monkeypatch.setattr( spack.package_base.PackageBase, "fetcher", URLFetchStrategy(url=mock_archive.url) ) class MockResourceFetcherGenerator: def __init__(self, url): self.url = url def _generate_fetchers(self, *args, **kwargs): return [URLFetchStrategy(url=self.url)] @pytest.fixture() def mock_resource_fetch(mock_archive, monkeypatch): """Fake fetcher generator that works with resource stages to redirect to a file.""" mfg = MockResourceFetcherGenerator(mock_archive.url) monkeypatch.setattr(spack.stage.ResourceStage, "_generate_fetchers", mfg._generate_fetchers) class MockLayout: def __init__(self, root): self.root = root def path_for_spec(self, spec): return os.path.sep.join([self.root, spec.name + "-" + spec.dag_hash()]) def ensure_installed(self, spec): pass @pytest.fixture() def gen_mock_layout(tmp_path: Path): # Generate a MockLayout in a temporary directory. In general the prefixes # specified by MockLayout should never be written to, but this ensures # that even if they are, that it causes no harm def create_layout(root): subroot = tmp_path / root subroot.mkdir(parents=True, exist_ok=True) return MockLayout(str(subroot)) yield create_layout class MockConfig: def __init__(self, configuration, writer_key): self._configuration = configuration self.writer_key = writer_key def configuration(self, module_set_name): return self._configuration def writer_configuration(self, module_set_name): return self.configuration(module_set_name)[self.writer_key] class ConfigUpdate: def __init__(self, root_for_conf, writer_mod, writer_key, monkeypatch): self.root_for_conf = root_for_conf self.writer_mod = writer_mod self.writer_key = writer_key self.monkeypatch = monkeypatch def __call__(self, filename): file = os.path.join(self.root_for_conf, filename + ".yaml") with open(file, encoding="utf-8") as f: config_settings = syaml.load_config(f) spack.config.set("modules:default", config_settings) mock_config = MockConfig(config_settings, self.writer_key) self.monkeypatch.setattr(spack.modules.common, "configuration", mock_config.configuration) self.monkeypatch.setattr( self.writer_mod, "configuration", mock_config.writer_configuration ) self.monkeypatch.setattr(self.writer_mod, "configuration_registry", {}) @pytest.fixture() def module_configuration(monkeypatch, request, mutable_config): """Reads the module configuration file from the mock ones prepared for tests and monkeypatches the right classes to hook it in. """ # Class of the module file writer writer_cls = getattr(request.module, "writer_cls") # Module where the module file writer is defined writer_mod = inspect.getmodule(writer_cls) # Key for specific settings relative to this module type writer_key = str(writer_mod.__name__).split(".")[-1] # Root folder for configuration root_for_conf = os.path.join(spack.paths.test_path, "data", "modules", writer_key) # ConfigUpdate, when called, will modify configuration, so we need to use # the mutable_config fixture return ConfigUpdate(root_for_conf, writer_mod, writer_key, monkeypatch) @pytest.fixture() def mock_gnupghome(monkeypatch): # GNU PGP can't handle paths longer than 108 characters (wtf!@#$) so we # have to make our own tmp_path with a shorter name than pytest's. # This comes up because tmp paths on macOS are already long-ish, and # pytest makes them longer. try: spack.util.gpg.init() except spack.util.gpg.SpackGPGError: if not spack.util.gpg.GPG: pytest.skip("This test requires gpg") short_name_tmpdir = tempfile.mkdtemp() with spack.util.gpg.gnupghome_override(short_name_tmpdir): yield short_name_tmpdir # clean up, since we are doing this manually # Ignore errors cause we seem to be hitting a bug similar to # https://bugs.python.org/issue29699 in CI (FileNotFoundError: [Errno 2] No such # file or directory: 'S.gpg-agent.extra'). shutil.rmtree(short_name_tmpdir, ignore_errors=True) ########## # Fake archives and repositories ########## @pytest.fixture(scope="session", params=[(".tar.gz", "z")]) def mock_archive(request, tmp_path_factory: pytest.TempPathFactory): """Creates a very simple archive directory with a configure script and a makefile that installs to a prefix. Tars it up into an archive. """ try: tar = spack.util.executable.which("tar", required=True) except spack.util.executable.CommandNotFoundError: pytest.skip("requires tar to be installed") tmpdir = tmp_path_factory.mktemp("mock-archive-dir") source_dir = tmpdir / spack.stage._source_path_subdir source_dir.mkdir() repodir = source_dir # Create the configure script configure_path = str(source_dir / "configure") with open(configure_path, "w", encoding="utf-8") as f: f.write( "#!/bin/sh\n" "prefix=$(echo $1 | sed 's/--prefix=//')\n" "cat > Makefile < bunch. Each bunch includes; all the args # that must be specified as part of a version() declaration (used to # manufacture a version for the 'git-test' package); the associated # revision for the version; a file associated with (and particular to) # that revision/branch. checks = { "default": Bunch(revision=default_branch, file=r0_file, args={"git": url}), "branch": Bunch(revision=branch, file=branch_file, args={"git": url, "branch": branch}), "tag-branch": Bunch( revision=tag_branch, file=tag_file, args={"git": url, "branch": tag_branch} ), "tag": Bunch(revision=tag, file=tag_file, args={"git": url, "tag": tag}), "commit": Bunch( revision=r1, file=r1_file, args={"git": url, "branch": branch, "commit": r1} ), "annotated-tag": Bunch(revision=a_tag, file=r2_file, args={"git": url, "tag": a_tag}), # In this case, the version() args do not include a 'git' key: # this is the norm for packages, so this tests how the fetching logic # would most-commonly assemble a Git fetcher "default-no-per-version-git": Bunch( revision=default_branch, file=r0_file, args={"branch": default_branch} ), "many-directories": Bunch( revision=multiple_directories_branch, file=dir_files[0], args={"git": url, "branch": multiple_directories_branch}, ), } t = Bunch( checks=checks, url=url, hash=rev_hash, path=str(repodir), git_exe=git, unversioned_commit=r2, ) yield t @pytest.fixture(scope="function") def mock_git_test_package(mock_git_repository, mutable_mock_repo, monkeypatch): # install a fake git version in the package class pkg_class = spack.repo.PATH.get_pkg_class("git-test") monkeypatch.delattr(pkg_class, "git") monkeypatch.setitem(pkg_class.versions, spack.version.Version("git"), mock_git_repository.url) return pkg_class @pytest.fixture(scope="session") def mock_hg_repository(tmp_path_factory: pytest.TempPathFactory): """Creates a very simple hg repository with two commits.""" try: hg = spack.util.executable.which("hg", required=True) except spack.util.executable.CommandNotFoundError: pytest.skip("requires mercurial to be installed") tmpdir = tmp_path_factory.mktemp("mock-hg-repo-dir") source_dir = tmpdir / spack.stage._source_path_subdir source_dir.mkdir() repodir = source_dir get_rev = lambda: hg("id", "-i", output=str).strip() # Initialize the repository with working_dir(str(repodir)): url = url_util.path_to_file_url(str(repodir)) hg("init") # Commit file r0 r0_file = "r0_file" (repodir / r0_file).touch() hg("add", r0_file) hg("commit", "-m", "revision 0", "-u", "test") r0 = get_rev() # Commit file r1 r1_file = "r1_file" (repodir / r1_file).touch() hg("add", r1_file) hg("commit", "-m", "revision 1", "-u", "test") r1 = get_rev() checks = { "default": Bunch(revision=r1, file=r1_file, args={"hg": str(repodir)}), "rev0": Bunch(revision=r0, file=r0_file, args={"hg": str(repodir), "revision": r0}), } t = Bunch(checks=checks, url=url, hash=get_rev, path=str(repodir)) yield t @pytest.fixture(scope="session") def mock_svn_repository(tmp_path_factory: pytest.TempPathFactory): """Creates a very simple svn repository with two commits.""" try: svn = spack.util.executable.which("svn", required=True) svnadmin = spack.util.executable.which("svnadmin", required=True) except spack.util.executable.CommandNotFoundError: pytest.skip("requires svn to be installed") tmpdir = tmp_path_factory.mktemp("mock-svn-stage") source_dir = tmpdir / spack.stage._source_path_subdir source_dir.mkdir() repodir = source_dir url = url_util.path_to_file_url(str(repodir)) # Initialize the repository with working_dir(str(repodir)): # NOTE: Adding --pre-1.5-compatible works for NERSC # Unknown if this is also an issue at other sites. svnadmin("create", "--pre-1.5-compatible", str(repodir)) # Import a structure (first commit) r0_file = "r0_file" tmp_path = tmpdir / "tmp-path" tmp_path.mkdir() (tmp_path / r0_file).touch() svn("import", str(tmp_path), url, "-m", "Initial import r0") shutil.rmtree(tmp_path, onerror=onerror) # Second commit r1_file = "r1_file" svn("checkout", url, str(tmp_path)) (tmp_path / r1_file).touch() with working_dir(str(tmp_path)): svn("add", str(tmp_path / r1_file)) svn("ci", "-m", "second revision r1") shutil.rmtree(tmp_path, onerror=onerror) r0 = "1" r1 = "2" checks = { "default": Bunch(revision=r1, file=r1_file, args={"svn": url}), "rev0": Bunch(revision=r0, file=r0_file, args={"svn": url, "revision": r0}), } def get_rev(): output = svn("info", "--xml", output=str) info = xml.etree.ElementTree.fromstring(output) return info.find("entry/commit").get("revision") t = Bunch(checks=checks, url=url, hash=get_rev, path=str(repodir)) yield t @pytest.fixture(scope="function") def mutable_mock_env_path(tmp_path: Path, mutable_config, monkeypatch): """Fixture for mocking the internal spack environments directory.""" mock_path = tmp_path / "mock-env-path" mutable_config.set("config:environments_root", str(mock_path)) monkeypatch.setattr(ev.environment, "default_env_path", str(mock_path)) return mock_path @pytest.fixture() def installation_dir_with_headers(tmp_path_factory: pytest.TempPathFactory): """Mock installation tree with a few headers placed in different subdirectories. Shouldn't be modified by tests as it is session scoped. """ root = tmp_path_factory.mktemp("prefix") # Create a few header files: # # # |-- include # | |--boost # | | |-- ex3.h # | |-- ex3.h # |-- path # |-- to # |-- ex1.h # |-- subdir # |-- ex2.h # (root / "include" / "boost").mkdir(parents=True) (root / "include" / "boost" / "ex3.h").touch() (root / "include" / "ex3.h").touch() (root / "path" / "to").mkdir(parents=True) (root / "path" / "to" / "ex1.h").touch() (root / "path" / "to" / "subdir").mkdir() (root / "path" / "to" / "subdir" / "ex2.h").touch() return root ########## # Specs of various kind ########## @pytest.fixture(params=["conflict+foo%clang", "conflict-parent@0.9^conflict~foo"]) def conflict_spec(request): """Specs which violate constraints specified with the "conflicts" directive in the "conflict" package. """ return request.param @pytest.fixture(scope="module") def mock_test_repo(tmp_path_factory: pytest.TempPathFactory): """Create an empty repository.""" repo_namespace = "mock_test_repo" repodir = tmp_path_factory.mktemp(repo_namespace) packages_dir = repodir / spack.repo.packages_dir_name packages_dir.mkdir() yaml_path = repodir / "repo.yaml" yaml_path.write_text( """ repo: namespace: mock_test_repo """ ) with spack.repo.use_repositories(str(repodir)) as repo: yield repo, repodir shutil.rmtree(str(repodir)) @pytest.fixture(scope="function") def mock_clone_repo(tmp_path_factory: pytest.TempPathFactory): """Create a cloned repository.""" repo_namespace = "mock_clone_repo" repodir = tmp_path_factory.mktemp(repo_namespace) yaml_path = repodir / "repo.yaml" yaml_path.write_text( """ repo: namespace: mock_clone_repo """ ) shutil.copytree( os.path.join(spack.paths.mock_packages_path, spack.repo.packages_dir_name), os.path.join(str(repodir), spack.repo.packages_dir_name), ) with spack.repo.use_repositories(str(repodir)) as repo: yield repo, repodir shutil.rmtree(str(repodir)) ########## # Class and fixture to work around problems raising exceptions in directives, # which cause tests like test_from_list_url to hang for Python 2.x metaclass # processing. # # At this point only version and patch directive handling has been addressed. ########## class MockBundle: has_code = False name = "mock-bundle" @pytest.fixture def mock_directive_bundle(): """Return a mock bundle package for directive tests.""" return MockBundle() @pytest.fixture def clear_directive_functions(): """Clear all overridden directive functions for subsequent tests.""" yield # Make sure any directive functions overridden by tests are cleared before # proceeding with subsequent tests that may depend on the original # functions. spack.directives_meta.DirectiveMeta._directives_to_be_executed.clear() @pytest.fixture def mock_executable(tmp_path: Path): """Factory to create a mock executable in a temporary directory that output a custom string when run. """ shebang = "#!/bin/sh\n" if sys.platform != "win32" else "@ECHO OFF\n" def _factory(name, output, subdir=("bin",)): executable_dir = tmp_path.joinpath(*subdir) executable_dir.mkdir(parents=True, exist_ok=True) executable_path = executable_dir / name if sys.platform == "win32": executable_path = executable_dir / (name + ".bat") executable_path.write_text(f"{shebang}{output}\n") executable_path.chmod(0o755) return executable_path return _factory @pytest.fixture() def mock_test_stage(mutable_config, tmp_path: Path): # NOTE: This fixture MUST be applied after any fixture that uses # the config fixture under the hood # No need to unset because we use mutable_config tmp_stage = str(tmp_path / "test_stage") mutable_config.set("config:test_stage", tmp_stage) yield tmp_stage @pytest.fixture(autouse=True) def inode_cache(): spack.llnl.util.lock.FILE_TRACKER.purge() yield # TODO: it is a bug when the file tracker is non-empty after a test, # since it means a lock was not released, or the inode was not purged # when acquiring the lock failed. So, we could assert that here, but # currently there are too many issues to fix, so look for the more # serious issue of having a closed file descriptor in the cache. assert not any(f.fh.closed for f in spack.llnl.util.lock.FILE_TRACKER._descriptors.values()) spack.llnl.util.lock.FILE_TRACKER.purge() @pytest.fixture(autouse=True) def brand_new_binary_cache(): yield spack.binary_distribution.BINARY_INDEX = spack.llnl.util.lang.Singleton( spack.binary_distribution.BinaryCacheIndex ) def _trivial_package_hash(spec: spack.spec.Spec) -> str: """Return a trivial package hash for tests to avoid expensive AST parsing.""" # Pad package name to consistent length and cap at 32 chars for realistic hash length return base64.b32encode(f"{spec.name:<32}".encode()[:32]).decode().lower() @pytest.fixture(autouse=True) def mock_package_hash_for_tests(request, monkeypatch): """Replace expensive package hash computation with trivial one for tests. Tests can force the real package hash by using the @pytest.mark.use_package_hash marker.""" if "use_package_hash" in request.keywords: yield return pkg_hash = spack.hash_types.package_hash idx = spack.hash_types.HASHES.index(pkg_hash) mock_pkg_hash = spack.hash_types.SpecHashDescriptor( depflag=0, package_hash=True, name="package_hash", override=_trivial_package_hash ) monkeypatch.setattr(spack.hash_types, "package_hash", mock_pkg_hash) try: spack.hash_types.HASHES[idx] = mock_pkg_hash yield finally: spack.hash_types.HASHES[idx] = pkg_hash @pytest.fixture() def noncyclical_dir_structure(tmp_path: Path): """ Create some non-trivial directory structure with symlinks to dirs and dangling symlinks, but no cycles:: . |-- a/ | |-- d/ | |-- file_1 | |-- to_file_1 -> file_1 | `-- to_c -> ../c |-- b -> a |-- c/ | |-- dangling_link -> nowhere | `-- file_2 `-- file_3 """ d = tmp_path / "nontrivial-dir" d.mkdir() j = os.path.join with working_dir(str(d)): os.mkdir(j("a")) os.mkdir(j("a", "d")) with open(j("a", "file_1"), "wb"): pass os.symlink(j("file_1"), j("a", "to_file_1")) os.symlink(j("..", "c"), j("a", "to_c")) os.symlink(j("a"), j("b")) os.mkdir(j("c")) os.symlink(j("nowhere"), j("c", "dangling_link")) with open(j("c", "file_2"), "wb"): pass with open(j("file_3"), "wb"): pass yield d @pytest.fixture(scope="function") def mock_config_data(): config_data_dir = os.path.join(spack.paths.test_path, "data", "config") return config_data_dir, os.listdir(config_data_dir) @pytest.fixture(scope="function") def mock_curl_configs(mock_config_data, monkeypatch): """ Mock curl-based retrieval of configuration files from the web by grabbing them from the test data configuration directory. Fetches a single (configuration) file if the name matches one in the test data directory. """ config_data_dir, config_files = mock_config_data class MockCurl: def __init__(self): self.returncode = None def __call__(self, *args, **kwargs): url = [a for a in args if a.startswith("http")][0] basename = os.path.basename(url) if os.path.splitext(url)[1]: if basename in config_files: filename = os.path.join(config_data_dir, basename) with open(filename, "r", encoding="utf-8") as f: lines = f.readlines() write_file(os.path.basename(filename), "".join(lines)) self.returncode = 0 else: # This is a "404" and is technically only returned if -f # flag is provided to curl. tty.msg("curl: (22) The requested URL returned error: 404") self.returncode = 22 monkeypatch.setattr(spack.util.web, "require_curl", MockCurl) @pytest.fixture(scope="function") def mock_fetch_url_text(mock_config_data, monkeypatch): """Mock spack.util.web.fetch_url_text.""" stage_dir, config_files = mock_config_data def _fetch_text_file(url, dest_dir): raw_url = raw_github_gitlab_url(url) mkdirp(dest_dir) basename = os.path.basename(raw_url) src = join_path(stage_dir, basename) dest = join_path(dest_dir, basename) copy(src, dest) return dest monkeypatch.setattr(spack.util.web, "fetch_url_text", _fetch_text_file) @pytest.fixture(scope="function") def mock_tty_stdout(monkeypatch): """Make sys.stdout.isatty() return True, while forcing no color output.""" monkeypatch.setattr(sys.stdout, "isatty", lambda: True) with spack.llnl.util.tty.color.color_when("never"): yield @pytest.fixture def prefix_like(): return "package-0.0.0.a1-hashhashhashhashhashhashhashhash" @pytest.fixture() def prefix_tmpdir(tmp_path: Path, prefix_like: str): prefix_dir = tmp_path / prefix_like prefix_dir.mkdir() return prefix_dir @pytest.fixture() def binary_with_rpaths(prefix_tmpdir: Path): """Factory fixture that compiles an ELF binary setting its RPATH. Relative paths are encoded with `$ORIGIN` prepended. """ def _factory(rpaths, message="Hello world!", dynamic_linker="/lib64/ld-linux.so.2"): source = prefix_tmpdir / "main.c" source.write_text( """ #include int main(){{ printf("{0}"); }} """.format(message) ) gcc = spack.util.executable.which("gcc", required=True) executable = source.parent / "main.x" # Encode relative RPATHs using `$ORIGIN` as the root prefix rpaths = [x if os.path.isabs(x) else os.path.join("$ORIGIN", x) for x in rpaths] opts = [ "-Wl,--disable-new-dtags", f"-Wl,-rpath={':'.join(rpaths)}", f"-Wl,--dynamic-linker,{dynamic_linker}", str(source), "-o", str(executable), ] gcc(*opts) return executable return _factory @pytest.fixture(scope="session") def concretized_specs_cache(): """Cache for mock concrete specs""" return {} @pytest.fixture def default_mock_concretization( config, mock_packages, concretized_specs_cache ) -> Callable[[str], spack.spec.Spec]: """Return the default mock concretization of a spec literal, obtained using the mock repository and the mock configuration. This fixture is unsafe to call in a test when either the default configuration or mock repository are not used or have been modified. """ def _func(spec_str, tests=False): key = spec_str, tests if key not in concretized_specs_cache: concretized_specs_cache[key] = spack.concretize.concretize_one( spack.spec.Spec(spec_str), tests=tests ) return concretized_specs_cache[key].copy() return _func @pytest.fixture def shell_as(shell): if sys.platform != "win32": yield return if shell not in ("pwsh", "bat"): raise RuntimeError("Shell must be one of supported Windows shells (pwsh|bat)") try: # fetch and store old shell type _shell = os.environ.get("SPACK_SHELL", None) os.environ["SPACK_SHELL"] = shell yield finally: # restore old shell if one was set if _shell: os.environ["SPACK_SHELL"] = _shell @pytest.fixture() def nullify_globals(request, monkeypatch): ensure_configuration_fixture_run_before(request) monkeypatch.setattr(spack.config, "CONFIG", None) monkeypatch.setattr(spack.caches, "MISC_CACHE", None) monkeypatch.setattr(spack.caches, "FETCH_CACHE", None) monkeypatch.setattr(spack.repo, "PATH", None) monkeypatch.setattr(spack.store, "STORE", None) def pytest_runtest_setup(item): # Skip test marked "not_on_windows" if they're run on Windows not_on_windows_marker = item.get_closest_marker(name="not_on_windows") if not_on_windows_marker and sys.platform == "win32": pytest.skip(*not_on_windows_marker.args) # Skip items marked "only windows" if they're run anywhere but Windows only_windows_marker = item.get_closest_marker(name="only_windows") if only_windows_marker and sys.platform != "win32": pytest.skip(*only_windows_marker.args) @pytest.fixture(autouse=True) def disable_parallelism(monkeypatch, request): """Disable process pools in tests. Enabled by default to avoid oversubscription when running under pytest-xdist. Can be overridden with `@pytest.mark.enable_parallelism`.""" if "enable_parallelism" not in request.keywords: monkeypatch.setattr(spack.util.parallel, "ENABLE_PARALLELISM", False) def _root_path(x, y, *, path): return path @pytest.fixture def mock_modules_root(tmp_path: Path, monkeypatch): """Sets the modules root to a temporary directory, to avoid polluting configuration scopes.""" fn = functools.partial(_root_path, path=str(tmp_path)) monkeypatch.setattr(spack.modules.common, "root_path", fn) @pytest.fixture() def compiler_factory(): """Factory for a compiler dict, taking a spec and an OS as arguments.""" def _factory(*, spec): return { "spec": f"{spec}", "prefix": "/path", "extra_attributes": {"compilers": {"c": "/path/bin/cc", "cxx": "/path/bin/cxx"}}, } return _factory @pytest.fixture() def host_architecture_str(): """Returns the broad architecture family (x86_64, aarch64, etc.)""" return str(spack.vendor.archspec.cpu.host().family) def _true(x): return True def _libc_from_python(self): return spack.spec.Spec("glibc@=2.28", external_path="/some/path") @pytest.fixture() def do_not_check_runtimes_on_reuse(monkeypatch): monkeypatch.setattr(spack.solver.reuse, "_has_runtime_dependencies", _true) @pytest.fixture(autouse=True, scope="session") def _c_compiler_always_exists(): fn = spack.solver.asp.c_compiler_runs spack.solver.asp.c_compiler_runs = _true mthd = spack.compilers.libraries.CompilerPropertyDetector.default_libc spack.compilers.libraries.CompilerPropertyDetector.default_libc = _libc_from_python yield spack.solver.asp.c_compiler_runs = fn spack.compilers.libraries.CompilerPropertyDetector.default_libc = mthd @pytest.fixture(scope="session") def mock_test_cache(tmp_path_factory: pytest.TempPathFactory): cache_dir = tmp_path_factory.mktemp("cache") return spack.util.file_cache.FileCache(cache_dir) class MockHTTPResponse(io.IOBase): """This is a mock HTTP response, which implements part of http.client.HTTPResponse""" def __init__(self, status, reason, headers=None, body=None): self.msg = None self.version = 11 self.url = None self.headers = email.message.EmailMessage() self.status = status self.code = status self.reason = reason self.debuglevel = 0 self._body = body if headers is not None: for key, value in headers.items(): self.headers[key] = value @classmethod def with_json(cls, status, reason, headers=None, body=None): """Create a mock HTTP response with JSON string as body""" body = io.BytesIO(json.dumps(body).encode("utf-8")) return cls(status, reason, headers, body) def readable(self): return True def read(self, *args, **kwargs): return self._body.read(*args, **kwargs) def getheader(self, name, default=None): self.headers.get(name, default) def getheaders(self): return self.headers.items() def fileno(self): return 0 def getcode(self): return self.status def info(self): return self.headers @pytest.fixture() def mock_runtimes(config, mock_packages): return mock_packages.packages_with_tags("runtime") @pytest.fixture() def write_config_file(tmp_path: Path): """Returns a function that writes a config file.""" def _write(config, data, scope): config_dir = tmp_path / scope config_dir.mkdir(parents=True, exist_ok=True) config_yaml = config_dir / (config + ".yaml") with config_yaml.open("w") as f: syaml.dump_config(data, f) return config_yaml return _write def _include_cache_root(): return join_path(str(tempfile.mkdtemp()), "user_cache", "includes") @pytest.fixture() def wrapper_dir(install_mockery): """Installs the compiler wrapper and returns the prefix where the script is installed.""" wrapper = spack.concretize.concretize_one("compiler-wrapper") wrapper_pkg = wrapper.package PackageInstaller([wrapper_pkg], explicit=True).install() return wrapper_pkg.bin_dir() def _noop(*args, **kwargs): pass @pytest.fixture(autouse=True) def no_compilers_init(monkeypatch): """Disables automatic compiler initialization""" monkeypatch.setattr(spack.compilers.config, "_init_packages_yaml", _noop) @pytest.fixture(autouse=True) def skip_provenance_check(monkeypatch, request): """Skip binary provenance check for git versions Binary provenance checks require querying git repositories and mirrors. The infrastructure for this is complex and a heavy lift for simple things like spec syntax checks. This fixture defaults to skipping this check, but can be overridden with the @pytest.mark.require_provenance decorator """ if "require_provenance" not in request.keywords: monkeypatch.setattr(spack.package_base.PackageBase, "_resolve_git_provenance", _noop) @pytest.fixture(scope="function") def config_two_gccs(mutable_config): # Configure two gcc compilers that could be concretized to extra_attributes_block = { "compilers": {"c": "/path/to/gcc", "cxx": "/path/to/g++", "fortran": "/path/to/fortran"} } mutable_config.set( "packages:gcc:externals::", [ { "spec": "gcc@12.3.1 languages=c,c++,fortran", "prefix": "/path", "extra_attributes": extra_attributes_block, }, { "spec": "gcc@10.3.1 languages=c,c++,fortran", "prefix": "/path", "extra_attributes": extra_attributes_block, }, ], ) @pytest.fixture(scope="function") def mock_util_executable(monkeypatch): logger = [] should_fail = [] registered_reponses = {} def mock_call(self, *args, **kwargs): cmd = self.exe + list(args) str_cmd = " ".join(map(str, cmd)) logger.append(str_cmd) for failure_key in should_fail: if failure_key in str_cmd: self.returncode = 1 if kwargs.get("fail_on_error", True): raise spack.util.executable.ProcessError(f"Failed: {str_cmd}") return for key, value in registered_reponses.items(): if key in str_cmd: return value self.returncode = 0 monkeypatch.setattr(spack.util.executable.Executable, "__call__", mock_call) yield logger, should_fail, registered_reponses @pytest.fixture() def reset_extension_paths(): """Clears the cache used for entry points, both in setup and tear-down. Needed if a test stresses parts related to computing paths for Spack extensions """ spack.extensions.extension_paths_from_entry_points.cache_clear() yield spack.extensions.extension_paths_from_entry_points.cache_clear() @pytest.fixture(params=["old", "new"]) def installer_variant(request): """Parametrize a test over the old and new installer.""" if request.param == "new" and sys.platform == "win32": pytest.skip("New installer not supported on Windows") with spack.config.override("config:installer", request.param): yield request.param ================================================ FILE: lib/spack/spack/test/container/cli.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.container.images import spack.llnl.util.filesystem as fs import spack.main containerize = spack.main.SpackCommand("containerize") def test_command(default_config, container_config_dir): with fs.working_dir(container_config_dir): output = containerize() assert "FROM spack/ubuntu-jammy" in output def test_listing_possible_os(): output = containerize("--list-os") for expected_os in spack.container.images.all_bootstrap_os(): assert expected_os in output @pytest.mark.maybeslow @pytest.mark.requires_executables("git") def test_bootstrap_phase(minimal_configuration, config_dumper): minimal_configuration["spack"]["container"]["images"] = { "os": "amazonlinux:2", "spack": {"resolve_sha": False}, } spack_yaml_dir = config_dumper(minimal_configuration) with fs.working_dir(spack_yaml_dir): output = containerize() # Check for the presence of the Git commands assert "git init" in output assert "git fetch" in output assert "git checkout" in output ================================================ FILE: lib/spack/spack/test/container/conftest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.util.spack_yaml as syaml @pytest.fixture() def minimal_configuration(): return { "spack": { "specs": ["gromacs", "mpich", "fftw precision=float"], "container": { "format": "docker", "images": {"os": "ubuntu:22.04", "spack": "develop"}, }, } } @pytest.fixture() def config_dumper(tmp_path: pathlib.Path): """Function that dumps an environment config in a temporary folder.""" def dumper(configuration): content = syaml.dump(configuration, default_flow_style=False) (tmp_path / "spack.yaml").write_text(content or "", encoding="utf-8") return str(tmp_path) return dumper @pytest.fixture() def container_config_dir(minimal_configuration, config_dumper): return config_dumper(minimal_configuration) ================================================ FILE: lib/spack/spack/test/container/docker.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import pytest import spack.container.writers as writers def test_manifest(minimal_configuration): writer = writers.create(minimal_configuration) manifest_str = writer.manifest for line in manifest_str.split("\n"): assert "echo" in line def test_build_and_run_images(minimal_configuration): writer = writers.create(minimal_configuration) # Test the output of run property run = writer.run assert run.image == "ubuntu:22.04" # Test the output of the build property build = writer.build assert build.image == "spack/ubuntu-jammy:develop" def test_packages(minimal_configuration): # In this minimal configuration we don't have packages writer = writers.create(minimal_configuration) assert writer.os_packages_build is None assert writer.os_packages_final is None # If we add them a list should be returned pkgs = ["libgomp1"] minimal_configuration["spack"]["container"]["os_packages"] = {"final": pkgs} writer = writers.create(minimal_configuration) p = writer.os_packages_final assert p.update assert p.install assert p.clean assert p.list == pkgs def test_container_os_packages_command(minimal_configuration): # In this minimal configuration we don't have packages writer = writers.create(minimal_configuration) assert writer.os_packages_build is None assert writer.os_packages_final is None # If we add them a list should be returned minimal_configuration["spack"]["container"]["images"] = { "build": "custom-build:latest", "final": "custom-final:latest", } minimal_configuration["spack"]["container"]["os_packages"] = { "command": "zypper", "final": ["libgomp1"], } writer = writers.create(minimal_configuration) p = writer.os_packages_final assert "zypper update -y" in p.update assert "zypper install -y" in p.install assert "zypper clean -a" in p.clean def test_ensure_render_works(minimal_configuration, default_config): # Here we just want to ensure that nothing is raised writer = writers.create(minimal_configuration) writer() def test_strip_is_set_from_config(minimal_configuration): writer = writers.create(minimal_configuration) assert writer.strip is True minimal_configuration["spack"]["container"]["strip"] = False writer = writers.create(minimal_configuration) assert writer.strip is False def test_custom_base_images(minimal_configuration): """Test setting custom base images from configuration file""" minimal_configuration["spack"]["container"]["images"] = { "build": "custom-build:latest", "final": "custom-final:latest", } writer = writers.create(minimal_configuration) assert writer.bootstrap.image is None assert writer.build.image == "custom-build:latest" assert writer.run.image == "custom-final:latest" @pytest.mark.parametrize( "images_cfg,expected", [ ( {"os": "amazonlinux:2", "spack": "develop"}, { "bootstrap_image": "amazonlinux:2", "build_image": "bootstrap", "final_image": "amazonlinux:2", }, ) ], ) def test_base_images_with_bootstrap(minimal_configuration, images_cfg, expected): """Check that base images are computed correctly when a bootstrap phase is present """ minimal_configuration["spack"]["container"]["images"] = images_cfg writer = writers.create(minimal_configuration) for property_name, value in expected.items(): assert getattr(writer, property_name) == value def test_error_message_invalid_os(minimal_configuration): minimal_configuration["spack"]["container"]["images"]["os"] = "invalid:1" with pytest.raises(ValueError, match="invalid operating system"): writers.create(minimal_configuration) @pytest.mark.regression("34629,18030") def test_not_stripping_all_symbols(minimal_configuration): """Tests that we are not stripping all symbols, so that libraries can still be used for linking. """ minimal_configuration["spack"]["container"]["strip"] = True content = writers.create(minimal_configuration)() assert "xargs strip" in content assert "xargs strip -s" not in content @pytest.mark.regression("22341") def test_using_single_quotes_in_dockerfiles(minimal_configuration): """Tests that Dockerfiles written by Spack use single quotes in manifest, to avoid issues with shell substitution. This may happen e.g. when users have "definitions:" they want to expand in dockerfiles. """ manifest_in_docker = writers.create(minimal_configuration).manifest assert not re.search(r"echo\s*\"", manifest_in_docker, flags=re.MULTILINE) assert re.search(r"echo\s*'", manifest_in_docker) ================================================ FILE: lib/spack/spack/test/container/images.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pytest import spack.container import spack.container.images @pytest.mark.parametrize( "image,spack_version,expected", [ ("ubuntu:22.04", "develop", ("spack/ubuntu-jammy", "develop")), ("ubuntu:22.04", "0.14.0", ("spack/ubuntu-jammy", "0.14.0")), ], ) def test_build_info(image, spack_version, expected): output = spack.container.images.build_info(image, spack_version) assert output == expected @pytest.mark.parametrize("image", ["ubuntu:22.04"]) def test_package_info(image): pkg_manager = spack.container.images.os_package_manager_for(image) update, install, clean = spack.container.images.commands_for(pkg_manager) assert update assert install assert clean @pytest.mark.parametrize( "extra_config,expected_msg", [ ({"modules": {"enable": ["tcl"]}}, 'the subsection "modules" in'), ({"concretizer": {"unify": False}}, '"concretizer:unify" is not set to "true"'), ( {"config": {"install_tree": {"root": "/some/dir"}}}, 'the "config:install_tree" attribute has been set', ), ({"view": "/some/dir"}, 'the "view" attribute has been set'), ], ) def test_validate(extra_config, expected_msg, minimal_configuration, config_dumper): minimal_configuration["spack"].update(extra_config) spack_yaml_dir = config_dumper(minimal_configuration) spack_yaml = os.path.join(spack_yaml_dir, "spack.yaml") with pytest.warns(UserWarning) as w: spack.container.validate(spack_yaml) # Tests are designed to raise only one warning assert len(w) == 1 assert expected_msg in str(w.pop().message) ================================================ FILE: lib/spack/spack/test/container/singularity.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.container.writers as writers @pytest.fixture def singularity_configuration(minimal_configuration): minimal_configuration["spack"]["container"]["format"] = "singularity" return minimal_configuration def test_ensure_render_works(default_config, singularity_configuration): container_config = singularity_configuration["spack"]["container"] assert container_config["format"] == "singularity" # Here we just want to ensure that nothing is raised writer = writers.create(singularity_configuration) writer() @pytest.mark.parametrize( "properties,expected", [ ( {"runscript": "/opt/view/bin/h5ls"}, {"runscript": "/opt/view/bin/h5ls", "startscript": "", "test": "", "help": ""}, ) ], ) def test_singularity_specific_properties(properties, expected, singularity_configuration): # Set the property in the configuration container_config = singularity_configuration["spack"]["container"] for name, value in properties.items(): container_config.setdefault("singularity", {})[name] = value # Assert the properties return the expected values writer = writers.create(singularity_configuration) for name, value in expected.items(): assert getattr(writer, name) == value @pytest.mark.regression("34629,18030") def test_not_stripping_all_symbols(singularity_configuration): """Tests that we are not stripping all symbols, so that libraries can still be used for linking. """ singularity_configuration["spack"]["container"]["strip"] = True content = writers.create(singularity_configuration)() assert "xargs strip" in content assert "xargs strip -s" not in content ================================================ FILE: lib/spack/spack/test/cray_manifest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ Note that where possible, this should produce specs using `entries_to_specs` rather than `spec_from_entry`, since the former does additional work to establish dependency relationships (and in general the manifest-parsing logic needs to consume all related specs in a single pass). """ import json import pathlib import pytest import spack.vendor.archspec.cpu import spack.cmd import spack.cmd.external import spack.compilers.config import spack.concretize import spack.cray_manifest import spack.platforms import spack.platforms.test import spack.solver.reuse import spack.spec import spack.store from spack.cray_manifest import compiler_from_entry, entries_to_specs pytestmark = [ pytest.mark.skipif( str(spack.platforms.host()) != "linux", reason="Cray manifest files are only for linux" ), pytest.mark.usefixtures("mutable_config", "mock_packages"), ] class JsonSpecEntry: def __init__(self, name, hash, prefix, version, arch, compiler, dependencies, parameters): self.name = name self.hash = hash self.prefix = prefix self.version = version self.arch = arch self.compiler = compiler self.dependencies = dependencies self.parameters = parameters def to_dict(self): return { "name": self.name, "hash": self.hash, "prefix": self.prefix, "version": self.version, "arch": self.arch, "compiler": self.compiler, "dependencies": self.dependencies, "parameters": self.parameters, } def as_dependency(self, deptypes): return (self.name, {"hash": self.hash, "type": list(deptypes)}) class JsonArchEntry: def __init__(self, platform, os, target): self.platform = platform self.os = os self.target = target def spec_json(self): return {"platform": self.platform, "platform_os": self.os, "target": {"name": self.target}} def compiler_json(self): return {"os": self.os, "target": self.target} class JsonCompilerEntry: def __init__(self, *, name, version, arch=None, executables=None, prefix=None): self.name = name self.version = version self.arch = arch or JsonArchEntry("anyplatform", "anyos", "anytarget") self.executables = executables or {"cc": "cc", "cxx": "cxx", "fc": "fc"} self.prefix = prefix def compiler_json(self): result = { "name": self.name, "version": self.version, "arch": self.arch.compiler_json(), "executables": self.executables, } # See https://github.com/spack/spack/pull/40061 if self.prefix is not None: result["prefix"] = self.prefix return result def spec_json(self): """The compiler spec only lists the name/version, not arch/executables. """ return {"name": self.name, "version": self.version} @pytest.fixture def _common_arch(test_platform): generic = spack.vendor.archspec.cpu.TARGETS[test_platform.default].family return JsonArchEntry(platform=test_platform.name, os="redhat6", target=generic.name) @pytest.fixture def _common_compiler(_common_arch): return JsonCompilerEntry( name="gcc", version="10.2.0.2112", arch=_common_arch, executables={ "cc": "/path/to/compiler/cc", "cxx": "/path/to/compiler/cxx", "fc": "/path/to/compiler/fc", }, ) @pytest.fixture def _other_compiler(_common_arch): return JsonCompilerEntry( name="clang", version="3.0.0", arch=_common_arch, executables={ "cc": "/path/to/compiler/clang", "cxx": "/path/to/compiler/clang++", "fc": "/path/to/compiler/flang", }, ) @pytest.fixture def _raw_json_x(_common_arch): return { "name": "packagex", "hash": "hash-of-x", "prefix": "/path/to/packagex-install/", "version": "1.0", "arch": _common_arch.spec_json(), "compiler": {"name": "gcc", "version": "10.2.0.2112"}, "dependencies": {"packagey": {"hash": "hash-of-y", "type": ["link"]}}, "parameters": {"precision": ["double", "float"]}, } def test_manifest_compatibility(_common_arch, _common_compiler, _raw_json_x): """Make sure that JsonSpecEntry outputs the expected JSON structure by comparing it with JSON parsed from an example string. This ensures that the testing objects like JsonSpecEntry produce the same JSON structure as the expected file format. """ y = JsonSpecEntry( name="packagey", hash="hash-of-y", prefix="/path/to/packagey-install/", version="1.0", arch=_common_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies={}, parameters={}, ) x = JsonSpecEntry( name="packagex", hash="hash-of-x", prefix="/path/to/packagex-install/", version="1.0", arch=_common_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies=dict([y.as_dependency(deptypes=["link"])]), parameters={"precision": ["double", "float"]}, ) x_from_entry = x.to_dict() assert x_from_entry == _raw_json_x def test_compiler_from_entry(mock_executable): """Tests that we can detect a compiler from a valid entry in the Cray manifest""" cc = mock_executable("gcc", output="echo 7.5.0") cxx = mock_executable("g++", output="echo 7.5.0") fc = mock_executable("gfortran", output="echo 7.5.0") compiler = compiler_from_entry( JsonCompilerEntry( name="gcc", version="7.5.0", arch=JsonArchEntry(platform="linux", os="centos8", target="x86_64"), prefix=str(cc.parent), executables={"cc": "gcc", "cxx": "g++", "fc": "gfortran"}, ).compiler_json(), manifest_path="/example/file", ) assert compiler.satisfies("gcc@7.5.0 target=x86_64 os=centos8") assert compiler.extra_attributes["compilers"]["c"] == str(cc) assert compiler.extra_attributes["compilers"]["cxx"] == str(cxx) assert compiler.extra_attributes["compilers"]["fortran"] == str(fc) @pytest.fixture def generate_openmpi_entries(_common_arch, _common_compiler): """Generate two example JSON entries that refer to an OpenMPI installation and a hwloc dependency. """ # The hashes need to be padded with 'a' at the end to align with 8-byte # boundaries (for base-32 decoding) hwloc = JsonSpecEntry( name="hwloc", hash="hwlocfakehashaaa", prefix="/path/to/hwloc-install/", version="2.0.3", arch=_common_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies={}, parameters={}, ) # This includes a variant which is guaranteed not to appear in the # OpenMPI package: we need to make sure we can use such package # descriptions. openmpi = JsonSpecEntry( name="openmpi", hash="openmpifakehasha", prefix="/path/to/openmpi-install/", version="4.1.0", arch=_common_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies=dict([hwloc.as_dependency(deptypes=["link"])]), parameters={"internal-hwloc": False, "fabrics": ["psm"], "missing_variant": True}, ) return list(x.to_dict() for x in [openmpi, hwloc]) def test_generate_specs_from_manifest(generate_openmpi_entries): """Given JSON entries, check that we can form a set of Specs including dependency references. """ specs = entries_to_specs(generate_openmpi_entries) (openmpi_spec,) = list(x for x in specs.values() if x.name == "openmpi") assert openmpi_spec["hwloc"] def test_translate_cray_platform_to_linux(monkeypatch, _common_compiler): """Manifests might list specs on newer Cray platforms as being "cray", but Spack identifies such platforms as "linux". Make sure we automatically transform these entries. """ test_linux_platform = spack.platforms.test.Test("linux") def the_host_is_linux(): return test_linux_platform monkeypatch.setattr(spack.platforms, "host", the_host_is_linux) cray_arch = JsonArchEntry(platform="cray", os="rhel8", target="x86_64") spec_json = JsonSpecEntry( name="mpich", hash="craympichfakehashaaa", prefix="/path/to/cray-mpich/", version="1.0.0", arch=cray_arch.spec_json(), compiler=_common_compiler.spec_json(), dependencies={}, parameters={}, ).to_dict() (spec,) = entries_to_specs([spec_json]).values() assert spec.architecture.platform == "linux" @pytest.mark.parametrize( "name_in_manifest,expected_name", [("nvidia", "nvhpc"), ("rocm", "llvm-amdgpu"), ("clang", "llvm")], ) def test_translated_compiler_name(name_in_manifest, expected_name): assert spack.cray_manifest.translated_compiler_name(name_in_manifest) == expected_name def test_failed_translate_compiler_name(_common_arch): unknown_compiler = JsonCompilerEntry(name="unknown", version="1.0") with pytest.raises(spack.compilers.config.UnknownCompilerError): compiler_from_entry(unknown_compiler.compiler_json(), manifest_path="/example/file") spec_json = JsonSpecEntry( name="packagey", hash="hash-of-y", prefix="/path/to/packagey-install/", version="1.0", arch=_common_arch.spec_json(), compiler=unknown_compiler.spec_json(), dependencies={}, parameters={}, ).to_dict() with pytest.raises(spack.compilers.config.UnknownCompilerError): entries_to_specs([spec_json]) @pytest.fixture def manifest_content(generate_openmpi_entries, _common_compiler, _other_compiler): return { "_meta": { "file-type": "cray-pe-json", "system-type": "EX", "schema-version": "1.3", "cpe-version": "22.06", }, "specs": generate_openmpi_entries, "compilers": [_common_compiler.compiler_json(), _other_compiler.compiler_json()], } def test_read_cray_manifest(temporary_store, manifest_file): """Check that (a) we can read the cray manifest and add it to the Spack Database and (b) we can concretize specs based on that. """ spack.cray_manifest.read(str(manifest_file), True) query_specs = temporary_store.db.query("openmpi") assert any(x.dag_hash() == "openmpifakehasha" for x in query_specs) concretized_spec = spack.concretize.concretize_one("depends-on-openmpi ^/openmpifakehasha") assert concretized_spec["hwloc"].dag_hash() == "hwlocfakehashaaa" def test_read_cray_manifest_add_compiler_failure(temporary_store, manifest_file, monkeypatch): """Tests the Cray manifest can be read even if some compilers cannot be added.""" def _mock(entry, *, manifest_path): if entry["name"] == "clang": raise RuntimeError("cannot determine the compiler") return spack.spec.Spec(f"{entry['name']}@{entry['version']}") monkeypatch.setattr(spack.cray_manifest, "compiler_from_entry", _mock) spack.cray_manifest.read(str(manifest_file), True) query_specs = spack.store.STORE.db.query("openmpi") assert any(x.dag_hash() == "openmpifakehasha" for x in query_specs) def test_read_cray_manifest_twice_no_duplicates( mutable_config, temporary_store, manifest_file, monkeypatch, tmp_path: pathlib.Path ): def _mock(entry, *, manifest_path): return spack.spec.Spec(f"{entry['name']}@{entry['version']}", external_path=str(tmp_path)) monkeypatch.setattr(spack.cray_manifest, "compiler_from_entry", _mock) # Read the manifest twice spack.cray_manifest.read(str(manifest_file), True) spack.cray_manifest.read(str(manifest_file), True) config_data = mutable_config.get("packages")["gcc"] assert "externals" in config_data specs = [spack.spec.Spec(x["spec"]) for x in config_data["externals"]] assert len(specs) == len(set(specs)) assert len([c for c in specs if c.satisfies("gcc@10.2.0.2112")]) == 1 def test_read_old_manifest_v1_2(tmp_path: pathlib.Path, temporary_store): """Test reading a file using the older format ('version' instead of 'schema-version').""" manifest = tmp_path / "manifest_dir" / "test.json" manifest.parent.mkdir(parents=True) manifest.write_text( """\ { "_meta": { "file-type": "cray-pe-json", "system-type": "EX", "version": "1.3" }, "specs": [] } """ ) spack.cray_manifest.read(str(manifest), True) def test_convert_validation_error( tmp_path: pathlib.Path, mutable_config, mock_packages, temporary_store ): manifest_dir = tmp_path / "manifest_dir" manifest_dir.mkdir() # Does not parse as valid JSON invalid_json_path = manifest_dir / "invalid-json.json" with open(invalid_json_path, "w", encoding="utf-8") as f: f.write( """\ { """ ) with pytest.raises(spack.cray_manifest.ManifestValidationError) as e: spack.cray_manifest.read(invalid_json_path, True) str(e) # Valid JSON, but does not conform to schema (schema-version is not a string # of length > 0) invalid_schema_path = manifest_dir / "invalid-schema.json" with open(invalid_schema_path, "w", encoding="utf-8") as f: f.write( """\ { "_meta": { "file-type": "cray-pe-json", "system-type": "EX", "schema-version": "" }, "specs": [] } """ ) with pytest.raises(spack.cray_manifest.ManifestValidationError) as e: spack.cray_manifest.read(invalid_schema_path, True) @pytest.fixture def manifest_file(tmp_path: pathlib.Path, manifest_content): """Create a manifest file in a directory. Used by 'spack external'.""" filename = tmp_path / "external-db.json" with open(filename, "w", encoding="utf-8") as db_file: json.dump(manifest_content, db_file) return filename def test_find_external_nonempty_default_manifest_dir( temporary_store, mutable_mock_repo, monkeypatch, manifest_file ): """The user runs 'spack external find'; the default manifest directory contains a manifest file. Ensure that the specs are read. """ monkeypatch.setenv("PATH", "") monkeypatch.setattr(spack.cray_manifest, "default_path", str(manifest_file.parent)) spack.cmd.external._collect_and_consume_cray_manifest_files(ignore_default_dir=False) specs = temporary_store.db.query("hwloc") assert any(x.dag_hash() == "hwlocfakehashaaa" for x in specs) def test_reusable_externals_cray_manifest(temporary_store, manifest_file): """The concretizer should be able to reuse specs imported from a manifest without a externals config entry in packages.yaml""" spack.cray_manifest.read(path=str(manifest_file), apply_updates=True) # Get any imported spec spec = temporary_store.db.query_local()[0] # Reusable if imported locally assert spack.solver.reuse._is_reusable(spec, packages_with_externals={}, local=True) # If cray manifest entries end up in a build cache somehow, they are not reusable assert not spack.solver.reuse._is_reusable(spec, packages_with_externals={}, local=False) ================================================ FILE: lib/spack/spack/test/cvs_fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.concretize from spack.fetch_strategy import CvsFetchStrategy from spack.llnl.util.filesystem import mkdirp, touch, working_dir from spack.stage import Stage from spack.util.executable import which from spack.version import Version pytestmark = pytest.mark.skipif(not which("cvs"), reason="requires CVS to be installed") @pytest.mark.parametrize("type_of_test", ["default", "branch", "date"]) def test_fetch(type_of_test, mock_cvs_repository, config, mutable_mock_repo): """Tries to: 1. Fetch the repo using a fetch strategy constructed with supplied args (they depend on type_of_test). 2. Check whether the checkout is on the correct branch or date 3. Check if the test_file is in the checked out repository. 4. Add and remove some files, then reset the repo, and ensure it's all there again. CVS does not have the notion of a unique branch; branches and revisions are managed separately for every file. """ # Retrieve the right test parameters test = mock_cvs_repository.checks[type_of_test] get_branch = mock_cvs_repository.get_branch get_date = mock_cvs_repository.get_date # Construct the package under test spec = spack.concretize.concretize_one("cvs-test") spec.package.versions[Version("cvs")] = test.args # Enter the stage directory and check some properties with spec.package.stage: spec.package.do_stage() with working_dir(spec.package.stage.source_path): # Check branch if test.branch is not None: assert get_branch() == test.branch # Check date if test.date is not None: assert get_date() <= test.date file_path = os.path.join(spec.package.stage.source_path, test.file) assert os.path.isdir(spec.package.stage.source_path) assert os.path.isfile(file_path) os.unlink(file_path) assert not os.path.isfile(file_path) untracked_file = "foobarbaz" touch(untracked_file) assert os.path.isfile(untracked_file) spec.package.do_restage() assert not os.path.isfile(untracked_file) assert os.path.isdir(spec.package.stage.source_path) assert os.path.isfile(file_path) def test_cvs_extra_fetch(tmp_path: pathlib.Path): """Ensure a fetch after downloading is effectively a no-op.""" testpath = str(tmp_path) fetcher = CvsFetchStrategy(cvs=":pserver:not-a-real-cvs-repo%module=not-a-real-module") assert fetcher is not None with Stage(fetcher, path=testpath) as stage: assert stage is not None source_path = stage.source_path mkdirp(source_path) # TODO: This doesn't look as if it was testing what this function's # comment says it is testing. However, the other `test_*_extra_fetch` # functions (for svn, git, hg) use equivalent code. # # We're calling `fetcher.fetch` twice as this might be what we want to # do, and it can't hurt. See # for a discussion on this. # Fetch once fetcher.fetch() # Fetch a second time fetcher.fetch() ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt ================================================ rm foo /opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld /usr/lib64//crt1.o /usr/lib64//crti.o /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtbeginT.o /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtfastmath.o /opt/cray/pe/cce/8.6.5/cce/x86_64/lib/no_mmap.o foo.o -Bstatic -rpath=/opt/cray/pe/cce/8.6.5/cce/x86_64/lib -L /opt/gcc/6.1.0/snos/lib64 -rpath=/opt/cray/pe/gcc-libs -L /usr/lib64 -L /lib64 -L /opt/cray/dmapp/default/lib64 -L /opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L /opt/cray/dmapp/default/lib64 -L /opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L /opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib -L /opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64 -L /opt/cray/pe/pmi/5.0.13/lib64 -L /opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64 -L /opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64 -L /opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64 -L /opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64 -L /opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64 -L /opt/cray/pe/atp/2.1.1/libApp -L /opt/cray/pe/cce/8.6.5/cce/x86_64/lib/pkgconfig/../ -L /opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64 --no-as-needed -lAtpSigHandler -lAtpSigHCommData --undefined=_ATP_Data_Globals --undefined=__atpHandlerInstall -lpthread -lmpichcxx_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mpi_mp -lm -lf -lsci_cray_mp -lmpich_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mp -lcraymp -lm -lpthread -lf -lhugetlbfs -lpgas-dmapp -lfi -lu -lrt --undefined=dmapp_get_flag_nbi -ldmapp -lugni -ludreg -lpthread -lm -lcray-c++-rts -lstdc++ -lxpmem -ldmapp -lpthread -lpmi -lpthread -lalpslli -lpthread -lwlm_detect -lugni -lpthread -lalpsutil -lpthread -lrca -ludreg -lquadmath -lm -lomp -lcraymp -lpthread -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lmodules -lm -lfi -lm -lquadmath -lcraymath -lm -lgfortran -lquadmath -lf -lm -lpthread -lu -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lcsup --as-needed -latomic --no-as-needed -lcray-c++-rts -lstdc++ -lsupc++ -lstdc++ -lpthread --start-group -lc -lcsup -lgcc_eh -lm -lgcc --end-group -T/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/2.23.1.cce.ld -L /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0 -L /opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -EL -o foo --undefined=__pthread_initialize_minimal /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtend.o /usr/lib64//crtn.o /opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/objcopy --remove-section=.note.ftn_module_data foo rm /tmp/pe_27645//pldir/PL_path rm /tmp/pe_27645//pldir/PL_module_list rm /tmp/pe_27645//pldir/PL_global_data rmdir /tmp/pe_27645//pldir rmdir /tmp/pe_27645/ ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt ================================================ clang version 4.0.1 (tags/RELEASE_401/final) Target: x86_64-unknown-linux-gnu Thread model: posix InstalledDir: /usr/bin Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-redhat-linux/7 Found candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7 Selected GCC installation: /usr/bin/../lib/gcc/x86_64-redhat-linux/7 Candidate multilib: .;@m64 Candidate multilib: 32;@m32 Selected multilib: .;@m64 "/usr/bin/clang-4.0" -cc1 -triple x86_64-unknown-linux-gnu -emit-obj -mrelax-all -disable-free -disable-llvm-verifier -discard-value-names -main-file-name main.c -mrelocation-model static -mthread-model posix -mdisable-fp-elim -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -v -dwarf-column-info -debugger-tuning=gdb -resource-dir /usr/bin/../lib64/clang/4.0.1 -internal-isystem /usr/local/include -internal-isystem /usr/bin/../lib64/clang/4.0.1/include -internal-externc-isystem /include -internal-externc-isystem /usr/include -fdebug-compilation-dir /tmp/spack-test -ferror-limit 19 -fmessage-length 0 -fobjc-runtime=gcc -fdiagnostics-show-option -o /tmp/main-bf64f0.o -x c main.c clang -cc1 version 4.0.1 based upon LLVM 4.0.1 default target x86_64-unknown-linux-gnu ignoring nonexistent directory "/include" #include "..." search starts here: #include <...> search starts here: /usr/local/include /usr/bin/../lib64/clang/4.0.1/include /usr/include End of search list. "/usr/bin/ld" --hash-style=gnu --no-add-needed --build-id --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crt1.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crti.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/crtbegin.o -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7 -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64 -L/usr/bin/../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../.. -L/usr/bin/../lib -L/lib -L/usr/lib /tmp/main-bf64f0.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/bin/../lib/gcc/x86_64-redhat-linux/7/crtend.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crtn.o ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt ================================================ @(#)PROGRAM:ld PROJECT:ld64-305 configured to support archs: armv6 armv7 armv7s arm64 i386 x86_64 x86_64h armv6m armv7k armv7m armv7em (tvOS) Library search paths: /usr/local/lib /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib Framework search paths: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/System/Library/Frameworks/ ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/collect2-6.3.0-gnu-ld.txt ================================================ collect2 version 6.5.0 /usr/bin/ld -plugin /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/liblto_plugin.so -plugin-opt=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper -plugin-opt=-fresolution=/tmp/ccbFmewQ.res -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -rpath /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64 --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/x86_64-linux-gnu/crt1.o /usr/lib/x86_64-linux-gnu/crti.o /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtbegin.o -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0 -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../.. -v /tmp/ccxz6i1I.o -lstdc++ -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtend.o /usr/lib/x86_64-linux-gnu/crtn.o GNU ld (GNU Binutils for Debian) 2.28 ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt ================================================ Using built-in specs. COLLECT_GCC=/usr/bin/gcc COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper OFFLOAD_TARGET_NAMES=nvptx-none OFFLOAD_TARGET_DEFAULT=1 Target: x86_64-redhat-linux Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux Thread model: posix gcc version 7.3.1 20180130 (Red Hat 7.3.1-2) (GCC) COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64' /usr/libexec/gcc/x86_64-redhat-linux/7/cc1 -quiet -v main.c -quiet -dumpbase main.c -mtune=generic -march=x86-64 -auxbase main -version -o /tmp/ccM76aqK.s GNU C11 (GCC) version 7.3.1 20180130 (Red Hat 7.3.1-2) (x86_64-redhat-linux) compiled by GNU C version 7.3.1 20180130 (Red Hat 7.3.1-2), GMP version 6.1.2, MPFR version 3.1.5, MPC version 1.0.2, isl version isl-0.16.1-GMP GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072 ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/7/include-fixed" ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/7/../../../../x86_64-redhat-linux/include" #include "..." search starts here: #include <...> search starts here: /usr/lib/gcc/x86_64-redhat-linux/7/include /usr/local/include /usr/include End of search list. GNU C11 (GCC) version 7.3.1 20180130 (Red Hat 7.3.1-2) (x86_64-redhat-linux) compiled by GNU C version 7.3.1 20180130 (Red Hat 7.3.1-2), GMP version 6.1.2, MPFR version 3.1.5, MPC version 1.0.2, isl version isl-0.16.1-GMP GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072 Compiler executable checksum: ad7c3a488cf591743af375264d348c5c COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64' as -v --64 -o /tmp/ccYFphwj.o /tmp/ccM76aqK.s GNU assembler version 2.27 (x86_64-redhat-linux) using BFD version version 2.27-28.fc26 COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/7/:/usr/libexec/gcc/x86_64-redhat-linux/7/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/7/:/usr/lib/gcc/x86_64-redhat-linux/ LIBRARY_PATH=/usr/lib/gcc/x86_64-redhat-linux/7/:/usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/7/../../../:/lib/:/usr/lib/ COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64' /usr/libexec/gcc/x86_64-redhat-linux/7/collect2 -plugin /usr/libexec/gcc/x86_64-redhat-linux/7/liblto_plugin.so -plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper -plugin-opt=-fresolution=/tmp/ccw0b6CS.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s --build-id --no-add-needed --eh-frame-hdr --hash-style=gnu -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux/7/crtbegin.o -L/usr/lib/gcc/x86_64-redhat-linux/7 -L/usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib/gcc/x86_64-redhat-linux/7/../../.. /tmp/ccYFphwj.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/lib/gcc/x86_64-redhat-linux/7/crtend.o /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crtn.o COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64' ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt ================================================ icc.orig version 16.0.3 (gcc version 4.9.3 compatibility) ld /lib/../lib64/crt1.o /lib/../lib64/crti.o /usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/crtbegin.o --eh-frame-hdr --build-id -dynamic-linker /lib64/ld-linux-x86-64.so.2 -m elf_x86_64 -o blah -L/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/ -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../../lib64 -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../../lib64/ -L/lib/../lib64 -L/lib/../lib64/ -L/usr/lib/../lib64 -L/usr/lib/../lib64/ -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../ -L/lib64 -L/lib/ -L/usr/lib64 -L/usr/lib -rpath /usr/tce/packages/intel/intel-16.0.3/lib/intel64 -rpath=/usr/tce/packages/gcc/default/lib64 -Bdynamic -Bstatic -limf -lsvml -lirng -Bdynamic -lm -Bstatic -lipgo -ldecimal --as-needed -Bdynamic -lcilkrts -lstdc++ --no-as-needed -lgcc -lgcc_s -Bstatic -lirc -lsvml -Bdynamic -lc -lgcc -lgcc_s -Bstatic -lirc_s -Bdynamic -ldl -lc /usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/crtend.o /lib/../lib64/crtn.o /lib/../lib64/crt1.o: In function `_start': (.text+0x20): undefined reference to `main' ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/nag-6.2-gcc-6.5.0.txt ================================================ NAG Fortran Compiler Release 6.2(Chiyoda) Build 6223 Reading specs from /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/specs COLLECT_GCC=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/bin/gcc COLLECT_LTO_WRAPPER=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper Target: x86_64-pc-linux-gnu Configured with: /tmp/m300488/spack-stage/spack-stage-gcc-6.5.0-4sdjgrsboy3lowtq3t7pmp7rx3ogkqtz/spack-src/configure --prefix=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs --with-pkgversion='Spack GCC' --with-bugurl=https://github.com/spack/spack/issues --disable-multilib --enable-languages=c,c++,fortran --disable-nls --with-mpfr=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/mpfr-3.1.6-w63rspk --with-gmp=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gmp-6.1.2-et64cuj --with-system-zlib --with-mpc=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/mpc-1.1.0-en66k4t --with-isl=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/isl-0.18-62v4uyg Thread model: posix gcc version 6.5.0 (Spack GCC) COMPILER_PATH=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/ LIBRARY_PATH=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64/:/lib/x86_64-linux-gnu/:/lib/../lib64/:/usr/lib/x86_64-linux-gnu/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../:/lib/:/usr/lib/ COLLECT_GCC_OPTIONS='-m64' '-o' 'output' '-v' '-mtune=generic' '-march=x86-64' /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/collect2 -plugin /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/liblto_plugin.so -plugin-opt=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper -plugin-opt=-fresolution=/tmp/ccBpU203.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -rpath /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64 --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/x86_64-linux-gnu/crt1.o /usr/lib/x86_64-linux-gnu/crti.o /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtbegin.o -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0 -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../.. /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/f62init.o /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/quickfit.o /tmp/main.000786.o -rpath /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/libf62rts.so /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/libf62rts.a -lm -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtend.o /usr/lib/x86_64-linux-gnu/crtn.o COLLECT_GCC_OPTIONS='-m64' '-o' 'output' '-v' '-mtune=generic' '-march=x86-64' ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt ================================================ This is synthetic data to test parsing cases for which I could not find compiler output ld -LIBPATH:/first/path /LIBPATH:/second/path -libpath:/third/path collect2 version ld -LIBPATH:/skip/path -L/skip/this/too ================================================ FILE: lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt ================================================ export XL_CONFIG=/opt/ibm/xlC/13.1.5/etc/xlc.cfg.centos.7.gcc.4.8.5:xlc /usr/bin/ld --eh-frame-hdr -Qy -melf64lppc /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crt1.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crti.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/crtbegin.o -L/opt/ibm/xlsmp/4.1.5/lib -L/opt/ibm/xlmass/8.1.5/lib -L/opt/ibm/xlC/13.1.5/lib -R/opt/ibm/lib -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5 -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../.. --no-toc-optimize -o foo foo.o -dynamic-linker /lib64/ld64.so.2 --enable-new-dtags -lxlopt -lxl --as-needed -ldl --no-as-needed -lgcc_s --as-needed -lpthread --no-as-needed -lgcc -lm -lc -lgcc_s -lgcc /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/crtend.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crtn.o rm /tmp/xlcW0iQ4uI8 rm /tmp/xlcW1aPLBFY rm /tmp/xlcW2ALFICO ================================================ FILE: lib/spack/spack/test/data/compression/Foo ================================================ TEST ================================================ FILE: lib/spack/spack/test/data/compression/Foo.cxx ================================================ ================================================ FILE: lib/spack/spack/test/data/config/base/config.yaml ================================================ # This file is here strictly so that the base include directory will work config: dirty: false ================================================ FILE: lib/spack/spack/test/data/config/bootstrap.yaml ================================================ bootstrap: sources: - name: 'github-actions' metadata: $spack/share/spack/bootstrap/github-actions-v2 trusted: {} ================================================ FILE: lib/spack/spack/test/data/config/concretizer.yaml ================================================ concretizer: reuse: true targets: granularity: microarchitectures host_compatible: false duplicates: strategy: minimal max_dupes: default: 1 # Virtuals c: 2 cxx: 2 fortran: 1 # Regular packages cmake: 2 gmake: 2 python: 2 python-venv: 2 py-cython: 2 py-flit-core: 2 py-pip: 2 py-setuptools: 2 py-versioneer: 2 py-wheel: 2 xcb-proto: 2 # Compilers gcc: 2 llvm: 2 concretization_cache: enable: false ================================================ FILE: lib/spack/spack/test/data/config/config.yaml ================================================ config: install_tree: root: {0} template_dirs: - $spack/share/spack/templates - $spack/lib/spack/spack/test/data/templates - $spack/lib/spack/spack/test/data/templates_again build_stage: - $tempdir/$user/spack-stage source_cache: $user_cache_path/source misc_cache: $user_cache_path/cache verify_ssl: true ssl_certs: $SSL_CERT_FILE checksum: true installer: old # many tests are based on stdout from old installer dirty: false locks: {1} ================================================ FILE: lib/spack/spack/test/data/config/include.yaml ================================================ include: - path: base ================================================ FILE: lib/spack/spack/test/data/config/modules.yaml ================================================ # ------------------------------------------------------------------------- # This is the default configuration for Spack's module file generation. # # Settings here are versioned with Spack and are intended to provide # sensible defaults out of the box. Spack maintainers should edit this # file to keep it current. # # Users can override these settings by editing the following files. # # Per-spack-instance settings (overrides defaults): # $SPACK_ROOT/etc/spack/modules.yaml # # Per-user settings (overrides default and site settings): # ~/.spack/modules.yaml # ------------------------------------------------------------------------- modules: prefix_inspections: ./bin: [PATH] ./man: [MANPATH] ./share/man: [MANPATH] ./share/aclocal: [ACLOCAL_PATH] ./lib/pkgconfig: [PKG_CONFIG_PATH] ./lib64/pkgconfig: [PKG_CONFIG_PATH] ./share/pkgconfig: [PKG_CONFIG_PATH] ./: [CMAKE_PREFIX_PATH] default: roots: tcl: {0} lmod: {1} enable: [] tcl: all: autoload: direct lmod: all: autoload: direct hierarchy: - mpi ================================================ FILE: lib/spack/spack/test/data/config/packages.yaml ================================================ packages: all: providers: c: [gcc, llvm] cxx: [gcc, llvm] fortran: [gcc] fortran-rt: [gcc-runtime] libc: [glibc] libgfortran: [gcc-runtime] mpi: [mpich, zmpi] lapack: [openblas-with-lapack] blas: [openblas] externaltool: buildable: False externals: - spec: externaltool@1.0 prefix: /path/to/external_tool - spec: externaltool@0.9 prefix: /usr - spec: externaltool@0_8 prefix: /usr externalvirtual: buildable: False externals: - spec: externalvirtual@2.0 prefix: /path/to/external_virtual_clang - spec: externalvirtual@1.0 prefix: /path/to/external_virtual_gcc externalmodule: buildable: False externals: - spec: externalmodule@1.0 modules: - external-module 'requires-virtual': buildable: False externals: - spec: requires-virtual@2.0 prefix: /usr 'external-buildable-with-variant': buildable: True externals: - spec: external-buildable-with-variant@1.1.special +baz prefix: /usr - spec: external-buildable-with-variant@0.9 +baz prefix: /usr 'old-external': buildable: True externals: - spec: old-external@1.0.0 prefix: /usr 'external-non-default-variant': buildable: True externals: - spec: external-non-default-variant@3.8.7~foo~bar prefix: /usr version-test-dependency-preferred: version: ['5.2.5'] # Compilers gcc: externals: - spec: "gcc@9.4.0 languages='c,c++' os={linux_os.name}{linux_os.version}" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ - spec: "gcc@9.4.1 languages='c,c++' os=redhat6" prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ - spec: "gcc@10.2.1 languages='c,c++,fortran' os={linux_os.name}{linux_os.version}" prefix: /path extra_attributes: compilers: c: /path/bin/gcc-10 cxx: /path/bin/g++-10 fortran: /path/bin/gfortran-10 llvm: externals: - spec: "llvm@15.0.0 +clang~flang os={linux_os.name}{linux_os.version}" prefix: /path extra_attributes: compilers: c: /path/bin/clang cxx: /path/bin/clang++ glibc: buildable: false ================================================ FILE: lib/spack/spack/test/data/config/repos.yaml ================================================ repos: builtin_mock: $spack/var/spack/test_repos/spack_repo/builtin_mock ================================================ FILE: lib/spack/spack/test/data/conftest/diff-test/package-0.txt ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package_base import PackageBase from spack.package import * class DiffTest(PackageBase): """zlib replacement with optimizations for next generation systems.""" homepage = "https://github.com/zlib-ng/zlib-ng" url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" git = "https://github.com/zlib-ng/zlib-ng.git" license("Zlib") version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8") version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") ================================================ FILE: lib/spack/spack/test/data/conftest/diff-test/package-1.txt ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package_base import PackageBase from spack.package import * class DiffTest(PackageBase): """zlib replacement with optimizations for next generation systems.""" homepage = "https://github.com/zlib-ng/zlib-ng" url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" git = "https://github.com/zlib-ng/zlib-ng.git" license("Zlib") version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04") version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8") ================================================ FILE: lib/spack/spack/test/data/conftest/diff-test/package-2.txt ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package_base import PackageBase from spack.package import * class DiffTest(PackageBase): """zlib replacement with optimizations for next generation systems.""" homepage = "https://github.com/zlib-ng/zlib-ng" url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" git = "https://github.com/zlib-ng/zlib-ng.git" license("Zlib") version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325") version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04") version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8") ================================================ FILE: lib/spack/spack/test/data/conftest/diff-test/package-3.txt ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package_base import PackageBase from spack.package import * class DiffTest(PackageBase): """zlib replacement with optimizations for next generation systems.""" homepage = "https://github.com/zlib-ng/zlib-ng" url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" git = "https://github.com/zlib-ng/zlib-ng.git" license("Zlib") version("2.1.8", sha256="59e68f67cbb16999842daeb517cdd86fc25b177b4affd335cd72b76ddc2a46d8") version("2.1.7", sha256="59e68f67cbb16999842daeb517cdd86fc25b177b4affd335cd72b76ddc2a46d8") version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325") version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04") version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8") ================================================ FILE: lib/spack/spack/test/data/directory_search/README.txt ================================================ This directory tree is made up to test that search functions will return a stable ordered sequence. ================================================ FILE: lib/spack/spack/test/data/directory_search/a/c.h ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/a/foobar.txt ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/a/libc.a ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/a/libc.lib ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/b/b.h ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/b/bar.txp ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/b/d.h ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/b/liba.a ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/b/liba.lib ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/b/libd.a ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/b/libd.lib ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/c/a.h ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/c/bar.txt ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/c/libb.a ================================================ ================================================ FILE: lib/spack/spack/test/data/directory_search/c/libb.lib ================================================ ================================================ FILE: lib/spack/spack/test/data/filter_file/start_stop.txt ================================================ A B C D ================================================ FILE: lib/spack/spack/test/data/filter_file/x86_cpuid_info.c ================================================ /****************************/ /* THIS IS OPEN SOURCE CODE */ /****************************/ /* * File: x86_cpuid_info.c * Author: Dan Terpstra * terpstra@eecs.utk.edu * complete rewrite of linux-memory.c to conform to latest docs * and convert Intel to a table driven implementation. * Now also supports multiple TLB descriptors */ #include #include #include #include "papi.h" #include "papi_internal.h" static void init_mem_hierarchy( PAPI_mh_info_t * mh_info ); static int init_amd( PAPI_mh_info_t * mh_info, int *levels ); static short int _amd_L2_L3_assoc( unsigned short int pattern ); static int init_intel( PAPI_mh_info_t * mh_info , int *levels); #if defined( __amd64__ ) || defined (__x86_64__) static inline void cpuid( unsigned int *a, unsigned int *b, unsigned int *c, unsigned int *d ) { unsigned int op = *a; __asm__("cpuid;" : "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d) : "a" (op) ); } #else static inline void cpuid( unsigned int *a, unsigned int *b, unsigned int *c, unsigned int *d ) { unsigned int op = *a; // .byte 0x53 == push ebx. it's universal for 32 and 64 bit // .byte 0x5b == pop ebx. // Some gcc's (4.1.2 on Core2) object to pairing push/pop and ebx in 64 bit mode. // Using the opcode directly avoids this problem. __asm__ __volatile__( ".byte 0x53\n\tcpuid\n\tmovl %%ebx, %%esi\n\t.byte 0x5b":"=a"( *a ), "=S"( *b ), "=c"( *c ), "=d" ( *d ) : "a"( op ) ); } #endif int _x86_cache_info( PAPI_mh_info_t * mh_info ) { int retval = 0; union { struct { unsigned int ax, bx, cx, dx; } e; char vendor[20]; /* leave room for terminator bytes */ } reg; /* Don't use cpu_type to determine the processor. * get the information directly from the chip. */ reg.e.ax = 0; /* function code 0: vendor string */ /* The vendor string is composed of EBX:EDX:ECX. * by swapping the register addresses in the call below, * the string is correctly composed in the char array. */ cpuid( ®.e.ax, ®.e.bx, ®.e.dx, ®.e.cx ); reg.vendor[16] = 0; MEMDBG( "Vendor: %s\n", ®.vendor[4] ); init_mem_hierarchy( mh_info ); if ( !strncmp( "GenuineIntel", ®.vendor[4], 12 ) ) { init_intel( mh_info, &mh_info->levels); } else if ( !strncmp( "AuthenticAMD", ®.vendor[4], 12 ) ) { init_amd( mh_info, &mh_info->levels ); } else { MEMDBG( "Unsupported cpu type; Not Intel or AMD x86\n" ); return PAPI_ENOIMPL; } /* This works only because an empty cache element is initialized to 0 */ MEMDBG( "Detected L1: %d L2: %d L3: %d\n", mh_info->level[0].cache[0].size + mh_info->level[0].cache[1].size, mh_info->level[1].cache[0].size + mh_info->level[1].cache[1].size, mh_info->level[2].cache[0].size + mh_info->level[2].cache[1].size ); return retval; } static void init_mem_hierarchy( PAPI_mh_info_t * mh_info ) { int i, j; PAPI_mh_level_t *L = mh_info->level; /* initialize entire memory hierarchy structure to benign values */ for ( i = 0; i < PAPI_MAX_MEM_HIERARCHY_LEVELS; i++ ) { for ( j = 0; j < PAPI_MH_MAX_LEVELS; j++ ) { L[i].tlb[j].type = PAPI_MH_TYPE_EMPTY; L[i].tlb[j].num_entries = 0; L[i].tlb[j].associativity = 0; L[i].cache[j].type = PAPI_MH_TYPE_EMPTY; L[i].cache[j].size = 0; L[i].cache[j].line_size = 0; L[i].cache[j].num_lines = 0; L[i].cache[j].associativity = 0; } } } static short int _amd_L2_L3_assoc( unsigned short int pattern ) { /* From "CPUID Specification" #25481 Rev 2.28, April 2008 */ short int assoc[16] = { 0, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, SHRT_MAX }; if ( pattern > 0xF ) return -1; return ( assoc[pattern] ); } /* Cache configuration for AMD Athlon/Duron */ static int init_amd( PAPI_mh_info_t * mh_info, int *num_levels ) { union { struct { unsigned int ax, bx, cx, dx; } e; unsigned char byt[16]; } reg; int i, j, levels = 0; PAPI_mh_level_t *L = mh_info->level; /* * Layout of CPU information taken from : * "CPUID Specification" #25481 Rev 2.28, April 2008 for most current info. */ MEMDBG( "Initializing AMD memory info\n" ); /* AMD level 1 cache info */ reg.e.ax = 0x80000005; /* extended function code 5: L1 Cache and TLB Identifiers */ cpuid( ®.e.ax, ®.e.bx, ®.e.cx, ®.e.dx ); MEMDBG( "e.ax=%#8.8x e.bx=%#8.8x e.cx=%#8.8x e.dx=%#8.8x\n", reg.e.ax, reg.e.bx, reg.e.cx, reg.e.dx ); MEMDBG ( ":\neax: %#x %#x %#x %#x\nebx: %#x %#x %#x %#x\necx: %#x %#x %#x %#x\nedx: %#x %#x %#x %#x\n", reg.byt[0], reg.byt[1], reg.byt[2], reg.byt[3], reg.byt[4], reg.byt[5], reg.byt[6], reg.byt[7], reg.byt[8], reg.byt[9], reg.byt[10], reg.byt[11], reg.byt[12], reg.byt[13], reg.byt[14], reg.byt[15] ); /* NOTE: We assume L1 cache and TLB always exists */ /* L1 TLB info */ /* 4MB memory page information; half the number of entries as 2MB */ L[0].tlb[0].type = PAPI_MH_TYPE_INST; L[0].tlb[0].num_entries = reg.byt[0] / 2; L[0].tlb[0].page_size = 4096 << 10; L[0].tlb[0].associativity = reg.byt[1]; L[0].tlb[1].type = PAPI_MH_TYPE_DATA; L[0].tlb[1].num_entries = reg.byt[2] / 2; L[0].tlb[1].page_size = 4096 << 10; L[0].tlb[1].associativity = reg.byt[3]; /* 2MB memory page information */ L[0].tlb[2].type = PAPI_MH_TYPE_INST; L[0].tlb[2].num_entries = reg.byt[0]; L[0].tlb[2].page_size = 2048 << 10; L[0].tlb[2].associativity = reg.byt[1]; L[0].tlb[3].type = PAPI_MH_TYPE_DATA; L[0].tlb[3].num_entries = reg.byt[2]; L[0].tlb[3].page_size = 2048 << 10; L[0].tlb[3].associativity = reg.byt[3]; /* 4k page information */ L[0].tlb[4].type = PAPI_MH_TYPE_INST; L[0].tlb[4].num_entries = reg.byt[4]; L[0].tlb[4].page_size = 4 << 10; L[0].tlb[4].associativity = reg.byt[5]; L[0].tlb[5].type = PAPI_MH_TYPE_DATA; L[0].tlb[5].num_entries = reg.byt[6]; L[0].tlb[5].page_size = 4 << 10; L[0].tlb[5].associativity = reg.byt[7]; for ( i = 0; i < PAPI_MH_MAX_LEVELS; i++ ) { if ( L[0].tlb[i].associativity == 0xff ) L[0].tlb[i].associativity = SHRT_MAX; } /* L1 D-cache info */ L[0].cache[0].type = PAPI_MH_TYPE_DATA | PAPI_MH_TYPE_WB | PAPI_MH_TYPE_PSEUDO_LRU; L[0].cache[0].size = reg.byt[11] << 10; L[0].cache[0].associativity = reg.byt[10]; L[0].cache[0].line_size = reg.byt[8]; /* Byt[9] is "Lines per tag" */ /* Is that == lines per cache? */ /* L[0].cache[1].num_lines = reg.byt[9]; */ if ( L[0].cache[0].line_size ) L[0].cache[0].num_lines = L[0].cache[0].size / L[0].cache[0].line_size; MEMDBG( "D-Cache Line Count: %d; Computed: %d\n", reg.byt[9], L[0].cache[0].num_lines ); /* L1 I-cache info */ L[0].cache[1].type = PAPI_MH_TYPE_INST; L[0].cache[1].size = reg.byt[15] << 10; L[0].cache[1].associativity = reg.byt[14]; L[0].cache[1].line_size = reg.byt[12]; /* Byt[13] is "Lines per tag" */ /* Is that == lines per cache? */ /* L[0].cache[1].num_lines = reg.byt[13]; */ if ( L[0].cache[1].line_size ) L[0].cache[1].num_lines = L[0].cache[1].size / L[0].cache[1].line_size; MEMDBG( "I-Cache Line Count: %d; Computed: %d\n", reg.byt[13], L[0].cache[1].num_lines ); for ( i = 0; i < 2; i++ ) { if ( L[0].cache[i].associativity == 0xff ) L[0].cache[i].associativity = SHRT_MAX; } /* AMD L2/L3 Cache and L2 TLB info */ /* NOTE: For safety we assume L2 and L3 cache and TLB may not exist */ reg.e.ax = 0x80000006; /* extended function code 6: L2/L3 Cache and L2 TLB Identifiers */ cpuid( ®.e.ax, ®.e.bx, ®.e.cx, ®.e.dx ); MEMDBG( "e.ax=%#8.8x e.bx=%#8.8x e.cx=%#8.8x e.dx=%#8.8x\n", reg.e.ax, reg.e.bx, reg.e.cx, reg.e.dx ); MEMDBG ( ":\neax: %#x %#x %#x %#x\nebx: %#x %#x %#x %#x\necx: %#x %#x %#x %#x\nedx: %#x %#x %#x %#x\n", reg.byt[0], reg.byt[1], reg.byt[2], reg.byt[3], reg.byt[4], reg.byt[5], reg.byt[6], reg.byt[7], reg.byt[8], reg.byt[9], reg.byt[10], reg.byt[11], reg.byt[12], reg.byt[13], reg.byt[14], reg.byt[15] ); /* L2 TLB info */ if ( reg.byt[0] | reg.byt[1] ) { /* Level 2 ITLB exists */ /* 4MB ITLB page information; half the number of entries as 2MB */ L[1].tlb[0].type = PAPI_MH_TYPE_INST; L[1].tlb[0].num_entries = ( ( ( short ) ( reg.byt[1] & 0xF ) << 8 ) + reg.byt[0] ) / 2; L[1].tlb[0].page_size = 4096 << 10; L[1].tlb[0].associativity = _amd_L2_L3_assoc( ( reg.byt[1] & 0xF0 ) >> 4 ); /* 2MB ITLB page information */ L[1].tlb[2].type = PAPI_MH_TYPE_INST; L[1].tlb[2].num_entries = L[1].tlb[0].num_entries * 2; L[1].tlb[2].page_size = 2048 << 10; L[1].tlb[2].associativity = L[1].tlb[0].associativity; } if ( reg.byt[2] | reg.byt[3] ) { /* Level 2 DTLB exists */ /* 4MB DTLB page information; half the number of entries as 2MB */ L[1].tlb[1].type = PAPI_MH_TYPE_DATA; L[1].tlb[1].num_entries = ( ( ( short ) ( reg.byt[3] & 0xF ) << 8 ) + reg.byt[2] ) / 2; L[1].tlb[1].page_size = 4096 << 10; L[1].tlb[1].associativity = _amd_L2_L3_assoc( ( reg.byt[3] & 0xF0 ) >> 4 ); /* 2MB DTLB page information */ L[1].tlb[3].type = PAPI_MH_TYPE_DATA; L[1].tlb[3].num_entries = L[1].tlb[1].num_entries * 2; L[1].tlb[3].page_size = 2048 << 10; L[1].tlb[3].associativity = L[1].tlb[1].associativity; } /* 4k page information */ if ( reg.byt[4] | reg.byt[5] ) { /* Level 2 ITLB exists */ L[1].tlb[4].type = PAPI_MH_TYPE_INST; L[1].tlb[4].num_entries = ( ( short ) ( reg.byt[5] & 0xF ) << 8 ) + reg.byt[4]; L[1].tlb[4].page_size = 4 << 10; L[1].tlb[4].associativity = _amd_L2_L3_assoc( ( reg.byt[5] & 0xF0 ) >> 4 ); } if ( reg.byt[6] | reg.byt[7] ) { /* Level 2 DTLB exists */ L[1].tlb[5].type = PAPI_MH_TYPE_DATA; L[1].tlb[5].num_entries = ( ( short ) ( reg.byt[7] & 0xF ) << 8 ) + reg.byt[6]; L[1].tlb[5].page_size = 4 << 10; L[1].tlb[5].associativity = _amd_L2_L3_assoc( ( reg.byt[7] & 0xF0 ) >> 4 ); } /* AMD Level 2 cache info */ if ( reg.e.cx ) { L[1].cache[0].type = PAPI_MH_TYPE_UNIFIED | PAPI_MH_TYPE_WT | PAPI_MH_TYPE_PSEUDO_LRU; L[1].cache[0].size = ( int ) ( ( reg.e.cx & 0xffff0000 ) >> 6 ); /* right shift by 16; multiply by 2^10 */ L[1].cache[0].associativity = _amd_L2_L3_assoc( ( reg.byt[9] & 0xF0 ) >> 4 ); L[1].cache[0].line_size = reg.byt[8]; /* L[1].cache[0].num_lines = reg.byt[9]&0xF; */ if ( L[1].cache[0].line_size ) L[1].cache[0].num_lines = L[1].cache[0].size / L[1].cache[0].line_size; MEMDBG( "U-Cache Line Count: %d; Computed: %d\n", reg.byt[9] & 0xF, L[1].cache[0].num_lines ); } /* AMD Level 3 cache info (shared across cores) */ if ( reg.e.dx ) { L[2].cache[0].type = PAPI_MH_TYPE_UNIFIED | PAPI_MH_TYPE_WT | PAPI_MH_TYPE_PSEUDO_LRU; L[2].cache[0].size = ( int ) ( reg.e.dx & 0xfffc0000 ) << 1; /* in blocks of 512KB (2^19) */ L[2].cache[0].associativity = _amd_L2_L3_assoc( ( reg.byt[13] & 0xF0 ) >> 4 ); L[2].cache[0].line_size = reg.byt[12]; /* L[2].cache[0].num_lines = reg.byt[13]&0xF; */ if ( L[2].cache[0].line_size ) L[2].cache[0].num_lines = L[2].cache[0].size / L[2].cache[0].line_size; MEMDBG( "U-Cache Line Count: %d; Computed: %d\n", reg.byt[13] & 0xF, L[1].cache[0].num_lines ); } for ( i = 0; i < PAPI_MAX_MEM_HIERARCHY_LEVELS; i++ ) { for ( j = 0; j < PAPI_MH_MAX_LEVELS; j++ ) { /* Compute the number of levels of hierarchy actually used */ if ( L[i].tlb[j].type != PAPI_MH_TYPE_EMPTY || L[i].cache[j].type != PAPI_MH_TYPE_EMPTY ) levels = i + 1; } } *num_levels = levels; return PAPI_OK; } /* * The data from this table now comes from figure 3-17 in * the Intel Architectures Software Reference Manual 2A * (cpuid instruction section) * * Pretviously the information was provided by * "Intel Processor Identification and the CPUID Instruction", * Application Note, AP-485, Nov 2008, 241618-033 * Updated to AP-485, Aug 2009, 241618-036 * * The following data structure and its instantiation trys to * capture all the information in Section 2.1.3 of the above * document. Not all of it is used by PAPI, but it could be. * As the above document is revised, this table should be * updated. */ #define TLB_SIZES 3 /* number of different page sizes for a single TLB descriptor */ struct _intel_cache_info { int descriptor; /* 0x00 - 0xFF: register descriptor code */ int level; /* 1 to PAPI_MH_MAX_LEVELS */ int type; /* Empty, instr, data, vector, unified | TLB */ int size[TLB_SIZES]; /* cache or TLB page size(s) in kB */ int associativity; /* SHRT_MAX == fully associative */ int sector; /* 1 if cache is sectored; else 0 */ int line_size; /* for cache */ int entries; /* for TLB */ }; static struct _intel_cache_info intel_cache[] = { // 0x01 {.descriptor = 0x01, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size[0] = 4, .associativity = 4, .entries = 32, }, // 0x02 {.descriptor = 0x02, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size[0] = 4096, .associativity = SHRT_MAX, .entries = 2, }, // 0x03 {.descriptor = 0x03, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4, .associativity = 4, .entries = 64, }, // 0x04 {.descriptor = 0x04, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4096, .associativity = 4, .entries = 8, }, // 0x05 {.descriptor = 0x05, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4096, .associativity = 4, .entries = 32, }, // 0x06 {.descriptor = 0x06, .level = 1, .type = PAPI_MH_TYPE_INST, .size[0] = 8, .associativity = 4, .line_size = 32, }, // 0x08 {.descriptor = 0x08, .level = 1, .type = PAPI_MH_TYPE_INST, .size[0] = 16, .associativity = 4, .line_size = 32, }, // 0x09 {.descriptor = 0x09, .level = 1, .type = PAPI_MH_TYPE_INST, .size[0] = 32, .associativity = 4, .line_size = 64, }, // 0x0A {.descriptor = 0x0A, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 8, .associativity = 2, .line_size = 32, }, // 0x0B {.descriptor = 0x0B, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size[0] = 4096, .associativity = 4, .entries = 4, }, // 0x0C {.descriptor = 0x0C, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 16, .associativity = 4, .line_size = 32, }, // 0x0D {.descriptor = 0x0D, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 16, .associativity = 4, .line_size = 64, }, // 0x0E {.descriptor = 0x0E, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 24, .associativity = 6, .line_size = 64, }, // 0x21 {.descriptor = 0x21, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 256, .associativity = 8, .line_size = 64, }, // 0x22 {.descriptor = 0x22, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 4, .sector = 1, .line_size = 64, }, // 0x23 {.descriptor = 0x23, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1024, .associativity = 8, .sector = 1, .line_size = 64, }, // 0x25 {.descriptor = 0x25, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 2048, .associativity = 8, .sector = 1, .line_size = 64, }, // 0x29 {.descriptor = 0x29, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 4096, .associativity = 8, .sector = 1, .line_size = 64, }, // 0x2C {.descriptor = 0x2C, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 32, .associativity = 8, .line_size = 64, }, // 0x30 {.descriptor = 0x30, .level = 1, .type = PAPI_MH_TYPE_INST, .size[0] = 32, .associativity = 8, .line_size = 64, }, // 0x39 {.descriptor = 0x39, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 128, .associativity = 4, .sector = 1, .line_size = 64, }, // 0x3A {.descriptor = 0x3A, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 192, .associativity = 6, .sector = 1, .line_size = 64, }, // 0x3B {.descriptor = 0x3B, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 128, .associativity = 2, .sector = 1, .line_size = 64, }, // 0x3C {.descriptor = 0x3C, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 256, .associativity = 4, .sector = 1, .line_size = 64, }, // 0x3D {.descriptor = 0x3D, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 384, .associativity = 6, .sector = 1, .line_size = 64, }, // 0x3E {.descriptor = 0x3E, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 4, .sector = 1, .line_size = 64, }, // 0x40: no last level cache (??) // 0x41 {.descriptor = 0x41, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 128, .associativity = 4, .line_size = 32, }, // 0x42 {.descriptor = 0x42, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 256, .associativity = 4, .line_size = 32, }, // 0x43 {.descriptor = 0x43, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 4, .line_size = 32, }, // 0x44 {.descriptor = 0x44, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1024, .associativity = 4, .line_size = 32, }, // 0x45 {.descriptor = 0x45, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 2048, .associativity = 4, .line_size = 32, }, // 0x46 {.descriptor = 0x46, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 4096, .associativity = 4, .line_size = 64, }, // 0x47 {.descriptor = 0x47, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 8192, .associativity = 8, .line_size = 64, }, // 0x48 {.descriptor = 0x48, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 3072, .associativity = 12, .line_size = 64, }, // 0x49 NOTE: for family 0x0F model 0x06 this is level 3 {.descriptor = 0x49, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 4096, .associativity = 16, .line_size = 64, }, // 0x4A {.descriptor = 0x4A, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 6144, .associativity = 12, .line_size = 64, }, // 0x4B {.descriptor = 0x4B, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 8192, .associativity = 16, .line_size = 64, }, // 0x4C {.descriptor = 0x4C, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 12288, .associativity = 12, .line_size = 64, }, // 0x4D {.descriptor = 0x4D, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 16384, .associativity = 16, .line_size = 64, }, // 0x4E {.descriptor = 0x4E, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 6144, .associativity = 24, .line_size = 64, }, // 0x4F {.descriptor = 0x4F, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size[0] = 4, .associativity = SHRT_MAX, .entries = 32, }, // 0x50 {.descriptor = 0x50, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size = {4, 2048, 4096}, .associativity = SHRT_MAX, .entries = 64, }, // 0x51 {.descriptor = 0x51, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size = {4, 2048, 4096}, .associativity = SHRT_MAX, .entries = 128, }, // 0x52 {.descriptor = 0x52, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size = {4, 2048, 4096}, .associativity = SHRT_MAX, .entries = 256, }, // 0x55 {.descriptor = 0x55, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size = {2048, 4096, 0}, .associativity = SHRT_MAX, .entries = 7, }, // 0x56 {.descriptor = 0x56, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4096, .associativity = 4, .entries = 16, }, // 0x57 {.descriptor = 0x57, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4, .associativity = 4, .entries = 16, }, // 0x59 {.descriptor = 0x59, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4, .associativity = SHRT_MAX, .entries = 16, }, // 0x5A {.descriptor = 0x5A, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size = {2048, 4096, 0}, .associativity = 4, .entries = 32, }, // 0x5B {.descriptor = 0x5B, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size = {4, 4096, 0}, .associativity = SHRT_MAX, .entries = 64, }, // 0x5C {.descriptor = 0x5C, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size = {4, 4096, 0}, .associativity = SHRT_MAX, .entries = 128, }, // 0x5D {.descriptor = 0x5D, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size = {4, 4096, 0}, .associativity = SHRT_MAX, .entries = 256, }, // 0x60 {.descriptor = 0x60, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 16, .associativity = 8, .sector = 1, .line_size = 64, }, // 0x66 {.descriptor = 0x66, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 8, .associativity = 4, .sector = 1, .line_size = 64, }, // 0x67 {.descriptor = 0x67, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 16, .associativity = 4, .sector = 1, .line_size = 64, }, // 0x68 {.descriptor = 0x68, .level = 1, .type = PAPI_MH_TYPE_DATA, .size[0] = 32, .associativity = 4, .sector = 1, .line_size = 64, }, // 0x70 {.descriptor = 0x70, .level = 1, .type = PAPI_MH_TYPE_TRACE, .size[0] = 12, .associativity = 8, }, // 0x71 {.descriptor = 0x71, .level = 1, .type = PAPI_MH_TYPE_TRACE, .size[0] = 16, .associativity = 8, }, // 0x72 {.descriptor = 0x72, .level = 1, .type = PAPI_MH_TYPE_TRACE, .size[0] = 32, .associativity = 8, }, // 0x73 {.descriptor = 0x73, .level = 1, .type = PAPI_MH_TYPE_TRACE, .size[0] = 64, .associativity = 8, }, // 0x78 {.descriptor = 0x78, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1024, .associativity = 4, .line_size = 64, }, // 0x79 {.descriptor = 0x79, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 128, .associativity = 8, .sector = 1, .line_size = 64, }, // 0x7A {.descriptor = 0x7A, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 256, .associativity = 8, .sector = 1, .line_size = 64, }, // 0x7B {.descriptor = 0x7B, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 8, .sector = 1, .line_size = 64, }, // 0x7C {.descriptor = 0x7C, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1024, .associativity = 8, .sector = 1, .line_size = 64, }, // 0x7D {.descriptor = 0x7D, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 2048, .associativity = 8, .line_size = 64, }, // 0x7F {.descriptor = 0x7F, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 2, .line_size = 64, }, // 0x80 {.descriptor = 0x80, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 8, .line_size = 64, }, // 0x82 {.descriptor = 0x82, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 256, .associativity = 8, .line_size = 32, }, // 0x83 {.descriptor = 0x83, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 8, .line_size = 32, }, // 0x84 {.descriptor = 0x84, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1024, .associativity = 8, .line_size = 32, }, // 0x85 {.descriptor = 0x85, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 2048, .associativity = 8, .line_size = 32, }, // 0x86 {.descriptor = 0x86, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 4, .line_size = 64, }, // 0x87 {.descriptor = 0x87, .level = 2, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1024, .associativity = 8, .line_size = 64, }, // 0xB0 {.descriptor = 0xB0, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size[0] = 4, .associativity = 4, .entries = 128, }, // 0xB1 NOTE: This is currently the only instance where .entries // is dependent on .size. It's handled as a code exception. // If other instances appear in the future, the structure // should probably change to accomodate it. {.descriptor = 0xB1, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size = {2048, 4096, 0}, .associativity = 4, .entries = 8, /* or 4 if size = 4096 */ }, // 0xB2 {.descriptor = 0xB2, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST, .size[0] = 4, .associativity = 4, .entries = 64, }, // 0xB3 {.descriptor = 0xB3, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4, .associativity = 4, .entries = 128, }, // 0xB4 {.descriptor = 0xB4, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4, .associativity = 4, .entries = 256, }, // 0xBA {.descriptor = 0xBA, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size[0] = 4, .associativity = 4, .entries = 64, }, // 0xC0 {.descriptor = 0xBA, .level = 1, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA, .size = {4,4096}, .associativity = 4, .entries = 8, }, // 0xCA {.descriptor = 0xCA, .level = 2, .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_UNIFIED, .size[0] = 4, .associativity = 4, .entries = 512, }, // 0xD0 {.descriptor = 0xD0, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 512, .associativity = 4, .line_size = 64, }, // 0xD1 {.descriptor = 0xD1, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1024, .associativity = 4, .line_size = 64, }, // 0xD2 {.descriptor = 0xD2, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 2048, .associativity = 4, .line_size = 64, }, // 0xD6 {.descriptor = 0xD6, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1024, .associativity = 8, .line_size = 64, }, // 0xD7 {.descriptor = 0xD7, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 2048, .associativity = 8, .line_size = 64, }, // 0xD8 {.descriptor = 0xD8, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 4096, .associativity = 8, .line_size = 64, }, // 0xDC {.descriptor = 0xDC, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 1536, .associativity = 12, .line_size = 64, }, // 0xDD {.descriptor = 0xDD, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 3072, .associativity = 12, .line_size = 64, }, // 0xDE {.descriptor = 0xDE, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 6144, .associativity = 12, .line_size = 64, }, // 0xE2 {.descriptor = 0xE2, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 2048, .associativity = 16, .line_size = 64, }, // 0xE3 {.descriptor = 0xE3, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 4096, .associativity = 16, .line_size = 64, }, // 0xE4 {.descriptor = 0xE4, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 8192, .associativity = 16, .line_size = 64, }, // 0xEA {.descriptor = 0xEA, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 12288, .associativity = 24, .line_size = 64, }, // 0xEB {.descriptor = 0xEB, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 18432, .associativity = 24, .line_size = 64, }, // 0xEC {.descriptor = 0xEC, .level = 3, .type = PAPI_MH_TYPE_UNIFIED, .size[0] = 24576, .associativity = 24, .line_size = 64, }, // 0xF0 {.descriptor = 0xF0, .level = 1, .type = PAPI_MH_TYPE_PREF, .size[0] = 64, }, // 0xF1 {.descriptor = 0xF1, .level = 1, .type = PAPI_MH_TYPE_PREF, .size[0] = 128, }, }; #ifdef DEBUG static void print_intel_cache_table( ) { int i, j, k = ( int ) ( sizeof ( intel_cache ) / sizeof ( struct _intel_cache_info ) ); for ( i = 0; i < k; i++ ) { printf( "%d.\tDescriptor: %#x\n", i, intel_cache[i].descriptor ); printf( "\t Level: %d\n", intel_cache[i].level ); printf( "\t Type: %d\n", intel_cache[i].type ); printf( "\t Size(s): " ); for ( j = 0; j < TLB_SIZES; j++ ) printf( "%d, ", intel_cache[i].size[j] ); printf( "\n" ); printf( "\t Assoc: %d\n", intel_cache[i].associativity ); printf( "\t Sector: %d\n", intel_cache[i].sector ); printf( "\t Line Size: %d\n", intel_cache[i].line_size ); printf( "\t Entries: %d\n", intel_cache[i].entries ); printf( "\n" ); } } #endif /* Given a specific cache descriptor, this routine decodes the information from a table * of such descriptors and fills out one or more records in a PAPI data structure. * Called only by init_intel() */ static void intel_decode_descriptor( struct _intel_cache_info *d, PAPI_mh_level_t * L ) { int i, next; int level = d->level - 1; PAPI_mh_tlb_info_t *t; PAPI_mh_cache_info_t *c; if ( d->descriptor == 0x49 ) { /* special case */ unsigned int r_eax, r_ebx, r_ecx, r_edx; r_eax = 0x1; /* function code 1: family & model */ cpuid( &r_eax, &r_ebx, &r_ecx, &r_edx ); /* override table for Family F, model 6 only */ if ( ( r_eax & 0x0FFF3FF0 ) == 0xF60 ) level = 3; } if ( d->type & PAPI_MH_TYPE_TLB ) { for ( next = 0; next < PAPI_MH_MAX_LEVELS - 1; next++ ) { if ( L[level].tlb[next].type == PAPI_MH_TYPE_EMPTY ) break; } /* expand TLB entries for multiple possible page sizes */ for ( i = 0; i < TLB_SIZES && next < PAPI_MH_MAX_LEVELS && d->size[i]; i++, next++ ) { // printf("Level %d Descriptor: %#x TLB type %#x next: %d, i: %d\n", level, d->descriptor, d->type, next, i); t = &L[level].tlb[next]; t->type = PAPI_MH_CACHE_TYPE( d->type ); t->num_entries = d->entries; t->page_size = d->size[i] << 10; /* minimum page size in KB */ t->associativity = d->associativity; /* another special case */ if ( d->descriptor == 0xB1 && d->size[i] == 4096 ) t->num_entries = d->entries / 2; } } else { for ( next = 0; next < PAPI_MH_MAX_LEVELS - 1; next++ ) { if ( L[level].cache[next].type == PAPI_MH_TYPE_EMPTY ) break; } // printf("Level %d Descriptor: %#x Cache type %#x next: %d\n", level, d->descriptor, d->type, next); c = &L[level].cache[next]; c->type = PAPI_MH_CACHE_TYPE( d->type ); c->size = d->size[0] << 10; /* convert from KB to bytes */ c->associativity = d->associativity; if ( d->line_size ) { c->line_size = d->line_size; c->num_lines = c->size / c->line_size; } } } #if defined(__amd64__) || defined(__x86_64__) static inline void cpuid2( unsigned int*eax, unsigned int* ebx, unsigned int*ecx, unsigned int *edx, unsigned int index, unsigned int ecx_in ) { __asm__ __volatile__ ("cpuid;" : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) : "0" (index), "2"(ecx_in) ); } #else static inline void cpuid2 ( unsigned int* eax, unsigned int* ebx, unsigned int* ecx, unsigned int* edx, unsigned int index, unsigned int ecx_in ) { unsigned int a,b,c,d; __asm__ __volatile__ (".byte 0x53\n\tcpuid\n\tmovl %%ebx, %%esi\n\t.byte 0x5b" : "=a" (a), "=S" (b), "=c" (c), "=d" (d) \ : "0" (index), "2"(ecx_in) ); *eax = a; *ebx = b; *ecx = c; *edx = d; } #endif static int init_intel_leaf4( PAPI_mh_info_t * mh_info, int *num_levels ) { unsigned int eax, ebx, ecx, edx; unsigned int maxidx, ecx_in; int next; int cache_type,cache_level,cache_selfinit,cache_fullyassoc; int cache_linesize,cache_partitions,cache_ways,cache_sets; PAPI_mh_cache_info_t *c; *num_levels=0; cpuid2(&eax,&ebx,&ecx,&edx, 0, 0); maxidx = eax; if (maxidx<4) { MEMDBG("Warning! CPUID Index 4 not supported!\n"); return PAPI_ENOSUPP; } ecx_in=0; while(1) { cpuid2(&eax,&ebx,&ecx,&edx, 4, ecx_in); /* decoded as per table 3-12 in Intel Software Developer's Manual Volume 2A */ cache_type=eax&0x1f; if (cache_type==0) break; cache_level=(eax>>5)&0x3; cache_selfinit=(eax>>8)&0x1; cache_fullyassoc=(eax>>9)&0x1; cache_linesize=(ebx&0xfff)+1; cache_partitions=((ebx>>12)&0x3ff)+1; cache_ways=((ebx>>22)&0x3ff)+1; cache_sets=(ecx)+1; /* should we export this info? cache_maxshare=((eax>>14)&0xfff)+1; cache_maxpackage=((eax>>26)&0x3f)+1; cache_wb=(edx)&1; cache_inclusive=(edx>>1)&1; cache_indexing=(edx>>2)&1; */ if (cache_level>*num_levels) *num_levels=cache_level; /* find next slot available to hold cache info */ for ( next = 0; next < PAPI_MH_MAX_LEVELS - 1; next++ ) { if ( mh_info->level[cache_level-1].cache[next].type == PAPI_MH_TYPE_EMPTY ) break; } c=&(mh_info->level[cache_level-1].cache[next]); switch(cache_type) { case 1: MEMDBG("L%d Data Cache\n",cache_level); c->type=PAPI_MH_TYPE_DATA; break; case 2: MEMDBG("L%d Instruction Cache\n",cache_level); c->type=PAPI_MH_TYPE_INST; break; case 3: MEMDBG("L%d Unified Cache\n",cache_level); c->type=PAPI_MH_TYPE_UNIFIED; break; } if (cache_selfinit) { MEMDBG("\tSelf-init\n"); } if (cache_fullyassoc) { MEMDBG("\tFully Associtative\n"); } //MEMDBG("\tMax logical processors sharing cache: %d\n",cache_maxshare); //MEMDBG("\tMax logical processors sharing package: %d\n",cache_maxpackage); MEMDBG("\tCache linesize: %d\n",cache_linesize); MEMDBG("\tCache partitions: %d\n",cache_partitions); MEMDBG("\tCache associaticity: %d\n",cache_ways); MEMDBG("\tCache sets: %d\n",cache_sets); MEMDBG("\tCache size = %dkB\n", (cache_ways*cache_partitions*cache_linesize*cache_sets)/1024); //MEMDBG("\tWBINVD/INVD acts on lower caches: %d\n",cache_wb); //MEMDBG("\tCache is not inclusive: %d\n",cache_inclusive); //MEMDBG("\tComplex cache indexing: %d\n",cache_indexing); c->line_size=cache_linesize; if (cache_fullyassoc) { c->associativity=SHRT_MAX; } else { c->associativity=cache_ways; } c->size=(cache_ways*cache_partitions*cache_linesize*cache_sets); c->num_lines=cache_ways*cache_partitions*cache_sets; ecx_in++; } return PAPI_OK; } static int init_intel_leaf2( PAPI_mh_info_t * mh_info , int *num_levels) { /* cpuid() returns memory copies of 4 32-bit registers * this union allows them to be accessed as either registers * or individual bytes. Remember that Intel is little-endian. */ union { struct { unsigned int ax, bx, cx, dx; } e; unsigned char descrip[16]; } reg; int r; /* register boundary index */ int b; /* byte index into a register */ int i; /* byte index into the descrip array */ int t; /* table index into the static descriptor table */ int count; /* how many times to call cpuid; from eax:lsb */ int size; /* size of the descriptor table */ int last_level = 0; /* how many levels in the cache hierarchy */ /* All of Intel's cache info is in 1 call to cpuid * however it is a table lookup :( */ MEMDBG( "Initializing Intel Cache and TLB descriptors\n" ); #ifdef DEBUG if ( ISLEVEL( DEBUG_MEMORY ) ) print_intel_cache_table( ); #endif reg.e.ax = 0x2; /* function code 2: cache descriptors */ cpuid( ®.e.ax, ®.e.bx, ®.e.cx, ®.e.dx ); MEMDBG( "e.ax=%#8.8x e.bx=%#8.8x e.cx=%#8.8x e.dx=%#8.8x\n", reg.e.ax, reg.e.bx, reg.e.cx, reg.e.dx ); MEMDBG ( ":\nd0: %#x %#x %#x %#x\nd1: %#x %#x %#x %#x\nd2: %#x %#x %#x %#x\nd3: %#x %#x %#x %#x\n", reg.descrip[0], reg.descrip[1], reg.descrip[2], reg.descrip[3], reg.descrip[4], reg.descrip[5], reg.descrip[6], reg.descrip[7], reg.descrip[8], reg.descrip[9], reg.descrip[10], reg.descrip[11], reg.descrip[12], reg.descrip[13], reg.descrip[14], reg.descrip[15] ); count = reg.descrip[0]; /* # times to repeat CPUID call. Not implemented. */ /* Knights Corner at least returns 0 here */ if (count==0) goto early_exit; size = ( sizeof ( intel_cache ) / sizeof ( struct _intel_cache_info ) ); /* # descriptors */ MEMDBG( "Repeat cpuid(2,...) %d times. If not 1, code is broken.\n", count ); if (count!=1) { fprintf(stderr,"Warning: Unhandled cpuid count of %d\n",count); } for ( r = 0; r < 4; r++ ) { /* walk the registers */ if ( ( reg.descrip[r * 4 + 3] & 0x80 ) == 0 ) { /* only process if high order bit is 0 */ for ( b = 3; b >= 0; b-- ) { /* walk the descriptor bytes from high to low */ i = r * 4 + b; /* calculate an index into the array of descriptors */ if ( i ) { /* skip the low order byte in eax [0]; it's the count (see above) */ if ( reg.descrip[i] == 0xff ) { MEMDBG("Warning! PAPI x86_cache: must implement cpuid leaf 4\n"); return PAPI_ENOSUPP; /* we might continue instead */ /* in order to get TLB info */ /* continue; */ } for ( t = 0; t < size; t++ ) { /* walk the descriptor table */ if ( reg.descrip[i] == intel_cache[t].descriptor ) { /* find match */ if ( intel_cache[t].level > last_level ) last_level = intel_cache[t].level; intel_decode_descriptor( &intel_cache[t], mh_info->level ); } } } } } } early_exit: MEMDBG( "# of Levels: %d\n", last_level ); *num_levels=last_level; return PAPI_OK; } static int init_intel( PAPI_mh_info_t * mh_info, int *levels ) { int result; int num_levels; /* try using the oldest leaf2 method first */ result=init_intel_leaf2(mh_info, &num_levels); if (result!=PAPI_OK) { /* All Core2 and newer also support leaf4 detection */ /* Starting with Westmere *only* leaf4 is supported */ result=init_intel_leaf4(mh_info, &num_levels); } *levels=num_levels; return PAPI_OK; } /* Returns 1 if hypervisor detected */ /* Returns 0 if none found. */ int _x86_detect_hypervisor(char *vendor_name) { unsigned int eax, ebx, ecx, edx; char hyper_vendor_id[13]; cpuid2(&eax, &ebx, &ecx, &edx,0x1,0); /* This is the hypervisor bit, ecx bit 31 */ if (ecx&0x80000000) { /* There are various values in the 0x4000000X range */ /* It is questionable how standard they are */ /* For now we just return the name. */ cpuid2(&eax, &ebx, &ecx, &edx, 0x40000000,0); memcpy(hyper_vendor_id + 0, &ebx, 4); memcpy(hyper_vendor_id + 4, &ecx, 4); memcpy(hyper_vendor_id + 8, &edx, 4); hyper_vendor_id[12] = '\0'; strncpy(vendor_name,hyper_vendor_id,PAPI_MAX_STR_LEN); return 1; } else { strncpy(vendor_name,"none",PAPI_MAX_STR_LEN); } return 0; } ================================================ FILE: lib/spack/spack/test/data/make/affirmative/capital_makefile/Makefile ================================================ # Tests that Spack checks for Makefile check: ================================================ FILE: lib/spack/spack/test/data/make/affirmative/check_test/Makefile ================================================ # Tests that Spack detects target when it is the first of two targets check test: ================================================ FILE: lib/spack/spack/test/data/make/affirmative/expansion/Makefile ================================================ # Tests that Spack can handle variable expansion targets TARGETS = check $(TARGETS): ================================================ FILE: lib/spack/spack/test/data/make/affirmative/gnu_makefile/GNUmakefile ================================================ # Tests that Spack checks for GNUmakefile check: ================================================ FILE: lib/spack/spack/test/data/make/affirmative/include/Makefile ================================================ # Tests that Spack detects targets in include files include make.mk ================================================ FILE: lib/spack/spack/test/data/make/affirmative/include/make.mk ================================================ check: ================================================ FILE: lib/spack/spack/test/data/make/affirmative/lowercase_makefile/makefile ================================================ # Tests that Spack checks for makefile check: ================================================ FILE: lib/spack/spack/test/data/make/affirmative/prerequisites/Makefile ================================================ # Tests that Spack detects a target even if it is followed by prerequisites check: check-recursive check-recursive: ================================================ FILE: lib/spack/spack/test/data/make/affirmative/spaces/Makefile ================================================ # Tests that Spack allows spaces following the target name check : ================================================ FILE: lib/spack/spack/test/data/make/affirmative/test_check/Makefile ================================================ # Tests that Spack detects target when it is the second of two targets test check: ================================================ FILE: lib/spack/spack/test/data/make/affirmative/three_targets/Makefile ================================================ # Tests that Spack detects a target if it is in the middle of a list foo check bar: ================================================ FILE: lib/spack/spack/test/data/make/negative/no_makefile/readme.txt ================================================ # Tests that Spack ignores directories without a Makefile check: ================================================ FILE: lib/spack/spack/test/data/make/negative/partial_match/Makefile ================================================ # Tests that Spack ignores targets that contain a partial match checkinstall: installcheck: foo-check-bar: foo_check_bar: foo/check/bar: ================================================ FILE: lib/spack/spack/test/data/make/negative/variable/Makefile ================================================ # Tests that Spack ignores variable definitions check = FOO check := BAR ================================================ FILE: lib/spack/spack/test/data/microarchitectures/microarchitectures.json ================================================ { "microarchitectures": { "x86": { "from": null, "vendor": "generic", "features": [] }, "i686": { "from": "x86", "vendor": "GenuineIntel", "features": [] }, "pentium2": { "from": "i686", "vendor": "GenuineIntel", "features": [ "mmx" ] }, "pentium3": { "from": "pentium2", "vendor": "GenuineIntel", "features": [ "mmx", "sse" ] }, "pentium4": { "from": "pentium3", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2" ] }, "prescott": { "from": "pentium4", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "sse3" ] }, "x86_64": { "from": null, "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "4.2.0:", "name": "x86-64", "flags": "-march={name} -mtune=generic" }, { "versions": ":4.1.2", "name": "x86-64", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "0.0.0-apple:", "name": "x86-64", "flags": "-march={name}" }, { "versions": ":", "name": "x86-64", "flags": "-march={name} -mtune=generic" } ], "intel": { "versions": ":", "name": "pentium4", "flags": "-march={name} -mtune=generic" } } }, "nocona": { "from": "x86_64", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "sse3" ], "compilers": { "gcc": { "versions": "4.0.4:", "flags": "-march={name} -mtune={name}" }, "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", "name": "pentium4", "flags": "-march={name} -mtune=generic" } } }, "core2": { "from": "nocona", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3" ], "compilers": { "gcc": { "versions": "4.3.0:", "flags": "-march={name} -mtune={name}" }, "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", "flags": "-march={name} -mtune={name}}" } } }, "nehalem": { "from": "core2", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, { "versions": "4.6:4.8.5", "name": "corei7", "flags": "-march={name} -mtune={name}" } ], "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", "name": "corei7", "flags": "-march={name} -mtune={name}" } } }, "westmere": { "from": "nehalem", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "aes", "pclmulqdq" ], "compilers": { "gcc": { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "16.0:", "name": "corei7", "flags": "-march={name} -mtune={name}" } } }, "sandybridge": { "from": "westmere", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "aes", "pclmulqdq", "avx" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, { "versions": "4.6:4.8.5", "name": "corei7-avx", "flags": "-march={name} -mtune={name}" } ], "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": [ { "versions": "16.0:17.9.0", "name": "corei7-avx", "flags": "-march={name} -mtune={name}" }, { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ] } }, "ivybridge": { "from": "sandybridge", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "aes", "pclmulqdq", "avx", "rdrand", "f16c" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, { "versions": "4.6:4.8.5", "name": "core-avx-i", "flags": "-march={name} -mtune={name}" } ], "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": [ { "versions": "16.0:17.9.0", "name": "core-avx-i", "flags": "-march={name} -mtune={name}" }, { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ] } }, "haswell": { "from": "ivybridge", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, { "versions": "4.8:4.8.5", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": [ { "versions": "16.0:17.9.0", "name": "core-avx2", "flags": "-march={name} -mtune={name}" }, { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ] } }, "broadwell": { "from": "haswell", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx" ], "compilers": { "gcc": { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } } }, "skylake": { "from": "broadwell", "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx", "clflushopt", "xsavec", "xsaveopt" ], "compilers": { "gcc": { "versions": "6.0:", "flags": "-march={name} -mtune={name}" }, "clang": { "versions": "3.9:", "flags": "-march={name} -mtune={name}" }, "intel": { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } } } }, "feature_aliases": { "sse3": { "reason": "ssse3 is a superset of sse3 and might be the only one listed", "any_of": [ "ssse3" ] }, "avx512": { "reason": "avx512 indicates generic support for any of the avx512 instruction sets", "any_of": [ "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512cd" ] }, "fma": { "reason": "FMA has been supported by PowerISA since Power1, but might not be listed in features", "families": [ "ppc64le", "ppc64" ] }, "sse4.1": { "reason": "permits to refer to sse4_1 also as sse4.1", "any_of": [ "sse4_1" ] }, "sse4.2": { "reason": "permits to refer to sse4_2 also as sse4.2", "any_of": [ "sse4_2" ] } }, "conversions": { "description": "Conversions that map some platform specific values to canonical values", "arm_vendors": { "0x41": "ARM", "0x42": "Broadcom", "0x43": "Cavium", "0x44": "DEC", "0x46": "Fujitsu", "0x48": "HiSilicon", "0x49": "Infineon Technologies AG", "0x4d": "Motorola", "0x4e": "Nvidia", "0x50": "APM", "0x51": "Qualcomm", "0x53": "Samsung", "0x56": "Marvell", "0x61": "Apple", "0x66": "Faraday", "0x68": "HXT", "0x69": "Intel" }, "darwin_flags": { "sse4.1": "sse4_1", "sse4.2": "sse4_2", "avx1.0": "avx", "clfsopt": "clflushopt", "xsave": "xsavec xsaveopt" } } } ================================================ FILE: lib/spack/spack/test/data/mirrors/legacy_yaml/build_cache/test-debian6-core2-gcc-4.5.0-zlib-1.2.11-t5mczux3tfqpxwmg7egp7axy2jvyulqk.spec.yaml ================================================ spec: - zlib: version: 1.2.11 arch: platform: test platform_os: debian6 target: name: core2 vendor: GenuineIntel features: - mmx - sse - sse2 - ssse3 generation: 0 parents: - nocona compiler: name: gcc version: 4.5.0 namespace: builtin_mock parameters: optimize: true pic: true shared: true cflags: [] cppflags: [] cxxflags: [] fflags: [] ldflags: [] ldlibs: [] package_hash: eukp6mqxxlfuxslsodbwbqtsznajielhh4avm2vgteo4ifdsjgjq==== hash: t5mczux3tfqpxwmg7egp7axy2jvyulqk full_hash: 6j4as6r3qd4qhf77yu44reyn2u6ggbuq build_hash: t5mczux3tfqpxwmg7egp7axy2jvyulqk binary_cache_checksum: hash_algorithm: sha256 hash: a62b50aee38bb5d6d1cbf9cd2b0badaf3eaa282cd6db0472b4468ff968a5e7f2 buildinfo: relative_prefix: test-debian6-core2/gcc-4.5.0/zlib-1.2.11-t5mczux3tfqpxwmg7egp7axy2jvyulqk relative_rpaths: false ================================================ FILE: lib/spack/spack/test/data/mirrors/signed_json/linux-ubuntu18.04-haswell-gcc-8.4.0-zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb.spec.json.sig ================================================ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 { "spec": { "_meta": { "version": 2 }, "nodes": [ { "name": "zlib", "version": "1.2.12", "arch": { "platform": "linux", "platform_os": "ubuntu18.04", "target": { "name": "haswell", "vendor": "GenuineIntel", "features": [ "aes", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "mmx", "movbe", "pclmulqdq", "popcnt", "rdrand", "sse", "sse2", "sse4_1", "sse4_2", "ssse3" ], "generation": 0, "parents": [ "ivybridge", "x86_64_v3" ] } }, "compiler": { "name": "gcc", "version": "8.4.0" }, "namespace": "builtin", "parameters": { "optimize": true, "patches": [ "0d38234384870bfd34dfcb738a9083952656f0c766a0f5990b1893076b084b76" ], "pic": true, "shared": true, "cflags": [], "cppflags": [], "cxxflags": [], "fflags": [], "ldflags": [], "ldlibs": [] }, "patches": [ "0d38234384870bfd34dfcb738a9083952656f0c766a0f5990b1893076b084b76" ], "package_hash": "bm7rut622h3yt5mpm4kvf7pmh7tnmueezgk5yquhr2orbmixwxuq====", "hash": "g7otk5dra3hifqxej36m5qzm7uyghqgb", "full_hash": "fx2fyri7bv3vpz2rhke6g3l3dwxda4t6", "build_hash": "g7otk5dra3hifqxej36m5qzm7uyghqgb" } ] }, "binary_cache_checksum": { "hash_algorithm": "sha256", "hash": "5b9a180f14e0d04b17b1b0c2a26cf3beae448d77d1bda4279283ca4567d0be90" }, "buildinfo": { "relative_prefix": "linux-ubuntu18.04-haswell/gcc-8.4.0/zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb", "relative_rpaths": false } } -----BEGIN PGP SIGNATURE----- iQEzBAEBCgAdFiEEz8AGj4zHZe4OaI2OQ0Tg92UAr50FAmJ5lD4ACgkQQ0Tg92UA r52LEggAl/wXlOlHDnjWvqBlqAn3gaJEZ5PDVPczk6k0w+SNfDGrHfWJnL2c23Oq CssbHylSgAFvaPT1frbiLfZj6L4j4Ym1qsxIlGNsVfW7Pbc4yNF0flqYMdWKXbgY 2sQoPegIKK7EBtpjDf0+VRYfJTMqjsSgjT/o+nTkg9oAnvU23EqXI5uiY84Z5z6l CKLBm0GWg7MzI0u8NdiQMVNYVatvvZ8EQpblEUQ7jD4Bo0yoSr33Qdq8uvu4ZdlW bvbIgeY3pTPF13g9uNznHLxW4j9BWQnOtHFI5UKQnYRner504Yoz9k+YxhwuDlaY TrOxvHe9hG1ox1AP4tqQc+HsNpm5Kg== =gi2R -----END PGP SIGNATURE----- ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/_pgp/CBAB2C1032C6FF5078049EC0FA61D50C12CAD37E.pub ================================================ -----BEGIN PGP PUBLIC KEY BLOCK----- mQINBGgnhhYBEAC5LOSkJlxL4rRDBLDatswpzAw7NQnONW37hwOauEf6rlw/wk6J 2D1l/jjmGwyo1iHOEu1/26fMuXMmG0vAxOQJFrkoKAgxDUD9nL0GqTJyg0+yTCN6 xsWsrIZi+8oNDXYzLiejICZorc+ri11kcZdA+WE2hWPRStmJH75afpSd7XfNijqb MPfDZBcr+pLeARSH11BTfb8Dtm9qN//+X+pNIUqeHL9hLu/W9hb3GCfXqnsCQJA1 WMFTrbCcPYm0R7EevMnscFvS8xbhocBPDwZ12f4W5CugrL29X4Vx9SaUlIyy/+SC 2Gwi8Yq78Y4dTN7N5aA8L169/uqy4Tx7/966wMkUYXk7UxmH9E0ol5EZYnY9SCj6 xLtMNKA+NLwESj0azaWEzxfztyNdTYfG8Eaa/QGFs1YVGhYdmcEp8KDbQg5FBeCA I6MUcH0XWOTJaZI/oEtukMYHzBt9jyyq6Gp45TiQvOou0wE+w/zJcd9Td23R81KW GfMh5r80NET/bx88vee4NNHkWCphhqs53rIrhWV3y3WKaWp7DfP3WMiTBJ+Yc+PI 0vMIHKYNy+OqwTjmwgKdN1w1xZhLG7hx0sAdcZGP7q0A6381HtucgS/fucDogMnW H3anE8UGx4HBRjyXsuOaOAgNw2K4IwancUSf67WSzji3AiP46sUun5ERNQARAQAB tBlTcGFjayA8c3BhY2tAc3BhY2suc3BhY2s+iQJXBBMBCgBBFiEEy6ssEDLG/1B4 BJ7A+mHVDBLK034FAmgnhhYCGwMFCQWjmoAFCwkIBwICIgIGFQoJCAsCBBYCAwEC HgcCF4AACgkQ+mHVDBLK034zWhAAtjm802qaTSCvB9WvY1RM65/B1GUK3ZEv3fw/ Dvt3xd3mh+rzWBTJ8t7+/cPaOq7qOGnfUateHgou+0T6lgCLkrwr4lFa6yZSUATb xcnopcA0Dal218UcIRb20PjPtoKu3Tt9JFceXJGCTYoGz5HbkOemwkR8B+4qMRPW sn1IhV32eig2HUzrUXVOv6WomMtk2qUpND0WnTlZo3EoInJeTzdlXkOR3lRLADM9 yPM6Rp8AV/ykM9DztL4SinzyZjqEM7o1H7EFITZSlkjcBPvqDlvowZGN8TVbG9TQ 8Nfz8BYF3SVaPduwXwhbE9D8jqtNt652IZ1+1KbMii1l4deu0UYx8BSfJjNANTTU jFDiyNaGnn5OsZXNllsyAHWky6ApyBD9qFxxNr0kiWbVrrN6s2u4ghm5Hgtdx40v hA9+kvB2mtV/HklUkwDTJ6Ytgp5veh8GKvBD9eAWIitl6w153Rba5LkZbk2ijK6k oyN9Ge/YloSMwXpIEnE7/SRE1o5vye294BZjyqnr+U+wzbEYbC7eXJ0peDCbpbZc 0kxMDDbrhmHeEaHeWF30hm6WBaUT4SUcPj5BiV3mt3BhtRgAwA3SvuSenk2yRzR8 tBES4b/RBmOczfs4w4m5rAmfVNkNwykry4M2jPCJhVA2qG8q1gLxf+AvaPcAvQ8D kmDeNLI= =CYuA -----END PGP PUBLIC KEY BLOCK----- ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/_pgp/index.json ================================================ {"keys":{"CBAB2C1032C6FF5078049EC0FA61D50C12CAD37E":{}}} ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/index.json ================================================ {"database":{"version":"8","installs":{"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez":{"spec":{"name":"libelf","version":"0.8.13","arch":{"platform":"test","platform_os":"debian6","target":{"name":"core2","vendor":"GenuineIntel","features":["mmx","sse","sse2","ssse3"],"generation":0,"parents":["nocona"],"cpupart":""}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====","annotations":{"original_specfile_version":4,"compiler":"gcc@=10.2.1"},"hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez"},"ref_count":1,"in_buildcache":true},"sk2gqqz4n5njmvktycnd25wq25jxiqkr":{"spec":{"name":"libdwarf","version":"20130729","arch":{"platform":"test","platform_os":"debian6","target":{"name":"core2","vendor":"GenuineIntel","features":["mmx","sse","sse2","ssse3"],"generation":0,"parents":["nocona"],"cpupart":""}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====","dependencies":[{"name":"libelf","hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":4,"compiler":"gcc@=10.2.1"},"hash":"sk2gqqz4n5njmvktycnd25wq25jxiqkr"},"ref_count":0,"in_buildcache":true},"qeehcxyvluwnihsc2qxstmpomtxo3lrc":{"spec":{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"test","platform_os":"debian6","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====","annotations":{"original_specfile_version":5},"hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc"},"ref_count":2,"in_buildcache":true},"vd7v4ssgnoqdplgxyig3orum67n4vmhq":{"spec":{"name":"gcc","version":"10.2.1","arch":{"platform":"test","platform_os":"debian6","target":"aarch64"},"namespace":"builtin_mock","parameters":{"build_system":"generic","languages":["c","c++","fortran"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/path","module":null,"extra_attributes":{"compilers":{"c":"/path/bin/gcc-10","cxx":"/path/bin/g++-10","fortran":"/path/bin/gfortran-10"}}},"package_hash":"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====","annotations":{"original_specfile_version":5},"hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq"},"ref_count":3,"in_buildcache":false},"izgzpzeljwairalfjm3k6fntbb64nt6n":{"spec":{"name":"gcc-runtime","version":"10.2.1","arch":{"platform":"test","platform_os":"debian6","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====","dependencies":[{"name":"gcc","hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"izgzpzeljwairalfjm3k6fntbb64nt6n"},"ref_count":2,"in_buildcache":true},"jr3yipyxyjulcdvckwwwjrrumis7glpa":{"spec":{"name":"libelf","version":"0.8.13","arch":{"platform":"test","platform_os":"debian6","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====","dependencies":[{"name":"compiler-wrapper","hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"izgzpzeljwairalfjm3k6fntbb64nt6n","parameters":{"deptypes":["link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa"},"ref_count":1,"in_buildcache":true},"u5uz3dcch5if4eve4sef67o2rf2lbfgh":{"spec":{"name":"libdwarf","version":"20130729","arch":{"platform":"test","platform_os":"debian6","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====","dependencies":[{"name":"compiler-wrapper","hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"izgzpzeljwairalfjm3k6fntbb64nt6n","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"libelf","hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"u5uz3dcch5if4eve4sef67o2rf2lbfgh"},"ref_count":0,"in_buildcache":true}}}} ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/index.json.hash ================================================ 81a5add9d75b27fc4d16a4f72685b54903973366531b98c65e8cf5376758a817 ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-core2-gcc-10.2.1-libdwarf-20130729-sk2gqqz4n5njmvktycnd25wq25jxiqkr.spec.json.sig ================================================ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 { "spec":{ "_meta":{ "version":4 }, "nodes":[ { "name":"libdwarf", "version":"20130729", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"core2", "vendor":"GenuineIntel", "features":[ "mmx", "sse", "sse2", "ssse3" ], "generation":0, "parents":[ "nocona" ], "cpupart":"" } }, "compiler":{ "name":"gcc", "version":"10.2.1" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====", "dependencies":[ { "name":"libelf", "hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez", "parameters":{ "deptypes":[ "build", "link" ], "virtuals":[] } } ], "hash":"sk2gqqz4n5njmvktycnd25wq25jxiqkr" }, { "name":"libelf", "version":"0.8.13", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"core2", "vendor":"GenuineIntel", "features":[ "mmx", "sse", "sse2", "ssse3" ], "generation":0, "parents":[ "nocona" ], "cpupart":"" } }, "compiler":{ "name":"gcc", "version":"10.2.1" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====", "hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"811f500a89ae7d2f61e2c0ef6f56e352dfbac245ae88275809088a1481489d5b" } } -----BEGIN PGP SIGNATURE----- iQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK 0373kg/+Iy7pfWoAa465XtWUyf87KcjmJ1hE4OmfMc9sA7kdKNYPfmttxfp8jCU5 gRc8RnQ5K+h4GWGl9nd6bFOT3oZSBH9WnH33gcnStHubwvHzhY05ZmlKjXKKTJmG rcQ8+vVv/e8KfMatydPuXQmAzbJ0pr2bGnicT8fs/W35hgcyygDZvDqJo3m+q4H7 uu4C3LnaixAf7kCZefdxReYvFBNz9Qovws3+LqVFPxWgqo4zYt1PcI24UhCpL2YJ 6XJySW7e0rR64bwCZR/owy504aUC64wr8kM19MMJAoB0R4zciJ0YyY8xLfRMI3Tr JTPetuTN7ncKJ2kZJ5L+KbeYnr4+CA5ZYmjyAM5NSJ3fTXuEu477H+1XovcJtP1s IZS10UWX452QEBXE5nWAludmiw4BenyR2Lccg2QfER8jbiZf3U3do43aGoI5U8rg qf1kQ/dMcIX6oSrbxMKymdsuf6e8UCSys3KNwb44UdSBiihgYFtiMfGtQ6Ixsvky TB+EwweUY6LtBuep1fh+M1tHgo9qCxUH79duor0JRDgQ/VLeO6e1RCptc7EHnQZQ mZK7YjVtHYWzyOZ4KsWuLYBSAMvKDhrTxI8cxp816NNGUfj1jmBQR/5vn6d7nMwX PmWrQV9O2e899Mv30VVR9XDf6tJoT+BPvS4Kc5hw/LxjaBbAxXo= =Zprh -----END PGP SIGNATURE----- ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-core2-gcc-10.2.1-libelf-0.8.13-rqh2vuf6fqwkmipzgi2wjx352mq7y7ez.spec.json.sig ================================================ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 { "spec":{ "_meta":{ "version":4 }, "nodes":[ { "name":"libelf", "version":"0.8.13", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"core2", "vendor":"GenuineIntel", "features":[ "mmx", "sse", "sse2", "ssse3" ], "generation":0, "parents":[ "nocona" ], "cpupart":"" } }, "compiler":{ "name":"gcc", "version":"10.2.1" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====", "hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"48c8aa769a62535f9d9f613722e3d3f5a48b91fde3c99a644b22f277a4502d75" } } -----BEGIN PGP SIGNATURE----- iQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK 036q+Q//XkOoRoZ5g3uyQTXTV3w6YCUezkvGv+WRV4oZfj0CElKf4KoW5bhdtWEM EBRC4UuFturk7m1KrgztKsEFq7vx0TxvbWjj5R64swrwczKkD7i5xjMhWZn0nrpk kzeKJw8zCr+o+qAHUoqTZAAf1GaMOwCKN8rZ5zrulbkrugPY783UKJtfyJc8+BPT dixOerTC5cvzFNHENIKXMTh7Pbww2jdnFCn2eGA1kmyJGkRFhKKQ9kerlUcfOdQB w51jMfgZRoG/hvSnrlrYHJQx1hpUiBV5eyEcLHnlbiJj7cNTvqcrt2nHpy/1Co1H 5uiQou5I8ETTvTQrtWNgCtUBg1ZqaKZw8tanSY4cHXoeP5s4uQl1yTEGCEDDFB9y E/yO9xTfak3Avv1h6FZ2Lw+ipVLnlurtpo/jGmr4UgoKV4MZ1hFSseIEWQVyXJ+4 kP2gZ/LZF84eYqRKANYGWbKp/fKJQgnn/nhKgySfx4dKHJFRpVNgiGzNYyYwOtOC BWrLIqgvETl+MZZPMPwt8T7ZCYIR5fzQ1itGM3ffmsh9DIvRyu32DRWBcqgiDE7o 866L+C6Kk2RyCS8dB3Ep4LW7kO42k0Rq6cvkO8wV+CjbTF/i8OQEclDMxr+ruoN0 IKEp2thRZA39iDHGAIPyCsryrZhpEJ+uOfMykWKc0j957CpXLck= =Qmpp -----END PGP SIGNATURE----- ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-m1-gcc-10.2.1-libdwarf-20130729-u5uz3dcch5if4eve4sef67o2rf2lbfgh.spec.json.sig ================================================ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 { "spec":{ "_meta":{ "version":5 }, "nodes":[ { "name":"libdwarf", "version":"20130729", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====", "dependencies":[ { "name":"compiler-wrapper", "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } }, { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[ "c", "cxx" ] } }, { "name":"gcc-runtime", "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n", "parameters":{ "deptypes":[ "link" ], "virtuals":[] } }, { "name":"libelf", "hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa", "parameters":{ "deptypes":[ "build", "link" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"u5uz3dcch5if4eve4sef67o2rf2lbfgh" }, { "name":"compiler-wrapper", "version":"1.0", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====", "annotations":{ "original_specfile_version":5 }, "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc" }, { "name":"gcc", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":"aarch64" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "languages":[ "c", "c++", "fortran" ], "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "external":{ "path":"/path", "module":null, "extra_attributes":{ "compilers":{ "c":"/path/bin/gcc-10", "cxx":"/path/bin/g++-10", "fortran":"/path/bin/gfortran-10" } } }, "package_hash":"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====", "annotations":{ "original_specfile_version":5 }, "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq" }, { "name":"gcc-runtime", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====", "dependencies":[ { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n" }, { "name":"libelf", "version":"0.8.13", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====", "dependencies":[ { "name":"compiler-wrapper", "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } }, { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[ "c" ] } }, { "name":"gcc-runtime", "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n", "parameters":{ "deptypes":[ "link" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"0898457b4cc4b18d71059ea254667fb6690f5933c82e1627f9fed3606488dbca" } } -----BEGIN PGP SIGNATURE----- iQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK 035oXBAAj12qztxIYhTbNRq0jpk7/ZfCLRDz/XyqzKx2JbS+p3DfZruVZV/OMZ9I Hlj9GYxQEwLGVsEMXoZDWtUytcte3m6sCG6H8fZGKw6IWQ6eiDR5i7TJWSuPvWGU NMH57kvSJlICLP9x6NWjQeyLAI4I3kASk+Ei/WHAGqIiP9CR1O5IXheMusPDAEjd 2IR7khPvJTwpD6rzMHPou9BWk0Jqefb9qHhaJnc0Ga1D5HCS2VdGltViQ0XCX7/7 nkWV9ad9NOvbO9oQYIW1jRY8D9Iw9vp2d77Dv5eUzI8or5c5x0VFAHpQL0FUxIR9 LpHWUohDiAp3M4kmZqLBPl1Qf2jAXFXiSmcrLhKD5eWhdiwn3Bkhs2JiSiJpHt6K Sa970evIFcGw6sUBGznsuFxmXFfp84LYvzIVjacuzkm9WDvbEE/5pa2b5Pxr7BmH d2xDmAYmZVOso6INf3ZEXOyMBPWyGyq9Hy/8Nyg/+7w2d4ICEG/z/N13VsTqRoXc rb8I0xDE9iCXCelQJYlJcJ2UMZk9E76zd3Bd2WcgCTrrnHsg0fBjmNeyPJcBN8hA am5Lq/Cxqm2Jo2qnjoVmCt8/TBkvT2w8PTpR5uTEbLDl2ghyzxyBkX7a8ldKx55f aL8/OxN+u0pyISTDs5AoZ1YbhgDMiBiZV8ZDIB8PzU8pE78De3Q= =YbRr -----END PGP SIGNATURE----- ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-m1-gcc-10.2.1-libelf-0.8.13-jr3yipyxyjulcdvckwwwjrrumis7glpa.spec.json.sig ================================================ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 { "spec":{ "_meta":{ "version":5 }, "nodes":[ { "name":"libelf", "version":"0.8.13", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====", "dependencies":[ { "name":"compiler-wrapper", "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } }, { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[ "c" ] } }, { "name":"gcc-runtime", "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n", "parameters":{ "deptypes":[ "link" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa" }, { "name":"compiler-wrapper", "version":"1.0", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====", "annotations":{ "original_specfile_version":5 }, "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc" }, { "name":"gcc", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":"aarch64" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "languages":[ "c", "c++", "fortran" ], "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "external":{ "path":"/path", "module":null, "extra_attributes":{ "compilers":{ "c":"/path/bin/gcc-10", "cxx":"/path/bin/g++-10", "fortran":"/path/bin/gfortran-10" } } }, "package_hash":"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====", "annotations":{ "original_specfile_version":5 }, "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq" }, { "name":"gcc-runtime", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====", "dependencies":[ { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"c068bcd1a27a3081c07ba775d83e90228e340bb6a7f0d55deb18a462760c4bcf" } } -----BEGIN PGP SIGNATURE----- iQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK 0356nQ//aVMUZU8Ly8/b1H4nvKM8Vyd275aFK64rvO89mERDNiYIOKk1pmYSMldU +ltx2iIfVTUCEWYYJb/4UXWmw6SLAXIZ5mtrkALDAeDSih4wqIdevM3yii7pn8Oh /OEyDX8N5k05pnxFLYqR/2gA6vvdxHFd9/h4/zy2Z5w6m1hXb5jtS2ECyYN72nYN 8QnnkXWZYturOhb4GawWY1l/rHIBqAseCQXSGR6UyrHTEGLUgT0+VQZwgxLNM4uG xj4xCDTgKiOesa5+3WE8Ug2wDIm48Prvg4qFmNrofguRNiIsNrl5k7wRiJWdfkjc gzs9URYddoCTRR2wpN0CaAQ268UlwZUCjPSrxgCNeqRi4Ob9Q4n37TKXNcVw46Ud MXRezAf+wyPGkq4vudh7cu11mHUcTeev82GM5bYQa6dSna3WvPpie/rx0TZYRkKE hesDW/41ZtFDANfXa7r011ngS5zZwak3zUaoqOdLNhN/xL4TFsZ19uSUdSZHAgSk 9Sr3xodwV2D5H6gDuOtAo1vRod1Fx+yoi3BubX0sI5QuFgvtJrHVZmVj2bnGMBKI gR17q1ZHOmp3yPhVE9ZsiLKn9r3yIsfVhoTB6mXOnvq2q1fBxyrEpIGzIUmWfuTm vLn4nXt7PD78msiG/GZt6fShYBAwVfuvG+M1AQrsyGGoW2Bty7M= =hLvB -----END PGP SIGNATURE----- ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-m1-none-none-compiler-wrapper-1.0-qeehcxyvluwnihsc2qxstmpomtxo3lrc.spec.json.sig ================================================ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 { "spec":{ "_meta":{ "version":5 }, "nodes":[ { "name":"compiler-wrapper", "version":"1.0", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====", "annotations":{ "original_specfile_version":5 }, "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"2c1c5576e30b7063aa02a22111eb24b3f2a93c35ac0f64b4e491c7078706c0ea" } } -----BEGIN PGP SIGNATURE----- iQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK 037BIQ//U30gx1qTt5cQs+I6fwqQSase8DT7Hi8VdYxMuBTVbEpnPScNpcH03ITC KWVbXvEAPBdoWEfAHpuOJr2pm013dYXaWp1k0G6pLSvnR17LEDTJs0ixAurH4vDr 4VXPerPR57sMi0WYomi1+dJhvA3S85+m6KBPLhXgi9Y28leDrFpjBwxVoIN7yUP2 tenMI9jAoGh/hts1pIPbALmKbeGUKC2MPu9MF0CtkbbE1VOkeJ6jkZLGki7AAYZ0 TSWAeWDk6EG90TZ6ls2anUPI1mNc7JdPqq8L0+jWAwLJi3i/JiDAGUM99hpu9cCF NvZn+eQFOKrE0WG1KsF4vQilOAuE3P+QLomcfZdf2UNi73XPWIF5j46r50oPmXZE +mVUyw7CUbHMZlXvWml0pdugEER1Kyc2nLZdLZYAT92AsPbAcDBQKsm1xf66lOB+ FPPLc97oybcFFldrjmUJAASJBeAihZG1aDm6dYBxtynMzzRGdq2+R1chHMOQ5Wej 8ZvyRv+TOPUTtRkAxrUpq6wA+BUoq+OBDltOs9mXUIcV3rpOq5nTjKZ5FLMtGaDw No0E5gwceDDLeshT9nAHaqcmSY1LK+/5+aDxOFRm4yRTI+GLJzg8FZCJbJRLstrD Ts4zKdcb0kukKdE9raqWw7xuhbjz2ORiEicZzckzvB1Lx38bG2s= =T5l5 -----END PGP SIGNATURE----- ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-m1-none-none-gcc-runtime-10.2.1-izgzpzeljwairalfjm3k6fntbb64nt6n.spec.json.sig ================================================ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 { "spec":{ "_meta":{ "version":5 }, "nodes":[ { "name":"gcc-runtime", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====", "dependencies":[ { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n" }, { "name":"gcc", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":"aarch64" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "languages":[ "c", "c++", "fortran" ], "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "external":{ "path":"/path", "module":null, "extra_attributes":{ "compilers":{ "c":"/path/bin/gcc-10", "cxx":"/path/bin/g++-10", "fortran":"/path/bin/gfortran-10" } } }, "package_hash":"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====", "annotations":{ "original_specfile_version":5 }, "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"f33e7a6798a5fb2db6e538d3a530cc79b298e36d56a1df385d93889a9ba431d0" } } -----BEGIN PGP SIGNATURE----- iQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK 037zHBAAsqy4wItctMqauuna+JjxT1HM7YJElXzqjOWmxyuAzUzjXlhR2DBd/2TI ZEN2q3Z3XY9sCjhZ/4c9wDfMNYLUBLMHuenyV3fOqsfIVL8NprrkGc5mOiJ8HbRk u00qXWogsYSEmbGrlfDKf4HmZtgPNs82+Li1MD5udDUzyApuVbObJumSRh6/1QHm BcQZgMlSCd8xsTxJudXKAnfpemqE41LF0znuU0x5Hj/hU1A3CELynQrLEYnJpzpR ja2l341cBQKNy86kX1/eHQtBJverjFoD3Nx4per8/qUc+xTH0ejMuseyd9P3RLnd WShY8Uk72f1OLGzq5RvayP1M/dBWedajKz5gYOD19pCuFEdQm1LkZhxRWJ35PYMV CqzY/uJgs33zyYkNJKO8CKG5j7Y8zOuZ3YFN8DKmoWa+lC4gFIsXm42BttqiQ5+x Q65YkX/DdPYO6dcUety1j3NuNr70W6PsLyqKBny1WOzKCx25nmzftS0OA76F6UZA hDneqltGrYEQTowU5I7V14f3SMeO8xje3BcqhOAn956/JJObd5VbwqcHwcslwEJA tL3361qbpkc7xURnhciV1eL3RYR9Q4xDnvI1i/k8J8E8W373TviK3r2MG/oKZ6N9 n+ehBZhSIT+QUgqylATekoMQfohNVbDQEsQhj96Ky1CC2Iqo1/c= =UIyv -----END PGP SIGNATURE----- ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/index.json ================================================ {"database":{"version":"8","installs":{"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez":{"spec":{"name":"libelf","version":"0.8.13","arch":{"platform":"test","platform_os":"debian6","target":{"name":"core2","vendor":"GenuineIntel","features":["mmx","sse","sse2","ssse3"],"generation":0,"parents":["nocona"],"cpupart":""}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====","annotations":{"original_specfile_version":4,"compiler":"gcc@=10.2.1"},"hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez"},"ref_count":1,"in_buildcache":true},"qeehcxyvluwnihsc2qxstmpomtxo3lrc":{"spec":{"name":"compiler-wrapper","version":"1.0","arch":{"platform":"test","platform_os":"debian6","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====","annotations":{"original_specfile_version":5},"hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc"},"ref_count":2,"in_buildcache":true},"vd7v4ssgnoqdplgxyig3orum67n4vmhq":{"spec":{"name":"gcc","version":"10.2.1","arch":{"platform":"test","platform_os":"debian6","target":"aarch64"},"namespace":"builtin_mock","parameters":{"build_system":"generic","languages":["c","c++","fortran"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/path","module":null,"extra_attributes":{"compilers":{"c":"/path/bin/gcc-10","cxx":"/path/bin/g++-10","fortran":"/path/bin/gfortran-10"}}},"package_hash":"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====","annotations":{"original_specfile_version":5},"hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq"},"ref_count":3,"in_buildcache":false},"izgzpzeljwairalfjm3k6fntbb64nt6n":{"spec":{"name":"gcc-runtime","version":"10.2.1","arch":{"platform":"test","platform_os":"debian6","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====","dependencies":[{"name":"gcc","hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq","parameters":{"deptypes":["build"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"izgzpzeljwairalfjm3k6fntbb64nt6n"},"ref_count":2,"in_buildcache":true},"jr3yipyxyjulcdvckwwwjrrumis7glpa":{"spec":{"name":"libelf","version":"0.8.13","arch":{"platform":"test","platform_os":"debian6","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====","dependencies":[{"name":"compiler-wrapper","hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq","parameters":{"deptypes":["build"],"virtuals":["c"]}},{"name":"gcc-runtime","hash":"izgzpzeljwairalfjm3k6fntbb64nt6n","parameters":{"deptypes":["link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa"},"ref_count":1,"in_buildcache":true},"u5uz3dcch5if4eve4sef67o2rf2lbfgh":{"spec":{"name":"libdwarf","version":"20130729","arch":{"platform":"test","platform_os":"debian6","target":{"name":"m1","vendor":"Apple","features":["aes","asimd","asimddp","asimdfhm","asimdhp","asimdrdm","atomics","cpuid","crc32","dcpodp","dcpop","dit","evtstrm","fcma","flagm","flagm2","fp","fphp","frint","ilrcpc","jscvt","lrcpc","paca","pacg","pmull","sb","sha1","sha2","sha3","sha512","ssbs","uscat"],"generation":0,"parents":["armv8.4a"],"cpupart":"0x022"}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====","dependencies":[{"name":"compiler-wrapper","hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gcc","hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq","parameters":{"deptypes":["build"],"virtuals":["c","cxx"]}},{"name":"gcc-runtime","hash":"izgzpzeljwairalfjm3k6fntbb64nt6n","parameters":{"deptypes":["link"],"virtuals":[]}},{"name":"libelf","hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":5},"hash":"u5uz3dcch5if4eve4sef67o2rf2lbfgh"},"ref_count":0,"in_buildcache":true},"sk2gqqz4n5njmvktycnd25wq25jxiqkr":{"spec":{"name":"libdwarf","version":"20130729","arch":{"platform":"test","platform_os":"debian6","target":{"name":"core2","vendor":"GenuineIntel","features":["mmx","sse","sse2","ssse3"],"generation":0,"parents":["nocona"],"cpupart":""}},"namespace":"builtin_mock","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====","dependencies":[{"name":"libelf","hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"annotations":{"original_specfile_version":4,"compiler":"gcc@=10.2.1"},"hash":"sk2gqqz4n5njmvktycnd25wq25jxiqkr"},"ref_count":0,"in_buildcache":true}}}} ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/index.json.hash ================================================ fc129b8fab649ab4c5623c874c73bd998a76fd30d2218b9d99340d045c1ec759 ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-core2-gcc-10.2.1-libdwarf-20130729-sk2gqqz4n5njmvktycnd25wq25jxiqkr.spec.json ================================================ { "spec":{ "_meta":{ "version":4 }, "nodes":[ { "name":"libdwarf", "version":"20130729", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"core2", "vendor":"GenuineIntel", "features":[ "mmx", "sse", "sse2", "ssse3" ], "generation":0, "parents":[ "nocona" ], "cpupart":"" } }, "compiler":{ "name":"gcc", "version":"10.2.1" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====", "dependencies":[ { "name":"libelf", "hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez", "parameters":{ "deptypes":[ "build", "link" ], "virtuals":[] } } ], "hash":"sk2gqqz4n5njmvktycnd25wq25jxiqkr" }, { "name":"libelf", "version":"0.8.13", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"core2", "vendor":"GenuineIntel", "features":[ "mmx", "sse", "sse2", "ssse3" ], "generation":0, "parents":[ "nocona" ], "cpupart":"" } }, "compiler":{ "name":"gcc", "version":"10.2.1" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====", "hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"f31bce1bfdcaa9b11cd02b869dd07a843db9819737399b99ac614ab3552bd4b6" } } ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-core2-gcc-10.2.1-libelf-0.8.13-rqh2vuf6fqwkmipzgi2wjx352mq7y7ez.spec.json ================================================ { "spec":{ "_meta":{ "version":4 }, "nodes":[ { "name":"libelf", "version":"0.8.13", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"core2", "vendor":"GenuineIntel", "features":[ "mmx", "sse", "sse2", "ssse3" ], "generation":0, "parents":[ "nocona" ], "cpupart":"" } }, "compiler":{ "name":"gcc", "version":"10.2.1" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====", "hash":"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"523ebc3691b0687cf93fcd477002972ea7d6da5d3e8d46636b30d4f1052fcbf2" } } ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-m1-gcc-10.2.1-libdwarf-20130729-u5uz3dcch5if4eve4sef67o2rf2lbfgh.spec.json ================================================ { "spec":{ "_meta":{ "version":5 }, "nodes":[ { "name":"libdwarf", "version":"20130729", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====", "dependencies":[ { "name":"compiler-wrapper", "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } }, { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[ "c", "cxx" ] } }, { "name":"gcc-runtime", "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n", "parameters":{ "deptypes":[ "link" ], "virtuals":[] } }, { "name":"libelf", "hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa", "parameters":{ "deptypes":[ "build", "link" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"u5uz3dcch5if4eve4sef67o2rf2lbfgh" }, { "name":"compiler-wrapper", "version":"1.0", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====", "annotations":{ "original_specfile_version":5 }, "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc" }, { "name":"gcc", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":"aarch64" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "languages":[ "c", "c++", "fortran" ], "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "external":{ "path":"/path", "module":null, "extra_attributes":{ "compilers":{ "c":"/path/bin/gcc-10", "cxx":"/path/bin/g++-10", "fortran":"/path/bin/gfortran-10" } } }, "package_hash":"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====", "annotations":{ "original_specfile_version":5 }, "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq" }, { "name":"gcc-runtime", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====", "dependencies":[ { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n" }, { "name":"libelf", "version":"0.8.13", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====", "dependencies":[ { "name":"compiler-wrapper", "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } }, { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[ "c" ] } }, { "name":"gcc-runtime", "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n", "parameters":{ "deptypes":[ "link" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"2a6f045996e4998ec37679e1aa8a245795fbbffaf9844692ba2de6eeffcbc722" } } ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-m1-gcc-10.2.1-libelf-0.8.13-jr3yipyxyjulcdvckwwwjrrumis7glpa.spec.json ================================================ { "spec":{ "_meta":{ "version":5 }, "nodes":[ { "name":"libelf", "version":"0.8.13", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====", "dependencies":[ { "name":"compiler-wrapper", "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } }, { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[ "c" ] } }, { "name":"gcc-runtime", "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n", "parameters":{ "deptypes":[ "link" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"jr3yipyxyjulcdvckwwwjrrumis7glpa" }, { "name":"compiler-wrapper", "version":"1.0", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====", "annotations":{ "original_specfile_version":5 }, "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc" }, { "name":"gcc", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":"aarch64" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "languages":[ "c", "c++", "fortran" ], "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "external":{ "path":"/path", "module":null, "extra_attributes":{ "compilers":{ "c":"/path/bin/gcc-10", "cxx":"/path/bin/g++-10", "fortran":"/path/bin/gfortran-10" } } }, "package_hash":"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====", "annotations":{ "original_specfile_version":5 }, "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq" }, { "name":"gcc-runtime", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====", "dependencies":[ { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"59141e49bd05abe40639360cd9422020513781270a3461083fee0eba2af62ca0" } } ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-m1-none-none-compiler-wrapper-1.0-qeehcxyvluwnihsc2qxstmpomtxo3lrc.spec.json ================================================ { "spec":{ "_meta":{ "version":5 }, "nodes":[ { "name":"compiler-wrapper", "version":"1.0", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====", "annotations":{ "original_specfile_version":5 }, "hash":"qeehcxyvluwnihsc2qxstmpomtxo3lrc" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"f27ddff0ef4268acbe816c51f6f1fc907dc1010d31f2d6556b699c80f026c47d" } } ================================================ FILE: lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-m1-none-none-gcc-runtime-10.2.1-izgzpzeljwairalfjm3k6fntbb64nt6n.spec.json ================================================ { "spec":{ "_meta":{ "version":5 }, "nodes":[ { "name":"gcc-runtime", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":{ "name":"m1", "vendor":"Apple", "features":[ "aes", "asimd", "asimddp", "asimdfhm", "asimdhp", "asimdrdm", "atomics", "cpuid", "crc32", "dcpodp", "dcpop", "dit", "evtstrm", "fcma", "flagm", "flagm2", "fp", "fphp", "frint", "ilrcpc", "jscvt", "lrcpc", "paca", "pacg", "pmull", "sb", "sha1", "sha2", "sha3", "sha512", "ssbs", "uscat" ], "generation":0, "parents":[ "armv8.4a" ], "cpupart":"0x022" } }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "package_hash":"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====", "dependencies":[ { "name":"gcc", "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq", "parameters":{ "deptypes":[ "build" ], "virtuals":[] } } ], "annotations":{ "original_specfile_version":5 }, "hash":"izgzpzeljwairalfjm3k6fntbb64nt6n" }, { "name":"gcc", "version":"10.2.1", "arch":{ "platform":"test", "platform_os":"debian6", "target":"aarch64" }, "namespace":"builtin_mock", "parameters":{ "build_system":"generic", "languages":[ "c", "c++", "fortran" ], "cflags":[], "cppflags":[], "cxxflags":[], "fflags":[], "ldflags":[], "ldlibs":[] }, "external":{ "path":"/path", "module":null, "extra_attributes":{ "compilers":{ "c":"/path/bin/gcc-10", "cxx":"/path/bin/g++-10", "fortran":"/path/bin/gfortran-10" } } }, "package_hash":"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====", "annotations":{ "original_specfile_version":5 }, "hash":"vd7v4ssgnoqdplgxyig3orum67n4vmhq" } ] }, "buildcache_layout_version":2, "binary_cache_checksum":{ "hash_algorithm":"sha256", "hash":"348f23717c5641fa6bbb90862e62bf632367511c53e9c6450584f0d000841320" } } ================================================ FILE: lib/spack/spack/test/data/modules/lmod/alter_environment.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@3.3' hierarchy: - mpi all: autoload: none filter: exclude_env_vars: - CMAKE_PREFIX_PATH environment: set: '{name}_ROOT': '{prefix}' 'platform=test target=x86_64': environment: set: FOO: 'foo' unset: - BAR 'platform=test target=core2': load: - 'foo/bar' ================================================ FILE: lib/spack/spack/test/data/modules/lmod/autoload_all.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@3.3' hierarchy: - mpi verbose: true all: autoload: all ================================================ FILE: lib/spack/spack/test/data/modules/lmod/autoload_direct.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@3.3' hierarchy: - mpi all: autoload: direct ================================================ FILE: lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml ================================================ enable: - lmod lmod: hash_length: 0 core_compilers: - 'clang@15.0.0' core_specs: - 'mpich@3.0.1' hierarchy: - lapack - blas - mpi - python filter_hierarchy_specs: 'mpileaks@:2.1': [mpi] verbose: false all: autoload: all ================================================ FILE: lib/spack/spack/test/data/modules/lmod/conflicts.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@3.3' all: autoload: none conflict: - '{name}' - 'intel/14.0.1' ================================================ FILE: lib/spack/spack/test/data/modules/lmod/core_compilers.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@15.0.0' ================================================ FILE: lib/spack/spack/test/data/modules/lmod/core_compilers_at_equal.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@=15.0.0' ================================================ FILE: lib/spack/spack/test/data/modules/lmod/core_compilers_empty.yaml ================================================ enable: - lmod lmod: all: autoload: none core_compilers: [] hierarchy: - mpi ================================================ FILE: lib/spack/spack/test/data/modules/lmod/exclude.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@3.3' hierarchy: - mpi exclude: - callpath all: autoload: direct ================================================ FILE: lib/spack/spack/test/data/modules/lmod/hide_implicits.yaml ================================================ enable: - lmod lmod: hide_implicits: true hash_length: 0 core_compilers: - 'clang@3.3' hierarchy: - mpi all: autoload: direct ================================================ FILE: lib/spack/spack/test/data/modules/lmod/missing_core_compilers.yaml ================================================ enable: - lmod lmod: all: autoload: none hierarchy: - mpi ================================================ FILE: lib/spack/spack/test/data/modules/lmod/module_path_separator.yaml ================================================ enable: - lmod lmod: all: autoload: none core_compilers: - 'clang@3.3' ================================================ FILE: lib/spack/spack/test/data/modules/lmod/no_arch.yaml ================================================ enable: - lmod arch_folder: false lmod: all: autoload: none core_compilers: - 'clang@3.3' ================================================ FILE: lib/spack/spack/test/data/modules/lmod/no_hash.yaml ================================================ enable: - lmod lmod: all: autoload: none hash_length: 0 core_compilers: - 'clang@3.3' hierarchy: - mpi ================================================ FILE: lib/spack/spack/test/data/modules/lmod/override_template.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@3.3' hierarchy: - mpi all: template: 'override_from_modules.txt' autoload: none ================================================ FILE: lib/spack/spack/test/data/modules/lmod/projections.yaml ================================================ enable: - lmod lmod: all: autoload: none projections: all: '{name}/v{version}' mpileaks: '{name}-mpiprojection' ================================================ FILE: lib/spack/spack/test/data/modules/lmod/with_view.yaml ================================================ enable: - lmod use_view: default lmod: all: autoload: none core_compilers: - 'clang@3.3' ================================================ FILE: lib/spack/spack/test/data/modules/lmod/wrong_conflicts.yaml ================================================ enable: - lmod lmod: core_compilers: - 'clang@3.3' projections: all: '{name}/{version}-{compiler.name}' all: autoload: none conflict: - '{name}/{compiler.name}' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/alter_environment.yaml ================================================ enable: - tcl tcl: all: autoload: none filter: exclude_env_vars: - CMAKE_PREFIX_PATH environment: set: '{name}_ROOT': '{prefix}' 'platform=test target=x86_64': environment: set: FOO: 'foo' OMPI_MCA_mpi_leave_pinned: '1' unset: - BAR 'platform=test target=core2': load: - 'foo/bar' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/autoload_all.yaml ================================================ enable: - tcl tcl: verbose: true all: autoload: all ================================================ FILE: lib/spack/spack/test/data/modules/tcl/autoload_direct.yaml ================================================ enable: - tcl tcl: all: autoload: direct ================================================ FILE: lib/spack/spack/test/data/modules/tcl/autoload_with_constraints.yaml ================================================ enable: - tcl tcl: all: autoload: none ^mpich2: autoload: direct ^python: autoload: direct ================================================ FILE: lib/spack/spack/test/data/modules/tcl/conflicts.yaml ================================================ enable: - tcl tcl: projections: all: '{name}/{version}-{compiler.name}' all: autoload: none conflict: - '{name}' - 'intel/14.0.1' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/exclude.yaml ================================================ enable: - tcl tcl: include: - zmpi exclude: - callpath - mpi all: autoload: direct ================================================ FILE: lib/spack/spack/test/data/modules/tcl/exclude_implicits.yaml ================================================ # DEPRECATED: remove this in ? # See `hide_implicits.yaml` for the new syntax enable: - tcl tcl: exclude_implicits: true hash_length: 0 all: autoload: direct ================================================ FILE: lib/spack/spack/test/data/modules/tcl/hide_implicits.yaml ================================================ enable: - tcl tcl: hide_implicits: true hash_length: 0 all: autoload: direct ================================================ FILE: lib/spack/spack/test/data/modules/tcl/invalid_naming_scheme.yaml ================================================ enable: - tcl tcl: all: autoload: none # {variants} is not allowed in the naming scheme, see #2884 projections: all: '{name}/{version}-{compiler.name}-{variants}' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml ================================================ enable: - tcl tcl: all: autoload: none filter: exclude_env_vars: - CMAKE_PREFIX_PATH environment: set: '{name}_ROOT_{prefix}': '{prefix}' 'platform=test target=x86_64': environment: set: FOO_{variants}: 'foo' unset: - BAR 'platform=test target=x86': load: - 'foo/bar' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/module_path_separator.yaml ================================================ enable: - tcl tcl: all: autoload: none ================================================ FILE: lib/spack/spack/test/data/modules/tcl/naming_scheme.yaml ================================================ enable: - tcl tcl: all: autoload: none naming_scheme: '{name}/{version}-{compiler.name}' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/no_arch.yaml ================================================ enable: - tcl arch_folder: false tcl: all: autoload: none projections: all: '' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/override_config.yaml ================================================ enable: - tcl tcl: all: autoload: none suffixes: '^mpich': mpich mpileaks: suffixes: '+static': static mpileaks+opt:: suffixes: '~debug': over '^mpich': ridden ================================================ FILE: lib/spack/spack/test/data/modules/tcl/override_template.yaml ================================================ enable: - tcl tcl: all: autoload: none template: 'override_from_modules.txt' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/prerequisites_all.yaml ================================================ enable: - tcl tcl: all: autoload: none prerequisites: all ================================================ FILE: lib/spack/spack/test/data/modules/tcl/prerequisites_direct.yaml ================================================ enable: - tcl tcl: all: autoload: none prerequisites: direct ================================================ FILE: lib/spack/spack/test/data/modules/tcl/projections.yaml ================================================ enable: - tcl tcl: all: autoload: none projections: all: '{name}/{version}-{compiler.name}' mpileaks: '{name}-mpiprojection' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/suffix-format.yaml ================================================ enable: - tcl tcl: all: autoload: none mpileaks: suffixes: mpileaks: 'debug={variants.debug.value}' '^mpi': 'mpi={^mpi.name}-v{^mpi.version}' ================================================ FILE: lib/spack/spack/test/data/modules/tcl/suffix.yaml ================================================ enable: - tcl tcl: all: autoload: none mpileaks: suffixes: '+opt': baz '+debug': foo '^mpich': foo '~debug': bar ================================================ FILE: lib/spack/spack/test/data/modules/tcl/wrong_conflicts.yaml ================================================ enable: - tcl tcl: projections: all: '{name}/{version}-{compiler.name}' all: autoload: none conflict: - '{name}/{compiler.name}' ================================================ FILE: lib/spack/spack/test/data/ninja/.gitignore ================================================ .ninja_deps .ninja_log ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/check_test/build.ninja ================================================ # Tests that Spack detects target when it is the first of two targets rule cc command = true build check test: cc ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/include/build.ninja ================================================ # Tests that Spack can handle targets in include files include include.ninja ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/include/include.ninja ================================================ rule cc command = true build check: cc ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/simple/build.ninja ================================================ # Tests that Spack can handle a simple Ninja build script rule cc command = true build check: cc ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/spaces/build.ninja ================================================ # Tests that Spack allows spaces following the target name rule cc command = true build check : cc ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/subninja/build.ninja ================================================ # Tests that Spack can handle targets in subninja files subninja subninja.ninja ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/subninja/subninja.ninja ================================================ rule cc command = true build check: cc ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/test_check/build.ninja ================================================ # Tests that Spack detects target when it is the second of two targets rule cc command = true build test check: cc ================================================ FILE: lib/spack/spack/test/data/ninja/affirmative/three_targets/build.ninja ================================================ # Tests that Spack detects a target if it is in the middle of a list rule cc command = true build foo check bar: cc ================================================ FILE: lib/spack/spack/test/data/ninja/negative/no_ninja/readme.txt ================================================ # Tests that Spack ignores directories without a Ninja build script cflags = -Wall rule cc command = gcc $cflags -c $in -o $out build check: cc foo.c ================================================ FILE: lib/spack/spack/test/data/ninja/negative/partial_match/build.ninja ================================================ # Tests that Spack ignores targets that contain a partial match cflags = -Wall rule cc command = gcc $cflags -c $in -o $out build installcheck: cc foo.c build checkinstall: cc foo.c build foo-check-bar: cc foo.c build foo_check_bar: cc foo.c build foo/check/bar: cc foo.c ================================================ FILE: lib/spack/spack/test/data/ninja/negative/rule/build.ninja ================================================ # Tests that Spack ignores rule names cflags = -Wall rule check command = gcc $cflags -c $in -o $out build foo: check foo.c ================================================ FILE: lib/spack/spack/test/data/ninja/negative/variable/build.ninja ================================================ # Tests that Spack ignores variable definitions check = -Wall rule cc command = gcc $check -c $in -o $out build foo: cc foo.c ================================================ FILE: lib/spack/spack/test/data/patch/foo.patch ================================================ --- a/foo.txt 2017-09-25 21:24:33.000000000 -0700 +++ b/foo.txt 2017-09-25 14:31:17.000000000 -0700 @@ -1,2 +1,3 @@ +zeroth line first line -second line +third line ================================================ FILE: lib/spack/spack/test/data/sourceme_first.bat ================================================ @echo off rem C:\lib\spack\spack\test\data rem rem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other rem Spack Project Developers. See the top-level COPYRIGHT file for details. rem rem SPDX-License-Identifier: (Apache-2.0 OR MIT) set NEW_VAR=new set UNSET_ME=overridden ================================================ FILE: lib/spack/spack/test/data/sourceme_first.sh ================================================ #!/usr/bin/env bash # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) export NEW_VAR='new' export UNSET_ME='overridden' ================================================ FILE: lib/spack/spack/test/data/sourceme_lmod.sh ================================================ #!/usr/bin/env bash # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) export LMOD_VARIABLE=foo export LMOD_ANOTHER_VARIABLE=bar export NEW_VAR=new ================================================ FILE: lib/spack/spack/test/data/sourceme_modules.bat ================================================ @echo off setlocal rem C:\lib\spack\spack\test\data rem rem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other rem Spack Project Developers. See the top-level COPYRIGHT file for details. rem rem SPDX-License-Identifier: (Apache-2.0 OR MIT) :_module_raw val_1 exit /b 0 :module exit /b 0 :ml exit /b 0 set "_module_raw=call :_module_raw" set "mod=call :mod" set "ml=call :ml" set MODULES_AUTO_HANDLING=1 set __MODULES_LMCONFLICT=bar^&foo set NEW_VAR=new ================================================ FILE: lib/spack/spack/test/data/sourceme_modules.sh ================================================ #!/usr/bin/env bash # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) _module_raw() { return 1; }; module() { return 1; }; ml() { return 1; }; export -f _module_raw; export -f module; export -f ml; export MODULES_AUTO_HANDLING=1 export __MODULES_LMCONFLICT=bar&foo export NEW_VAR=new ================================================ FILE: lib/spack/spack/test/data/sourceme_parameters.bat ================================================ @echo off rem "C:\lib\spack\spack\test\data rem rem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other rem Spack Project Developers. See the top-level COPYRIGHT file for details. rem rem SPDX-License-Identifier: (Apache-2.0 OR MIT) if "%1" == "intel64" ( set FOO=intel64 ) else ( set FOO=default ) ================================================ FILE: lib/spack/spack/test/data/sourceme_parameters.sh ================================================ #!/usr/bin/env bash # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) if [[ "$1" == "intel64" ]] ; then export FOO='intel64' else export FOO='default' fi ================================================ FILE: lib/spack/spack/test/data/sourceme_second.bat ================================================ @echo off rem "C:\lib\spack\spack\test\data rem rem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other rem Spack Project Developers. See the top-level COPYRIGHT file for details. rem rem SPDX-License-Identifier: (Apache-2.0 OR MIT) set PATH_LIST=C:\path\first;C:\path\second;C:\path\fourth set EMPTY_PATH_LIST= ================================================ FILE: lib/spack/spack/test/data/sourceme_second.sh ================================================ #!/usr/bin/env bash # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) export PATH_LIST='/path/first:/path/second:/path/fourth' unset EMPTY_PATH_LIST ================================================ FILE: lib/spack/spack/test/data/sourceme_unicode.bat ================================================ @echo off rem "C:\lib\spack\spack\test\data rem rem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other rem Spack Project Developers. See the top-level COPYRIGHT file for details. rem rem SPDX-License-Identifier: (Apache-2.0 OR MIT) rem Set an environment variable with some unicode in it to ensure that rem Spack can decode it. rem rem This has caused squashed commits on develop to break, as some rem committers use unicode in their messages, and Travis sets the rem current commit message in an environment variable. chcp 65001 > nul set UNICODE_VAR=don\xe2\x80\x99t chcp 437 > nul ================================================ FILE: lib/spack/spack/test/data/sourceme_unicode.sh ================================================ #!/usr/bin/env bash # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # Set an environment variable with some unicode in it to ensure that # Spack can decode it. # # This has caused squashed commits on develop to break, as some # committers use unicode in their messages, and Travis sets the # current commit message in an environment variable. export UNICODE_VAR='don\xe2\x80\x99t' ================================================ FILE: lib/spack/spack/test/data/sourceme_unset.bat ================================================ @echo off rem C:\lib\spack\spack\test\data rem rem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other rem Spack Project Developers. See the top-level COPYRIGHT file for details. rem rem SPDX-License-Identifier: (Apache-2.0 OR MIT) set UNSET_ME= ================================================ FILE: lib/spack/spack/test/data/sourceme_unset.sh ================================================ #!/usr/bin/env bash # # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) unset UNSET_ME ================================================ FILE: lib/spack/spack/test/data/style/broken.dummy ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import sys import os def this_is_a_function(): """This is a docstring.""" def this_should_be_offset(): sys.stdout.write(os.name) ================================================ FILE: lib/spack/spack/test/data/style/fixed.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import sys def this_is_a_function(): """This is a docstring.""" def this_should_be_offset(): sys.stdout.write(os.name) ================================================ FILE: lib/spack/spack/test/data/templates/a.txt ================================================ Hello {{ word }}! ================================================ FILE: lib/spack/spack/test/data/templates/extension.tcl ================================================ {% extends "modules/modulefile.tcl" %} {% block footer %} puts stderr "{{ sentence }}" {% endblock %} ================================================ FILE: lib/spack/spack/test/data/templates/override.txt ================================================ Override successful! ================================================ FILE: lib/spack/spack/test/data/templates_again/b.txt ================================================ Howdy {{ word }}! ================================================ FILE: lib/spack/spack/test/data/templates_again/override_from_modules.txt ================================================ Override even better! ================================================ FILE: lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-test-out.txt ================================================ ==> Testing package printing-package-1.0-hzgcoow ==> [2022-12-06-20:21:46.550943] test: test_print: Test python print example. ==> [2022-12-06-20:21:46.553219] '/usr/tce/bin/python' '-c' 'print("Running test_print")' Running test_print ==> [2022-12-06-20:21:46.721077] '/usr/tce/bin/python' '-c' 'print("Running test_print")' PASSED: test_print ==> [2022-12-06-20:21:46.822608] Completed testing ================================================ FILE: lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-tested.txt ================================================ ================================================ FILE: lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/results.txt ================================================ printing-package-1.0-hzgcoow PASSED ================================================ FILE: lib/spack/spack/test/data/unparse/README.md ================================================ # Test data for unparser These are test packages for testing Spack's unparser. They are used to ensure that the canonical unparser used for Spack's package hash remains consistent across Python versions. All of these were copied from mainline Spack packages, and they have been renamed with `.txt` suffixes so that they're not considered proper source files by the various checkers used in Spack CI. These packages were chosen for various reasons, but mainly because: 1. They're some of the more complex packages in Spack, and they exercise more unparser features than other packages. 2. Each of these packages has some interesting feature that was hard to unparse consistently across Python versions. See docstrings in packages for details. ================================================ FILE: lib/spack/spack/test/data/unparse/amdfftw.txt ================================================ # -*- python -*- # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This is an unparser test package. ``amdfftw`` was chosen for its complexity and because it uses a negative array index that was not being unparsed consistently from Python 2 to 3. """ import os from spack import * from spack.pkg.builtin.fftw import FftwBase class Amdfftw(FftwBase): """FFTW (AMD Optimized version) is a comprehensive collection of fast C routines for computing the Discrete Fourier Transform (DFT) and various special cases thereof. It is an open-source implementation of the Fast Fourier transform algorithm. It can compute transforms of real and complex-values arrays of arbitrary size and dimension. AMD Optimized FFTW is the optimized FFTW implementation targeted for AMD CPUs. For single precision build, please use precision value as float. Example : spack install amdfftw precision=float """ _name = "amdfftw" homepage = "https://developer.amd.com/amd-aocl/fftw/" url = "https://github.com/amd/amd-fftw/archive/3.0.tar.gz" git = "https://github.com/amd/amd-fftw.git" maintainers("amd-toolchain-support") version("3.1", sha256="3e777f3acef13fa1910db097e818b1d0d03a6a36ef41186247c6ab1ab0afc132") version("3.0.1", sha256="87030c6bbb9c710f0a64f4f306ba6aa91dc4b182bb804c9022b35aef274d1a4c") version("3.0", sha256="a69deaf45478a59a69f77c4f7e9872967f1cfe996592dd12beb6318f18ea0bcd") version("2.2", sha256="de9d777236fb290c335860b458131678f75aa0799c641490c644c843f0e246f8") variant("shared", default=True, description="Builds a shared version of the library") variant("openmp", default=True, description="Enable OpenMP support") variant("threads", default=False, description="Enable SMP threads support") variant("debug", default=False, description="Builds a debug version of the library") variant( "amd-fast-planner", default=False, description="Option to reduce the planning time without much" "tradeoff in the performance. It is supported for" "Float and double precisions only.", ) variant("amd-top-n-planner", default=False, description="Build with amd-top-n-planner support") variant( "amd-mpi-vader-limit", default=False, description="Build with amd-mpi-vader-limit support" ) variant("static", default=False, description="Build with static suppport") variant("amd-trans", default=False, description="Build with amd-trans suppport") variant("amd-app-opt", default=False, description="Build with amd-app-opt suppport") depends_on("texinfo") provides("fftw-api@3", when="@2:") conflicts( "precision=quad", when="@2.2 %aocc", msg="Quad precision is not supported by AOCC clang version 2.2", ) conflicts( "+debug", when="@2.2 %aocc", msg="debug mode is not supported by AOCC clang version 2.2" ) conflicts("%gcc@:7.2", when="@2.2:", msg="GCC version above 7.2 is required for AMDFFTW") conflicts( "+amd-fast-planner ", when="+mpi", msg="mpi thread is not supported with amd-fast-planner" ) conflicts( "+amd-fast-planner", when="@2.2", msg="amd-fast-planner is supported from 3.0 onwards" ) conflicts( "+amd-fast-planner", when="precision=quad", msg="Quad precision is not supported with amd-fast-planner", ) conflicts( "+amd-fast-planner", when="precision=long_double", msg="long_double precision is not supported with amd-fast-planner", ) conflicts( "+amd-top-n-planner", when="@:3.0.0", msg="amd-top-n-planner is supported from 3.0.1 onwards", ) conflicts( "+amd-top-n-planner", when="precision=long_double", msg="long_double precision is not supported with amd-top-n-planner", ) conflicts( "+amd-top-n-planner", when="precision=quad", msg="Quad precision is not supported with amd-top-n-planner", ) conflicts( "+amd-top-n-planner", when="+amd-fast-planner", msg="amd-top-n-planner cannot be used with amd-fast-planner", ) conflicts( "+amd-top-n-planner", when="+threads", msg="amd-top-n-planner works only for single thread" ) conflicts( "+amd-top-n-planner", when="+mpi", msg="mpi thread is not supported with amd-top-n-planner" ) conflicts( "+amd-top-n-planner", when="+openmp", msg="openmp thread is not supported with amd-top-n-planner", ) conflicts( "+amd-mpi-vader-limit", when="@:3.0.0", msg="amd-mpi-vader-limit is supported from 3.0.1 onwards", ) conflicts( "+amd-mpi-vader-limit", when="precision=quad", msg="Quad precision is not supported with amd-mpi-vader-limit", ) conflicts("+amd-trans", when="+threads", msg="amd-trans works only for single thread") conflicts("+amd-trans", when="+mpi", msg="mpi thread is not supported with amd-trans") conflicts("+amd-trans", when="+openmp", msg="openmp thread is not supported with amd-trans") conflicts( "+amd-trans", when="precision=long_double", msg="long_double precision is not supported with amd-trans", ) conflicts( "+amd-trans", when="precision=quad", msg="Quad precision is not supported with amd-trans" ) conflicts("+amd-app-opt", when="@:3.0.1", msg="amd-app-opt is supported from 3.1 onwards") conflicts("+amd-app-opt", when="+mpi", msg="mpi thread is not supported with amd-app-opt") conflicts( "+amd-app-opt", when="precision=long_double", msg="long_double precision is not supported with amd-app-opt", ) conflicts( "+amd-app-opt", when="precision=quad", msg="Quad precision is not supported with amd-app-opt", ) def configure(self, spec, prefix): """Configure function""" # Base options options = ["--prefix={0}".format(prefix), "--enable-amd-opt"] # Check if compiler is AOCC if "%aocc" in spec: options.append("CC={0}".format(os.path.basename(spack_cc))) options.append("FC={0}".format(os.path.basename(spack_fc))) options.append("F77={0}".format(os.path.basename(spack_fc))) if "+debug" in spec: options.append("--enable-debug") if "+mpi" in spec: options.append("--enable-mpi") options.append("--enable-amd-mpifft") else: options.append("--disable-mpi") options.append("--disable-amd-mpifft") options.extend(self.enable_or_disable("shared")) options.extend(self.enable_or_disable("openmp")) options.extend(self.enable_or_disable("threads")) options.extend(self.enable_or_disable("amd-fast-planner")) options.extend(self.enable_or_disable("amd-top-n-planner")) options.extend(self.enable_or_disable("amd-mpi-vader-limit")) options.extend(self.enable_or_disable("static")) options.extend(self.enable_or_disable("amd-trans")) options.extend(self.enable_or_disable("amd-app-opt")) if not self.compiler.f77 or not self.compiler.fc: options.append("--disable-fortran") # Cross compilation is supported in amd-fftw by making use of target # variable to set AMD_ARCH configure option. # Spack user can not directly use AMD_ARCH for this purpose but should # use target variable to set appropriate -march option in AMD_ARCH. arch = spec.architecture options.append( "AMD_ARCH={0}".format(arch.target.optimization_flags(spec.compiler).split("=")[-1]) ) # Specific SIMD support. # float and double precisions are supported simd_features = ["sse2", "avx", "avx2"] simd_options = [] for feature in simd_features: msg = "--enable-{0}" if feature in spec.target else "--disable-{0}" simd_options.append(msg.format(feature)) # When enabling configure option "--enable-amd-opt", do not use the # configure option "--enable-generic-simd128" or # "--enable-generic-simd256" # Double is the default precision, for all the others we need # to enable the corresponding option. enable_precision = { "float": ["--enable-float"], "double": None, "long_double": ["--enable-long-double"], "quad": ["--enable-quad-precision"], } # Different precisions must be configured and compiled one at a time configure = Executable("../configure") for precision in self.selected_precisions: opts = (enable_precision[precision] or []) + options[:] # SIMD optimizations are available only for float and double if precision in ("float", "double"): opts += simd_options with working_dir(precision, create=True): configure(*opts) ================================================ FILE: lib/spack/spack/test/data/unparse/grads.txt ================================================ # -*- python -*- # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This is an unparser test package. ``grads`` was chosen because it has an embedded comment that looks like a docstring, which should be removed when doing canonical unparsing. """ from spack import * class Grads(AutotoolsPackage): """The Grid Analysis and Display System (GrADS) is an interactive desktop tool that is used for easy access, manipulation, and visualization of earth science data. GrADS has two data models for handling gridded and station data. GrADS supports many data file formats, including binary (stream or sequential), GRIB (version 1 and 2), NetCDF, HDF (version 4 and 5), and BUFR (for station data).""" homepage = "http://cola.gmu.edu/grads/grads.php" url = "ftp://cola.gmu.edu/grads/2.2/grads-2.2.1-src.tar.gz" version('2.2.1', sha256='695e2066d7d131720d598bac0beb61ac3ae5578240a5437401dc0ffbbe516206') variant('geotiff', default=True, description="Enable GeoTIFF support") variant('shapefile', default=True, description="Enable Shapefile support") """ # FIXME: Fails with undeclared functions (tdefi, tdef, ...) in gauser.c variant('hdf5', default=False, description="Enable HDF5 support") variant('hdf4', default=False, description="Enable HDF4 support") variant('netcdf', default=False, description="Enable NetCDF support") depends_on('hdf5', when='+hdf5') depends_on('hdf', when='+hdf4') depends_on('netcdf-c', when='+netcdf') """ depends_on('libgeotiff', when='+geotiff') depends_on('shapelib', when='+shapefile') depends_on('udunits') depends_on('libgd') depends_on('libxmu') depends_on('cairo +X +pdf +fc +ft') depends_on('readline') depends_on('pkgconfig', type='build') def setup_build_environment(self, env: EnvironmentModifications) -> None: env.set('SUPPLIBS', '/') def setup_run_environment(self, env: EnvironmentModifications) -> None: env.set('GADDIR', self.prefix.data) @run_after('install') def copy_data(self): with working_dir(self.build_directory): install_tree('data', self.prefix.data) with working_dir(self.package_dir): install('udpt', self.prefix.data) filter_file( r'({lib})', self.prefix.lib, self.prefix.data.udpt ) def configure_args(self): args = [] args.extend(self.with_or_without('geotiff')) return args ================================================ FILE: lib/spack/spack/test/data/unparse/legion.txt ================================================ # -*- python -*- # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This is an unparser test package. ``legion`` was chosen because it was very complex and because it was the only package in Spack that used a multi-argument print statement, which needs to be handled consistently across python versions *despite* the fact that it produces different ASTs and different semantics for Python 2 and 3. """ import os from spack import * class Legion(CMakePackage): """Legion is a data-centric parallel programming system for writing portable high performance programs targeted at distributed heterogeneous architectures. Legion presents abstractions which allow programmers to describe properties of program data (e.g. independence, locality). By making the Legion programming system aware of the structure of program data, it can automate many of the tedious tasks programmers currently face, including correctly extracting task- and data-level parallelism and moving data around complex memory hierarchies. A novel mapping interface provides explicit programmer controlled placement of data in the memory hierarchy and assignment of tasks to processors in a way that is orthogonal to correctness, thereby enabling easy porting and tuning of Legion applications to new architectures.""" homepage = "https://legion.stanford.edu/" git = "https://github.com/StanfordLegion/legion.git" maintainers('pmccormick', 'streichler') tags = ['e4s'] version('21.03.0', tag='legion-21.03.0') version('stable', branch='stable') version('master', branch='master') version('cr', branch='control_replication') depends_on("cmake@3.16:", type='build') # TODO: Need to spec version of MPI v3 for use of the low-level MPI transport # layer. At present the MPI layer is still experimental and we discourge its # use for general (not legion development) use cases. depends_on('mpi', when='network=mpi') depends_on('mpi', when='network=gasnet') # MPI is required to build gasnet (needs mpicc). depends_on('ucx', when='conduit=ucx') depends_on('mpi', when='conduit=mpi') depends_on('cuda@10.0:11.9', when='+cuda_unsupported_compiler') depends_on('cuda@10.0:11.9', when='+cuda') depends_on('hdf5', when='+hdf5') depends_on('hwloc', when='+hwloc') # cuda-centric # reminder for arch numbers to names: 60=pascal, 70=volta, 75=turing, 80=ampere # TODO: we could use a map here to clean up and use naming vs. numbers. cuda_arch_list = ('60', '70', '75', '80') for nvarch in cuda_arch_list: depends_on('kokkos@3.3.01+cuda+cuda_lambda+wrapper cuda_arch={0}'.format(nvarch), when='%gcc+kokkos+cuda cuda_arch={0}'.format(nvarch)) depends_on("kokkos@3.3.01+cuda+cuda_lambda~wrapper cuda_arch={0}".format(nvarch), when="%clang+kokkos+cuda cuda_arch={0}".format(nvarch)) depends_on('kokkos@3.3.01~cuda', when='+kokkos~cuda') depends_on("kokkos@3.3.01~cuda+openmp", when='+kokkos+openmp') depends_on('python@3', when='+python') depends_on('papi', when='+papi') depends_on('zlib', when='+zlib') # TODO: Need a AMD/HIP variant to match support landing in 21.03.0. # Network transport layer: the underlying data transport API should be used for # distributed data movement. For Legion, gasnet is the currently the most # mature. We have many users that default to using no network layer for # day-to-day development thus we default to 'none'. MPI support is new and # should be considered as a beta release. variant('network', default='none', values=('gasnet', 'mpi', 'none'), description="The network communications/transport layer to use.", multi=False) # Add Gasnet tarball dependency in spack managed manner # TODO: Provide less mutable tag instead of branch resource(name='stanfordgasnet', git='https://github.com/StanfordLegion/gasnet.git', destination='stanfordgasnet', branch='master', when='network=gasnet') # We default to automatically embedding a gasnet build. To override this # point the package a pre-installed version of GASNet-Ex via the gasnet_root # variant. # # make sure we have a valid directory provided for gasnet_root... def validate_gasnet_root(value): if value == 'none': return True if not os.path.isdir(value): print(("gasnet_root:", value, "-- no such directory.")) print("gasnet_root:", value, "-- no such directory.") return False else: return True variant('gasnet_root', default='none', values=validate_gasnet_root, description="Path to a pre-installed version of GASNet (prefix directory).", multi=False) conflicts('gasnet_root', when="network=mpi") variant('conduit', default='none', values=('aries', 'ibv', 'udp', 'mpi', 'ucx', 'none'), description="The gasnet conduit(s) to enable.", multi=False) conflicts('conduit=none', when='network=gasnet', msg="a conduit must be selected when 'network=gasnet'") gasnet_conduits = ('aries', 'ibv', 'udp', 'mpi', 'ucx') for c in gasnet_conduits: conflict_str = 'conduit=%s' % c conflicts(conflict_str, when='network=mpi', msg="conduit attribute requires 'network=gasnet'.") conflicts(conflict_str, when='network=none', msg="conduit attribute requires 'network=gasnet'.") variant('gasnet_debug', default=False, description="Build gasnet with debugging enabled.") conflicts('+gasnet_debug', when='network=mpi') conflicts('+gasnet_debug', when='network=none') variant('shared', default=False, description="Build shared libraries.") variant('bounds_checks', default=False, description="Enable bounds checking in Legion accessors.") variant('privilege_checks', default=False, description="Enable runtime privildge checks in Legion accessors.") variant('enable_tls', default=False, description="Enable thread-local-storage of the Legion context.") variant('output_level', default='warning', # Note: these values are dependent upon those used in the cmake config. values=("spew", "debug", "info", "print", "warning", "error", "fatal", "none"), description="Set the compile-time logging level.", multi=False) variant('spy', default=False, description="Enable detailed logging for Legion Spy debugging.") # note: we will be dependent upon spack's latest-and-greatest cuda version... variant('cuda', default=False, description="Enable CUDA support.") variant('cuda_hijack', default=False, description="Hijack application calls into the CUDA runtime (+cuda).") variant('cuda_arch', default='70', values=cuda_arch_list, description="GPU/CUDA architecture to build for.", multi=False) variant('cuda_unsupported_compiler', default=False, description="Disable nvcc version check (--allow-unsupported-compiler).") conflicts('+cuda_hijack', when='~cuda') variant('fortran', default=False, description="Enable Fortran bindings.") variant('hdf5', default=False, description="Enable support for HDF5.") variant('hwloc', default=False, description="Use hwloc for topology awareness.") variant('kokkos', default=False, description="Enable support for interoperability with Kokkos.") variant('bindings', default=False, description="Build runtime language bindings (excl. Fortran).") variant('libdl', default=True, description="Enable support for dynamic object/library loading.") variant('openmp', default=False, description="Enable support for OpenMP within Legion tasks.") variant('papi', default=False, description="Enable PAPI performance measurements.") variant('python', default=False, description="Enable Python support.") variant('zlib', default=True, description="Enable zlib support.") variant('redop_complex', default=False, description="Use reduction operators for complex types.") variant('max_dims', values=int, default=3, description="Set max number of dimensions for logical regions.") variant('max_fields', values=int, default=512, description="Maximum number of fields allowed in a logical region.") def cmake_args(self): spec = self.spec cmake_cxx_flags = [] options = [] if 'network=gasnet' in spec: options.append('-DLegion_NETWORKS=gasnetex') if spec.variants['gasnet_root'].value != 'none': gasnet_dir = spec.variants['gasnet_root'].value options.append('-DGASNet_ROOT_DIR=%s' % gasnet_dir) else: gasnet_dir = join_path(self.stage.source_path, "stanfordgasnet", "gasnet") options.append('-DLegion_EMBED_GASNet=ON') options.append('-DLegion_EMBED_GASNet_LOCALSRC=%s' % gasnet_dir) gasnet_conduit = spec.variants['conduit'].value options.append('-DGASNet_CONDUIT=%s' % gasnet_conduit) if '+gasnet_debug' in spec: options.append('-DLegion_EMBED_GASNet_CONFIGURE_ARGS=--enable-debug') elif 'network=mpi' in spec: options.append('-DLegion_NETWORKS=mpi') if spec.variants['gasnet_root'].value != 'none': raise InstallError("'gasnet_root' is only valid when 'network=gasnet'.") else: if spec.variants['gasnet_root'].value != 'none': raise InstallError("'gasnet_root' is only valid when 'network=gasnet'.") options.append('-DLegion_EMBED_GASNet=OFF') if '+shared' in spec: options.append('-DBUILD_SHARED_LIBS=ON') else: options.append('-DBUILD_SHARED_LIBS=OFF') if '+bounds_checks' in spec: # default is off. options.append('-DLegion_BOUNDS_CHECKS=ON') if '+privilege_checks' in spec: # default is off. options.append('-DLegion_PRIVILEGE_CHECKS=ON') if '+enable_tls' in spec: # default is off. options.append('-DLegion_ENABLE_TLS=ON') if 'output_level' in spec: level = str.upper(spec.variants['output_level'].value) options.append('-DLegion_OUTPUT_LEVEL=%s' % level) if '+spy' in spec: # default is off. options.append('-DLegion_SPY=ON') if '+cuda' in spec: cuda_arch = spec.variants['cuda_arch'].value options.append('-DLegion_USE_CUDA=ON') options.append('-DLegion_GPU_REDUCTIONS=ON') options.append('-DLegion_CUDA_ARCH=%s' % cuda_arch) if '+cuda_hijack' in spec: options.append('-DLegion_HIJACK_CUDART=ON') else: options.append('-DLegion_HIJACK_CUDART=OFF') if '+cuda_unsupported_compiler' in spec: options.append('-DCUDA_NVCC_FLAGS:STRING=--allow-unsupported-compiler') if '+fortran' in spec: # default is off. options.append('-DLegion_USE_Fortran=ON') if '+hdf5' in spec: # default is off. options.append('-DLegion_USE_HDF5=ON') if '+hwloc' in spec: # default is off. options.append('-DLegion_USE_HWLOC=ON') if '+kokkos' in spec: # default is off. options.append('-DLegion_USE_Kokkos=ON') os.environ['KOKKOS_CXX_COMPILER'] = spec['kokkos'].kokkos_cxx if '+libdl' in spec: # default is on. options.append('-DLegion_USE_LIBDL=ON') else: options.append('-DLegion_USE_LIBDL=OFF') if '+openmp' in spec: # default is off. options.append('-DLegion_USE_OpenMP=ON') if '+papi' in spec: # default is off. options.append('-DLegion_USE_PAPI=ON') if '+python' in spec: # default is off. options.append('-DLegion_USE_Python=ON') if '+zlib' in spec: # default is on. options.append('-DLegion_USE_ZLIB=ON') else: options.append('-DLegion_USE_ZLIB=OFF') if '+redop_complex' in spec: # default is off. options.append('-DLegion_REDOP_COMPLEX=ON') if '+bindings' in spec: # default is off. options.append('-DLegion_BUILD_BINDINGS=ON') options.append('-DLegion_REDOP_COMPLEX=ON') # required for bindings options.append('-DLegion_USE_Fortran=ON') if spec.variants['build_type'].value == 'Debug': cmake_cxx_flags.extend([ '-DDEBUG_REALM', '-DDEBUG_LEGION', '-ggdb', ]) maxdims = int(spec.variants['max_dims'].value) # TODO: sanity check if maxdims < 0 || > 9??? options.append('-DLegion_MAX_DIM=%d' % maxdims) maxfields = int(spec.variants['max_fields'].value) if (maxfields <= 0): maxfields = 512 # make sure maxfields is a power of two. if not, # find the next largest power of two and use that... if (maxfields & (maxfields - 1) != 0): while maxfields & maxfields - 1: maxfields = maxfields & maxfields - 1 maxfields = maxfields << 1 options.append('-DLegion_MAX_FIELDS=%d' % maxfields) # This disables Legion's CMake build system's logic for targeting the native # CPU architecture in favor of Spack-provided compiler flags options.append('-DBUILD_MARCH:STRING=') return options @run_after('install') def cache_test_sources(self): """Copy the example source files after the package is installed to an install test subdirectory for use during `spack test run`.""" cache_extra_test_sources(self, [join_path('examples', 'local_function_tasks')]) def test_run_local_function_tasks(self): """Build and run external application example""" test_dir = join_path( self.test_suite.current_test_cache_dir, "examples", "local_function_tasks" ) if not os.path.exists(test_dir): raise SkipTest(f"{test_dir} must exist") cmake_args = [ f"-DCMAKE_C_COMPILER={self.compiler.cc}", f"-DCMAKE_CXX_COMPILER={self.compiler.cxx}", f"-DLegion_DIR={join_path(self.prefix, 'share', 'Legion', 'cmake')}", ] with working_dir(test_dir): cmake = self.spec["cmake"].command cmake(*cmake_args) make = which("make") make() exe = which("local_function_tasks") exe() ================================================ FILE: lib/spack/spack/test/data/unparse/llvm.txt ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import os.path import re import sys import llnl.util.tty as tty import spack.build_environment import spack.util.executable from spack.package import * class Llvm(CMakePackage, CudaPackage): """The LLVM Project is a collection of modular and reusable compiler and toolchain technologies. Despite its name, LLVM has little to do with traditional virtual machines, though it does provide helpful libraries that can be used to build them. The name "LLVM" itself is not an acronym; it is the full name of the project. """ homepage = "https://llvm.org/" url = "https://github.com/llvm/llvm-project/archive/llvmorg-7.1.0.tar.gz" list_url = "https://releases.llvm.org/download.html" git = "https://github.com/llvm/llvm-project" maintainers("trws", "haampie") tags = ["e4s"] generator = "Ninja" family = "compiler" # Used by lmod # fmt: off version('main', branch='main') version('14.0.6', sha256='98f15f842700bdb7220a166c8d2739a03a72e775b67031205078f39dd756a055') version('14.0.5', sha256='a4a57f029cb81f04618e05853f05fc2d21b64353c760977d8e7799bf7218a23a') version('14.0.4', sha256='1333236f9bee38658762076be4236cb5ebf15ae9b7f2bfce6946b96ae962dc73') version('14.0.3', sha256='0e1d049b050127ecf6286107e9a4400b0550f841d5d2288b9d31fd32ed0683d5') version('14.0.2', sha256='ca52232b3451c8e017f00eb882277707c13e30fac1271ec97015f6d0eeb383d1') version('14.0.1', sha256='c8be00406e872c8a24f8571cf6f5517b73ae707104724b1fd1db2f0af9544019') version('14.0.0', sha256='87b1a068b370df5b79a892fdb2935922a8efb1fddec4cc506e30fe57b6a1d9c4') version('13.0.1', sha256='09c50d558bd975c41157364421820228df66632802a4a6a7c9c17f86a7340802') version('13.0.0', sha256='a1131358f1f9f819df73fa6bff505f2c49d176e9eef0a3aedd1fdbce3b4630e8') version('12.0.1', sha256='66b64aa301244975a4aea489f402f205cde2f53dd722dad9e7b77a0459b4c8df') version('12.0.0', sha256='8e6c99e482bb16a450165176c2d881804976a2d770e0445af4375e78a1fbf19c') version('11.1.0', sha256='53a0719f3f4b0388013cfffd7b10c7d5682eece1929a9553c722348d1f866e79') version('11.0.1', sha256='9c7ad8e8ec77c5bde8eb4afa105a318fd1ded7dff3747d14f012758719d7171b') version('11.0.0', sha256='8ad4ddbafac4f2c8f2ea523c2c4196f940e8e16f9e635210537582a48622a5d5') version('10.0.1', sha256='c7ccb735c37b4ec470f66a6c35fbae4f029c0f88038f6977180b1a8ddc255637') version('10.0.0', sha256='b81c96d2f8f40dc61b14a167513d87c0d813aae0251e06e11ae8a4384ca15451') version('9.0.1', sha256='be7b034641a5fda51ffca7f5d840b1a768737779f75f7c4fd18fe2d37820289a') version('9.0.0', sha256='7807fac25330e24e9955ca46cd855dd34bbc9cc4fdba8322366206654d1036f2') version('8.0.1', sha256='5b18f6111c7aee7c0933c355877d4abcfe6cb40c1a64178f28821849c725c841') version('8.0.0', sha256='d81238b4a69e93e29f74ce56f8107cbfcf0c7d7b40510b7879e98cc031e25167') version('7.1.0', sha256='71c93979f20e01f1a1cc839a247945f556fa5e63abf2084e8468b238080fd839') version('7.0.1', sha256='f17a6cd401e8fd8f811fbfbb36dcb4f455f898c9d03af4044807ad005df9f3c0') version('6.0.1', sha256='aefadceb231f4c195fe6d6cd3b1a010b269c8a22410f339b5a089c2e902aa177') version('6.0.0', sha256='1946ec629c88d30122afa072d3c6a89cc5d5e4e2bb28dc63b2f9ebcc7917ee64') version('5.0.2', sha256='fe87aa11558c08856739bfd9bd971263a28657663cb0c3a0af01b94f03b0b795') version('5.0.1', sha256='84ca454abf262579814a2a2b846569f6e0cb3e16dc33ca3642b4f1dff6fbafd3') version('5.0.0', sha256='1f1843315657a4371d8ca37f01265fa9aae17dbcf46d2d0a95c1fdb3c6a4bab6') version('4.0.1', sha256='cd664fb3eec3208c08fb61189c00c9118c290b3be5adb3215a97b24255618be5') version('4.0.0', sha256='28ca4b2fc434cb1f558e8865386c233c2a6134437249b8b3765ae745ffa56a34') version('3.9.1', sha256='f5b6922a5c65f9232f83d89831191f2c3ccf4f41fdd8c63e6645bbf578c4ab92') version('3.9.0', sha256='9c6563a72c8b5b79941c773937d997dd2b1b5b3f640136d02719ec19f35e0333') version('3.8.1', sha256='69360f0648fde0dc3d3c4b339624613f3bc2a89c4858933bc3871a250ad02826') version('3.8.0', sha256='b5cc5974cc2fd4e9e49e1bbd0700f872501a8678bd9694fa2b36c65c026df1d1') version('3.7.1', sha256='d2cb0eb9b8eb21e07605bfe5e7a5c6c5f5f8c2efdac01ec1da6ffacaabe4195a') version('3.7.0', sha256='dc00bc230be2006fb87b84f6fe4800ca28bc98e6692811a98195da53c9cb28c6') version('3.6.2', sha256='f75d703a388ba01d607f9cf96180863a5e4a106827ade17b221d43e6db20778a') version('3.5.1', sha256='5d739684170d5b2b304e4fb521532d5c8281492f71e1a8568187bfa38eb5909d') # fmt: on # NOTE: The debug version of LLVM is an order of magnitude larger than # the release version, and may take up 20-30 GB of space. If you want # to save space, build with `build_type=Release`. variant( "clang", default=True, description="Build the LLVM C/C++/Objective-C compiler frontend" ) variant( "flang", default=False, when="@11: +clang", description="Build the LLVM Fortran compiler frontend " "(experimental - parser only, needs GCC)", ) variant( "omp_debug", default=False, description="Include debugging code in OpenMP runtime libraries", ) variant("lldb", default=True, when="+clang", description="Build the LLVM debugger") variant("lld", default=True, description="Build the LLVM linker") variant("mlir", default=False, when="@10:", description="Build with MLIR support") variant( "internal_unwind", default=True, when="+clang", description="Build the libcxxabi libunwind" ) variant( "polly", default=True, description="Build the LLVM polyhedral optimization plugin, " "only builds for 3.7.0+", ) variant( "libcxx", default=True, when="+clang", description="Build the LLVM C++ standard library" ) variant( "compiler-rt", when="+clang", default=True, description="Build LLVM compiler runtime, including sanitizers", ) variant( "gold", default=(sys.platform != "darwin"), description="Add support for LTO with the gold linker plugin", ) variant("split_dwarf", default=False, description="Build with split dwarf information") variant( "llvm_dylib", default=True, description="Build a combined LLVM shared library with all components", ) variant( "link_llvm_dylib", default=False, when="+llvm_dylib", description="Link LLVM tools against the LLVM shared library", ) variant( "targets", default="none", description=( "What targets to build. Spack's target family is always added " "(e.g. X86 is automatically enabled when targeting znver2)." ), values=( "all", "none", "aarch64", "amdgpu", "arm", "avr", "bpf", "cppbackend", "hexagon", "lanai", "mips", "msp430", "nvptx", "powerpc", "riscv", "sparc", "systemz", "webassembly", "x86", "xcore", ), multi=True, ) variant( "build_type", default="Release", description="CMake build type", values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel"), ) variant( "omp_tsan", default=False, when="@6:", description="Build with OpenMP capable thread sanitizer", ) variant( "omp_as_runtime", default=True, when="+clang @12:", description="Build OpenMP runtime via ENABLE_RUNTIME by just-built Clang", ) variant( "code_signing", default=False, when="+lldb platform=darwin", description="Enable code-signing on macOS", ) variant("python", default=False, description="Install python bindings") variant("version_suffix", default="none", description="Add a symbol suffix") variant( "shlib_symbol_version", default="none", description="Add shared library symbol version", when="@13:", ) variant( "z3", default=False, when="+clang @8:", description="Use Z3 for the clang static analyzer" ) provides("libllvm@14", when="@14.0.0:14") provides("libllvm@13", when="@13.0.0:13") provides("libllvm@12", when="@12.0.0:12") provides("libllvm@11", when="@11.0.0:11") provides("libllvm@10", when="@10.0.0:10") provides("libllvm@9", when="@9.0.0:9") provides("libllvm@8", when="@8.0.0:8") provides("libllvm@7", when="@7.0.0:7") provides("libllvm@6", when="@6.0.0:6") provides("libllvm@5", when="@5.0.0:5") provides("libllvm@4", when="@4.0.0:4") provides("libllvm@3", when="@3.0.0:3") extends("python", when="+python") # Build dependency depends_on("cmake@3.4.3:", type="build") depends_on("cmake@3.13.4:", type="build", when="@12:") depends_on("ninja", type="build") depends_on("python@2.7:2.8", when="@:4 ~python", type="build") depends_on("python", when="@5: ~python", type="build") depends_on("pkgconfig", type="build") # Universal dependency depends_on("python@2.7:2.8", when="@:4+python") depends_on("python", when="@5:+python") # clang and clang-tools dependencies depends_on("z3@4.7.1:", when="+z3") # openmp dependencies depends_on("perl-data-dumper", type=("build")) depends_on("hwloc") depends_on("libelf", when="+cuda") # libomptarget depends_on("libffi", when="+cuda") # libomptarget # llvm-config --system-libs libraries. depends_on("zlib-api") # lldb dependencies depends_on("swig", when="+lldb") depends_on("libedit", when="+lldb") depends_on("ncurses", when="+lldb") depends_on("py-six", when="@5.0.0: +lldb +python") # gold support, required for some features depends_on("binutils+gold+ld+plugins", when="+gold") # polly plugin depends_on("gmp", when="@:3.6 +polly") depends_on("isl", when="@:3.6 +polly") # Older LLVM do not build with newer compilers, and vice versa conflicts("%gcc@8:", when="@:5") conflicts("%gcc@:5.0", when="@8:") # clang/lib: a lambda parameter cannot shadow an explicitly captured entity conflicts("%clang@8:", when="@:4") # Internal compiler error on gcc 8.4 on aarch64 https://bugzilla.redhat.com/show_bug.cgi?id=1958295 conflicts("%gcc@8.4:8.4.9", when="@12: target=aarch64:") # When these versions are concretized, but not explicitly with +libcxx, these # conflicts will enable clingo to set ~libcxx, making the build successful: # libc++ of LLVM13, see https://libcxx.llvm.org/#platform-and-compiler-support # @13 does not support %gcc@:10 https://bugs.llvm.org/show_bug.cgi?id=51359#c1 # GCC 11 - latest stable release per GCC release page # Clang: 11, 12 - latest two stable releases per LLVM release page # AppleClang 12 - latest stable release per Xcode release page conflicts("%gcc@:10", when="@13:+libcxx") conflicts("%clang@:10", when="@13:+libcxx") conflicts("%apple-clang@:11", when="@13:+libcxx") # libcxx-4 and compiler-rt-4 fail to build with "newer" clang and gcc versions: conflicts("%gcc@7:", when="@:4+libcxx") conflicts("%clang@6:", when="@:4+libcxx") conflicts("%apple-clang@6:", when="@:4+libcxx") conflicts("%gcc@7:", when="@:4+compiler-rt") conflicts("%clang@6:", when="@:4+compiler-rt") conflicts("%apple-clang@6:", when="@:4+compiler-rt") # cuda_arch value must be specified conflicts("cuda_arch=none", when="+cuda", msg="A value for cuda_arch must be specified.") # LLVM bug https://bugs.llvm.org/show_bug.cgi?id=48234 # CMake bug: https://gitlab.kitware.com/cmake/cmake/-/issues/21469 # Fixed in upstream versions of both conflicts("^cmake@3.19.0", when="@6:11.0.0") # Github issue #4986 patch("llvm_gcc7.patch", when="@4.0.0:4.0.1+lldb %gcc@7.0:") # sys/ustat.h has been removed in favour of statfs from glibc-2.28. Use fixed sizes: patch("llvm5-sanitizer-ustat.patch", when="@4:6.0.0+compiler-rt") # Fix lld templates: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=230463 patch("llvm4-lld-ELF-Symbols.patch", when="@4+lld%clang@6:") patch("llvm5-lld-ELF-Symbols.patch", when="@5+lld%clang@7:") # Fix missing std:size_t in 'llvm@4:5' when built with '%clang@7:' patch("xray_buffer_queue-cstddef.patch", when="@4:5+compiler-rt%clang@7:") # https://github.com/llvm/llvm-project/commit/947f9692440836dcb8d88b74b69dd379d85974ce patch("sanitizer-ipc_perm_mode.patch", when="@5:7+compiler-rt%clang@11:") patch("sanitizer-ipc_perm_mode.patch", when="@5:9+compiler-rt%gcc@9:") # github.com/spack/spack/issues/24270: MicrosoftDemangle for %gcc@10: and %clang@13: patch("missing-includes.patch", when="@8") # Backport from llvm master + additional fix # see https://bugs.llvm.org/show_bug.cgi?id=39696 # for a bug report about this problem in llvm master. patch("constexpr_longdouble.patch", when="@6:8+libcxx") patch("constexpr_longdouble_9.0.patch", when="@9:10.0.0+libcxx") # Backport from llvm master; see # https://bugs.llvm.org/show_bug.cgi?id=38233 # for a bug report about this problem in llvm master. patch("llvm_py37.patch", when="@4:6 ^python@3.7:") # https://bugs.llvm.org/show_bug.cgi?id=39696 patch("thread-p9.patch", when="@:10 +libcxx") # https://github.com/spack/spack/issues/19625, # merged in llvm-11.0.0_rc2, but not found in 11.0.1 patch("lldb_external_ncurses-10.patch", when="@10.0.0:11.0.1+lldb") # https://github.com/spack/spack/issues/19908 # merged in llvm main prior to 12.0.0 patch("llvm_python_path.patch", when="@:11") # Workaround for issue https://github.com/spack/spack/issues/18197 patch("llvm7_intel.patch", when="@7 %intel@18.0.2,19.0.0:19.1.99") # Remove cyclades support to build against newer kernel headers # https://reviews.llvm.org/D102059 patch("no_cyclades.patch", when="@10:12.0.0") patch("no_cyclades9.patch", when="@6:9") patch("llvm-gcc11.patch", when="@9:11%gcc@11:") # add -lpthread to build OpenMP libraries with Fujitsu compiler patch("llvm12-thread.patch", when="@12 %fj") patch("llvm13-thread.patch", when="@13 %fj") # avoid build failed with Fujitsu compiler patch("llvm13-fujitsu.patch", when="@13 %fj") # patch for missing hwloc.h include for libompd patch("llvm14-hwloc-ompd.patch", when="@14") # make libflags a list in openmp subproject when ~omp_as_runtime patch("libomp-libflags-as-list.patch", when="@3.7:") # The functions and attributes below implement external package # detection for LLVM. See: # # https://spack.readthedocs.io/en/latest/packaging_guide.html#making-a-package-discoverable-with-spack-external-find executables = ["clang", "flang", "ld.lld", "lldb"] @classmethod def filter_detected_exes(cls, prefix, exes_in_prefix): result = [] for exe in exes_in_prefix: # Executables like lldb-vscode-X are daemon listening # on some port and would hang Spack during detection. # clang-cl and clang-cpp are dev tools that we don't # need to test if any(x in exe for x in ("vscode", "cpp", "-cl", "-gpu")): continue result.append(exe) return result @classmethod def determine_version(cls, exe): version_regex = re.compile( # Normal clang compiler versions are left as-is r"clang version ([^ )\n]+)-svn[~.\w\d-]*|" # Don't include hyphenated patch numbers in the version # (see https://github.com/spack/spack/pull/14365 for details) r"clang version ([^ )\n]+?)-[~.\w\d-]*|" r"clang version ([^ )\n]+)|" # LLDB r"lldb version ([^ )\n]+)|" # LLD r"LLD ([^ )\n]+) \(compatible with GNU linkers\)" ) try: compiler = Executable(exe) output = compiler("--version", output=str, error=str) if "Apple" in output: return None match = version_regex.search(output) if match: return match.group(match.lastindex) except spack.util.executable.ProcessError: pass except Exception as e: tty.debug(e) return None @classmethod def determine_variants(cls, exes, version_str): variants, compilers = ["+clang"], {} lld_found, lldb_found = False, False for exe in exes: if "clang++" in exe: compilers["cxx"] = exe elif "clang" in exe: compilers["c"] = exe elif "flang" in exe: variants.append("+flang") compilers["fc"] = exe compilers["f77"] = exe elif "ld.lld" in exe: lld_found = True compilers["ld"] = exe elif "lldb" in exe: lldb_found = True compilers["lldb"] = exe variants.append("+lld" if lld_found else "~lld") variants.append("+lldb" if lldb_found else "~lldb") return "".join(variants), {"compilers": compilers} @classmethod def validate_detected_spec(cls, spec, extra_attributes): # For LLVM 'compilers' is a mandatory attribute msg = 'the extra attribute "compilers" must be set for ' 'the detected spec "{0}"'.format( spec ) assert "compilers" in extra_attributes, msg compilers = extra_attributes["compilers"] for key in ("c", "cxx"): msg = "{0} compiler not found for {1}" assert key in compilers, msg.format(key, spec) @property def cc(self): msg = "cannot retrieve C compiler [spec is not concrete]" assert self.spec.concrete, msg if self.spec.external: return self.spec.extra_attributes["compilers"].get("c", None) result = None if "+clang" in self.spec: result = os.path.join(self.spec.prefix.bin, "clang") return result @property def cxx(self): msg = "cannot retrieve C++ compiler [spec is not concrete]" assert self.spec.concrete, msg if self.spec.external: return self.spec.extra_attributes["compilers"].get("cxx", None) result = None if "+clang" in self.spec: result = os.path.join(self.spec.prefix.bin, "clang++") return result @property def fc(self): msg = "cannot retrieve Fortran compiler [spec is not concrete]" assert self.spec.concrete, msg if self.spec.external: return self.spec.extra_attributes["compilers"].get("fc", None) result = None if "+flang" in self.spec: result = os.path.join(self.spec.prefix.bin, "flang") return result @property def f77(self): msg = "cannot retrieve Fortran 77 compiler [spec is not concrete]" assert self.spec.concrete, msg if self.spec.external: return self.spec.extra_attributes["compilers"].get("f77", None) result = None if "+flang" in self.spec: result = os.path.join(self.spec.prefix.bin, "flang") return result @property def libs(self): return LibraryList(self.llvm_config("--libfiles", "all", result="list")) @run_before("cmake") def codesign_check(self): if self.spec.satisfies("+code_signing"): codesign = which("codesign") mkdir("tmp") llvm_check_file = join_path("tmp", "llvm_check") copy("/usr/bin/false", llvm_check_file) try: codesign("-f", "-s", "lldb_codesign", "--dryrun", llvm_check_file) except ProcessError: # Newer LLVM versions have a simple script that sets up # automatically when run with sudo priviliges setup = Executable("./lldb/scripts/macos-setup-codesign.sh") try: setup() except Exception: raise RuntimeError( "spack was unable to either find or set up" "code-signing on your system. Please refer to" "https://lldb.llvm.org/resources/build.html#" "code-signing-on-macos for details on how to" "create this identity." ) def flag_handler(self, name, flags): if name == "cxxflags": flags.append(self.compiler.cxx11_flag) return (None, flags, None) elif name == "ldflags" and self.spec.satisfies("%intel"): flags.append("-shared-intel") return (None, flags, None) return (flags, None, None) def setup_build_environment(self, env: EnvironmentModifications) -> None: """When using %clang, add only its ld.lld-$ver and/or ld.lld to our PATH""" if self.compiler.name in ["clang", "apple-clang"]: for lld in "ld.lld-{0}".format(self.compiler.version.version[0]), "ld.lld": bin = os.path.join(os.path.dirname(self.compiler.cc), lld) sym = os.path.join(self.stage.path, "ld.lld") if os.path.exists(bin) and not os.path.exists(sym): mkdirp(self.stage.path) os.symlink(bin, sym) env.prepend_path("PATH", self.stage.path) def setup_run_environment(self, env: EnvironmentModifications) -> None: if "+clang" in self.spec: env.set("CC", join_path(self.spec.prefix.bin, "clang")) env.set("CXX", join_path(self.spec.prefix.bin, "clang++")) if "+flang" in self.spec: env.set("FC", join_path(self.spec.prefix.bin, "flang")) env.set("F77", join_path(self.spec.prefix.bin, "flang")) root_cmakelists_dir = "llvm" def cmake_args(self): spec = self.spec define = CMakePackage.define from_variant = self.define_from_variant python = spec["python"] cmake_args = [ define("LLVM_REQUIRES_RTTI", True), define("LLVM_ENABLE_RTTI", True), define("LLVM_ENABLE_EH", True), define("LLVM_ENABLE_TERMINFO", False), define("LLVM_ENABLE_LIBXML2", False), define("CLANG_DEFAULT_OPENMP_RUNTIME", "libomp"), define("PYTHON_EXECUTABLE", python.command.path), define("LIBOMP_USE_HWLOC", True), define("LIBOMP_HWLOC_INSTALL_DIR", spec["hwloc"].prefix), ] version_suffix = spec.variants["version_suffix"].value if version_suffix != "none": cmake_args.append(define("LLVM_VERSION_SUFFIX", version_suffix)) shlib_symbol_version = spec.variants.get("shlib_symbol_version", None) if shlib_symbol_version is not None and shlib_symbol_version.value != "none": cmake_args.append(define("LLVM_SHLIB_SYMBOL_VERSION", shlib_symbol_version.value)) if python.version >= Version("3"): cmake_args.append(define("Python3_EXECUTABLE", python.command.path)) else: cmake_args.append(define("Python2_EXECUTABLE", python.command.path)) projects = [] runtimes = [] if "+cuda" in spec: cmake_args.extend( [ define("CUDA_TOOLKIT_ROOT_DIR", spec["cuda"].prefix), define( "LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES", ",".join(spec.variants["cuda_arch"].value), ), define( "CLANG_OPENMP_NVPTX_DEFAULT_ARCH", "sm_{0}".format(spec.variants["cuda_arch"].value[-1]), ), ] ) if "+omp_as_runtime" in spec: cmake_args.extend( [ define("LIBOMPTARGET_NVPTX_ENABLE_BCLIB", True), # work around bad libelf detection in libomptarget define( "LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR", spec["libelf"].prefix.include ), ] ) else: # still build libomptarget but disable cuda cmake_args.extend( [ define("CUDA_TOOLKIT_ROOT_DIR", "IGNORE"), define("CUDA_SDK_ROOT_DIR", "IGNORE"), define("CUDA_NVCC_EXECUTABLE", "IGNORE"), define("LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES", "IGNORE"), ] ) cmake_args.append(from_variant("LIBOMPTARGET_ENABLE_DEBUG", "omp_debug")) if "+lldb" in spec: projects.append("lldb") cmake_args.append(define("LLDB_ENABLE_LIBEDIT", True)) cmake_args.append(define("LLDB_ENABLE_NCURSES", True)) cmake_args.append(define("LLDB_ENABLE_LIBXML2", False)) if spec.version >= Version("10"): cmake_args.append(from_variant("LLDB_ENABLE_PYTHON", "python")) else: cmake_args.append(define("LLDB_DISABLE_PYTHON", "~python" in spec)) if spec.satisfies("@5.0.0: +python"): cmake_args.append(define("LLDB_USE_SYSTEM_SIX", True)) if "+gold" in spec: cmake_args.append(define("LLVM_BINUTILS_INCDIR", spec["binutils"].prefix.include)) if "+clang" in spec: projects.append("clang") projects.append("clang-tools-extra") if "+omp_as_runtime" in spec: runtimes.append("openmp") else: projects.append("openmp") if "@8" in spec: cmake_args.append(from_variant("CLANG_ANALYZER_ENABLE_Z3_SOLVER", "z3")) elif "@9:" in spec: cmake_args.append(from_variant("LLVM_ENABLE_Z3_SOLVER", "z3")) if "+flang" in spec: projects.append("flang") if "+lld" in spec: projects.append("lld") if "+compiler-rt" in spec: projects.append("compiler-rt") if "+libcxx" in spec: projects.append("libcxx") projects.append("libcxxabi") if "+mlir" in spec: projects.append("mlir") if "+internal_unwind" in spec: projects.append("libunwind") if "+polly" in spec: projects.append("polly") cmake_args.append(define("LINK_POLLY_INTO_TOOLS", True)) cmake_args.extend( [ define("BUILD_SHARED_LIBS", False), from_variant("LLVM_BUILD_LLVM_DYLIB", "llvm_dylib"), from_variant("LLVM_LINK_LLVM_DYLIB", "link_llvm_dylib"), from_variant("LLVM_USE_SPLIT_DWARF", "split_dwarf"), # By default on Linux, libc++.so is a ldscript. CMake fails to add # CMAKE_INSTALL_RPATH to it, which fails. Statically link libc++abi.a # into libc++.so, linking with -lc++ or -stdlib=libc++ is enough. define("LIBCXX_ENABLE_STATIC_ABI_LIBRARY", True), ] ) cmake_args.append(define("LLVM_TARGETS_TO_BUILD", get_llvm_targets_to_build(spec))) cmake_args.append(from_variant("LIBOMP_TSAN_SUPPORT", "omp_tsan")) if self.compiler.name == "gcc": compiler = Executable(self.compiler.cc) gcc_output = compiler("-print-search-dirs", output=str, error=str) for line in gcc_output.splitlines(): if line.startswith("install:"): # Get path and strip any whitespace # (causes oddity with ancestor) gcc_prefix = line.split(":")[1].strip() gcc_prefix = ancestor(gcc_prefix, 4) break cmake_args.append(define("GCC_INSTALL_PREFIX", gcc_prefix)) if self.spec.satisfies("~code_signing platform=darwin"): cmake_args.append(define("LLDB_USE_SYSTEM_DEBUGSERVER", True)) # Semicolon seperated list of projects to enable cmake_args.append(define("LLVM_ENABLE_PROJECTS", projects)) # Semicolon seperated list of runtimes to enable if runtimes: cmake_args.append(define("LLVM_ENABLE_RUNTIMES", runtimes)) return cmake_args @run_after("install") def post_install(self): spec = self.spec define = CMakePackage.define # unnecessary if we build openmp via LLVM_ENABLE_RUNTIMES if "+cuda ~omp_as_runtime" in self.spec: ompdir = "build-bootstrapped-omp" prefix_paths = spack.build_environment.get_cmake_prefix_path(self) prefix_paths.append(str(spec.prefix)) # rebuild libomptarget to get bytecode runtime library files with working_dir(ompdir, create=True): cmake_args = [ "-G", "Ninja", define("CMAKE_BUILD_TYPE", spec.variants["build_type"].value), define("CMAKE_C_COMPILER", spec.prefix.bin + "/clang"), define("CMAKE_CXX_COMPILER", spec.prefix.bin + "/clang++"), define("CMAKE_INSTALL_PREFIX", spec.prefix), define("CMAKE_PREFIX_PATH", prefix_paths), ] cmake_args.extend(self.cmake_args()) cmake_args.extend( [ define("LIBOMPTARGET_NVPTX_ENABLE_BCLIB", True), define( "LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR", spec["libelf"].prefix.include ), self.stage.source_path + "/openmp", ] ) cmake(*cmake_args) ninja() ninja("install") if "+python" in self.spec: install_tree("llvm/bindings/python", python_platlib) if "+clang" in self.spec: install_tree("clang/bindings/python", python_platlib) with working_dir(self.build_directory): install_tree("bin", join_path(self.prefix, "libexec", "llvm")) def llvm_config(self, *args, **kwargs): lc = Executable(self.prefix.bin.join("llvm-config")) if not kwargs.get("output"): kwargs["output"] = str ret = lc(*args, **kwargs) if kwargs.get("result") == "list": return ret.split() else: return ret def get_llvm_targets_to_build(spec): targets = spec.variants["targets"].value # Build everything? if "all" in targets: return "all" # Convert targets variant values to CMake LLVM_TARGETS_TO_BUILD array. spack_to_cmake = { "aarch64": "AArch64", "amdgpu": "AMDGPU", "arm": "ARM", "avr": "AVR", "bpf": "BPF", "cppbackend": "CppBackend", "hexagon": "Hexagon", "lanai": "Lanai", "mips": "Mips", "msp430": "MSP430", "nvptx": "NVPTX", "powerpc": "PowerPC", "riscv": "RISCV", "sparc": "Sparc", "systemz": "SystemZ", "webassembly": "WebAssembly", "x86": "X86", "xcore": "XCore", } if "none" in targets: llvm_targets = set() else: llvm_targets = set(spack_to_cmake[target] for target in targets) if spec.target.family in ("x86", "x86_64"): llvm_targets.add("X86") elif spec.target.family == "arm": llvm_targets.add("ARM") elif spec.target.family == "aarch64": llvm_targets.add("AArch64") elif spec.target.family in ("sparc", "sparc64"): llvm_targets.add("Sparc") elif spec.target.family in ("ppc64", "ppc64le", "ppc", "ppcle"): llvm_targets.add("PowerPC") return list(llvm_targets) ================================================ FILE: lib/spack/spack/test/data/unparse/mfem.txt ================================================ # -*- python -*- # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This is an unparser test package. ``mfem`` was chosen because it's one of the most complex packages in Spack, because it uses ``@when`` functions, because it has ``configure()`` calls with star-args in different locations, and beacuse it has a function with embedded unicode that needs to be unparsed consistently between Python versions. """ import os import shutil import sys from spack import * class Mfem(Package, CudaPackage, ROCmPackage): """Free, lightweight, scalable C++ library for finite element methods.""" tags = ['fem', 'finite-elements', 'high-order', 'amr', 'hpc', 'radiuss', 'e4s'] homepage = 'http://www.mfem.org' git = 'https://github.com/mfem/mfem.git' maintainers('v-dobrev', 'tzanio', 'acfisher', 'goxberry', 'markcmiller86') test_requires_compiler = True # Recommended mfem builds to test when updating this file: see the shell # script 'test_builds.sh' in the same directory as this file. # mfem is downloaded from a URL shortener at request of upstream # author Tzanio Kolev . See here: # https://github.com/mfem/mfem/issues/53 # # The following procedure should be used to verify security when a # new version is added: # # 1. Verify that no checksums on old versions have changed. # # 2. Verify that the shortened URL for the new version is listed at: # https://mfem.org/download/ # # 3. Use http://getlinkinfo.com or similar to verify that the # underling download link for the latest version comes has the # prefix: http://mfem.github.io/releases # # If this quick verification procedure fails, additional discussion # will be required to verify the new version. # 'develop' is a special version that is always larger (or newer) than any # other version. version('develop', branch='master') version('4.3.0', sha256='3a495602121b986049286ea0b23512279cdbdfb43c15c42a1511b521051fbe38', url='https://bit.ly/mfem-4-3', extension='tar.gz') version('4.2.0', '4352a225b55948d2e73a5ee88cece0e88bdbe7ba6726a23d68b2736d3221a86d', url='https://bit.ly/mfem-4-2', extension='tar.gz') version('4.1.0', '4c83fdcf083f8e2f5b37200a755db843cdb858811e25a8486ad36b2cbec0e11d', url='https://bit.ly/mfem-4-1', extension='tar.gz') # Tagged development version used by xSDK version('4.0.1-xsdk', commit='c55c80d17b82d80de04b849dd526e17044f8c99a') version('4.0.0', 'df5bdac798ea84a263979f6fbf79de9013e1c55562f95f98644c3edcacfbc727', url='https://bit.ly/mfem-4-0', extension='tar.gz') # Tagged development version used by the laghos package: version('3.4.1-laghos-v2.0', tag='laghos-v2.0') version('3.4.0', sha256='4e73e4fe0482636de3c5dc983cd395839a83cb16f6f509bd88b053e8b3858e05', url='https://bit.ly/mfem-3-4', extension='tar.gz') version('3.3.2', sha256='b70fa3c5080b9ec514fc05f4a04ff74322b99ac4ecd6d99c229f0ed5188fc0ce', url='https://goo.gl/Kd7Jk8', extension='tar.gz') # Tagged development version used by the laghos package: version('3.3.1-laghos-v1.0', tag='laghos-v1.0') version('3.3', sha256='b17bd452593aada93dc0fee748fcfbbf4f04ce3e7d77fdd0341cc9103bcacd0b', url='http://goo.gl/Vrpsns', extension='tar.gz') version('3.2', sha256='2938c3deed4ec4f7fd5b5f5cfe656845282e86e2dcd477d292390058b7b94340', url='http://goo.gl/Y9T75B', extension='tar.gz') version('3.1', sha256='841ea5cf58de6fae4de0f553b0e01ebaab9cd9c67fa821e8a715666ecf18fc57', url='http://goo.gl/xrScXn', extension='tar.gz') variant('static', default=True, description='Build static library') variant('shared', default=False, description='Build shared library') variant('mpi', default=True, description='Enable MPI parallelism') # Can we make the default value for 'metis' to depend on the 'mpi' value? variant('metis', default=True, description='Enable METIS support') variant('openmp', default=False, description='Enable OpenMP parallelism') # Note: '+cuda' and 'cuda_arch' variants are added by the CudaPackage # Note: '+rocm' and 'amdgpu_target' variants are added by the ROCmPackage variant('occa', default=False, description='Enable OCCA backend') variant('raja', default=False, description='Enable RAJA backend') variant('libceed', default=False, description='Enable libCEED backend') variant('umpire', default=False, description='Enable Umpire support') variant('amgx', default=False, description='Enable NVIDIA AmgX solver support') variant('threadsafe', default=False, description=('Enable thread safe features.' ' Required for OpenMP.' ' May cause minor performance issues.')) variant('superlu-dist', default=False, description='Enable MPI parallel, sparse direct solvers') variant('strumpack', default=False, description='Enable support for STRUMPACK') variant('suite-sparse', default=False, description='Enable serial, sparse direct solvers') variant('petsc', default=False, description='Enable PETSc solvers, preconditioners, etc.') variant('slepc', default=False, description='Enable SLEPc integration') variant('sundials', default=False, description='Enable Sundials time integrators') variant('pumi', default=False, description='Enable functionality based on PUMI') variant('gslib', default=False, description='Enable functionality based on GSLIB') variant('mpfr', default=False, description='Enable precise, 1D quadrature rules') variant('lapack', default=False, description='Use external blas/lapack routines') variant('debug', default=False, description='Build debug instead of optimized version') variant('netcdf', default=False, description='Enable Cubit/Genesis reader') variant('conduit', default=False, description='Enable binary data I/O using Conduit') variant('zlib', default=True, description='Support zip\'d streams for I/O') variant('gnutls', default=False, description='Enable secure sockets using GnuTLS') variant('libunwind', default=False, description='Enable backtrace on error support using Libunwind') # TODO: SIMD, Ginkgo, ADIOS2, HiOp, MKL CPardiso, Axom/Sidre variant('timer', default='auto', values=('auto', 'std', 'posix', 'mac', 'mpi'), description='Timing functions to use in mfem::StopWatch') variant('examples', default=False, description='Build and install examples') variant('miniapps', default=False, description='Build and install miniapps') conflicts('+shared', when='@:3.3.2') conflicts('~static~shared') conflicts('~threadsafe', when='@:3+openmp') conflicts('+cuda', when='@:3') conflicts('+rocm', when='@:4.1') conflicts('+cuda+rocm') conflicts('+netcdf', when='@:3.1') conflicts('+superlu-dist', when='@:3.1') # STRUMPACK support was added in mfem v3.3.2, however, here we allow only # strumpack v3+ support for which is available starting with mfem v4.0: conflicts('+strumpack', when='@:3') conflicts('+gnutls', when='@:3.1') conflicts('+zlib', when='@:3.2') conflicts('+mpfr', when='@:3.2') conflicts('+petsc', when='@:3.2') conflicts('+slepc', when='@:4.1') conflicts('+sundials', when='@:3.2') conflicts('+pumi', when='@:3.3.2') conflicts('+gslib', when='@:4.0') conflicts('timer=mac', when='@:3.3.0') conflicts('timer=mpi', when='@:3.3.0') conflicts('~metis+mpi', when='@:3.3.0') conflicts('+metis~mpi', when='@:3.3.0') conflicts('+conduit', when='@:3.3.2') conflicts('+occa', when='mfem@:3') conflicts('+raja', when='mfem@:3') conflicts('+libceed', when='mfem@:4.0') conflicts('+umpire', when='mfem@:4.0') conflicts('+amgx', when='mfem@:4.1') conflicts('+amgx', when='~cuda') conflicts('+mpi~cuda ^hypre+cuda') conflicts('+superlu-dist', when='~mpi') conflicts('+strumpack', when='~mpi') conflicts('+petsc', when='~mpi') conflicts('+slepc', when='~petsc') conflicts('+pumi', when='~mpi') conflicts('timer=mpi', when='~mpi') depends_on('mpi', when='+mpi') depends_on('hypre@2.10.0:2.13', when='@:3.3+mpi') depends_on('hypre@:2.20.0', when='@3.4:4.2+mpi') depends_on('hypre@:2.23.0', when='@4.3.0+mpi') depends_on('hypre', when='+mpi') depends_on('metis', when='+metis') depends_on('blas', when='+lapack') depends_on('lapack@3.0:', when='+lapack') depends_on('sundials@2.7.0', when='@:3.3.0+sundials~mpi') depends_on('sundials@2.7.0+mpi+hypre', when='@:3.3.0+sundials+mpi') depends_on('sundials@2.7.0:', when='@3.3.2:+sundials~mpi') depends_on('sundials@2.7.0:+mpi+hypre', when='@3.3.2:+sundials+mpi') depends_on('sundials@5.0.0:', when='@4.0.1-xsdk:+sundials~mpi') depends_on('sundials@5.0.0:+mpi+hypre', when='@4.0.1-xsdk:+sundials+mpi') for sm_ in CudaPackage.cuda_arch_values: depends_on('sundials@5.4.0:+cuda cuda_arch={0}'.format(sm_), when='@4.2.0:+sundials+cuda cuda_arch={0}'.format(sm_)) depends_on('pumi@2.2.3:', when='@4.2.0:+pumi') depends_on('pumi', when='+pumi~shared') depends_on('pumi+shared', when='+pumi+shared') depends_on('gslib@1.0.5:+mpi', when='+gslib+mpi') depends_on('gslib@1.0.5:~mpi~mpiio', when='+gslib~mpi') depends_on('suite-sparse', when='+suite-sparse') depends_on('superlu-dist', when='+superlu-dist') depends_on('strumpack@3.0.0:', when='+strumpack~shared') depends_on('strumpack@3.0.0:+shared', when='+strumpack+shared') for sm_ in CudaPackage.cuda_arch_values: depends_on('strumpack+cuda cuda_arch={0}'.format(sm_), when='+strumpack+cuda cuda_arch={0}'.format(sm_)) # The PETSc tests in MFEM will fail if PETSc is not configured with # SuiteSparse and MUMPS. On the other hand, if we require the variants # '+suite-sparse+mumps' of PETSc, the xsdk package concretization fails. depends_on('petsc@3.8:+mpi+double+hypre', when='+petsc') depends_on('slepc@3.8.0:', when='+slepc') # Recommended when building outside of xsdk: # depends_on('petsc@3.8:+mpi+double+hypre+suite-sparse+mumps', # when='+petsc') depends_on('mpfr', when='+mpfr') depends_on('netcdf-c@4.1.3:', when='+netcdf') depends_on('unwind', when='+libunwind') depends_on('zlib', when='+zlib') depends_on('gnutls', when='+gnutls') depends_on('conduit@0.3.1:,master:', when='+conduit') depends_on('conduit+mpi', when='+conduit+mpi') # The MFEM 4.0.0 SuperLU interface fails when using hypre@2.16.0 and # superlu-dist@6.1.1. See https://github.com/mfem/mfem/issues/983. # This issue was resolved in v4.1. conflicts('+superlu-dist', when='mfem@:4.0 ^hypre@2.16.0: ^superlu-dist@6:') # The STRUMPACK v3 interface in MFEM seems to be broken as of MFEM v4.1 # when using hypre version >= 2.16.0. # This issue is resolved in v4.2. conflicts('+strumpack', when='mfem@4.0.0:4.1 ^hypre@2.16.0:') conflicts('+strumpack ^strumpack+cuda', when='~cuda') depends_on('occa@1.0.8:', when='@:4.1+occa') depends_on('occa@1.1.0:', when='@4.2.0:+occa') depends_on('occa+cuda', when='+occa+cuda') # TODO: propagate '+rocm' variant to occa when it is supported depends_on('raja@0.10.0:', when='@4.0.1:+raja') depends_on('raja@0.7.0:0.9.0', when='@4.0.0+raja') for sm_ in CudaPackage.cuda_arch_values: depends_on('raja+cuda cuda_arch={0}'.format(sm_), when='+raja+cuda cuda_arch={0}'.format(sm_)) for gfx in ROCmPackage.amdgpu_targets: depends_on('raja+rocm amdgpu_target={0}'.format(gfx), when='+raja+rocm amdgpu_target={0}'.format(gfx)) depends_on('libceed@0.6:', when='@:4.1+libceed') depends_on('libceed@0.7:', when='@4.2.0:+libceed') for sm_ in CudaPackage.cuda_arch_values: depends_on('libceed+cuda cuda_arch={0}'.format(sm_), when='+libceed+cuda cuda_arch={0}'.format(sm_)) for gfx in ROCmPackage.amdgpu_targets: depends_on('libceed+rocm amdgpu_target={0}'.format(gfx), when='+libceed+rocm amdgpu_target={0}'.format(gfx)) depends_on('umpire@2.0.0:', when='+umpire') for sm_ in CudaPackage.cuda_arch_values: depends_on('umpire+cuda cuda_arch={0}'.format(sm_), when='+umpire+cuda cuda_arch={0}'.format(sm_)) for gfx in ROCmPackage.amdgpu_targets: depends_on('umpire+rocm amdgpu_target={0}'.format(gfx), when='+umpire+rocm amdgpu_target={0}'.format(gfx)) # AmgX: propagate the cuda_arch and mpi settings: for sm_ in CudaPackage.cuda_arch_values: depends_on('amgx+mpi cuda_arch={0}'.format(sm_), when='+amgx+mpi cuda_arch={0}'.format(sm_)) depends_on('amgx~mpi cuda_arch={0}'.format(sm_), when='+amgx~mpi cuda_arch={0}'.format(sm_)) patch('mfem_ppc_build.patch', when='@3.2:3.3.0 arch=ppc64le') patch('mfem-3.4.patch', when='@3.4.0') patch('mfem-3.3-3.4-petsc-3.9.patch', when='@3.3.0:3.4.0 +petsc ^petsc@3.9.0:') patch('mfem-4.2-umpire.patch', when='@4.2.0+umpire') patch('mfem-4.2-slepc.patch', when='@4.2.0+slepc') patch('mfem-4.2-petsc-3.15.0.patch', when='@4.2.0+petsc ^petsc@3.15.0:') patch('mfem-4.3-hypre-2.23.0.patch', when='@4.3.0') patch('mfem-4.3-cusparse-11.4.patch', when='@4.3.0+cuda') # Patch to fix MFEM makefile syntax error. See # https://github.com/mfem/mfem/issues/1042 for the bug report and # https://github.com/mfem/mfem/pull/1043 for the bugfix contributed # upstream. patch('mfem-4.0.0-makefile-syntax-fix.patch', when='@4.0.0') phases = ['configure', 'build', 'install'] def setup_build_environment(self, env: EnvironmentModifications) -> None: env.unset('MFEM_DIR') env.unset('MFEM_BUILD_DIR') # # Note: Although MFEM does support CMake configuration, MFEM # development team indicates that vanilla GNU Make is the # preferred mode of configuration of MFEM and the mode most # likely to be up to date in supporting *all* of MFEM's # configuration options. So, don't use CMake # def configure(self, spec, prefix): def yes_no(varstr): return 'YES' if varstr in self.spec else 'NO' # See also find_system_libraries in lib/spack/llnl/util/filesystem.py # where the same list of paths is used. sys_lib_paths = [ '/lib64', '/lib', '/usr/lib64', '/usr/lib', '/usr/local/lib64', '/usr/local/lib'] def is_sys_lib_path(dir): return dir in sys_lib_paths xcompiler = '' xlinker = '-Wl,' if '+cuda' in spec: xcompiler = '-Xcompiler=' xlinker = '-Xlinker=' cuda_arch = None if '~cuda' in spec else spec.variants['cuda_arch'].value # We need to add rpaths explicitly to allow proper export of link flags # from within MFEM. # Similar to spec[pkg].libs.ld_flags but prepends rpath flags too. # Also does not add system library paths as defined by 'sys_lib_paths' # above -- this is done to avoid issues like this: # https://github.com/mfem/mfem/issues/1088. def ld_flags_from_library_list(libs_list): flags = ['%s-rpath,%s' % (xlinker, dir) for dir in libs_list.directories if not is_sys_lib_path(dir)] flags += ['-L%s' % dir for dir in libs_list.directories if not is_sys_lib_path(dir)] flags += [libs_list.link_flags] return ' '.join(flags) def ld_flags_from_dirs(pkg_dirs_list, pkg_libs_list): flags = ['%s-rpath,%s' % (xlinker, dir) for dir in pkg_dirs_list if not is_sys_lib_path(dir)] flags += ['-L%s' % dir for dir in pkg_dirs_list if not is_sys_lib_path(dir)] flags += ['-l%s' % lib for lib in pkg_libs_list] return ' '.join(flags) def find_optional_library(name, prefix): for shared in [True, False]: for path in ['lib64', 'lib']: lib = find_libraries(name, join_path(prefix, path), shared=shared, recursive=False) if lib: return lib return LibraryList([]) # Determine how to run MPI tests, e.g. when using '--test=root', when # Spack is run inside a batch system job. mfem_mpiexec = 'mpirun' mfem_mpiexec_np = '-np' if 'SLURM_JOBID' in os.environ: mfem_mpiexec = 'srun' mfem_mpiexec_np = '-n' elif 'LSB_JOBID' in os.environ: if 'LLNL_COMPUTE_NODES' in os.environ: mfem_mpiexec = 'lrun' mfem_mpiexec_np = '-n' else: mfem_mpiexec = 'jsrun' mfem_mpiexec_np = '-p' metis5_str = 'NO' if ('+metis' in spec) and spec['metis'].satisfies('@5:'): metis5_str = 'YES' zlib_var = 'MFEM_USE_ZLIB' if (spec.satisfies('@4.1.0:')) else \ 'MFEM_USE_GZSTREAM' options = [ 'PREFIX=%s' % prefix, 'MFEM_USE_MEMALLOC=YES', 'MFEM_DEBUG=%s' % yes_no('+debug'), # NOTE: env['CXX'] is the spack c++ compiler wrapper. The real # compiler is defined by env['SPACK_CXX']. 'CXX=%s' % env['CXX'], 'MFEM_USE_LIBUNWIND=%s' % yes_no('+libunwind'), '%s=%s' % (zlib_var, yes_no('+zlib')), 'MFEM_USE_METIS=%s' % yes_no('+metis'), 'MFEM_USE_METIS_5=%s' % metis5_str, 'MFEM_THREAD_SAFE=%s' % yes_no('+threadsafe'), 'MFEM_USE_MPI=%s' % yes_no('+mpi'), 'MFEM_USE_LAPACK=%s' % yes_no('+lapack'), 'MFEM_USE_SUPERLU=%s' % yes_no('+superlu-dist'), 'MFEM_USE_STRUMPACK=%s' % yes_no('+strumpack'), 'MFEM_USE_SUITESPARSE=%s' % yes_no('+suite-sparse'), 'MFEM_USE_SUNDIALS=%s' % yes_no('+sundials'), 'MFEM_USE_PETSC=%s' % yes_no('+petsc'), 'MFEM_USE_SLEPC=%s' % yes_no('+slepc'), 'MFEM_USE_PUMI=%s' % yes_no('+pumi'), 'MFEM_USE_GSLIB=%s' % yes_no('+gslib'), 'MFEM_USE_NETCDF=%s' % yes_no('+netcdf'), 'MFEM_USE_MPFR=%s' % yes_no('+mpfr'), 'MFEM_USE_GNUTLS=%s' % yes_no('+gnutls'), 'MFEM_USE_OPENMP=%s' % yes_no('+openmp'), 'MFEM_USE_CONDUIT=%s' % yes_no('+conduit'), 'MFEM_USE_CUDA=%s' % yes_no('+cuda'), 'MFEM_USE_HIP=%s' % yes_no('+rocm'), 'MFEM_USE_OCCA=%s' % yes_no('+occa'), 'MFEM_USE_RAJA=%s' % yes_no('+raja'), 'MFEM_USE_AMGX=%s' % yes_no('+amgx'), 'MFEM_USE_CEED=%s' % yes_no('+libceed'), 'MFEM_USE_UMPIRE=%s' % yes_no('+umpire'), 'MFEM_MPIEXEC=%s' % mfem_mpiexec, 'MFEM_MPIEXEC_NP=%s' % mfem_mpiexec_np] cxxflags = spec.compiler_flags['cxxflags'] if cxxflags: # Add opt/debug flags if they are not present in global cxx flags opt_flag_found = any(f in self.compiler.opt_flags for f in cxxflags) debug_flag_found = any(f in self.compiler.debug_flags for f in cxxflags) if '+debug' in spec: if not debug_flag_found: cxxflags.append('-g') if not opt_flag_found: cxxflags.append('-O0') else: if not opt_flag_found: cxxflags.append('-O2') cxxflags = [(xcompiler + flag) for flag in cxxflags] if '+cuda' in spec: cxxflags += [ '-x=cu --expt-extended-lambda -arch=sm_%s' % cuda_arch, '-ccbin %s' % (spec['mpi'].mpicxx if '+mpi' in spec else env['CXX'])] if self.spec.satisfies('@4.0.0:'): cxxflags.append(self.compiler.cxx11_flag) # The cxxflags are set by the spack c++ compiler wrapper. We also # set CXXFLAGS explicitly, for clarity, and to properly export the # cxxflags in the variable MFEM_CXXFLAGS in config.mk. options += ['CXXFLAGS=%s' % ' '.join(cxxflags)] if '~static' in spec: options += ['STATIC=NO'] if '+shared' in spec: options += [ 'SHARED=YES', 'PICFLAG=%s' % (xcompiler + self.compiler.cxx_pic_flag)] if '+mpi' in spec: options += ['MPICXX=%s' % spec['mpi'].mpicxx] hypre = spec['hypre'] # The hypre package always links with 'blas' and 'lapack'. all_hypre_libs = hypre.libs + hypre['lapack'].libs + \ hypre['blas'].libs options += [ 'HYPRE_OPT=-I%s' % hypre.prefix.include, 'HYPRE_LIB=%s' % ld_flags_from_library_list(all_hypre_libs)] if '+metis' in spec: options += [ 'METIS_OPT=-I%s' % spec['metis'].prefix.include, 'METIS_LIB=%s' % ld_flags_from_library_list(spec['metis'].libs)] if '+lapack' in spec: lapack_blas = spec['lapack'].libs + spec['blas'].libs options += [ # LAPACK_OPT is not used 'LAPACK_LIB=%s' % ld_flags_from_library_list(lapack_blas)] if '+superlu-dist' in spec: lapack_blas = spec['lapack'].libs + spec['blas'].libs options += [ 'SUPERLU_OPT=-I%s -I%s' % (spec['superlu-dist'].prefix.include, spec['parmetis'].prefix.include), 'SUPERLU_LIB=%s %s' % (ld_flags_from_dirs([spec['superlu-dist'].prefix.lib, spec['parmetis'].prefix.lib], ['superlu_dist', 'parmetis']), ld_flags_from_library_list(lapack_blas))] if '+strumpack' in spec: strumpack = spec['strumpack'] sp_opt = ['-I%s' % strumpack.prefix.include] sp_lib = [ld_flags_from_library_list(strumpack.libs)] # Parts of STRUMPACK use fortran, so we need to link with the # fortran library and also the MPI fortran library: if '~shared' in strumpack: if os.path.basename(env['FC']) == 'gfortran': gfortran = Executable(env['FC']) libext = 'dylib' if sys.platform == 'darwin' else 'so' libfile = os.path.abspath(gfortran( '-print-file-name=libgfortran.%s' % libext, output=str).strip()) gfortran_lib = LibraryList(libfile) sp_lib += [ld_flags_from_library_list(gfortran_lib)] if ('^mpich' in strumpack) or ('^mvapich2' in strumpack): sp_lib += ['-lmpifort'] elif '^openmpi' in strumpack: sp_lib += ['-lmpi_mpifh'] elif '^spectrum-mpi' in strumpack: sp_lib += ['-lmpi_ibm_mpifh'] if '+openmp' in strumpack: # The '+openmp' in the spec means strumpack will TRY to find # OpenMP; if not found, we should not add any flags -- how do # we figure out if strumpack found OpenMP? if not self.spec.satisfies('%apple-clang'): sp_opt += [xcompiler + self.compiler.openmp_flag] if '^parmetis' in strumpack: parmetis = strumpack['parmetis'] sp_opt += [parmetis.headers.cpp_flags] sp_lib += [ld_flags_from_library_list(parmetis.libs)] if '^netlib-scalapack' in strumpack: scalapack = strumpack['scalapack'] sp_opt += ['-I%s' % scalapack.prefix.include] sp_lib += [ld_flags_from_dirs([scalapack.prefix.lib], ['scalapack'])] elif '^scalapack' in strumpack: scalapack = strumpack['scalapack'] sp_opt += [scalapack.headers.cpp_flags] sp_lib += [ld_flags_from_library_list(scalapack.libs)] if '+butterflypack' in strumpack: bp = strumpack['butterflypack'] sp_opt += ['-I%s' % bp.prefix.include] sp_lib += [ld_flags_from_dirs([bp.prefix.lib], ['dbutterflypack', 'zbutterflypack'])] if '+zfp' in strumpack: zfp = strumpack['zfp'] sp_opt += ['-I%s' % zfp.prefix.include] sp_lib += [ld_flags_from_dirs([zfp.prefix.lib], ['zfp'])] if '+cuda' in strumpack: # assuming also ('+cuda' in spec) sp_lib += ['-lcusolver', '-lcublas'] options += [ 'STRUMPACK_OPT=%s' % ' '.join(sp_opt), 'STRUMPACK_LIB=%s' % ' '.join(sp_lib)] if '+suite-sparse' in spec: ss_spec = 'suite-sparse:' + self.suitesparse_components options += [ 'SUITESPARSE_OPT=-I%s' % spec[ss_spec].prefix.include, 'SUITESPARSE_LIB=%s' % ld_flags_from_library_list(spec[ss_spec].libs)] if '+sundials' in spec: sun_spec = 'sundials:' + self.sundials_components options += [ 'SUNDIALS_OPT=%s' % spec[sun_spec].headers.cpp_flags, 'SUNDIALS_LIB=%s' % ld_flags_from_library_list(spec[sun_spec].libs)] if '+petsc' in spec: petsc = spec['petsc'] if '+shared' in petsc: options += [ 'PETSC_OPT=%s' % petsc.headers.cpp_flags, 'PETSC_LIB=%s' % ld_flags_from_library_list(petsc.libs)] else: options += ['PETSC_DIR=%s' % petsc.prefix] if '+slepc' in spec: slepc = spec['slepc'] options += [ 'SLEPC_OPT=%s' % slepc.headers.cpp_flags, 'SLEPC_LIB=%s' % ld_flags_from_library_list(slepc.libs)] if '+pumi' in spec: pumi_libs = ['pumi', 'crv', 'ma', 'mds', 'apf', 'pcu', 'gmi', 'parma', 'lion', 'mth', 'apf_zoltan', 'spr'] options += [ 'PUMI_OPT=-I%s' % spec['pumi'].prefix.include, 'PUMI_LIB=%s' % ld_flags_from_dirs([spec['pumi'].prefix.lib], pumi_libs)] if '+gslib' in spec: options += [ 'GSLIB_OPT=-I%s' % spec['gslib'].prefix.include, 'GSLIB_LIB=%s' % ld_flags_from_dirs([spec['gslib'].prefix.lib], ['gs'])] if '+netcdf' in spec: lib_flags = ld_flags_from_dirs([spec['netcdf-c'].prefix.lib], ['netcdf']) hdf5 = spec['hdf5:hl'] if hdf5.satisfies('~shared'): hdf5_libs = hdf5.libs hdf5_libs += LibraryList(find_system_libraries('libdl')) lib_flags += " " + ld_flags_from_library_list(hdf5_libs) options += [ 'NETCDF_OPT=-I%s' % spec['netcdf-c'].prefix.include, 'NETCDF_LIB=%s' % lib_flags] if '+zlib' in spec: if "@:3.3.2" in spec: options += ['ZLIB_DIR=%s' % spec['zlib'].prefix] else: options += [ 'ZLIB_OPT=-I%s' % spec['zlib'].prefix.include, 'ZLIB_LIB=%s' % ld_flags_from_library_list(spec['zlib'].libs)] if '+mpfr' in spec: options += [ 'MPFR_OPT=-I%s' % spec['mpfr'].prefix.include, 'MPFR_LIB=%s' % ld_flags_from_dirs([spec['mpfr'].prefix.lib], ['mpfr'])] if '+gnutls' in spec: options += [ 'GNUTLS_OPT=-I%s' % spec['gnutls'].prefix.include, 'GNUTLS_LIB=%s' % ld_flags_from_dirs([spec['gnutls'].prefix.lib], ['gnutls'])] if '+libunwind' in spec: libunwind = spec['unwind'] headers = find_headers('libunwind', libunwind.prefix.include) headers.add_macro('-g') libs = find_optional_library('libunwind', libunwind.prefix) # When mfem uses libunwind, it also needs 'libdl'. libs += LibraryList(find_system_libraries('libdl')) options += [ 'LIBUNWIND_OPT=%s' % headers.cpp_flags, 'LIBUNWIND_LIB=%s' % ld_flags_from_library_list(libs)] if '+openmp' in spec: options += [ 'OPENMP_OPT=%s' % (xcompiler + self.compiler.openmp_flag)] if '+cuda' in spec: options += [ 'CUDA_CXX=%s' % join_path(spec['cuda'].prefix, 'bin', 'nvcc'), 'CUDA_ARCH=sm_%s' % cuda_arch] if '+rocm' in spec: amdgpu_target = ','.join(spec.variants['amdgpu_target'].value) options += [ 'HIP_CXX=%s' % spec['hip'].hipcc, 'HIP_ARCH=%s' % amdgpu_target] if '+occa' in spec: options += ['OCCA_OPT=-I%s' % spec['occa'].prefix.include, 'OCCA_LIB=%s' % ld_flags_from_dirs([spec['occa'].prefix.lib], ['occa'])] if '+raja' in spec: options += ['RAJA_OPT=-I%s' % spec['raja'].prefix.include, 'RAJA_LIB=%s' % ld_flags_from_dirs([spec['raja'].prefix.lib], ['RAJA'])] if '+amgx' in spec: amgx = spec['amgx'] if '+shared' in amgx: options += ['AMGX_OPT=-I%s' % amgx.prefix.include, 'AMGX_LIB=%s' % ld_flags_from_library_list(amgx.libs)] else: options += ['AMGX_DIR=%s' % amgx.prefix] if '+libceed' in spec: options += ['CEED_OPT=-I%s' % spec['libceed'].prefix.include, 'CEED_LIB=%s' % ld_flags_from_dirs([spec['libceed'].prefix.lib], ['ceed'])] if '+umpire' in spec: options += ['UMPIRE_OPT=-I%s' % spec['umpire'].prefix.include, 'UMPIRE_LIB=%s' % ld_flags_from_library_list(spec['umpire'].libs)] timer_ids = {'std': '0', 'posix': '2', 'mac': '4', 'mpi': '6'} timer = spec.variants['timer'].value if timer != 'auto': options += ['MFEM_TIMER_TYPE=%s' % timer_ids[timer]] if '+conduit' in spec: conduit = spec['conduit'] headers = HeaderList(find(conduit.prefix.include, 'conduit.hpp', recursive=True)) conduit_libs = ['libconduit', 'libconduit_relay', 'libconduit_blueprint'] libs = find_libraries(conduit_libs, conduit.prefix.lib, shared=('+shared' in conduit)) libs += LibraryList(find_system_libraries('libdl')) if '+hdf5' in conduit: hdf5 = conduit['hdf5'] headers += find_headers('hdf5', hdf5.prefix.include) libs += hdf5.libs ################## # cyrush note: ################## # spack's HeaderList is applying too much magic, undermining us: # # It applies a regex to strip back to the last "include" dir # in the path. In our case we need to pass the following # as part of the CONDUIT_OPT flags: # # -I/include/conduit # # I tried several ways to present this path to the HeaderList, # but the regex always kills the trailing conduit dir # breaking build. # # To resolve the issue, we simply join our own string with # the headers results (which are important b/c they handle # hdf5 paths when enabled). ################## # construct proper include path conduit_include_path = conduit.prefix.include.conduit # add this path to the found flags conduit_opt_flags = "-I{0} {1}".format(conduit_include_path, headers.cpp_flags) options += [ 'CONDUIT_OPT=%s' % conduit_opt_flags, 'CONDUIT_LIB=%s' % ld_flags_from_library_list(libs)] make('config', *options, parallel=False) make('info', parallel=False) def build(self, spec, prefix): make('lib') @run_after('build') def check_or_test(self): # Running 'make check' or 'make test' may fail if MFEM_MPIEXEC or # MFEM_MPIEXEC_NP are not set appropriately. if not self.run_tests: # check we can build ex1 (~mpi) or ex1p (+mpi). make('-C', 'examples', 'ex1p' if ('+mpi' in self.spec) else 'ex1', parallel=False) # make('check', parallel=False) else: make('all') make('test', parallel=False) def install(self, spec, prefix): make('install', parallel=False) # TODO: The way the examples and miniapps are being installed is not # perfect. For example, the makefiles do not work. install_em = ('+examples' in spec) or ('+miniapps' in spec) if install_em and ('+shared' in spec): make('examples/clean', 'miniapps/clean') # This is a hack to get the examples and miniapps to link with the # installed shared mfem library: with working_dir('config'): os.rename('config.mk', 'config.mk.orig') copy(str(self.config_mk), 'config.mk') shutil.copystat('config.mk.orig', 'config.mk') prefix_share = join_path(prefix, 'share', 'mfem') if '+examples' in spec: make('examples') install_tree('examples', join_path(prefix_share, 'examples')) if '+miniapps' in spec: make('miniapps') install_tree('miniapps', join_path(prefix_share, 'miniapps')) if install_em: install_tree('data', join_path(prefix_share, 'data')) examples_src_dir = 'examples' examples_data_dir = 'data' @run_after('install') def cache_test_sources(self): """Copy the example source files after the package is installed to an install test subdirectory for use during `spack test run`.""" cache_extra_test_sources(self, [self.examples_src_dir, self.examples_data_dir]) def test_ex10(self): """build and run ex10(p)""" # MFEM has many examples to serve as a suitable smoke check. ex10 # was chosen arbitrarily among the examples that work both with # MPI and without it test_dir = join_path(self.test_suite.current_test_cache_dir, self.examples_src_dir) mesh = join_path("..", self.examples_data_dir, "beam-quad.mesh") test_exe = "ex10p" if ("+mpi" in self.spec) else "ex10" with working_dir(test_dir): make = which("make") make(f"CONFIG_MK={self.config_mk}", test_exe, "parallel=False") ex10 = which(test_exe) ex10("--mesh", mesh) # this patch is only needed for mfem 4.1, where a few # released files include byte order marks @when('@4.1.0') def patch(self): # Remove the byte order mark since it messes with some compilers files_with_bom = [ 'fem/gslib.hpp', 'fem/gslib.cpp', 'linalg/hiop.hpp', 'miniapps/gslib/field-diff.cpp', 'miniapps/gslib/findpts.cpp', 'miniapps/gslib/pfindpts.cpp'] bom = '\xef\xbb\xbf' if sys.version_info < (3,) else u'\ufeff' for f in files_with_bom: filter_file(bom, '', f) @property def suitesparse_components(self): """Return the SuiteSparse components needed by MFEM.""" ss_comps = 'umfpack,cholmod,colamd,amd,camd,ccolamd,suitesparseconfig' if self.spec.satisfies('@3.2:'): ss_comps = 'klu,btf,' + ss_comps return ss_comps @property def sundials_components(self): """Return the SUNDIALS components needed by MFEM.""" spec = self.spec sun_comps = 'arkode,cvodes,nvecserial,kinsol' if '+mpi' in spec: if spec.satisfies('@4.2:'): sun_comps += ',nvecparallel,nvecmpiplusx' else: sun_comps += ',nvecparhyp,nvecparallel' if '+cuda' in spec and '+cuda' in spec['sundials']: sun_comps += ',nveccuda' return sun_comps @property def headers(self): """Export the main mfem header, mfem.hpp. """ hdrs = HeaderList(find(self.prefix.include, 'mfem.hpp', recursive=False)) return hdrs or None @property def libs(self): """Export the mfem library file. """ libs = find_libraries('libmfem', root=self.prefix.lib, shared=('+shared' in self.spec), recursive=False) return libs or None @property def config_mk(self): """Export the location of the config.mk file. This property can be accessed using spec['mfem'].package.config_mk """ dirs = [self.prefix, self.prefix.share.mfem] for d in dirs: f = join_path(d, 'config.mk') if os.access(f, os.R_OK): return FileList(f) return FileList(find(self.prefix, 'config.mk', recursive=True)) @property def test_mk(self): """Export the location of the test.mk file. This property can be accessed using spec['mfem'].package.test_mk. In version 3.3.2 and newer, the location of test.mk is also defined inside config.mk, variable MFEM_TEST_MK. """ dirs = [self.prefix, self.prefix.share.mfem] for d in dirs: f = join_path(d, 'test.mk') if os.access(f, os.R_OK): return FileList(f) return FileList(find(self.prefix, 'test.mk', recursive=True)) ================================================ FILE: lib/spack/spack/test/data/unparse/py-torch.txt ================================================ # -*- python -*- # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This is an unparser test package. ``py-torch`` was chosen for its complexity and because it has an ``@when`` function that can be removed statically, as well as several decorated @run_after functions that should be preserved. """ import os import sys from spack import * class PyTorch(PythonPackage, CudaPackage): """Tensors and Dynamic neural networks in Python with strong GPU acceleration.""" homepage = "https://pytorch.org/" git = "https://github.com/pytorch/pytorch.git" # Exact set of modules is version- and variant-specific, just attempt to import the # core libraries to ensure that the package was successfully installed. import_modules = ["torch", "torch.autograd", "torch.nn", "torch.utils"] version("master", branch="master", submodules=True) version("1.10.1", tag="v1.10.1", submodules=True) version("1.10.0", tag="v1.10.0", submodules=True) version("1.9.1", tag="v1.9.1", submodules=True) version("1.9.0", tag="v1.9.0", submodules=True) version("1.8.2", tag="v1.8.2", submodules=True) version("1.8.1", tag="v1.8.1", submodules=True) version("1.8.0", tag="v1.8.0", submodules=True) version("1.7.1", tag="v1.7.1", submodules=True) version("1.7.0", tag="v1.7.0", submodules=True) version("1.6.0", tag="v1.6.0", submodules=True) version("1.5.1", tag="v1.5.1", submodules=True) version("1.5.0", tag="v1.5.0", submodules=True) version("1.4.1", tag="v1.4.1", submodules=True) version( "1.4.0", tag="v1.4.0", submodules=True, deprecated=True, submodules_delete=["third_party/fbgemm"], ) version("1.3.1", tag="v1.3.1", submodules=True) version("1.3.0", tag="v1.3.0", submodules=True) version("1.2.0", tag="v1.2.0", submodules=True) version("1.1.0", tag="v1.1.0", submodules=True) version("1.0.1", tag="v1.0.1", submodules=True) version("1.0.0", tag="v1.0.0", submodules=True) version( "0.4.1", tag="v0.4.1", submodules=True, deprecated=True, submodules_delete=["third_party/nervanagpu"], ) version("0.4.0", tag="v0.4.0", submodules=True, deprecated=True) version("0.3.1", tag="v0.3.1", submodules=True, deprecated=True) is_darwin = sys.platform == "darwin" # All options are defined in CMakeLists.txt. # Some are listed in setup.py, but not all. variant("caffe2", default=True, description="Build Caffe2") variant("test", default=False, description="Build C++ test binaries") variant("cuda", default=not is_darwin, description="Use CUDA") variant("rocm", default=False, description="Use ROCm") variant("cudnn", default=not is_darwin, description="Use cuDNN") variant("fbgemm", default=True, description="Use FBGEMM (quantized 8-bit server operators)") variant("kineto", default=True, description="Use Kineto profiling library") variant("magma", default=not is_darwin, description="Use MAGMA") variant("metal", default=is_darwin, description="Use Metal for Caffe2 iOS build") variant("nccl", default=not is_darwin, description="Use NCCL") variant("nnpack", default=True, description="Use NNPACK") variant("numa", default=not is_darwin, description="Use NUMA") variant("numpy", default=True, description="Use NumPy") variant("openmp", default=True, description="Use OpenMP for parallel code") variant("qnnpack", default=True, description="Use QNNPACK (quantized 8-bit operators)") variant("valgrind", default=not is_darwin, description="Use Valgrind") variant("xnnpack", default=True, description="Use XNNPACK") variant("mkldnn", default=True, description="Use MKLDNN") variant("distributed", default=not is_darwin, description="Use distributed") variant("mpi", default=not is_darwin, description="Use MPI for Caffe2") variant("gloo", default=not is_darwin, description="Use Gloo") variant("tensorpipe", default=not is_darwin, description="Use TensorPipe") variant("onnx_ml", default=True, description="Enable traditional ONNX ML API") variant("breakpad", default=True, description="Enable breakpad crash dump library") conflicts("+cuda", when="+rocm") conflicts("+cudnn", when="~cuda") conflicts("+magma", when="~cuda") conflicts("+nccl", when="~cuda~rocm") conflicts("+nccl", when="platform=darwin") conflicts("+numa", when="platform=darwin", msg="Only available on Linux") conflicts("+valgrind", when="platform=darwin", msg="Only available on Linux") conflicts("+mpi", when="~distributed") conflicts("+gloo", when="~distributed") conflicts("+tensorpipe", when="~distributed") conflicts("+kineto", when="@:1.7") conflicts("+valgrind", when="@:1.7") conflicts("~caffe2", when="@0.4.0:1.6") # no way to disable caffe2? conflicts("+caffe2", when="@:0.3.1") # caffe2 did not yet exist? conflicts("+tensorpipe", when="@:1.5") conflicts("+xnnpack", when="@:1.4") conflicts("~onnx_ml", when="@:1.4") # no way to disable ONNX? conflicts("+rocm", when="@:0.4") conflicts("+cudnn", when="@:0.4") conflicts("+fbgemm", when="@:0.4,1.4.0") conflicts("+qnnpack", when="@:0.4") conflicts("+mkldnn", when="@:0.4") conflicts("+breakpad", when="@:1.9") # Option appeared in 1.10.0 conflicts("+breakpad", when="target=ppc64:", msg="Unsupported") conflicts("+breakpad", when="target=ppc64le:", msg="Unsupported") conflicts( "cuda_arch=none", when="+cuda", msg="Must specify CUDA compute capabilities of your GPU, see " "https://developer.nvidia.com/cuda-gpus", ) # Required dependencies depends_on("cmake@3.5:", type="build") # Use Ninja generator to speed up build times, automatically used if found depends_on("ninja@1.5:", when="@1.1.0:", type="build") # See python_min_version in setup.py depends_on("python@3.6.2:", when="@1.7.1:", type=("build", "link", "run")) depends_on("python@3.6.1:", when="@1.6.0:1.7.0", type=("build", "link", "run")) depends_on("python@3.5:", when="@1.5.0:1.5", type=("build", "link", "run")) depends_on("python@2.7:2.8,3.5:", when="@1.4.0:1.4", type=("build", "link", "run")) depends_on("python@2.7:2.8,3.5:3.7", when="@:1.3", type=("build", "link", "run")) depends_on("py-setuptools", type=("build", "run")) depends_on("py-future", when="@1.5:", type=("build", "run")) depends_on("py-future", when="@1.1: ^python@:2", type=("build", "run")) depends_on("py-pyyaml", type=("build", "run")) depends_on("py-typing", when="@0.4: ^python@:3.4", type=("build", "run")) depends_on("py-typing-extensions", when="@1.7:", type=("build", "run")) depends_on("py-pybind11@2.6.2", when="@1.8.0:", type=("build", "link", "run")) depends_on("py-pybind11@2.3.0", when="@1.1.0:1.7", type=("build", "link", "run")) depends_on("py-pybind11@2.2.4", when="@1.0.0:1.0", type=("build", "link", "run")) depends_on("py-pybind11@2.2.2", when="@0.4.0:0.4", type=("build", "link", "run")) depends_on("py-dataclasses", when="@1.7: ^python@3.6.0:3.6", type=("build", "run")) depends_on("py-tqdm", type="run") depends_on("py-protobuf", when="@0.4:", type=("build", "run")) depends_on("protobuf", when="@0.4:") depends_on("blas") depends_on("lapack") depends_on("eigen", when="@0.4:") # https://github.com/pytorch/pytorch/issues/60329 # depends_on('cpuinfo@2020-12-17', when='@1.8.0:') # depends_on('cpuinfo@2020-06-11', when='@1.6.0:1.7') # https://github.com/shibatch/sleef/issues/427 # depends_on('sleef@3.5.1_2020-12-22', when='@1.8.0:') # https://github.com/pytorch/pytorch/issues/60334 # depends_on('sleef@3.4.0_2019-07-30', when='@1.6.0:1.7') # https://github.com/Maratyszcza/FP16/issues/18 # depends_on('fp16@2020-05-14', when='@1.6.0:') depends_on("pthreadpool@2021-04-13", when="@1.9.0:") depends_on("pthreadpool@2020-10-05", when="@1.8.0:1.8") depends_on("pthreadpool@2020-06-15", when="@1.6.0:1.7") depends_on("psimd@2020-05-17", when="@1.6.0:") depends_on("fxdiv@2020-04-17", when="@1.6.0:") depends_on("benchmark", when="@1.6:+test") # Optional dependencies depends_on("cuda@7.5:", when="+cuda", type=("build", "link", "run")) depends_on("cuda@9:", when="@1.1:+cuda", type=("build", "link", "run")) depends_on("cuda@9.2:", when="@1.6:+cuda", type=("build", "link", "run")) depends_on("cudnn@6.0:7", when="@:1.0+cudnn") depends_on("cudnn@7.0:7", when="@1.1.0:1.5+cudnn") depends_on("cudnn@7.0:", when="@1.6.0:+cudnn") depends_on("magma", when="+magma") depends_on("nccl", when="+nccl") depends_on("numactl", when="+numa") depends_on("py-numpy", when="+numpy", type=("build", "run")) depends_on("llvm-openmp", when="%apple-clang +openmp") depends_on("valgrind", when="+valgrind") # https://github.com/pytorch/pytorch/issues/60332 # depends_on('xnnpack@2021-02-22', when='@1.8.0:+xnnpack') # depends_on('xnnpack@2020-03-23', when='@1.6.0:1.7+xnnpack') depends_on("mpi", when="+mpi") # https://github.com/pytorch/pytorch/issues/60270 # depends_on('gloo@2021-05-04', when='@1.9.0:+gloo') # depends_on('gloo@2020-09-18', when='@1.7.0:1.8+gloo') # depends_on('gloo@2020-03-17', when='@1.6.0:1.6+gloo') # https://github.com/pytorch/pytorch/issues/60331 # depends_on('onnx@1.8.0_2020-11-03', when='@1.8.0:+onnx_ml') # depends_on('onnx@1.7.0_2020-05-31', when='@1.6.0:1.7+onnx_ml') depends_on("mkl", when="+mkldnn") # Test dependencies depends_on("py-hypothesis", type="test") depends_on("py-six", type="test") depends_on("py-psutil", type="test") # Fix BLAS being overridden by MKL # https://github.com/pytorch/pytorch/issues/60328 patch( "https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/59220.patch", sha256="e37afffe45cf7594c22050109942370e49983ad772d12ebccf508377dc9dcfc9", when="@1.2.0:", ) # Fixes build on older systems with glibc <2.12 patch( "https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/55063.patch", sha256="e17eaa42f5d7c18bf0d7c37d7b0910127a01ad53fdce3e226a92893356a70395", when="@1.1.0:1.8.1", ) # Fixes CMake configuration error when XNNPACK is disabled # https://github.com/pytorch/pytorch/pull/35607 # https://github.com/pytorch/pytorch/pull/37865 patch("xnnpack.patch", when="@1.5.0:1.5") # Fixes build error when ROCm is enabled for pytorch-1.5 release patch("rocm.patch", when="@1.5.0:1.5+rocm") # Fixes fatal error: sleef.h: No such file or directory # https://github.com/pytorch/pytorch/pull/35359 # https://github.com/pytorch/pytorch/issues/26555 # patch('sleef.patch', when='@1.0.0:1.5') # Fixes compilation with Clang 9.0.0 and Apple Clang 11.0.3 # https://github.com/pytorch/pytorch/pull/37086 patch( "https://github.com/pytorch/pytorch/commit/e921cd222a8fbeabf5a3e74e83e0d8dfb01aa8b5.patch", sha256="17561b16cd2db22f10c0fe1fdcb428aecb0ac3964ba022a41343a6bb8cba7049", when="@1.1:1.5", ) # Removes duplicate definition of getCusparseErrorString # https://github.com/pytorch/pytorch/issues/32083 patch("cusparseGetErrorString.patch", when="@0.4.1:1.0^cuda@10.1.243:") # Fixes 'FindOpenMP.cmake' # to detect openmp settings used by Fujitsu compiler. patch("detect_omp_of_fujitsu_compiler.patch", when="%fj") # Fix compilation of +distributed~tensorpipe # https://github.com/pytorch/pytorch/issues/68002 patch( "https://github.com/pytorch/pytorch/commit/c075f0f633fa0136e68f0a455b5b74d7b500865c.patch", sha256="e69e41b5c171bfb00d1b5d4ee55dd5e4c8975483230274af4ab461acd37e40b8", when="@1.10.0+distributed~tensorpipe", ) # Both build and install run cmake/make/make install # Only run once to speed up build times phases = ["install"] @property def libs(self): root = join_path( self.prefix, self.spec["python"].package.site_packages_dir, "torch", "lib" ) return find_libraries("libtorch", root) @property def headers(self): root = join_path( self.prefix, self.spec["python"].package.site_packages_dir, "torch", "include" ) headers = find_all_headers(root) headers.directories = [root] return headers @when("@1.5.0:") def patch(self): # https://github.com/pytorch/pytorch/issues/52208 filter_file( "torch_global_deps PROPERTIES LINKER_LANGUAGE C", "torch_global_deps PROPERTIES LINKER_LANGUAGE CXX", "caffe2/CMakeLists.txt", ) def setup_build_environment(self, env: EnvironmentModifications) -> None: """Set environment variables used to control the build. PyTorch's ``setup.py`` is a thin wrapper around ``cmake``. In ``tools/setup_helpers/cmake.py``, you can see that all environment variables that start with ``BUILD_``, ``USE_``, or ``CMAKE_``, plus a few more explicitly specified variable names, are passed directly to the ``cmake`` call. Therefore, most flags defined in ``CMakeLists.txt`` can be specified as environment variables. """ def enable_or_disable(variant, keyword="USE", var=None, newer=False): """Set environment variable to enable or disable support for a particular variant. Parameters: variant (str): the variant to check keyword (str): the prefix to use for enabling/disabling var (str): CMake variable to set. Defaults to variant.upper() newer (bool): newer variants that never used NO_* """ if var is None: var = variant.upper() # Version 1.1.0 switched from NO_* to USE_* or BUILD_* # But some newer variants have always used USE_* or BUILD_* if self.spec.satisfies("@1.1:") or newer: if "+" + variant in self.spec: env.set(keyword + "_" + var, "ON") else: env.set(keyword + "_" + var, "OFF") else: if "+" + variant in self.spec: env.unset("NO_" + var) else: env.set("NO_" + var, "ON") # Build in parallel to speed up build times env.set("MAX_JOBS", make_jobs) # Spack logs have trouble handling colored output env.set("COLORIZE_OUTPUT", "OFF") if self.spec.satisfies("@0.4:"): enable_or_disable("test", keyword="BUILD") if self.spec.satisfies("@1.7:"): enable_or_disable("caffe2", keyword="BUILD") enable_or_disable("cuda") if "+cuda" in self.spec: # cmake/public/cuda.cmake # cmake/Modules_CUDA_fix/upstream/FindCUDA.cmake env.unset("CUDA_ROOT") torch_cuda_arch = ";".join( "{0:.1f}".format(float(i) / 10.0) for i in self.spec.variants["cuda_arch"].value ) env.set("TORCH_CUDA_ARCH_LIST", torch_cuda_arch) enable_or_disable("rocm") enable_or_disable("cudnn") if "+cudnn" in self.spec: # cmake/Modules_CUDA_fix/FindCUDNN.cmake env.set("CUDNN_INCLUDE_DIR", self.spec["cudnn"].prefix.include) env.set("CUDNN_LIBRARY", self.spec["cudnn"].libs[0]) enable_or_disable("fbgemm") if self.spec.satisfies("@1.8:"): enable_or_disable("kineto") enable_or_disable("magma") enable_or_disable("metal") if self.spec.satisfies("@1.10:"): enable_or_disable("breakpad") enable_or_disable("nccl") if "+nccl" in self.spec: env.set("NCCL_LIB_DIR", self.spec["nccl"].libs.directories[0]) env.set("NCCL_INCLUDE_DIR", self.spec["nccl"].prefix.include) # cmake/External/nnpack.cmake enable_or_disable("nnpack") enable_or_disable("numa") if "+numa" in self.spec: # cmake/Modules/FindNuma.cmake env.set("NUMA_ROOT_DIR", self.spec["numactl"].prefix) # cmake/Modules/FindNumPy.cmake enable_or_disable("numpy") # cmake/Modules/FindOpenMP.cmake enable_or_disable("openmp", newer=True) enable_or_disable("qnnpack") if self.spec.satisfies("@1.3:"): enable_or_disable("qnnpack", var="PYTORCH_QNNPACK") if self.spec.satisfies("@1.8:"): enable_or_disable("valgrind") if self.spec.satisfies("@1.5:"): enable_or_disable("xnnpack") enable_or_disable("mkldnn") enable_or_disable("distributed") enable_or_disable("mpi") # cmake/Modules/FindGloo.cmake enable_or_disable("gloo", newer=True) if self.spec.satisfies("@1.6:"): enable_or_disable("tensorpipe") if "+onnx_ml" in self.spec: env.set("ONNX_ML", "ON") else: env.set("ONNX_ML", "OFF") if not self.spec.satisfies("@master"): env.set("PYTORCH_BUILD_VERSION", self.version) env.set("PYTORCH_BUILD_NUMBER", 0) # BLAS to be used by Caffe2 # Options defined in cmake/Dependencies.cmake and cmake/Modules/FindBLAS.cmake if self.spec["blas"].name == "atlas": env.set("BLAS", "ATLAS") env.set("WITH_BLAS", "atlas") elif self.spec["blas"].name in ["blis", "amdblis"]: env.set("BLAS", "BLIS") env.set("WITH_BLAS", "blis") elif self.spec["blas"].name == "eigen": env.set("BLAS", "Eigen") elif self.spec["lapack"].name in ["libflame", "amdlibflame"]: env.set("BLAS", "FLAME") env.set("WITH_BLAS", "FLAME") elif self.spec["blas"].name in ["intel-mkl", "intel-parallel-studio", "intel-oneapi-mkl"]: env.set("BLAS", "MKL") env.set("WITH_BLAS", "mkl") elif self.spec["blas"].name == "openblas": env.set("BLAS", "OpenBLAS") env.set("WITH_BLAS", "open") elif self.spec["blas"].name == "veclibfort": env.set("BLAS", "vecLib") env.set("WITH_BLAS", "veclib") else: env.set("BLAS", "Generic") env.set("WITH_BLAS", "generic") # Don't use vendored third-party libraries when possible env.set("BUILD_CUSTOM_PROTOBUF", "OFF") env.set("USE_SYSTEM_NCCL", "ON") env.set("USE_SYSTEM_EIGEN_INSTALL", "ON") if self.spec.satisfies("@0.4:"): env.set("pybind11_DIR", self.spec["py-pybind11"].prefix) env.set("pybind11_INCLUDE_DIR", self.spec["py-pybind11"].prefix.include) if self.spec.satisfies("@1.10:"): env.set("USE_SYSTEM_PYBIND11", "ON") # https://github.com/pytorch/pytorch/issues/60334 # if self.spec.satisfies('@1.8:'): # env.set('USE_SYSTEM_SLEEF', 'ON') if self.spec.satisfies("@1.6:"): # env.set('USE_SYSTEM_LIBS', 'ON') # https://github.com/pytorch/pytorch/issues/60329 # env.set('USE_SYSTEM_CPUINFO', 'ON') # https://github.com/pytorch/pytorch/issues/60270 # env.set('USE_SYSTEM_GLOO', 'ON') # https://github.com/Maratyszcza/FP16/issues/18 # env.set('USE_SYSTEM_FP16', 'ON') env.set("USE_SYSTEM_PTHREADPOOL", "ON") env.set("USE_SYSTEM_PSIMD", "ON") env.set("USE_SYSTEM_FXDIV", "ON") env.set("USE_SYSTEM_BENCHMARK", "ON") # https://github.com/pytorch/pytorch/issues/60331 # env.set('USE_SYSTEM_ONNX', 'ON') # https://github.com/pytorch/pytorch/issues/60332 # env.set('USE_SYSTEM_XNNPACK', 'ON') @run_before("install") def build_amd(self): if "+rocm" in self.spec: python(os.path.join("tools", "amd_build", "build_amd.py")) @run_after("install") @on_package_attributes(run_tests=True) def install_test(self): with working_dir("test"): python("run_test.py") # Tests need to be re-added since `phases` was overridden run_after("install")(PythonPackage._run_default_install_time_test_callbacks) run_after("install")(PythonPackage.sanity_check_prefix) ================================================ FILE: lib/spack/spack/test/data/unparse/trilinos.txt ================================================ # -*- python -*- # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This is an unparser test package. ``trilinos`` was chosen because it's one of the most complex packages in Spack, because it has a lot of nested ``with when():`` blocks, and because it has loops and nested logic at the package level. """ import os import sys from spack import * from spack.build_environment import dso_suffix from spack.error import NoHeadersError from spack.operating_systems.mac_os import macos_version from spack.pkg.builtin.kokkos import Kokkos # Trilinos is complicated to build, as an inspiration a couple of links to # other repositories which build it: # https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/t/trilinos.py#L111 # https://github.com/koecher/candi/blob/master/deal.II-toolchain/packages/trilinos.package # https://gitlab.com/configurations/cluster-config/blob/master/trilinos.sh # https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb and some # relevant documentation/examples: # https://github.com/trilinos/Trilinos/issues/175 class Trilinos(CMakePackage, CudaPackage): """The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. A unique design feature of Trilinos is its focus on packages. """ homepage = "https://trilinos.org/" url = "https://github.com/trilinos/Trilinos/archive/trilinos-release-12-12-1.tar.gz" git = "https://github.com/trilinos/Trilinos.git" maintainers("keitat", "sethrj", "kuberry") tags = ["e4s"] # ###################### Versions ########################## version("master", branch="master") version("develop", branch="develop") version( "13.2.0", commit="4a5f7906a6420ee2f9450367e9cc95b28c00d744" ) # tag trilinos-release-13-2-0 version( "13.0.1", commit="4796b92fb0644ba8c531dd9953e7a4878b05c62d", preferred=True ) # tag trilinos-release-13-0-1 version( "13.0.0", commit="9fec35276d846a667bc668ff4cbdfd8be0dfea08" ) # tag trilinos-release-13-0-0 version( "12.18.1", commit="55a75997332636a28afc9db1aee4ae46fe8d93e7" ) # tag trilinos-release-12-8-1 version("12.14.1", sha256="52a4406cca2241f5eea8e166c2950471dd9478ad6741cbb2a7fc8225814616f0") version("12.12.1", sha256="5474c5329c6309224a7e1726cf6f0d855025b2042959e4e2be2748bd6bb49e18") version("12.10.1", sha256="ab81d917196ffbc21c4927d42df079dd94c83c1a08bda43fef2dd34d0c1a5512") version("12.8.1", sha256="d20fe60e31e3ba1ef36edecd88226240a518f50a4d6edcc195b88ee9dda5b4a1") version("12.6.4", sha256="1c7104ba60ee8cc4ec0458a1c4f6a26130616bae7580a7b15f2771a955818b73") version("12.6.3", sha256="4d28298bb4074eef522db6cd1626f1a934e3d80f292caf669b8846c0a458fe81") version("12.6.2", sha256="8be7e3e1166cc05aea7f856cc8033182e8114aeb8f87184cb38873bfb2061779") version("12.6.1", sha256="4b38ede471bed0036dcb81a116fba8194f7bf1a9330da4e29c3eb507d2db18db") version("12.4.2", sha256="fd2c12e87a7cedc058bcb8357107ffa2474997aa7b17b8e37225a1f7c32e6f0e") version("12.2.1", sha256="088f303e0dc00fb4072b895c6ecb4e2a3ad9a2687b9c62153de05832cf242098") version("12.0.1", sha256="eee7c19ca108538fa1c77a6651b084e06f59d7c3307dae77144136639ab55980") version("11.14.3", sha256="e37fa5f69103576c89300e14d43ba77ad75998a54731008b25890d39892e6e60") version("11.14.2", sha256="f22b2b0df7b88e28b992e19044ba72b845292b93cbbb3a948488199647381119") version("11.14.1", sha256="f10fc0a496bf49427eb6871c80816d6e26822a39177d850cc62cf1484e4eec07") # ###################### Variants ########################## # Build options variant("complex", default=False, description="Enable complex numbers in Trilinos") variant("cuda_rdc", default=False, description="turn on RDC for CUDA build") variant("cxxstd", default="14", values=["11", "14", "17"], multi=False) variant("debug", default=False, description="Enable runtime safety and debug checks") variant( "explicit_template_instantiation", default=True, description="Enable explicit template instantiation (ETI)", ) variant( "float", default=False, description="Enable single precision (float) numbers in Trilinos" ) variant("fortran", default=True, description="Compile with Fortran support") variant( "gotype", default="long_long", values=("int", "long", "long_long", "all"), multi=False, description="global ordinal type for Tpetra", ) variant("openmp", default=False, description="Enable OpenMP") variant("python", default=False, description="Build PyTrilinos wrappers") variant("shared", default=True, description="Enables the build of shared libraries") variant("wrapper", default=False, description="Use nvcc-wrapper for CUDA build") # TPLs (alphabet order) variant("adios2", default=False, description="Enable ADIOS2") variant("boost", default=False, description="Compile with Boost") variant("hdf5", default=False, description="Compile with HDF5") variant("hypre", default=False, description="Compile with Hypre preconditioner") variant("mpi", default=True, description="Compile with MPI parallelism") variant("mumps", default=False, description="Compile with support for MUMPS solvers") variant("suite-sparse", default=False, description="Compile with SuiteSparse solvers") variant("superlu-dist", default=False, description="Compile with SuperluDist solvers") variant("superlu", default=False, description="Compile with SuperLU solvers") variant("strumpack", default=False, description="Compile with STRUMPACK solvers") variant("x11", default=False, description="Compile with X11 when +exodus") # Package options (alphabet order) variant("amesos", default=True, description="Compile with Amesos") variant("amesos2", default=True, description="Compile with Amesos2") variant("anasazi", default=True, description="Compile with Anasazi") variant("aztec", default=True, description="Compile with Aztec") variant("belos", default=True, description="Compile with Belos") variant("chaco", default=False, description="Compile with Chaco from SEACAS") variant("epetra", default=True, description="Compile with Epetra") variant("epetraext", default=True, description="Compile with EpetraExt") variant("exodus", default=False, description="Compile with Exodus from SEACAS") variant("ifpack", default=True, description="Compile with Ifpack") variant("ifpack2", default=True, description="Compile with Ifpack2") variant("intrepid", default=False, description="Enable Intrepid") variant("intrepid2", default=False, description="Enable Intrepid2") variant("isorropia", default=False, description="Compile with Isorropia") variant("gtest", default=False, description="Build vendored Googletest") variant("kokkos", default=True, description="Compile with Kokkos") variant("ml", default=True, description="Compile with ML") variant("minitensor", default=False, description="Compile with MiniTensor") variant("muelu", default=True, description="Compile with Muelu") variant("nox", default=False, description="Compile with NOX") variant("piro", default=False, description="Compile with Piro") variant("phalanx", default=False, description="Compile with Phalanx") variant("rol", default=False, description="Compile with ROL") variant("rythmos", default=False, description="Compile with Rythmos") variant("sacado", default=True, description="Compile with Sacado") variant("stk", default=False, description="Compile with STK") variant("shards", default=False, description="Compile with Shards") variant("shylu", default=False, description="Compile with ShyLU") variant("stokhos", default=False, description="Compile with Stokhos") variant("stratimikos", default=False, description="Compile with Stratimikos") variant("teko", default=False, description="Compile with Teko") variant("tempus", default=False, description="Compile with Tempus") variant("tpetra", default=True, description="Compile with Tpetra") variant("trilinoscouplings", default=False, description="Compile with TrilinosCouplings") variant("zoltan", default=False, description="Compile with Zoltan") variant("zoltan2", default=False, description="Compile with Zoltan2") # Internal package options (alphabetical order) variant("basker", default=False, description="Compile with the Basker solver in Amesos2") variant("epetraextbtf", default=False, description="Compile with BTF in EpetraExt") variant( "epetraextexperimental", default=False, description="Compile with experimental in EpetraExt", ) variant( "epetraextgraphreorderings", default=False, description="Compile with graph reorderings in EpetraExt", ) # External package options variant("dtk", default=False, description="Enable DataTransferKit (deprecated)") variant("scorec", default=False, description="Enable SCOREC") variant("mesquite", default=False, description="Enable Mesquite (deprecated)") resource( name="dtk", git="https://github.com/ornl-cees/DataTransferKit.git", commit="4fe4d9d56cfd4f8a61f392b81d8efd0e389ee764", # branch dtk-3.0 placement="DataTransferKit", when="+dtk @12.14.0:12.14", ) resource( name="dtk", git="https://github.com/ornl-cees/DataTransferKit.git", commit="edfa050cd46e2274ab0a0b7558caca0079c2e4ca", # tag 3.1-rc1 placement="DataTransferKit", submodules=True, when="+dtk @12.18.0:12.18", ) resource( name="scorec", git="https://github.com/SCOREC/core.git", commit="73c16eae073b179e45ec625a5abe4915bc589af2", # tag v2.2.5 placement="SCOREC", when="+scorec", ) resource( name="mesquite", url="https://github.com/trilinos/mesquite/archive/trilinos-release-12-12-1.tar.gz", sha256="e0d09b0939dbd461822477449dca611417316e8e8d8268fd795debb068edcbb5", placement="packages/mesquite", when="+mesquite @12.12.1:12.16", ) resource( name="mesquite", git="https://github.com/trilinos/mesquite.git", commit="20a679679b5cdf15bf573d66c5dc2b016e8b9ca1", # branch trilinos-release-12-12-1 placement="packages/mesquite", when="+mesquite @12.18.1:12.18", ) resource( name="mesquite", git="https://github.com/trilinos/mesquite.git", tag="develop", placement="packages/mesquite", when="+mesquite @master", ) # ###################### Conflicts ########################## # Epetra packages with when("~epetra"): conflicts("+amesos") conflicts("+aztec") conflicts("+epetraext") conflicts("+ifpack") conflicts("+isorropia") conflicts("+ml", when="@13.2:") with when("~epetraext"): conflicts("+isorropia") conflicts("+teko") conflicts("+epetraextbtf") conflicts("+epetraextexperimental") conflicts("+epetraextgraphreorderings") # Tpetra packages with when("~kokkos"): conflicts("+cuda") conflicts("+tpetra") conflicts("+intrepid2") conflicts("+phalanx") with when("~tpetra"): conflicts("+amesos2") conflicts("+dtk") conflicts("+ifpack2") conflicts("+muelu") conflicts("+teko") conflicts("+zoltan2") with when("+teko"): conflicts("~amesos") conflicts("~anasazi") conflicts("~aztec") conflicts("~ifpack") conflicts("~ml") conflicts("~stratimikos") conflicts("@:12 gotype=long") # Known requirements from tribits dependencies conflicts("+aztec", when="~fortran") conflicts("+basker", when="~amesos2") conflicts("+minitensor", when="~boost") conflicts("+ifpack2", when="~belos") conflicts("+intrepid", when="~sacado") conflicts("+intrepid", when="~shards") conflicts("+intrepid2", when="~shards") conflicts("+isorropia", when="~zoltan") conflicts("+phalanx", when="~sacado") conflicts("+scorec", when="~mpi") conflicts("+scorec", when="~shards") conflicts("+scorec", when="~stk") conflicts("+scorec", when="~zoltan") conflicts("+tempus", when="~nox") conflicts("+zoltan2", when="~zoltan") # Only allow DTK with Trilinos 12.14, 12.18 conflicts("+dtk", when="~boost") conflicts("+dtk", when="~intrepid2") conflicts("+dtk", when="@:12.12,13:") # Installed FindTrilinos are broken in SEACAS if Fortran is disabled # see https://github.com/trilinos/Trilinos/issues/3346 conflicts("+exodus", when="@:13.0.1 ~fortran") # Only allow Mesquite with Trilinos 12.12 and up, and master conflicts("+mesquite", when="@:12.10,master") # Strumpack is only available as of mid-2021 conflicts("+strumpack", when="@:13.0") # Can only use one type of SuperLU conflicts("+superlu-dist", when="+superlu") # For Trilinos v11 we need to force SuperLUDist=OFF, since only the # deprecated SuperLUDist v3.3 together with an Amesos patch is working. conflicts("+superlu-dist", when="@11.4.1:11.14.3") # see https://github.com/trilinos/Trilinos/issues/3566 conflicts( "+superlu-dist", when="+float+amesos2+explicit_template_instantiation^superlu-dist@5.3.0:" ) # Amesos, conflicting types of double and complex SLU_D # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html conflicts("+superlu-dist", when="+complex+amesos2") # https://github.com/trilinos/Trilinos/issues/2994 conflicts( "+shared", when="+stk platform=darwin", msg="Cannot build Trilinos with STK as a shared library on Darwin.", ) conflicts("+adios2", when="@:12.14.1") conflicts("cxxstd=11", when="@master:") conflicts("cxxstd=11", when="+wrapper ^cuda@6.5.14") conflicts("cxxstd=14", when="+wrapper ^cuda@6.5.14:8.0.61") conflicts("cxxstd=17", when="+wrapper ^cuda@6.5.14:10.2.89") # Multi-value gotype only applies to trilinos through 12.14 conflicts("gotype=all", when="@12.15:") # CUDA without wrapper requires clang for _compiler in spack.compilers.supported_compilers(): if _compiler != "clang": conflicts( "+cuda", when="~wrapper %" + _compiler, msg="trilinos~wrapper+cuda can only be built with the " "Clang compiler", ) conflicts("+cuda_rdc", when="~cuda") conflicts("+wrapper", when="~cuda") conflicts("+wrapper", when="%clang") # Old trilinos fails with new CUDA (see #27180) conflicts("@:13.0.1 +cuda", when="^cuda@11:") # stokhos fails on xl/xl_r conflicts("+stokhos", when="%xl") conflicts("+stokhos", when="%xl_r") # Fortran mangling fails on Apple M1 (see spack/spack#25900) conflicts("@:13.0.1 +fortran", when="target=m1") # ###################### Dependencies ########################## depends_on("adios2", when="+adios2") depends_on("blas") depends_on("boost", when="+boost") depends_on("cgns", when="+exodus") depends_on("hdf5+hl", when="+hdf5") depends_on("hypre~internal-superlu~int64", when="+hypre") depends_on("kokkos-nvcc-wrapper", when="+wrapper") depends_on("lapack") # depends_on('perl', type=('build',)) # TriBITS finds but doesn't use... depends_on("libx11", when="+x11") depends_on("matio", when="+exodus") depends_on("metis", when="+zoltan") depends_on("mpi", when="+mpi") depends_on("netcdf-c", when="+exodus") depends_on("parallel-netcdf", when="+exodus+mpi") depends_on("parmetis", when="+mpi +zoltan") depends_on("parmetis", when="+scorec") depends_on("py-mpi4py", when="+mpi+python", type=("build", "run")) depends_on("py-numpy", when="+python", type=("build", "run")) depends_on("python", when="+python") depends_on("python", when="@13.2: +ifpack +hypre", type="build") depends_on("python", when="@13.2: +ifpack2 +hypre", type="build") depends_on("scalapack", when="+mumps") depends_on("scalapack", when="+strumpack+mpi") depends_on("strumpack+shared", when="+strumpack") depends_on("suite-sparse", when="+suite-sparse") depends_on("superlu-dist", when="+superlu-dist") depends_on("superlu@4.3 +pic", when="+superlu") depends_on("swig", when="+python") depends_on("zlib", when="+zoltan") # Trilinos' Tribits config system is limited which makes it very tricky to # link Amesos with static MUMPS, see # https://trilinos.org/docs/dev/packages/amesos2/doc/html/classAmesos2_1_1MUMPS.html # One could work it out by getting linking flags from mpif90 --showme:link # (or alike) and adding results to -DTrilinos_EXTRA_LINK_FLAGS together # with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and it may # work at the end. But let's avoid all this by simply using shared libs depends_on("mumps@5.0:+shared", when="+mumps") for _flag in ("~mpi", "+mpi"): depends_on("hdf5" + _flag, when="+hdf5" + _flag) depends_on("mumps" + _flag, when="+mumps" + _flag) for _flag in ("~openmp", "+openmp"): depends_on("mumps" + _flag, when="+mumps" + _flag) depends_on("hwloc", when="@13: +kokkos") depends_on("hwloc+cuda", when="@13: +kokkos+cuda") depends_on("hypre@develop", when="@master: +hypre") depends_on("netcdf-c+mpi+parallel-netcdf", when="+exodus+mpi@12.12.1:") depends_on("superlu-dist@4.4:5.3", when="@12.6.2:12.12.1+superlu-dist") depends_on("superlu-dist@5.4:6.2.0", when="@12.12.2:13.0.0+superlu-dist") depends_on("superlu-dist@6.3.0:", when="@13.0.1:99 +superlu-dist") depends_on("superlu-dist@:4.3", when="@11.14.1:12.6.1+superlu-dist") depends_on("superlu-dist@develop", when="@master: +superlu-dist") # ###################### Patches ########################## patch("umfpack_from_suitesparse.patch", when="@11.14.1:12.8.1") for _compiler in ["xl", "xl_r", "clang"]: patch("xlf_seacas.patch", when="@12.10.1:12.12.1 %" + _compiler) patch("xlf_tpetra.patch", when="@12.12.1 %" + _compiler) patch("fix_clang_errors_12_18_1.patch", when="@12.18.1%clang") patch("cray_secas_12_12_1.patch", when="@12.12.1%cce") patch("cray_secas.patch", when="@12.14.1:%cce") # workaround an NVCC bug with c++14 (https://github.com/trilinos/Trilinos/issues/6954) # avoid calling deprecated functions with CUDA-11 patch("fix_cxx14_cuda11.patch", when="@13.0.0:13.0.1 cxxstd=14 ^cuda@11:") # Allow building with +teko gotype=long patch( "https://github.com/trilinos/Trilinos/commit/b17f20a0b91e0b9fc5b1b0af3c8a34e2a4874f3f.patch", sha256="dee6c55fe38eb7f6367e1896d6bc7483f6f9ab8fa252503050cc0c68c6340610", when="@13.0.0:13.0.1 +teko gotype=long", ) def flag_handler(self, name, flags): is_cce = self.spec.satisfies("%cce") if name == "cxxflags": spec = self.spec if "+mumps" in spec: # see https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS flags.append("-DMUMPS_5_0") if "+stk platform=darwin" in spec: flags.append("-DSTK_NO_BOOST_STACKTRACE") if "+stk%intel" in spec: # Workaround for Intel compiler segfaults with STK and IPO flags.append("-no-ipo") if "+wrapper" in spec: flags.append("--expt-extended-lambda") elif name == "ldflags" and is_cce: flags.append("-fuse-ld=gold") if is_cce: return (None, None, flags) return (flags, None, None) def url_for_version(self, version): url = "https://github.com/trilinos/Trilinos/archive/trilinos-release-{0}.tar.gz" return url.format(version.dashed) def setup_dependent_run_environment(self, env: EnvironmentModifications, dependent_spec: Spec) -> None: if "+cuda" in self.spec: # currently Trilinos doesn't perform the memory fence so # it relies on blocking CUDA kernel launch. This is needed # in case the dependent app also run a CUDA backend via Trilinos env.set("CUDA_LAUNCH_BLOCKING", "1") def setup_dependent_package(self, module, dependent_spec): if "+wrapper" in self.spec: self.spec.kokkos_cxx = self.spec["kokkos-nvcc-wrapper"].kokkos_cxx else: self.spec.kokkos_cxx = spack_cxx def setup_build_environment(self, env: EnvironmentModifications) -> None: spec = self.spec if "+cuda" in spec and "+wrapper" in spec: if "+mpi" in spec: env.set("OMPI_CXX", spec["kokkos-nvcc-wrapper"].kokkos_cxx) env.set("MPICH_CXX", spec["kokkos-nvcc-wrapper"].kokkos_cxx) env.set("MPICXX_CXX", spec["kokkos-nvcc-wrapper"].kokkos_cxx) else: env.set("CXX", spec["kokkos-nvcc-wrapper"].kokkos_cxx) def cmake_args(self): options = [] spec = self.spec define = CMakePackage.define define_from_variant = self.define_from_variant def _make_definer(prefix): def define_enable(suffix, value=None): key = prefix + suffix if value is None: # Default to lower-case spec value = suffix.lower() elif isinstance(value, bool): # Explicit true/false return define(key, value) return define_from_variant(key, value) return define_enable # Return "Trilinos_ENABLE_XXX" for spec "+xxx" or boolean value define_trilinos_enable = _make_definer("Trilinos_ENABLE_") # Same but for TPLs define_tpl_enable = _make_definer("TPL_ENABLE_") # #################### Base Settings ####################### options.extend( [ define("Trilinos_VERBOSE_CONFIGURE", False), define_from_variant("BUILD_SHARED_LIBS", "shared"), define_from_variant("CMAKE_CXX_STANDARD", "cxxstd"), define_trilinos_enable("ALL_OPTIONAL_PACKAGES", False), define_trilinos_enable("ALL_PACKAGES", False), define_trilinos_enable("CXX11", True), define_trilinos_enable("DEBUG", "debug"), define_trilinos_enable("EXAMPLES", False), define_trilinos_enable("SECONDARY_TESTED_CODE", True), define_trilinos_enable("TESTS", False), define_trilinos_enable("Fortran"), define_trilinos_enable("OpenMP"), define_trilinos_enable( "EXPLICIT_INSTANTIATION", "explicit_template_instantiation" ), ] ) # ################## Trilinos Packages ##################### options.extend( [ define_trilinos_enable("Amesos"), define_trilinos_enable("Amesos2"), define_trilinos_enable("Anasazi"), define_trilinos_enable("AztecOO", "aztec"), define_trilinos_enable("Belos"), define_trilinos_enable("Epetra"), define_trilinos_enable("EpetraExt"), define_trilinos_enable("FEI", False), define_trilinos_enable("Gtest"), define_trilinos_enable("Ifpack"), define_trilinos_enable("Ifpack2"), define_trilinos_enable("Intrepid"), define_trilinos_enable("Intrepid2"), define_trilinos_enable("Isorropia"), define_trilinos_enable("Kokkos"), define_trilinos_enable("MiniTensor"), define_trilinos_enable("Mesquite"), define_trilinos_enable("ML"), define_trilinos_enable("MueLu"), define_trilinos_enable("NOX"), define_trilinos_enable("Pamgen", False), define_trilinos_enable("Panzer", False), define_trilinos_enable("Pike", False), define_trilinos_enable("Piro"), define_trilinos_enable("Phalanx"), define_trilinos_enable("PyTrilinos", "python"), define_trilinos_enable("ROL"), define_trilinos_enable("Rythmos"), define_trilinos_enable("Sacado"), define_trilinos_enable("SCOREC"), define_trilinos_enable("Shards"), define_trilinos_enable("ShyLU"), define_trilinos_enable("STK"), define_trilinos_enable("Stokhos"), define_trilinos_enable("Stratimikos"), define_trilinos_enable("Teko"), define_trilinos_enable("Tempus"), define_trilinos_enable("Tpetra"), define_trilinos_enable("TrilinosCouplings"), define_trilinos_enable("Zoltan"), define_trilinos_enable("Zoltan2"), define_tpl_enable("Cholmod", False), define_from_variant("EpetraExt_BUILD_BTF", "epetraextbtf"), define_from_variant("EpetraExt_BUILD_EXPERIMENTAL", "epetraextexperimental"), define_from_variant( "EpetraExt_BUILD_GRAPH_REORDERINGS", "epetraextgraphreorderings" ), define_from_variant("Amesos2_ENABLE_Basker", "basker"), ] ) if "+dtk" in spec: options.extend( [ define("Trilinos_EXTRA_REPOSITORIES", "DataTransferKit"), define_trilinos_enable("DataTransferKit", True), ] ) if "+exodus" in spec: options.extend( [ define_trilinos_enable("SEACAS", True), define_trilinos_enable("SEACASExodus", True), define_trilinos_enable("SEACASIoss", True), define_trilinos_enable("SEACASEpu", True), define_trilinos_enable("SEACASExodiff", True), define_trilinos_enable("SEACASNemspread", True), define_trilinos_enable("SEACASNemslice", True), ] ) else: options.extend( [ define_trilinos_enable("SEACASExodus", False), define_trilinos_enable("SEACASIoss", False), ] ) if "+chaco" in spec: options.extend( [ define_trilinos_enable("SEACAS", True), define_trilinos_enable("SEACASChaco", True), ] ) else: # don't disable SEACAS, could be needed elsewhere options.extend( [ define_trilinos_enable("SEACASChaco", False), define_trilinos_enable("SEACASNemslice", False), ] ) if "+stratimikos" in spec: # Explicitly enable Thyra (ThyraCore is required). If you don't do # this, then you get "NOT setting ${pkg}_ENABLE_Thyra=ON since # Thyra is NOT enabled at this point!" leading to eventual build # errors if using MueLu because `Xpetra_ENABLE_Thyra` is set to # off. options.append(define_trilinos_enable("Thyra", True)) # Add thyra adapters based on package enables options.extend( define_trilinos_enable("Thyra" + pkg + "Adapters", pkg.lower()) for pkg in ["Epetra", "EpetraExt", "Tpetra"] ) # ######################### TPLs ############################# def define_tpl(trilinos_name, spack_name, have_dep): options.append(define("TPL_ENABLE_" + trilinos_name, have_dep)) if not have_dep: return depspec = spec[spack_name] libs = depspec.libs try: options.extend( [define(trilinos_name + "_INCLUDE_DIRS", depspec.headers.directories)] ) except NoHeadersError: # Handle case were depspec does not have headers pass options.extend( [ define(trilinos_name + "_ROOT", depspec.prefix), define(trilinos_name + "_LIBRARY_NAMES", libs.names), define(trilinos_name + "_LIBRARY_DIRS", libs.directories), ] ) # Enable these TPLs explicitly from variant options. # Format is (TPL name, variant name, Spack spec name) tpl_variant_map = [ ("ADIOS2", "adios2", "adios2"), ("Boost", "boost", "boost"), ("CUDA", "cuda", "cuda"), ("HDF5", "hdf5", "hdf5"), ("HYPRE", "hypre", "hypre"), ("MUMPS", "mumps", "mumps"), ("UMFPACK", "suite-sparse", "suite-sparse"), ("SuperLU", "superlu", "superlu"), ("SuperLUDist", "superlu-dist", "superlu-dist"), ("X11", "x11", "libx11"), ] if spec.satisfies("@13.0.2:"): tpl_variant_map.append(("STRUMPACK", "strumpack", "strumpack")) for tpl_name, var_name, spec_name in tpl_variant_map: define_tpl(tpl_name, spec_name, spec.variants[var_name].value) # Enable these TPLs based on whether they're in our spec; prefer to # require this way so that packages/features disable availability tpl_dep_map = [ ("BLAS", "blas"), ("CGNS", "cgns"), ("LAPACK", "lapack"), ("Matio", "matio"), ("METIS", "metis"), ("Netcdf", "netcdf-c"), ("SCALAPACK", "scalapack"), ("Zlib", "zlib"), ] if spec.satisfies("@12.12.1:"): tpl_dep_map.append(("Pnetcdf", "parallel-netcdf")) if spec.satisfies("@13:"): tpl_dep_map.append(("HWLOC", "hwloc")) for tpl_name, dep_name in tpl_dep_map: define_tpl(tpl_name, dep_name, dep_name in spec) # MPI settings options.append(define_tpl_enable("MPI")) if "+mpi" in spec: # Force Trilinos to use the MPI wrappers instead of raw compilers # to propagate library link flags for linkers that require fully # resolved symbols in shared libs (such as macOS and some newer # Ubuntu) options.extend( [ define("CMAKE_C_COMPILER", spec["mpi"].mpicc), define("CMAKE_CXX_COMPILER", spec["mpi"].mpicxx), define("CMAKE_Fortran_COMPILER", spec["mpi"].mpifc), define("MPI_BASE_DIR", spec["mpi"].prefix), ] ) # ParMETIS dependencies have to be transitive explicitly have_parmetis = "parmetis" in spec options.append(define_tpl_enable("ParMETIS", have_parmetis)) if have_parmetis: options.extend( [ define( "ParMETIS_LIBRARY_DIRS", [spec["parmetis"].prefix.lib, spec["metis"].prefix.lib], ), define("ParMETIS_LIBRARY_NAMES", ["parmetis", "metis"]), define( "TPL_ParMETIS_INCLUDE_DIRS", spec["parmetis"].headers.directories + spec["metis"].headers.directories, ), ] ) if spec.satisfies("^superlu-dist@4.0:"): options.extend([define("HAVE_SUPERLUDIST_LUSTRUCTINIT_2ARG", True)]) if spec.satisfies("^parallel-netcdf"): options.extend( [ define("TPL_Netcdf_Enables_Netcdf4", True), define("TPL_Netcdf_PARALLEL", True), define("PNetCDF_ROOT", spec["parallel-netcdf"].prefix), ] ) # ################# Explicit template instantiation ################# complex_s = spec.variants["complex"].value float_s = spec.variants["float"].value options.extend( [define("Teuchos_ENABLE_COMPLEX", complex_s), define("Teuchos_ENABLE_FLOAT", float_s)] ) if "+tpetra +explicit_template_instantiation" in spec: options.append(define_from_variant("Tpetra_INST_OPENMP", "openmp")) options.extend( [ define("Tpetra_INST_DOUBLE", True), define("Tpetra_INST_COMPLEX_DOUBLE", complex_s), define("Tpetra_INST_COMPLEX_FLOAT", float_s and complex_s), define("Tpetra_INST_FLOAT", float_s), define("Tpetra_INST_SERIAL", True), ] ) gotype = spec.variants["gotype"].value if gotype == "all": # default in older Trilinos versions to enable multiple GOs options.extend( [ define("Tpetra_INST_INT_INT", True), define("Tpetra_INST_INT_LONG", True), define("Tpetra_INST_INT_LONG_LONG", True), ] ) else: options.extend( [ define("Tpetra_INST_INT_INT", gotype == "int"), define("Tpetra_INST_INT_LONG", gotype == "long"), define("Tpetra_INST_INT_LONG_LONG", gotype == "long_long"), ] ) # ################# Kokkos ###################### if "+kokkos" in spec: arch = Kokkos.get_microarch(spec.target) if arch: options.append(define("Kokkos_ARCH_" + arch.upper(), True)) define_kok_enable = _make_definer("Kokkos_ENABLE_") options.extend( [ define_kok_enable("CUDA"), define_kok_enable("OPENMP" if spec.version >= Version("13") else "OpenMP"), ] ) if "+cuda" in spec: options.extend( [ define_kok_enable("CUDA_UVM", True), define_kok_enable("CUDA_LAMBDA", True), define_kok_enable("CUDA_RELOCATABLE_DEVICE_CODE", "cuda_rdc"), ] ) arch_map = Kokkos.spack_cuda_arch_map options.extend( define("Kokkos_ARCH_" + arch_map[arch].upper(), True) for arch in spec.variants["cuda_arch"].value ) # ################# System-specific ###################### # Fortran lib (assumes clang is built with gfortran!) if "+fortran" in spec and spec.compiler.name in ["gcc", "clang", "apple-clang"]: fc = Executable(spec["mpi"].mpifc) if ("+mpi" in spec) else Executable(spack_fc) libgfortran = fc("--print-file-name", "libgfortran." + dso_suffix, output=str).strip() # if libgfortran is equal to "libgfortran." then # print-file-name failed, use static library instead if libgfortran == "libgfortran." + dso_suffix: libgfortran = fc("--print-file-name", "libgfortran.a", output=str).strip() # -L -lgfortran required for OSX # https://github.com/spack/spack/pull/25823#issuecomment-917231118 options.append( define( "Trilinos_EXTRA_LINK_FLAGS", "-L%s/ -lgfortran" % os.path.dirname(libgfortran) ) ) if sys.platform == "darwin" and macos_version() >= Version("10.12"): # use @rpath on Sierra due to limit of dynamic loader options.append(define("CMAKE_MACOSX_RPATH", True)) else: options.append(define("CMAKE_INSTALL_NAME_DIR", self.prefix.lib)) return options @run_after("install") def filter_python(self): # When trilinos is built with Python, libpytrilinos is included # through cmake configure files. Namely, Trilinos_LIBRARIES in # TrilinosConfig.cmake contains pytrilinos. This leads to a # run-time error: Symbol not found: _PyBool_Type and prevents # Trilinos to be used in any C++ code, which links executable # against the libraries listed in Trilinos_LIBRARIES. See # https://github.com/trilinos/Trilinos/issues/569 and # https://github.com/trilinos/Trilinos/issues/866 # A workaround is to remove PyTrilinos from the COMPONENTS_LIST # and to remove -lpytrilonos from Makefile.export.Trilinos if "+python" in self.spec: filter_file( r"(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)", (r"\1\3"), "%s/cmake/Trilinos/TrilinosConfig.cmake" % self.prefix.lib, ) filter_file(r"-lpytrilinos", "", "%s/Makefile.export.Trilinos" % self.prefix.include) def setup_run_environment(self, env: EnvironmentModifications) -> None: if "+exodus" in self.spec: env.prepend_path("PYTHONPATH", self.prefix.lib) if "+cuda" in self.spec: # currently Trilinos doesn't perform the memory fence so # it relies on blocking CUDA kernel launch. env.set("CUDA_LAUNCH_BLOCKING", "1") ================================================ FILE: lib/spack/spack/test/data/web/1.html ================================================ This is page 1. list_depth=2 follows this. foo-1.0.0.tar.gz ================================================ FILE: lib/spack/spack/test/data/web/2.html ================================================ This is page 2. list_depth=3 follows this. list_depth=3 follows this too. foo-2.0.0.tar.gz foo-2.0.0b2.tar.gz ================================================ FILE: lib/spack/spack/test/data/web/3.html ================================================ This is page 3. This link is already visited. foo-3.0.tar.gz foo-3.0a1.tar.gz ================================================ FILE: lib/spack/spack/test/data/web/4.html ================================================ This is page 4. This page is terminal and has no links to other pages. foo-4.5.tar.gz. foo-4.1-rc5.tar.gz. foo-4.5.0.tar.gz. ================================================ FILE: lib/spack/spack/test/data/web/fragment.html ================================================ foo-5.0.0.tar.gz ================================================ FILE: lib/spack/spack/test/data/web/index.html ================================================ This is the root page. list_depth=1 follows this. foo-0.0.0.tar.gz ================================================ FILE: lib/spack/spack/test/data/web/index_with_fragment.html ================================================ This is the root page. This is a page with an include-fragment element.

Loading...

 ================================================ FILE: lib/spack/spack/test/data/web/index_with_javascript.html ================================================ This is the root page. This is a page with a Vue javascript drop down with links as used in GitLab.
================================================ FILE: lib/spack/spack/test/database.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Check the database is functioning properly, both in memory and in its file.""" import contextlib import datetime import functools import gzip import json import os import pathlib import re import shutil import sys import pytest import spack.config import spack.subprocess_context from spack.directory_layout import DirectoryLayoutError try: import uuid _use_uuid = True except ImportError: _use_uuid = False import spack.vendor.jsonschema import spack.concretize import spack.database import spack.deptypes as dt import spack.llnl.util.filesystem as fs import spack.llnl.util.lock as lk import spack.package_base import spack.paths import spack.repo import spack.spec import spack.store import spack.util.lock import spack.version as vn from spack.enums import InstallRecordStatus from spack.installer import PackageInstaller from spack.llnl.util.tty.colify import colify from spack.schema.database_index import schema from spack.test.conftest import RepoBuilder from spack.util.executable import Executable pytestmark = pytest.mark.db @contextlib.contextmanager def writable(database): """Allow a database to be written inside this context manager.""" old_lock, old_is_upstream = database.lock, database.is_upstream db_root = pathlib.Path(database.root) try: # this is safe on all platforms during tests (tests get their own tmpdirs) database.lock = spack.util.lock.Lock(str(database._lock_path), enable=False) database.is_upstream = False db_root.chmod(mode=0o755) with database.write_transaction(): yield finally: db_root.chmod(mode=0o555) database.lock = old_lock database.is_upstream = old_is_upstream @pytest.fixture() def upstream_and_downstream_db(tmp_path: pathlib.Path, gen_mock_layout): """Fixture for a pair of stores: upstream and downstream. Upstream API prohibits writing to an upstream, so we also return a writable version of the upstream DB for tests to use. """ mock_db_root = tmp_path / "mock_db_root" mock_db_root.mkdir() mock_db_root.chmod(0o555) upstream_db = spack.database.Database( str(mock_db_root), is_upstream=True, layout=gen_mock_layout("a") ) with writable(upstream_db): upstream_db._write() downstream_db_root = tmp_path / "mock_downstream_db_root" downstream_db_root.mkdir() downstream_db_root.chmod(0o755) downstream_db = spack.database.Database( str(downstream_db_root), upstream_dbs=[upstream_db], layout=gen_mock_layout("b") ) downstream_db._write() yield upstream_db, downstream_db @pytest.mark.parametrize( "install_tree,result", [ ("all", ["pkg-b", "pkg-c", "gcc-runtime", "gcc", "compiler-wrapper"]), ("upstream", ["pkg-c"]), ("local", ["pkg-b", "gcc-runtime", "gcc", "compiler-wrapper"]), ("{u}", ["pkg-c"]), ("{d}", ["pkg-b", "gcc-runtime", "gcc", "compiler-wrapper"]), ], ids=["all", "upstream", "local", "upstream_path", "downstream_path"], ) def test_query_by_install_tree( install_tree, result, upstream_and_downstream_db, mock_packages, monkeypatch, config ): up_db, down_db = upstream_and_downstream_db # Set the upstream DB to contain "pkg-c" and downstream to contain "pkg-b") b = spack.concretize.concretize_one("pkg-b") c = spack.concretize.concretize_one("pkg-c") with writable(up_db): up_db.add(c) up_db._read() down_db.add(b) specs = down_db.query(install_tree=install_tree.format(u=up_db.root, d=down_db.root)) assert {s.name for s in specs} == set(result) def test_spec_installed_upstream( upstream_and_downstream_db, mock_custom_repository, config, monkeypatch ): """Test whether Spec.installed_upstream() works.""" upstream_db, downstream_db = upstream_and_downstream_db # a known installed spec should say that it's installed with spack.repo.use_repositories(mock_custom_repository): spec = spack.concretize.concretize_one("pkg-c") assert not spec.installed assert not spec.installed_upstream with writable(upstream_db): upstream_db.add(spec) upstream_db._read() monkeypatch.setattr(spack.store.STORE, "db", downstream_db) assert spec.installed assert spec.installed_upstream assert spec.copy().installed # an abstract spec should say it's not installed spec = spack.spec.Spec("not-a-real-package") assert not spec.installed assert not spec.installed_upstream @pytest.mark.usefixtures("config") def test_installed_upstream(upstream_and_downstream_db, repo_builder: RepoBuilder): upstream_db, downstream_db = upstream_and_downstream_db repo_builder.add_package("x") repo_builder.add_package("z") repo_builder.add_package("y", dependencies=[("z", None, None)]) repo_builder.add_package("w", dependencies=[("x", None, None), ("y", None, None)]) with spack.repo.use_repositories(repo_builder.root): spec = spack.concretize.concretize_one("w") with writable(upstream_db): for dep in spec.traverse(root=False): upstream_db.add(dep) upstream_db._read() for dep in spec.traverse(root=False): record = downstream_db.get_by_hash(dep.dag_hash()) assert record is not None with pytest.raises(spack.database.ForbiddenLockError): upstream_db.get_by_hash(dep.dag_hash()) new_spec = spack.concretize.concretize_one("w") downstream_db.add(new_spec) for dep in new_spec.traverse(root=False): upstream, record = downstream_db.query_by_spec_hash(dep.dag_hash()) assert upstream assert record.path == upstream_db.layout.path_for_spec(dep) upstream, record = downstream_db.query_by_spec_hash(new_spec.dag_hash()) assert not upstream assert record.installed upstream_db._check_ref_counts() downstream_db._check_ref_counts() def test_missing_upstream_build_dep( upstream_and_downstream_db, tmp_path: pathlib.Path, monkeypatch, config, repo_builder: RepoBuilder, ): upstream_db, downstream_db = upstream_and_downstream_db z_y_prefix = str(tmp_path / "z-y") def fail_for_z(spec): if spec.prefix == z_y_prefix: raise DirectoryLayoutError("Fake layout error for z") upstream_db.layout.ensure_installed = fail_for_z repo_builder.add_package("z") repo_builder.add_package("y", dependencies=[("z", "build", None)]) monkeypatch.setattr(spack.store.STORE, "db", downstream_db) with spack.repo.use_repositories(repo_builder.root): y = spack.concretize.concretize_one("y") z_y = y["z"] z_y.set_prefix(z_y_prefix) with writable(upstream_db): upstream_db.add(y) upstream_db._read() upstream, record = downstream_db.query_by_spec_hash(z_y.dag_hash()) assert upstream assert not record.installed assert y.installed assert y.installed_upstream assert not z_y.installed assert not z_y.installed_upstream # Now add z to downstream with non-triggering prefix # and make sure z *is* installed z_new = z_y.copy() z_new.set_prefix(str(tmp_path / "z-new")) downstream_db.add(z_new) assert z_new.installed assert not z_new.installed_upstream def test_removed_upstream_dep( upstream_and_downstream_db, capfd, config, repo_builder: RepoBuilder ): upstream_db, downstream_db = upstream_and_downstream_db repo_builder.add_package("z") repo_builder.add_package("y", dependencies=[("z", None, None)]) with spack.repo.use_repositories(repo_builder.root): y = spack.concretize.concretize_one("y") z = y["z"] # add dependency to upstream, dependents to downstream with writable(upstream_db): upstream_db.add(z) upstream_db._read() downstream_db.add(y) # remove the dependency from the upstream DB with writable(upstream_db): upstream_db.remove(z) upstream_db._read() # then rereading the downstream DB should warn about the missing dep downstream_db._read_from_file(downstream_db._index_path) assert ( f"Missing dependency not in database: y/{y.dag_hash(7)} needs z" in capfd.readouterr().err ) @pytest.mark.usefixtures("config") def test_add_to_upstream_after_downstream(upstream_and_downstream_db, repo_builder: RepoBuilder): """An upstream DB can add a package after it is installed in the downstream DB. When a package is recorded as installed in both, the results should refer to the downstream DB. """ upstream_db, downstream_db = upstream_and_downstream_db repo_builder.add_package("x") with spack.repo.use_repositories(repo_builder.root): spec = spack.concretize.concretize_one("x") downstream_db.add(spec) with writable(upstream_db): upstream_db.add(spec) upstream_db._read() upstream, record = downstream_db.query_by_spec_hash(spec.dag_hash()) # Even though the package is recorded as installed in the upstream DB, # we prefer the locally-installed instance assert not upstream qresults = downstream_db.query("x") assert len(qresults) == 1 (queried_spec,) = qresults try: orig_db = spack.store.STORE.db spack.store.STORE.db = downstream_db assert queried_spec.prefix == downstream_db.layout.path_for_spec(spec) finally: spack.store.STORE.db = orig_db def test_cannot_write_upstream(tmp_path: pathlib.Path, mock_packages, config): # Instantiate the database that will be used as the upstream DB and make # sure it has an index file with spack.database.Database(str(tmp_path)).write_transaction(): pass # Create it as an upstream db = spack.database.Database(str(tmp_path), is_upstream=True) with pytest.raises(spack.database.ForbiddenLockError): db.add(spack.concretize.concretize_one("pkg-a")) @pytest.mark.usefixtures("config", "temporary_store") def test_recursive_upstream_dbs( tmp_path: pathlib.Path, gen_mock_layout, repo_builder: RepoBuilder ): roots = [str(tmp_path / x) for x in ["a", "b", "c"]] for root in roots: pathlib.Path(root).mkdir(parents=True, exist_ok=True) layouts = [gen_mock_layout(x) for x in ["ra", "rb", "rc"]] repo_builder.add_package("z") repo_builder.add_package("y", dependencies=[("z", None, None)]) repo_builder.add_package("x", dependencies=[("y", None, None)]) with spack.repo.use_repositories(repo_builder.root): spec = spack.concretize.concretize_one("x") db_c = spack.database.Database(roots[2], layout=layouts[2]) db_c.add(spec["z"]) db_b = spack.database.Database(roots[1], upstream_dbs=[db_c], layout=layouts[1]) db_b.add(spec["y"]) db_a = spack.database.Database(roots[0], upstream_dbs=[db_b, db_c], layout=layouts[0]) db_a.add(spec["x"]) upstream_dbs_from_scratch = spack.store._construct_upstream_dbs_from_install_roots( [roots[1], roots[2]] ) db_a_from_scratch = spack.database.Database( roots[0], upstream_dbs=upstream_dbs_from_scratch ) assert db_a_from_scratch.db_for_spec_hash(spec.dag_hash()) == (db_a_from_scratch) assert ( db_a_from_scratch.db_for_spec_hash(spec["y"].dag_hash()) == (upstream_dbs_from_scratch[0]) ) assert ( db_a_from_scratch.db_for_spec_hash(spec["z"].dag_hash()) == (upstream_dbs_from_scratch[1]) ) db_a_from_scratch._check_ref_counts() upstream_dbs_from_scratch[0]._check_ref_counts() upstream_dbs_from_scratch[1]._check_ref_counts() assert db_a_from_scratch.installed_relatives(spec) == set(spec.traverse(root=False)) assert db_a_from_scratch.installed_relatives(spec["z"], direction="parents") == set( [spec, spec["y"]] ) @pytest.fixture() def usr_folder_exists(monkeypatch): """The ``/usr`` folder is assumed to be existing in some tests. This fixture makes it such that its existence is mocked, so we have no requirements on the system running tests. """ isdir = os.path.isdir @functools.wraps(os.path.isdir) def mock_isdir(path): if path == "/usr": return True return isdir(path) monkeypatch.setattr(os.path, "isdir", mock_isdir) def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" recs = [] def add_rec(spec): cspecs = spack.store.STORE.db.query(spec, installed=InstallRecordStatus.ANY) if not cspecs: recs.append("[ %-7s ] %-20s-" % ("", spec)) else: key = cspecs[0].dag_hash() rec = spack.store.STORE.db.get_record(cspecs[0]) recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count)) with spack.store.STORE.db.read_transaction(): add_rec("mpileaks ^mpich") add_rec("callpath ^mpich") add_rec("mpich") add_rec("mpileaks ^mpich2") add_rec("callpath ^mpich2") add_rec("mpich2") add_rec("mpileaks ^zmpi") add_rec("callpath ^zmpi") add_rec("zmpi") add_rec("fake") add_rec("dyninst") add_rec("libdwarf") add_rec("libelf") colify(recs, cols=3) def _check_merkleiness(): """Ensure the spack database is a valid merkle graph.""" all_specs = spack.store.STORE.db.query(installed=InstallRecordStatus.ANY) seen = {} for spec in all_specs: for dep in spec.dependencies(): hash_key = dep.dag_hash() if hash_key not in seen: seen[hash_key] = id(dep) else: assert seen[hash_key] == id(dep) def _check_db_sanity(database): """Utility function to check db against install layout.""" pkg_in_layout = sorted(spack.store.STORE.layout.all_specs()) actual = sorted(database.query()) externals = sorted([x for x in actual if x.external]) nexpected = len(pkg_in_layout) + len(externals) assert nexpected == len(actual) non_external_in_db = sorted([x for x in actual if not x.external]) for e, a in zip(pkg_in_layout, non_external_in_db): assert e == a _check_merkleiness() def _check_remove_and_add_package(database: spack.database.Database, spec): """Remove a spec from the DB, then add it and make sure everything's still ok once it is added. This checks that it was removed, that it's back when added again, and that ref counts are consistent. """ original = database.query() database._check_ref_counts() # Remove spec concrete_spec = database.remove(spec) database._check_ref_counts() remaining = database.query() # ensure spec we removed is gone assert len(original) - 1 == len(remaining) assert all(s in original for s in remaining) assert concrete_spec not in remaining # add it back and make sure everything is ok. database.add(concrete_spec) installed = database.query() assert concrete_spec in installed assert installed == original # sanity check against directory layout and check ref counts. _check_db_sanity(database) database._check_ref_counts() def _mock_install(spec: str): s = spack.concretize.concretize_one(spec) PackageInstaller([s.package], fake=True, explicit=True).install() def _mock_remove(spec): specs = spack.store.STORE.db.query(spec) assert len(specs) == 1 spec = specs[0] spec.package.do_uninstall(spec) def test_default_queries(database): # Testing a package whose name *doesn't* start with 'lib' # to ensure the library has 'lib' prepended to the name rec = database.get_record("zmpi") spec = rec.spec libraries = spec["zmpi"].libs assert len(libraries) == 1 assert libraries.names[0] == "zmpi" headers = spec["zmpi"].headers assert len(headers) == 1 assert headers.names[0] == "zmpi" command = spec["zmpi"].command assert isinstance(command, Executable) assert command.name == "zmpi" assert os.path.exists(command.path) # Testing a package whose name *does* start with 'lib' # to ensure the library doesn't have a double 'lib' prefix rec = database.get_record("libelf") spec = rec.spec libraries = spec["libelf"].libs assert len(libraries) == 1 assert libraries.names[0] == "elf" headers = spec["libelf"].headers assert len(headers) == 1 assert headers.names[0] == "libelf" command = spec["libelf"].command assert isinstance(command, Executable) assert command.name == "libelf" assert os.path.exists(command.path) def test_005_db_exists(database): """Make sure db cache file exists after creating.""" index_file = os.path.join(database.root, ".spack-db", spack.database.INDEX_JSON_FILE) lock_file = os.path.join(database.root, ".spack-db", spack.database._LOCK_FILE) assert os.path.exists(str(index_file)) # Lockfiles not currently supported on Windows if sys.platform != "win32": assert os.path.exists(str(lock_file)) with open(index_file, encoding="utf-8") as fd: index_object = json.load(fd) spack.vendor.jsonschema.validate(index_object, schema) def test_010_all_install_sanity(database): """Ensure that the install layout reflects what we think it does.""" all_specs = spack.store.STORE.layout.all_specs() assert len(all_specs) == 17 # Query specs with multiple configurations mpileaks_specs = [s for s in all_specs if s.satisfies("mpileaks")] callpath_specs = [s for s in all_specs if s.satisfies("callpath")] mpi_specs = [s for s in all_specs if s.satisfies("mpi")] assert len(mpileaks_specs) == 3 assert len(callpath_specs) == 3 assert len(mpi_specs) == 3 # Query specs with single configurations dyninst_specs = [s for s in all_specs if s.satisfies("dyninst")] libdwarf_specs = [s for s in all_specs if s.satisfies("libdwarf")] libelf_specs = [s for s in all_specs if s.satisfies("libelf")] assert len(dyninst_specs) == 1 assert len(libdwarf_specs) == 1 assert len(libelf_specs) == 1 # Query by dependency assert len([s for s in all_specs if s.satisfies("mpileaks ^mpich")]) == 1 assert len([s for s in all_specs if s.satisfies("mpileaks ^mpich2")]) == 1 assert len([s for s in all_specs if s.satisfies("mpileaks ^zmpi")]) == 1 def test_015_write_and_read(mutable_database): # write and read DB with spack.store.STORE.db.write_transaction(): specs = spack.store.STORE.db.query() recs = [spack.store.STORE.db.get_record(s) for s in specs] for spec, rec in zip(specs, recs): new_rec = spack.store.STORE.db.get_record(spec) assert new_rec.ref_count == rec.ref_count assert new_rec.spec == rec.spec assert new_rec.path == rec.path assert new_rec.installed == rec.installed def test_016_roundtrip_spliced_spec(mutable_database): build_spec = spack.concretize.concretize_one("splice-t") replacement = spack.concretize.concretize_one("splice-h+foo") spec = build_spec.splice(replacement) spack.store.STORE.db.add(spec) spack.store.STORE.db._state_is_inconsistent = True # force re-read _, spec_record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash()) _, buildspec_record = spack.store.STORE.db.query_by_spec_hash(spec.build_spec.dag_hash()) assert spec_record.spec == spec assert spec_record.spec.build_spec == spec.build_spec assert buildspec_record # buildspec needs to be recorded in db def test_017_write_and_read_without_uuid(mutable_database, monkeypatch): monkeypatch.setattr(spack.database, "_use_uuid", False) # write and read DB with spack.store.STORE.db.write_transaction(): specs = spack.store.STORE.db.query() recs = [spack.store.STORE.db.get_record(s) for s in specs] for spec, rec in zip(specs, recs): new_rec = spack.store.STORE.db.get_record(spec) assert new_rec.ref_count == rec.ref_count assert new_rec.spec == rec.spec assert new_rec.path == rec.path assert new_rec.installed == rec.installed def test_020_db_sanity(database): """Make sure query() returns what's actually in the db.""" _check_db_sanity(database) def test_025_reindex(mutable_database): """Make sure reindex works and ref counts are valid.""" spack.store.STORE.reindex() _check_db_sanity(mutable_database) def test_026_reindex_after_deprecate(mutable_database): """Make sure reindex works and ref counts are valid after deprecation.""" mpich = mutable_database.query_one("mpich") zmpi = mutable_database.query_one("zmpi") mutable_database.deprecate(mpich, zmpi) spack.store.STORE.reindex() _check_db_sanity(mutable_database) class ReadModify: """Provide a function which can execute in a separate process that removes a spec from the database. """ def __call__(self): # check that other process can read DB _check_db_sanity(spack.store.STORE.db) with spack.store.STORE.db.write_transaction(): _mock_remove("mpileaks ^zmpi") def test_030_db_sanity_from_another_process(mutable_database): spack_process = spack.subprocess_context.SpackTestProcess(ReadModify()) p = spack_process.create() p.start() p.join() # ensure child process change is visible in parent process with mutable_database.read_transaction(): assert len(mutable_database.query("mpileaks ^zmpi")) == 0 def test_040_ref_counts(database): """Ensure that we got ref counts right when we read the DB.""" database._check_ref_counts() def test_041_ref_counts_deprecate(mutable_database): """Ensure that we have appropriate ref counts after deprecating""" mpich = mutable_database.query_one("mpich") zmpi = mutable_database.query_one("zmpi") mutable_database.deprecate(mpich, zmpi) mutable_database._check_ref_counts() def test_050_basic_query(database): """Ensure querying database is consistent with what is installed.""" # query everything total_specs = len(spack.store.STORE.db.query()) assert total_specs == 20 # query specs with multiple configurations mpileaks_specs = database.query("mpileaks") callpath_specs = database.query("callpath") mpi_specs = database.query("mpi") assert len(mpileaks_specs) == 3 assert len(callpath_specs) == 3 assert len(mpi_specs) == 3 # query specs with single configurations dyninst_specs = database.query("dyninst") libdwarf_specs = database.query("libdwarf") libelf_specs = database.query("libelf") assert len(dyninst_specs) == 1 assert len(libdwarf_specs) == 1 assert len(libelf_specs) == 1 # Query by dependency assert len(database.query("mpileaks ^mpich")) == 1 assert len(database.query("mpileaks ^mpich2")) == 1 assert len(database.query("mpileaks ^zmpi")) == 1 # Query by date assert len(database.query(start_date=datetime.datetime.min)) == total_specs assert len(database.query(start_date=datetime.datetime.max)) == 0 assert len(database.query(end_date=datetime.datetime.min)) == 0 assert len(database.query(end_date=datetime.datetime.max)) == total_specs def test_060_remove_and_add_root_package(mutable_database): _check_remove_and_add_package(mutable_database, "mpileaks ^mpich") def test_070_remove_and_add_dependency_package(mutable_database): _check_remove_and_add_package(mutable_database, "dyninst") def test_080_root_ref_counts(mutable_database): rec = mutable_database.get_record("mpileaks ^mpich") # Remove a top-level spec from the DB mutable_database.remove("mpileaks ^mpich") # record no longer in DB assert mutable_database.query("mpileaks ^mpich", installed=InstallRecordStatus.ANY) == [] # record's deps have updated ref_counts assert mutable_database.get_record("callpath ^mpich").ref_count == 0 assert mutable_database.get_record("mpich").ref_count == 1 # Put the spec back mutable_database.add(rec.spec) # record is present again assert len(mutable_database.query("mpileaks ^mpich", installed=InstallRecordStatus.ANY)) == 1 # dependencies have ref counts updated assert mutable_database.get_record("callpath ^mpich").ref_count == 1 assert mutable_database.get_record("mpich").ref_count == 2 def test_090_non_root_ref_counts(mutable_database): mutable_database.get_record("mpileaks ^mpich") mutable_database.get_record("callpath ^mpich") # "force remove" a non-root spec from the DB mutable_database.remove("callpath ^mpich") # record still in DB but marked uninstalled assert mutable_database.query("callpath ^mpich", installed=True) == [] assert len(mutable_database.query("callpath ^mpich", installed=InstallRecordStatus.ANY)) == 1 # record and its deps have same ref_counts assert ( mutable_database.get_record("callpath ^mpich", installed=InstallRecordStatus.ANY).ref_count == 1 ) assert mutable_database.get_record("mpich").ref_count == 2 # remove only dependent of uninstalled callpath record mutable_database.remove("mpileaks ^mpich") # record and parent are completely gone. assert mutable_database.query("mpileaks ^mpich", installed=InstallRecordStatus.ANY) == [] assert mutable_database.query("callpath ^mpich", installed=InstallRecordStatus.ANY) == [] # mpich ref count updated properly. mpich_rec = mutable_database.get_record("mpich") assert mpich_rec.ref_count == 0 def test_100_no_write_with_exception_on_remove(database): def fail_while_writing(): with database.write_transaction(): _mock_remove("mpileaks ^zmpi") raise Exception() with database.read_transaction(): assert len(database.query("mpileaks ^zmpi", installed=InstallRecordStatus.ANY)) == 1 with pytest.raises(Exception): fail_while_writing() # reload DB and make sure zmpi is still there. with database.read_transaction(): assert len(database.query("mpileaks ^zmpi", installed=InstallRecordStatus.ANY)) == 1 def test_110_no_write_with_exception_on_install(database): def fail_while_writing(): with database.write_transaction(): _mock_install("cmake") raise Exception() with database.read_transaction(): assert database.query("cmake", installed=InstallRecordStatus.ANY) == [] with pytest.raises(Exception): fail_while_writing() # reload DB and make sure cmake was not written. with database.read_transaction(): assert database.query("cmake", installed=InstallRecordStatus.ANY) == [] def test_115_reindex_with_packages_not_in_repo(mutable_database, repo_builder: RepoBuilder): # Dont add any package definitions to this repository, the idea is that # packages should not have to be defined in the repository once they # are installed with spack.repo.use_repositories(repo_builder.root): spack.store.STORE.reindex() _check_db_sanity(mutable_database) def test_external_entries_in_db(mutable_database): rec = mutable_database.get_record("mpileaks ^zmpi") assert rec.spec.external_path is None assert not rec.spec.external_modules rec = mutable_database.get_record("externaltool") assert rec.spec.external_path == os.path.sep + os.path.join("path", "to", "external_tool") assert not rec.spec.external_modules assert rec.explicit is False PackageInstaller([rec.spec.package], fake=True, explicit=True).install() rec = mutable_database.get_record("externaltool") assert rec.spec.external_path == os.path.sep + os.path.join("path", "to", "external_tool") assert not rec.spec.external_modules assert rec.explicit is True @pytest.mark.regression("8036") def test_regression_issue_8036(mutable_database, usr_folder_exists): # The test ensures that the external package prefix is treated as # existing. Even when the package prefix exists, the package should # not be considered installed until it is added to the database by # the installer with install(). s = spack.concretize.concretize_one("externaltool@0.9") assert not s.installed # Now install the external package and check again the `installed` property PackageInstaller([s.package], fake=True, explicit=True).install() assert s.installed @pytest.mark.regression("11118") def test_old_external_entries_prefix(mutable_database: spack.database.Database): with open(spack.store.STORE.db._index_path, "r", encoding="utf-8") as f: db_obj = json.loads(f.read()) spack.vendor.jsonschema.validate(db_obj, schema) s, *_ = mutable_database.query("externaltool") db_obj["database"]["installs"][s.dag_hash()]["path"] = "None" with open(spack.store.STORE.db._index_path, "w", encoding="utf-8") as f: f.write(json.dumps(db_obj)) if _use_uuid: with open(spack.store.STORE.db._verifier_path, "w", encoding="utf-8") as f: f.write(str(uuid.uuid4())) record = spack.store.STORE.db.get_record(s) assert record is not None assert record.path is None assert record.spec._prefix is None assert record.spec.prefix == record.spec.external_path def test_uninstall_by_spec(mutable_database): with mutable_database.write_transaction(): for spec in mutable_database.query(): if spec.installed: spack.package_base.PackageBase.uninstall_by_spec(spec, force=True) else: mutable_database.remove(spec) assert len(mutable_database.query()) == 0 def test_query_unused_specs(mutable_database): # This spec installs a fake cmake as a build only dependency s = spack.concretize.concretize_one("simple-inheritance") PackageInstaller([s.package], fake=True, explicit=True).install() si = s.dag_hash() ml_mpich = spack.store.STORE.db.query_one("mpileaks ^mpich").dag_hash() ml_mpich2 = spack.store.STORE.db.query_one("mpileaks ^mpich2").dag_hash() ml_zmpi = spack.store.STORE.db.query_one("mpileaks ^zmpi").dag_hash() externaltest = spack.store.STORE.db.query_one("externaltest").dag_hash() trivial_smoke_test = spack.store.STORE.db.query_one("trivial-smoke-test").dag_hash() def check_unused(roots, deptype, expected): unused = spack.store.STORE.db.unused_specs(root_hashes=roots, deptype=deptype) assert set(u.name for u in unused) == set(expected) default_dt = dt.LINK | dt.RUN check_unused(None, default_dt, ["cmake", "gcc", "compiler-wrapper"]) check_unused( [si, ml_mpich, ml_mpich2, ml_zmpi, externaltest], default_dt, ["trivial-smoke-test", "cmake", "gcc", "compiler-wrapper"], ) check_unused( [si, ml_mpich, ml_mpich2, ml_zmpi, externaltest], dt.LINK | dt.RUN | dt.BUILD, ["trivial-smoke-test"], ) check_unused( [si, ml_mpich, ml_mpich2, externaltest, trivial_smoke_test], dt.LINK | dt.RUN | dt.BUILD, ["mpileaks", "callpath", "zmpi", "fake"], ) check_unused( [si, ml_mpich, ml_mpich2, ml_zmpi], default_dt, [ "trivial-smoke-test", "cmake", "externaltest", "externaltool", "externalvirtual", "gcc", "compiler-wrapper", ], ) @pytest.mark.regression("10019") def test_query_spec_with_conditional_dependency(mutable_database): # The issue is triggered by having dependencies that are # conditional on a Boolean variant s = spack.concretize.concretize_one("hdf5~mpi") PackageInstaller([s.package], fake=True, explicit=True).install() results = spack.store.STORE.db.query_local("hdf5 ^mpich") assert not results @pytest.mark.regression("10019") def test_query_spec_with_non_conditional_virtual_dependency(database): # Ensure the same issue doesn't come up for virtual # dependency that are not conditional on variants results = spack.store.STORE.db.query_local("mpileaks ^mpich") assert len(results) == 1 def test_query_virtual_spec(database): """Make sure we can query for virtuals in the DB""" results = spack.store.STORE.db.query_local("mpi") assert len(results) == 3 names = [s.name for s in results] assert all(name in names for name in ["mpich", "mpich2", "zmpi"]) def test_failed_spec_path_error(mutable_database): """Ensure spec not concrete check is covered.""" s = spack.spec.Spec("pkg-a") with pytest.raises(AssertionError, match="concrete spec required"): spack.store.STORE.failure_tracker.mark(s) @pytest.mark.db def test_clear_failure_keep(mutable_database, monkeypatch, capfd): """Add test coverage for clear_failure operation when to be retained.""" def _is(self, spec): return True # Pretend the spec has been failure locked monkeypatch.setattr(spack.database.FailureTracker, "lock_taken", _is) s = spack.concretize.concretize_one("pkg-a") spack.store.STORE.failure_tracker.clear(s) out = capfd.readouterr()[0] assert "Retaining failure marking" in out @pytest.mark.db def test_clear_failure_forced(mutable_database, monkeypatch, capfd): """Add test coverage for clear_failure operation when force.""" def _is(self, spec): return True # Pretend the spec has been failure locked monkeypatch.setattr(spack.database.FailureTracker, "lock_taken", _is) # Ensure raise OSError when try to remove the non-existent marking monkeypatch.setattr(spack.database.FailureTracker, "persistent_mark", _is) s = spack.concretize.concretize_one("pkg-a") spack.store.STORE.failure_tracker.clear(s, force=True) out = capfd.readouterr()[1] assert "Removing failure marking despite lock" in out assert "Unable to remove failure marking" in out @pytest.mark.db def test_mark_failed(mutable_database, monkeypatch, tmp_path: pathlib.Path, capfd): """Add coverage to mark_failed.""" def _raise_exc(lock): raise lk.LockTimeoutError(lk.LockType.WRITE, "/mock-lock", 1.234, 10) with fs.working_dir(str(tmp_path)): s = spack.concretize.concretize_one("pkg-a") # Ensure attempt to acquire write lock on the mark raises the exception monkeypatch.setattr(lk.Lock, "acquire_write", _raise_exc) spack.store.STORE.failure_tracker.mark(s) out = str(capfd.readouterr()[1]) assert "Unable to mark pkg-a as failed" in out spack.store.STORE.failure_tracker.clear_all() @pytest.mark.db def test_prefix_failed(mutable_database, monkeypatch): """Add coverage to failed operation.""" s = spack.concretize.concretize_one("pkg-a") # Confirm the spec is not already marked as failed assert not spack.store.STORE.failure_tracker.has_failed(s) # Check that a failure entry is sufficient spack.store.STORE.failure_tracker.mark(s) assert spack.store.STORE.failure_tracker.has_failed(s) # Remove the entry and check again spack.store.STORE.failure_tracker.clear(s) assert not spack.store.STORE.failure_tracker.has_failed(s) # Now pretend that the prefix failure is locked monkeypatch.setattr(spack.database.FailureTracker, "lock_taken", lambda self, spec: True) assert spack.store.STORE.failure_tracker.has_failed(s) def test_prefix_write_lock_error(mutable_database, monkeypatch): """Cover the prefix write lock exception.""" def _raise(db, spec): raise lk.LockError("Mock lock error") s = spack.concretize.concretize_one("pkg-a") # Ensure subsequent lock operations fail monkeypatch.setattr(lk.Lock, "acquire_write", _raise) with pytest.raises(Exception): with spack.store.STORE.prefix_locker.write_lock(s): assert False @pytest.mark.regression("26600") def test_database_works_with_empty_dir(tmp_path: pathlib.Path): # Create the lockfile and failures directory otherwise # we'll get a permission error on Database creation db_dir = tmp_path / ".spack-db" db_dir.mkdir() (db_dir / spack.database._LOCK_FILE).touch() (db_dir / "failures").mkdir() tmp_path.chmod(mode=0o555) db = spack.database.Database(str(tmp_path)) with db.read_transaction(): db.query() # Check that reading an empty directory didn't create a new index.json assert not os.path.exists(db._index_path) @pytest.mark.parametrize( "query_arg,exc_type,msg_str", [ (["callpath"], spack.store.MatchError, "matches multiple packages"), (["tensorflow"], spack.store.MatchError, "does not match any"), ], ) def test_store_find_failures(database, query_arg, exc_type, msg_str): with pytest.raises(exc_type) as exc_info: spack.store.find(query_arg, multiple=False) assert msg_str in str(exc_info.value) def test_store_find_accept_string(database): result = spack.store.find("callpath", multiple=True) assert len(result) == 3 def test_reindex_removed_prefix_is_not_installed(mutable_database, mock_store, capfd): """When a prefix of a dependency is removed and the database is reindexed, the spec should still be added through the dependent, but should be listed as not installed.""" # Remove libelf from the filesystem prefix = mutable_database.query_one("libelf").prefix assert prefix.startswith(str(mock_store)) shutil.rmtree(prefix) # Reindex should pick up libelf as a dependency of libdwarf spack.store.STORE.reindex() # Reindexing should warn about libelf not found on the filesystem assert re.search( "libelf@0.8.13.+ was marked installed in the database " "but was not found on the file system", capfd.readouterr().err, ) # And we should still have libelf in the database, but not installed. assert not mutable_database.query_one("libelf", installed=True) assert mutable_database.query_one("libelf", installed=False) def test_reindex_when_all_prefixes_are_removed(mutable_database, mock_store): # Remove all non-external installations from the filesystem for spec in spack.store.STORE.db.query_local(): if not spec.external: assert spec.prefix.startswith(str(mock_store)) shutil.rmtree(spec.prefix) # Make sure we have some explicitly installed specs num = len(mutable_database.query_local(installed=True, explicit=True)) assert num > 0 # Reindex uses the current index to repopulate itself spack.store.STORE.reindex() # Make sure all explicit specs are still there, but are now uninstalled. specs = mutable_database.query_local(installed=False, explicit=True) assert len(specs) == num # And make sure they can be removed from the database (covers the case where # `ref_count == 0 and not installed`, which hits some obscure branches. for s in specs: mutable_database.remove(s) assert len(mutable_database.query_local(installed=False, explicit=True)) == 0 @pytest.mark.parametrize( "spec_str,parent_name,expected_nparents", [("dyninst", "callpath", 3), ("libelf", "dyninst", 1), ("libelf", "libdwarf", 1)], ) @pytest.mark.regression("11983") def test_check_parents(spec_str, parent_name, expected_nparents, database): """Check that a spec returns the correct number of parents.""" s = database.query_one(spec_str) parents = s.dependents(name=parent_name) assert len(parents) == expected_nparents edges = s.edges_from_dependents(name=parent_name) assert len(edges) == expected_nparents def test_db_all_hashes(database): # ensure we get the right number of hashes without a read transaction hashes = database.all_hashes() assert len(hashes) == 20 # and make sure the hashes match with database.read_transaction(): assert set(s.dag_hash() for s in database.query()) == set(hashes) def test_consistency_of_dependents_upon_remove(mutable_database): # Check the initial state s = mutable_database.query_one("dyninst") parents = s.dependents(name="callpath") assert len(parents) == 3 # Remove a dependent (and all its dependents) mutable_database.remove("mpileaks ^callpath ^mpich2") mutable_database.remove("callpath ^mpich2") # Check the final state s = mutable_database.query_one("dyninst") parents = s.dependents(name="callpath") assert len(parents) == 2 @pytest.mark.regression("30187") def test_query_installed_when_package_unknown(database, repo_builder: RepoBuilder): """Test that we can query the installation status of a spec when we don't know its package.py """ with spack.repo.use_repositories(repo_builder.root): specs = database.query("mpileaks") for s in specs: # Assert that we can query the installation methods even though we # don't have the package.py available assert s.installed assert not s.installed_upstream with pytest.raises(spack.repo.UnknownNamespaceError): s.package def test_error_message_when_using_too_new_db(database, monkeypatch): """Sometimes the database format needs to be bumped. When that happens, we have forward incompatibilities that need to be reported in a clear way to the user, in case we moved back to an older version of Spack. This test ensures that the error message for a too new database version stays comprehensible across refactoring of the database code. """ monkeypatch.setattr(spack.database, "_DB_VERSION", vn.Version("0")) with pytest.raises( spack.database.InvalidDatabaseVersionError, match="you need a newer Spack version" ): spack.database.Database(database.root)._read() @pytest.mark.parametrize( "lock_cfg", [spack.database.NO_LOCK, spack.database.NO_TIMEOUT, spack.database.DEFAULT_LOCK_CFG, None], ) def test_database_construction_doesnt_use_globals( tmp_path: pathlib.Path, config, nullify_globals, lock_cfg ): lock_cfg = lock_cfg or spack.database.lock_configuration(config) db = spack.database.Database(str(tmp_path), lock_cfg=lock_cfg) with db.write_transaction(): pass # ensure the DB is written assert os.path.exists(db.database_directory) def test_database_read_works_with_trailing_data( tmp_path: pathlib.Path, default_mock_concretization ): # Populate a database root = str(tmp_path) db = spack.database.Database(root, layout=None) spec = default_mock_concretization("pkg-a") db.add(spec) specs_in_db = db.query_local() assert spec in specs_in_db # Append anything to the end of the database file with open(db._index_path, "a", encoding="utf-8") as f: f.write(json.dumps({"hello": "world"})) # Read the database and check that it ignores the trailing data assert spack.database.Database(root).query_local() == specs_in_db def test_database_errors_with_just_a_version_key(mutable_database): next_version = f"{spack.database._DB_VERSION}.next" with open(mutable_database._index_path, "w", encoding="utf-8") as f: f.write(json.dumps({"database": {"version": next_version}})) with pytest.raises(spack.database.InvalidDatabaseVersionError): spack.database.Database(mutable_database.root).query_local() def test_reindex_with_upstreams(tmp_path: pathlib.Path, monkeypatch, mock_packages, config): # Reindexing should not put install records of upstream entries into the local database. Here # we install `mpileaks` locally with dependencies in the upstream. And we even install # `mpileaks` with the same hash in the upstream. After reindexing, `mpileaks` should still be # in the local db, and `callpath` should not. mpileaks = spack.concretize.concretize_one("mpileaks") callpath = mpileaks.dependencies("callpath")[0] upstream_store = spack.store.create( spack.config.create_from( spack.config.InternalConfigScope( "cfg", {"config": {"install_tree": {"root": str(tmp_path / "upstream")}}} ) ) ) monkeypatch.setattr(spack.store, "STORE", upstream_store) PackageInstaller([callpath.package], fake=True, explicit=True).install() local_store = spack.store.create( spack.config.create_from( spack.config.InternalConfigScope( "cfg", { "config": {"install_tree": {"root": str(tmp_path / "local")}}, "upstreams": {"my-upstream": {"install_tree": str(tmp_path / "upstream")}}, }, ) ) ) monkeypatch.setattr(spack.store, "STORE", local_store) PackageInstaller([mpileaks.package], fake=True, explicit=True).install() # Sanity check that callpath is from upstream. assert not local_store.db.query_local("callpath") assert local_store.db.query("callpath") # Install mpileaks also upstream with the same hash to ensure that determining upstreamness # checks local installs before upstream databases, even when the local database is being # reindexed. monkeypatch.setattr(spack.store, "STORE", upstream_store) PackageInstaller([mpileaks.package], fake=True, explicit=True).install() # Delete the local database shutil.rmtree(local_store.db.database_directory) # Create a new instance s.t. we don't have cached specs in memory reindexed_local_store = spack.store.create( spack.config.create_from( spack.config.InternalConfigScope( "cfg", { "config": {"install_tree": {"root": str(tmp_path / "local")}}, "upstreams": {"my-upstream": {"install_tree": str(tmp_path / "upstream")}}, }, ) ) ) reindexed_local_store.db.reindex() assert not reindexed_local_store.db.query_local("callpath") assert reindexed_local_store.db.query("callpath") == [callpath] assert reindexed_local_store.db.query_local("mpileaks") == [mpileaks] @pytest.mark.regression("47101") def test_query_with_predicate_fn(database): all_specs = database.query() # Name starts with a string specs = database.query(predicate_fn=lambda x: x.spec.name.startswith("mpil")) assert specs and all(x.name.startswith("mpil") for x in specs) assert len(specs) < len(all_specs) # Recipe is currently known/unknown specs = database.query(predicate_fn=lambda x: spack.repo.PATH.exists(x.spec.name)) assert specs == all_specs specs = database.query(predicate_fn=lambda x: not spack.repo.PATH.exists(x.spec.name)) assert not specs @pytest.mark.regression("49964") def test_querying_reindexed_database_specfilev5(tmp_path: pathlib.Path): """Tests that we can query a reindexed database from before compilers as dependencies, and get appropriate results for % and similar selections. """ test_path = pathlib.Path(spack.paths.test_path) zipfile = test_path / "data" / "database" / "index.json.v7_v8.json.gz" with gzip.open(str(zipfile), "rt", encoding="utf-8") as f: data = json.load(f) index_json = tmp_path / spack.database._DB_DIRNAME / spack.database.INDEX_JSON_FILE index_json.parent.mkdir(parents=True) index_json.write_text(json.dumps(data)) db = spack.database.Database(str(tmp_path)) specs = db.query("%gcc") assert len(specs) == 8 assert len([x for x in specs if x.external]) == 2 assert len([x for x in specs if x.original_spec_format() < 5]) == 8 ================================================ FILE: lib/spack/spack/test/detection.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import pathlib import pytest import spack.config import spack.detection import spack.detection.common import spack.detection.path import spack.repo import spack.spec def test_detection_update_config(mutable_config): # mock detected package detected_packages = collections.defaultdict(list) detected_packages["cmake"] = [spack.spec.Spec("cmake@3.27.5", external_path="/usr/bin")] # update config for new package spack.detection.common.update_configuration(detected_packages) # Check entries in 'packages.yaml' packages_yaml = spack.config.get("packages") assert "cmake" in packages_yaml assert "externals" in packages_yaml["cmake"] externals = packages_yaml["cmake"]["externals"] assert len(externals) == 1 external_gcc = externals[0] assert external_gcc["spec"] == "cmake@3.27.5" assert external_gcc["prefix"] == "/usr/bin" def test_dedupe_paths(tmp_path: pathlib.Path): """Test that ``dedupe_paths`` deals with symlinked directories, retaining the target""" x = tmp_path / "x" y = tmp_path / "y" z = tmp_path / "z" x.mkdir() y.mkdir() z.symlink_to("x", target_is_directory=True) # dedupe repeated dirs, should preserve order assert spack.detection.path.dedupe_paths([str(x), str(y), str(x)]) == [str(x), str(y)] assert spack.detection.path.dedupe_paths([str(y), str(x), str(y)]) == [str(y), str(x)] # dedupe repeated symlinks assert spack.detection.path.dedupe_paths([str(z), str(y), str(z)]) == [str(z), str(y)] assert spack.detection.path.dedupe_paths([str(y), str(z), str(y)]) == [str(y), str(z)] # when both symlink and target are present, only target is retained, and it comes at the # priority of the first occurrence. assert spack.detection.path.dedupe_paths([str(x), str(y), str(z)]) == [str(x), str(y)] assert spack.detection.path.dedupe_paths([str(z), str(y), str(x)]) == [str(x), str(y)] assert spack.detection.path.dedupe_paths([str(y), str(z), str(x)]) == [str(y), str(x)] @pytest.mark.usefixtures("mock_packages") def test_detect_specs_deduplicates_across_prefixes(tmp_path, monkeypatch): """Tests that the same spec detected at two different prefixes should yield only one result. Returning both causes duplicate externals in packages.yaml and non-deterministic hashes during concretization. """ # Create two independent bin/ directories, each containing the same executable name. prefix_a = tmp_path / "prefix_a" prefix_b = tmp_path / "prefix_b" (prefix_a / "bin").mkdir(parents=True) (prefix_b / "bin").mkdir(parents=True) exe_a = prefix_a / "bin" / "cmake" exe_b = prefix_b / "bin" / "cmake" exe_a.touch() exe_b.touch() cmake_cls = spack.repo.PATH.get_pkg_class("cmake") # Patch determine_spec_details to always return the same spec, regardless of prefix. @classmethod def _same_spec(cls, prefix, exes_in_prefix): return spack.spec.Spec("cmake@3.17.1") monkeypatch.setattr(cmake_cls, "determine_spec_details", _same_spec) finder = spack.detection.path.ExecutablesFinder() detected = finder.detect_specs( pkg=cmake_cls, paths=[str(exe_a), str(exe_b)], repo_path=spack.repo.PATH ) # Both prefixes produce cmake@3.17.1; only the first should be kept. assert len(detected) == 1 ================================================ FILE: lib/spack/spack/test/directives.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from collections import namedtuple import pytest import spack.concretize import spack.directives import spack.repo import spack.spec import spack.version from spack.directives import _make_when_spec, depends_on, extends, patch from spack.directives_meta import DirectiveDictDescriptor, DirectiveMeta from spack.spec import Spec def test_false_directives_do_not_exist(mock_packages): """Ensure directives that evaluate to False at import time are added to dicts on packages. """ cls = spack.repo.PATH.get_pkg_class("when-directives-false") assert not cls.dependencies assert not cls.resources assert not cls.patches def test_true_directives_exist(mock_packages): """Ensure directives that evaluate to True at import time are added to dicts on packages. """ cls = spack.repo.PATH.get_pkg_class("when-directives-true") assert cls.dependencies assert "extendee" in cls.dependencies[spack.spec.Spec()] assert "pkg-b" in cls.dependencies[spack.spec.Spec()] assert cls.resources assert spack.spec.Spec() in cls.resources assert cls.patches assert spack.spec.Spec() in cls.patches def test_constraints_from_context(mock_packages): pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met") assert pkg_cls.dependencies assert "pkg-b" in pkg_cls.dependencies[spack.spec.Spec("@1.0")] assert pkg_cls.conflicts assert (spack.spec.Spec("%gcc"), None) in pkg_cls.conflicts[spack.spec.Spec("+foo@1.0")] @pytest.mark.regression("26656") def test_constraints_from_context_are_merged(mock_packages): pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met") assert pkg_cls.dependencies assert "pkg-c" in pkg_cls.dependencies[spack.spec.Spec("@0.14:15 ^pkg-b@3.8:4.0")] @pytest.mark.regression("27754") def test_extends_spec(config, mock_packages): extender = spack.concretize.concretize_one("extends-spec") extendee = spack.concretize.concretize_one("extendee") assert extender.dependencies assert extender.package.extends(extendee) @pytest.mark.regression("48024") def test_conditionally_extends_transitive_dep(config, mock_packages): spec = spack.concretize.concretize_one("conditionally-extends-transitive-dep") assert not spec.package.extendee_spec @pytest.mark.regression("48025") def test_conditionally_extends_direct_dep(config, mock_packages): spec = spack.concretize.concretize_one("conditionally-extends-direct-dep") assert not spec.package.extendee_spec @pytest.mark.regression("34368") def test_error_on_anonymous_dependency(config, mock_packages): pkg = spack.repo.PATH.get_pkg_class("pkg-a") with pytest.raises(spack.directives.DependencyError): spack.directives._execute_depends_on(pkg, spack.spec.Spec("@4.5")) @pytest.mark.regression("34879") @pytest.mark.parametrize( "package_name,expected_maintainers", [ ("maintainers-1", ["user1", "user2"]), # Extends PythonPackage ("py-extension1", ["user1", "user2"]), # Extends maintainers-1 ("maintainers-3", ["user0", "user1", "user2", "user3"]), ], ) def test_maintainer_directive(config, mock_packages, package_name, expected_maintainers): pkg_cls = spack.repo.PATH.get_pkg_class(package_name) assert pkg_cls.maintainers == expected_maintainers @pytest.mark.parametrize( "package_name,expected_licenses", [("licenses-1", [("MIT", "+foo"), ("Apache-2.0", "~foo")])] ) def test_license_directive(config, mock_packages, package_name, expected_licenses): pkg_cls = spack.repo.PATH.get_pkg_class(package_name) for license in expected_licenses: assert spack.spec.Spec(license[1]) in pkg_cls.licenses assert license[0] == pkg_cls.licenses[spack.spec.Spec(license[1])] def test_duplicate_exact_range_license(): package = namedtuple("package", ["licenses", "name"]) package.licenses = {spack.spec.Spec("+foo"): "Apache-2.0"} package.name = "test_package" msg = ( r"test_package is specified as being licensed as MIT when \+foo, but it is also " r"specified as being licensed under Apache-2.0 when \+foo, which conflict." ) with pytest.raises(spack.directives.OverlappingLicenseError, match=msg): spack.directives._execute_license(package, "MIT", "+foo") def test_overlapping_duplicate_licenses(): package = namedtuple("package", ["licenses", "name"]) package.licenses = {spack.spec.Spec("+foo"): "Apache-2.0"} package.name = "test_package" msg = ( r"test_package is specified as being licensed as MIT when \+bar, but it is also " r"specified as being licensed under Apache-2.0 when \+foo, which conflict." ) with pytest.raises(spack.directives.OverlappingLicenseError, match=msg): spack.directives._execute_license(package, "MIT", "+bar") def test_version_type_validation(): # A version should be a string or an int, not a float, because it leads to subtle issues # such as 3.10 being interpreted as 3.1. package = namedtuple("package", ["name"]) msg = r"python: declared version '.+' in package should be a string or int\." # Pass a float with pytest.raises(spack.version.VersionError, match=msg): spack.directives._execute_version(package(name="python"), ver=3.10, kwargs={}) # Try passing a bogus type; it's just that we want a nice error message with pytest.raises(spack.version.VersionError, match=msg): spack.directives._execute_version(package(name="python"), ver={}, kwargs={}) @pytest.mark.parametrize( "spec_str,distribute_src,distribute_bin", [ ("redistribute-x@1.1~foo", False, False), ("redistribute-x@1.2+foo", False, False), ("redistribute-x@1.2~foo", False, True), ("redistribute-x@1.0~foo", False, True), ("redistribute-x@1.3+foo", True, True), ("redistribute-y@2.0", False, False), ("redistribute-y@2.1+bar", False, False), ], ) def test_redistribute_directive(mock_packages, spec_str, distribute_src, distribute_bin): spec = spack.spec.Spec(spec_str) assert spack.repo.PATH.get_pkg_class(spec.fullname).redistribute_source(spec) == distribute_src concretized_spec = spack.concretize.concretize_one(spec) assert concretized_spec.package.redistribute_binary == distribute_bin def test_redistribute_override_when(): """Allow a user to call `redistribute` twice to separately disable source and binary distribution for the same when spec. The second call should not undo the effect of the first. """ class MockPackage: name = "mock" disable_redistribute = {} cls = MockPackage spack.directives._execute_redistribute(cls, source=False, binary=None, when="@1.0") spec_key = spack.directives._make_when_spec("@1.0") assert not cls.disable_redistribute[spec_key].binary assert cls.disable_redistribute[spec_key].source spack.directives._execute_redistribute(cls, source=None, binary=False, when="@1.0") assert cls.disable_redistribute[spec_key].binary assert cls.disable_redistribute[spec_key].source @pytest.mark.regression("51248") def test_direct_dependencies_from_when_context_are_retained(mock_packages): """Tests that direct dependencies from the "when" context manager don't lose the "direct" attribute when turned into directives on the package class. """ pkg_cls = spack.repo.PATH.get_pkg_class("with-constraint-met") # Direct dependency in a "when" single context manager assert spack.spec.Spec("%pkg-b") in pkg_cls.dependencies # Direct dependency in a "when" nested context manager assert spack.spec.Spec("@2 %c=gcc %pkg-c %pkg-b@:4.0") in pkg_cls.dependencies # Nested ^foo followed by %foo assert spack.spec.Spec("%pkg-c") in pkg_cls.dependencies # Nested ^foo followed by ^foo %gcc assert spack.spec.Spec("^pkg-c %gcc") in pkg_cls.dependencies def test_directives_meta_combine_when(): x, y, z = "+x ^dep +a", "+y ^dep +b", "+z" assert _make_when_spec((x, y, z)) == Spec("+x +y +z ^dep +a +b") assert _make_when_spec((x, y)) == Spec("+x +y ^dep +a +b") assert _make_when_spec((x,)) == Spec("+x ^dep +a") def test_directive_descriptor_init(): # when `pkg.variants` is initialized, only the `variant` directive should run variants = DirectiveDictDescriptor("variants") assert variants.directives_to_run == ["variant"] assert variants.dicts_to_init == ["variants"] # when `pkg.dependencies` is initialized, `depends_on` and `extends` should run, and also # `pkg.extendees` should be initialized dependencies = DirectiveDictDescriptor("dependencies") assert dependencies.directives_to_run == ["depends_on", "extends"] assert dependencies.dicts_to_init == ["dependencies", "extendees"] # when `pkg.provided` is initialized, so should `pkg.provided_together`, and only the # provides directive should run provided = DirectiveDictDescriptor("provided") assert provided.directives_to_run == ["provides"] assert provided.dicts_to_init == ["provided", "provided_together"] # idem for `pkg.provided_together` provided_together = DirectiveDictDescriptor("provided_together") assert provided_together.directives_to_run == ["provides"] assert provided_together.dicts_to_init == ["provided", "provided_together"] # when specifying patches on dependencies with `depends_on` and `extends`, the `pkg.patches` # dict is not affects -- they are stored on a Dependency object. patches = DirectiveDictDescriptor("patches") assert patches.directives_to_run == ["patch"] assert patches.dicts_to_init == ["patches"] def test_directive_laziness(): class ExamplePackage(metaclass=DirectiveMeta): name = "example-package" depends_on("foo") extends("bar", when="+bar") # Initially, no directive dicts are initialized assert ExamplePackage._dependencies is None # type: ignore assert ExamplePackage._extendees is None # type: ignore assert ExamplePackage._variants is None # type: ignore # Only when we access the dependencies descriptor, the relevant dicts (dependencies, extendees) # are initialized, while others remain None dependencies = ExamplePackage.dependencies # type: ignore assert type(ExamplePackage._dependencies) is dict # type: ignore assert type(ExamplePackage._extendees) is dict # type: ignore assert ExamplePackage._variants is None # type: ignore # The dependencies dict is populated with the expected entries assert "foo" in dependencies[spack.spec.Spec()] assert "bar" in dependencies[spack.spec.Spec("+bar")] def test_patched_dependencies_sets_class_attribute(): sha256 = "a" * 64 class PatchesDependencies(metaclass=DirectiveMeta): name = "patches-dependencies" depends_on("dependency", patches=patch("https://example.com/diff.patch", sha256=sha256)) assert PatchesDependencies._patches_dependencies is True assert not PatchesDependencies.patches # type: ignore class DoesNotPatchDependencies(metaclass=DirectiveMeta): name = "does-not-patch-dependencies" fullname = "does-not-patch-dependencies" patch("https://example.com/diff.patch", sha256=sha256) assert DoesNotPatchDependencies._patches_dependencies is False assert DoesNotPatchDependencies.patches # type: ignore ================================================ FILE: lib/spack/spack/test/directory_layout.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This test verifies that the Spack directory layout works properly. """ import os import pathlib from pathlib import Path import pytest import spack.concretize import spack.hash_types import spack.paths import spack.repo import spack.util.file_cache from spack.directory_layout import DirectoryLayout, InvalidDirectoryLayoutParametersError from spack.llnl.path import path_to_os_path from spack.spec import Spec # number of packages to test (to reduce test time) max_packages = 10 def test_yaml_directory_layout_parameters(tmp_path: pathlib.Path, default_mock_concretization): """This tests the various parameters that can be used to configure the install location""" spec = default_mock_concretization("python") # Ensure default layout matches expected spec format layout_default = DirectoryLayout(str(tmp_path)) path_default = layout_default.relative_path_for_spec(spec) assert path_default == str( Path(spec.format("{architecture.platform}-{architecture.target}/{name}-{version}-{hash}")) ) # Test hash_length parameter works correctly layout_10 = DirectoryLayout(str(tmp_path), hash_length=10) path_10 = layout_10.relative_path_for_spec(spec) layout_7 = DirectoryLayout(str(tmp_path), hash_length=7) path_7 = layout_7.relative_path_for_spec(spec) assert len(path_default) - len(path_10) == 22 assert len(path_default) - len(path_7) == 25 # Test path_scheme arch, package7 = path_7.split(os.sep) projections_package7 = {"all": "{name}-{version}-{hash:7}"} layout_package7 = DirectoryLayout(str(tmp_path), projections=projections_package7) path_package7 = layout_package7.relative_path_for_spec(spec) assert package7 == path_package7 # Test separation of architecture or namespace spec2 = spack.concretize.concretize_one("libelf") arch_scheme = ( "{architecture.platform}/{architecture.target}/{architecture.os}/{name}/{version}/{hash:7}" ) ns_scheme = "{architecture}/{namespace}/{name}-{version}-{hash:7}" arch_ns_scheme_projections = {"all": arch_scheme, "python": ns_scheme} layout_arch_ns = DirectoryLayout(str(tmp_path), projections=arch_ns_scheme_projections) arch_path_spec2 = layout_arch_ns.relative_path_for_spec(spec2) assert arch_path_spec2 == str(Path(spec2.format(arch_scheme))) ns_path_spec = layout_arch_ns.relative_path_for_spec(spec) assert ns_path_spec == str(Path(spec.format(ns_scheme))) # Ensure conflicting parameters caught with pytest.raises(InvalidDirectoryLayoutParametersError): DirectoryLayout(str(tmp_path), hash_length=20, projections=projections_package7) def test_read_and_write_spec(temporary_store, config, mock_packages): """This goes through each package in spack and creates a directory for it. It then ensures that the spec for the directory's installed package can be read back in consistently, and finally that the directory can be removed by the directory layout. """ layout = temporary_store.layout pkg_names = list(spack.repo.PATH.all_package_names())[:max_packages] for name in pkg_names: if name.startswith("external"): # External package tests cannot be installed continue # If a spec fails to concretize, just skip it. If it is a # real error, it will be caught by concretization tests. try: spec = spack.concretize.concretize_one(name) except Exception: continue layout.create_install_directory(spec) install_dir = path_to_os_path(layout.path_for_spec(spec))[0] spec_path = layout.spec_file_path(spec) # Ensure directory has been created in right place. assert os.path.isdir(install_dir) assert install_dir.startswith(temporary_store.root) # Ensure spec file exists when directory is created assert os.path.isfile(spec_path) assert spec_path.startswith(install_dir) # Make sure spec file can be read back in to get the original spec spec_from_file = layout.read_spec(spec_path) stored_deptypes = spack.hash_types.dag_hash expected = spec.copy(deps=stored_deptypes) expected._mark_concrete() assert expected.concrete assert expected == spec_from_file assert expected.eq_dag(spec_from_file) assert spec_from_file.concrete # Ensure that specs that come out "normal" are really normal. with open(spec_path, encoding="utf-8") as spec_file: read_separately = Spec.from_yaml(spec_file.read()) # TODO: revise this when build deps are in dag_hash norm = read_separately.copy(deps=stored_deptypes) assert norm == spec_from_file assert norm.eq_dag(spec_from_file) # TODO: revise this when build deps are in dag_hash conc = spack.concretize.concretize_one(read_separately).copy(deps=stored_deptypes) assert conc == spec_from_file assert conc.eq_dag(spec_from_file) assert expected.dag_hash() == spec_from_file.dag_hash() # Ensure directories are properly removed layout.remove_install_directory(spec) assert not os.path.isdir(install_dir) assert not os.path.exists(install_dir) def test_handle_unknown_package(temporary_store, config, mock_packages, tmp_path: pathlib.Path): """This test ensures that spack can at least do *some* operations with packages that are installed but that it does not know about. This is actually not such an uncommon scenario with spack; it can happen when you switch from a git branch where you're working on a new package. This test ensures that the directory layout stores enough information about installed packages' specs to uninstall or query them again if the package goes away. """ layout = temporary_store.layout repo_cache = spack.util.file_cache.FileCache(tmp_path / "cache") mock_db = spack.repo.Repo(spack.paths.mock_packages_path, cache=repo_cache) not_in_mock = set.difference( set(spack.repo.all_package_names()), set(mock_db.all_package_names()) ) packages = list(not_in_mock)[:max_packages] # Create all the packages that are not in mock. installed_specs = {} for pkg_name in packages: # If a spec fails to concretize, just skip it. If it is a # real error, it will be caught by concretization tests. try: spec = spack.concretize.concretize_one(pkg_name) except Exception: continue layout.create_install_directory(spec) installed_specs[spec] = layout.path_for_spec(spec) with spack.repo.use_repositories(spack.paths.mock_packages_path): # Now check that even without the package files, we know # enough to read a spec from the spec file. for spec, path in installed_specs.items(): spec_from_file = layout.read_spec(os.path.join(path, ".spack", "spec.json")) # To satisfy these conditions, directory layouts need to # read in concrete specs from their install dirs somehow. assert path == layout.path_for_spec(spec_from_file) assert spec == spec_from_file assert spec.eq_dag(spec_from_file) assert spec.dag_hash() == spec_from_file.dag_hash() def test_find(temporary_store, config, mock_packages): """Test that finding specs within an install layout works.""" layout = temporary_store.layout package_names = list(spack.repo.PATH.all_package_names())[:max_packages] # Create install prefixes for all packages in the list installed_specs = {} for name in package_names: if name.startswith("external"): # External package tests cannot be installed continue spec = spack.concretize.concretize_one(name) installed_specs[spec.name] = spec layout.create_install_directory(spec) # Make sure all the installed specs appear in # DirectoryLayout.all_specs() found_specs = dict((s.name, s) for s in layout.all_specs()) for name, spec in found_specs.items(): assert name in found_specs assert found_specs[name].eq_dag(spec) def test_yaml_directory_layout_build_path(tmp_path: pathlib.Path, default_mock_concretization): """This tests build path method.""" spec = default_mock_concretization("python") layout = DirectoryLayout(str(tmp_path)) rel_path = os.path.join(layout.metadata_dir, layout.packages_dir) assert layout.build_packages_path(spec) == os.path.join(spec.prefix, rel_path) ================================================ FILE: lib/spack/spack/test/entry_points.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import sys import pytest import spack.config import spack.extensions import spack.llnl.util.lang class MockConfigEntryPoint: def __init__(self, tmp_path: pathlib.Path): self.dir = tmp_path self.name = "mypackage_config" def load(self): etc_path = self.dir.joinpath("spack/etc") etc_path.mkdir(exist_ok=True, parents=True) f = self.dir / "spack/etc/config.yaml" with open(f, "w", encoding="utf-8") as fh: fh.write("config:\n install_tree:\n root: /spam/opt\n") def ep(): return self.dir / "spack/etc" return ep class MockExtensionsEntryPoint: def __init__(self, tmp_path: pathlib.Path): self.dir = tmp_path self.name = "mypackage_extensions" def load(self): cmd_path = self.dir.joinpath("spack/spack-myext/myext/cmd") cmd_path.mkdir(exist_ok=True, parents=True) f = self.dir / "spack/spack-myext/myext/cmd/spam.py" with open(f, "w", encoding="utf-8") as fh: fh.write("description = 'hello world extension command'\n") fh.write("section = 'test command'\n") fh.write("level = 'long'\n") fh.write("def setup_parser(subparser):\n pass\n") fh.write("def spam(parser, args):\n print('spam for all!')\n") def ep(): return self.dir / "spack/spack-myext" return ep def entry_points_factory(tmp_path: pathlib.Path): def entry_points(group=None): if group == "spack.config": return (MockConfigEntryPoint(tmp_path),) elif group == "spack.extensions": return (MockExtensionsEntryPoint(tmp_path),) return () return entry_points @pytest.fixture() def mock_get_entry_points(tmp_path: pathlib.Path, reset_extension_paths, monkeypatch): entry_points = entry_points_factory(tmp_path) monkeypatch.setattr(spack.llnl.util.lang, "get_entry_points", entry_points) def test_spack_entry_point_config(tmp_path: pathlib.Path, mock_get_entry_points): """Test config scope entry point""" config_paths = dict(spack.config.config_paths_from_entry_points()) config_path = config_paths.get("plugin-mypackage_config") my_config_path = tmp_path / "spack/etc" if config_path is None: raise ValueError("Did not find entry point config in %s" % str(config_paths)) else: assert os.path.samefile(config_path, my_config_path) config = spack.config.create() assert config.get("config:install_tree:root", scope="plugin-mypackage_config") == "/spam/opt" def test_spack_entry_point_extension(tmp_path: pathlib.Path, mock_get_entry_points): """Test config scope entry point""" my_ext = tmp_path / "spack/spack-myext" extensions = spack.extensions.get_extension_paths() found = bool([ext for ext in extensions if os.path.samefile(ext, my_ext)]) if not found: raise ValueError("Did not find extension in %s" % ", ".join(extensions)) extensions = spack.extensions.extension_paths_from_entry_points() found = bool([ext for ext in extensions if os.path.samefile(ext, my_ext)]) if not found: raise ValueError("Did not find extension in %s" % ", ".join(extensions)) root = spack.extensions.load_extension("myext") assert os.path.samefile(root, my_ext) module = spack.extensions.get_module("spam") assert module is not None @pytest.mark.skipif(sys.version_info[:2] < (3, 8), reason="Python>=3.8 required") def test_llnl_util_lang_get_entry_points(tmp_path: pathlib.Path, monkeypatch): import importlib.metadata # type: ignore # novermin monkeypatch.setattr(importlib.metadata, "entry_points", entry_points_factory(tmp_path)) entry_points = list(spack.llnl.util.lang.get_entry_points(group="spack.config")) assert isinstance(entry_points[0], MockConfigEntryPoint) entry_points = list(spack.llnl.util.lang.get_entry_points(group="spack.extensions")) assert isinstance(entry_points[0], MockExtensionsEntryPoint) ================================================ FILE: lib/spack/spack/test/env.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test environment internals without CLI""" import filecmp import json import os import pathlib import pickle import pytest import spack.config import spack.environment as ev import spack.llnl.util.filesystem as fs import spack.platforms import spack.solver.asp import spack.spec import spack.spec_parser from spack.enums import ConfigScopePriority from spack.environment import SpackEnvironmentConfigError from spack.environment.environment import ( EnvironmentManifestFile, SpackEnvironmentViewError, _error_on_nonempty_view_dir, ) from spack.environment.list import UndefinedReferenceError from spack.traverse import traverse_nodes pytestmark = [ pytest.mark.not_on_windows("Envs are not supported on windows"), pytest.mark.usefixtures("mock_packages"), ] class TestDirectoryInitialization: def test_environment_dir_from_name(self, mutable_mock_env_path): """Test the function mapping a managed environment name to its folder.""" env = ev.create("test") environment_dir = ev.environment_dir_from_name("test") assert env.path == environment_dir with pytest.raises(ev.SpackEnvironmentError, match="environment already exists"): ev.environment_dir_from_name("test", exists_ok=False) def test_environment_dir_from_nested_name(self, mutable_mock_env_path): """Test the function mapping a nested managed environment name to its folder.""" env = ev.create("group/test") environment_dir = ev.environment_dir_from_name("group/test") assert env.path == environment_dir with pytest.raises(ev.SpackEnvironmentError, match="environment already exists"): ev.environment_dir_from_name("group/test", exists_ok=False) def test_hash_change_no_rehash_concrete(tmp_path: pathlib.Path, config): # create an environment env_path = tmp_path / "env_dir" env_path.mkdir(exist_ok=False) env = ev.create_in_dir(env_path) env.write() # add a spec with a rewritten build hash spec = spack.spec.Spec("mpileaks") env.add(spec) env.concretize() # rewrite the hash old_hash, new_hash = env.concretized_roots[0].hash, "abc" env.specs_by_hash[old_hash]._hash = new_hash # type: ignore[attr-defined] env.concretized_roots[0].hash = new_hash env.specs_by_hash[new_hash] = env.specs_by_hash[old_hash] del env.specs_by_hash[old_hash] env.write() # Read environment read_in = ev.Environment(env_path) # Ensure read hashes are used (rewritten hash seen on read) hashes = [x.hash for x in read_in.concretized_roots] assert hashes assert hashes[0] in read_in.specs_by_hash _hash = read_in.specs_by_hash[hashes[0]]._hash # type: ignore[attr-defined] assert _hash == new_hash def test_env_change_spec(tmp_path: pathlib.Path, config): env_path = tmp_path / "env_dir" env_path.mkdir(exist_ok=False) env = ev.create_in_dir(env_path) env.write() spec = spack.spec.Spec("mpileaks@2.1~shared+debug") env.add(spec) env.write() change_spec = spack.spec.Spec("mpileaks@2.2") env.change_existing_spec(change_spec) (spec,) = env.added_specs() assert spec == spack.spec.Spec("mpileaks@2.2~shared+debug") change_spec = spack.spec.Spec("mpileaks~debug") env.change_existing_spec(change_spec) (spec,) = env.added_specs() assert spec == spack.spec.Spec("mpileaks@2.2~shared~debug") _test_matrix_yaml = """\ spack: definitions: - compilers: ["%gcc", "%clang"] - desired_specs: ["mpileaks@2.1"] specs: - matrix: - [$compilers] - [$desired_specs] """ def test_env_change_spec_in_definition(tmp_path: pathlib.Path, mutable_mock_env_path): manifest_file = tmp_path / ev.manifest_name manifest_file.write_text(_test_matrix_yaml) e = ev.create("test", manifest_file) e.concretize() e.write() assert any(x.intersects("mpileaks@2.1%gcc") for x in e.user_specs) with e: e.change_existing_spec(spack.spec.Spec("mpileaks@2.2"), list_name="desired_specs") e.write() # Ensure changed specs are in memory assert any(x.intersects("mpileaks@2.2%gcc") for x in e.user_specs) assert not any(x.intersects("mpileaks@2.1%gcc") for x in e.user_specs) # Now make sure the changes can be read from the modified config e = ev.read("test") assert any(x.intersects("mpileaks@2.2%gcc") for x in e.user_specs) assert not any(x.intersects("mpileaks@2.1%gcc") for x in e.user_specs) def test_env_change_spec_in_matrix_raises_error(tmp_path: pathlib.Path, mutable_mock_env_path): manifest_file = tmp_path / ev.manifest_name manifest_file.write_text(_test_matrix_yaml) e = ev.create("test", manifest_file) e.concretize() e.write() with pytest.raises(ev.SpackEnvironmentError) as error: e.change_existing_spec(spack.spec.Spec("mpileaks@2.2")) assert "Cannot directly change specs in matrices" in str(error) def test_activate_should_require_an_env(): with pytest.raises(TypeError): ev.activate(env="name") with pytest.raises(TypeError): ev.activate(env=None) def test_user_view_path_is_not_canonicalized_in_yaml(tmp_path: pathlib.Path, config): # When spack.yaml files are checked into version control, we # don't want view: ./relative to get canonicalized on disk. # We create a view in /env_dir env_path = str(tmp_path / "env_dir") (tmp_path / "env_dir").mkdir() # And use a relative path to specify the view dir view = os.path.join(".", "view") # Which should always resolve to the following independent of cwd. absolute_view = os.path.join(env_path, "view") # Serialize environment with relative view path with fs.working_dir(str(tmp_path)): fst = ev.create_in_dir(env_path, with_view=view) fst.regenerate_views() # The view link should be created assert os.path.isdir(absolute_view) # Deserialize and check if the view path is still relative in yaml # and also check that the getter is pointing to the right dir. with fs.working_dir(str(tmp_path)): snd = ev.Environment(env_path) assert snd.manifest["spack"]["view"] == view assert os.path.samefile(snd.default_view.root, absolute_view) def test_environment_cant_modify_environments_root(tmp_path: pathlib.Path): filename = str(tmp_path / "spack.yaml") with open(filename, "w", encoding="utf-8") as f: f.write( """\ spack: config: environments_root: /a/black/hole view: false specs: [] """ ) with fs.working_dir(str(tmp_path)): with pytest.raises(ev.SpackEnvironmentError): e = ev.Environment(str(tmp_path)) ev.activate(e) @pytest.mark.regression("35420") @pytest.mark.parametrize( "original_content", [ """\ spack: specs: - matrix: # test - - a concretizer: unify: false""" ], ) def test_roundtrip_spack_yaml_with_comments(original_content, config, tmp_path: pathlib.Path): """Ensure that round-tripping a spack.yaml file doesn't change its content.""" spack_yaml = tmp_path / "spack.yaml" spack_yaml.write_text(original_content) e = ev.Environment(tmp_path) e.manifest.flush() content = spack_yaml.read_text() assert content == original_content def test_adding_anonymous_specs_to_env_fails(tmp_path: pathlib.Path): """Tests that trying to add an anonymous spec to the 'specs' section of an environment raises an exception """ env = ev.create_in_dir(tmp_path) with pytest.raises(ev.SpackEnvironmentError, match="cannot add anonymous"): env.add("%gcc") def test_removing_from_non_existing_list_fails(tmp_path: pathlib.Path): """Tests that trying to remove a spec from a non-existing definition fails.""" env = ev.create_in_dir(tmp_path) with pytest.raises(ev.SpackEnvironmentError, match="'bar' does not exist"): env.remove("%gcc", list_name="bar") @pytest.mark.parametrize( "init_view,update_value", [ (True, False), (True, "./view"), (False, True), ("./view", True), ("./view", False), (True, True), (False, False), ], ) def test_update_default_view(init_view, update_value, tmp_path: pathlib.Path, config): """Tests updating the default view with different values.""" env = ev.create_in_dir(tmp_path, with_view=init_view) env.update_default_view(update_value) env.write(regenerate=True) if not isinstance(update_value, bool): assert env.default_view.raw_root == update_value expected_value = update_value if isinstance(init_view, str) and update_value is True: expected_value = init_view assert env.manifest.yaml_content["spack"]["view"] == expected_value @pytest.mark.parametrize( "initial_content,update_value,expected_view", [ ( """ spack: specs: - mpileaks view: default: root: ./view-gcc select: ['%gcc'] link_type: symlink """, "./another-view", {"root": "./another-view", "select": ["%gcc"], "link_type": "symlink"}, ), ( """ spack: specs: - mpileaks view: default: root: ./view-gcc select: ['%gcc'] link_type: symlink """, True, {"root": "./view-gcc", "select": ["%gcc"], "link_type": "symlink"}, ), ], ) def test_update_default_complex_view( initial_content, update_value, expected_view, tmp_path: pathlib.Path, config ): spack_yaml = tmp_path / "spack.yaml" spack_yaml.write_text(initial_content) env = ev.Environment(tmp_path) env.update_default_view(update_value) env.write(regenerate=True) assert env.default_view.to_dict() == expected_view @pytest.mark.parametrize("filename", [ev.manifest_name, ev.lockfile_name]) def test_cannot_initialize_in_dir_with_init_file(tmp_path: pathlib.Path, filename): """Tests that initializing an environment in a directory with an already existing spack.yaml or spack.lock raises an exception. """ init_file = tmp_path / filename init_file.touch() with pytest.raises(ev.SpackEnvironmentError, match="cannot initialize"): ev.create_in_dir(tmp_path) def test_cannot_initiliaze_if_dirname_exists_as_a_file(tmp_path: pathlib.Path): """Tests that initializing an environment using as a location an existing file raises an error. """ dir_name = tmp_path / "dir" dir_name.touch() with pytest.raises(ev.SpackEnvironmentError, match="cannot initialize"): ev.create_in_dir(dir_name) def test_cannot_initialize_if_init_file_does_not_exist(tmp_path: pathlib.Path): """Tests that initializing an environment passing a non-existing init file raises an error.""" init_file = tmp_path / ev.manifest_name with pytest.raises(ev.SpackEnvironmentError, match="cannot initialize"): ev.create_in_dir(tmp_path, init_file=init_file) def test_environment_pickle(tmp_path: pathlib.Path): env1 = ev.create_in_dir(tmp_path) obj = pickle.dumps(env1) env2 = pickle.loads(obj) assert isinstance(env2, ev.Environment) def test_error_on_nonempty_view_dir(tmp_path: pathlib.Path): """Error when the target is not an empty dir""" with fs.working_dir(str(tmp_path)): os.mkdir("empty_dir") os.mkdir("nonempty_dir") with open(os.path.join("nonempty_dir", "file"), "wb"): pass os.symlink("empty_dir", "symlinked_empty_dir") os.symlink("does_not_exist", "broken_link") os.symlink("broken_link", "file") # This is OK. _error_on_nonempty_view_dir("empty_dir") # This is not OK. with pytest.raises(SpackEnvironmentViewError): _error_on_nonempty_view_dir("nonempty_dir") with pytest.raises(SpackEnvironmentViewError): _error_on_nonempty_view_dir("symlinked_empty_dir") with pytest.raises(SpackEnvironmentViewError): _error_on_nonempty_view_dir("broken_link") with pytest.raises(SpackEnvironmentViewError): _error_on_nonempty_view_dir("file") def test_can_add_specs_to_environment_without_specs_attribute(tmp_path: pathlib.Path, config): """Sometimes users have template manifest files, and save one line in the YAML file by removing the empty 'specs: []' attribute. This test ensures that adding a spec to an environment without the 'specs' attribute, creates the attribute first instead of returning an error. """ spack_yaml = tmp_path / "spack.yaml" spack_yaml.write_text( """ spack: view: true concretizer: unify: true """ ) env = ev.Environment(tmp_path) env.add("pkg-a") assert len(env.user_specs) == 1 assert env.manifest.yaml_content["spack"]["specs"] == ["pkg-a"] @pytest.mark.parametrize( "original_yaml,new_spec,expected_yaml", [ ( """spack: specs: # baz - zlib """, "libdwarf", """spack: specs: # baz - zlib - libdwarf """, ) ], ) def test_preserving_comments_when_adding_specs( original_yaml, new_spec, expected_yaml, config, tmp_path: pathlib.Path ): """Ensure that round-tripping a spack.yaml file doesn't change its content.""" spack_yaml = tmp_path / "spack.yaml" spack_yaml.write_text(original_yaml) e = ev.Environment(str(tmp_path)) e.add(new_spec) e.write() content = spack_yaml.read_text() assert content == expected_yaml @pytest.mark.parametrize("filename", [ev.lockfile_name, "as9582g54.lock", "m3ia54s.json"]) @pytest.mark.regression("37410") def test_initialize_from_lockfile(tmp_path: pathlib.Path, filename): """Some users have workflows where they store multiple lockfiles in the same directory, and pick one of them to create an environment depending on external parameters e.g. while running CI jobs. This test ensures that Spack can create environments from lockfiles that are not necessarily named 'spack.lock' and can thus coexist in the same directory. """ init_file = tmp_path / filename env_dir = tmp_path / "env_dir" init_file.write_text('{ "roots": [] }\n') ev.initialize_environment_dir(env_dir, init_file) assert os.path.exists(env_dir / ev.lockfile_name) assert filecmp.cmp(env_dir / ev.lockfile_name, init_file, shallow=False) def test_cannot_initialize_from_bad_lockfile(tmp_path: pathlib.Path): """Test that we fail on an incorrectly constructed lockfile""" init_file = tmp_path / ev.lockfile_name env_dir = tmp_path / "env_dir" init_file.write_text("Not a legal JSON file\n") with pytest.raises(ev.SpackEnvironmentError, match="from lockfile"): ev.initialize_environment_dir(env_dir, init_file) @pytest.mark.parametrize("filename", ["random.txt", "random.yaml", ev.manifest_name]) @pytest.mark.regression("37410") def test_initialize_from_random_file_as_manifest(tmp_path: pathlib.Path, filename): """Some users have workflows where they store multiple lockfiles in the same directory, and pick one of them to create an environment depending on external parameters e.g. while running CI jobs. This test ensures that Spack can create environments from manifest that are not necessarily named 'spack.yaml' and can thus coexist in the same directory. """ init_file = tmp_path / filename env_dir = tmp_path / "env_dir" init_file.write_text( """\ spack: view: true concretizer: unify: true specs: [] """ ) ev.create_in_dir(env_dir, init_file) assert not os.path.exists(env_dir / ev.lockfile_name) assert os.path.exists(env_dir / ev.manifest_name) assert filecmp.cmp(env_dir / ev.manifest_name, init_file, shallow=False) def test_error_message_when_using_too_new_lockfile(tmp_path: pathlib.Path): """Sometimes the lockfile format needs to be bumped. When that happens, we have forward incompatibilities that need to be reported in a clear way to the user, in case we moved back to an older version of Spack. This test ensures that the error message for a too new lockfile version stays comprehensible across refactoring of the environment code. """ init_file = tmp_path / ev.lockfile_name env_dir = tmp_path / "env_dir" init_file.write_text( """ { "_meta": { "file-type": "spack-lockfile", "lockfile-version": 100, "specfile-version": 3 }, "roots": [], "concrete_specs": {} }\n """ ) ev.initialize_environment_dir(env_dir, init_file) with pytest.raises(ev.SpackEnvironmentError, match="You need to use a newer Spack version."): ev.Environment(env_dir) @pytest.mark.regression("38240") @pytest.mark.parametrize( "unify_in_lower_scope,unify_in_spack_yaml", [ (True, False), (True, "when_possible"), (False, True), (False, "when_possible"), ("when_possible", False), ("when_possible", True), ], ) def test_environment_concretizer_scheme_used( tmp_path: pathlib.Path, mutable_config, unify_in_lower_scope, unify_in_spack_yaml ): """Tests that "unify" settings in spack.yaml always take precedence over settings in lower configuration scopes. """ manifest = tmp_path / "spack.yaml" manifest.write_text( f"""\ spack: specs: - mpileaks concretizer: unify: {str(unify_in_spack_yaml).lower()} """ ) mutable_config.set("concretizer:unify", unify_in_lower_scope) assert mutable_config.get("concretizer:unify") == unify_in_lower_scope with ev.Environment(manifest.parent): assert mutable_config.get("concretizer:unify") == unify_in_spack_yaml @pytest.mark.parametrize("unify_in_config", [True, False, "when_possible"]) def test_environment_config_scheme_used(tmp_path: pathlib.Path, unify_in_config): """Tests that "unify" settings in lower configuration scopes is taken into account, if absent in spack.yaml. """ manifest = tmp_path / "spack.yaml" manifest.write_text( """\ spack: specs: - mpileaks """ ) with spack.config.override("concretizer:unify", unify_in_config): with ev.Environment(manifest.parent): assert spack.config.CONFIG.get("concretizer:unify") == unify_in_config @pytest.mark.parametrize( "spec_str,expected_raise,expected_spec", [ # vendorsb vendors "b" only when @=1.1 ("vendorsb", False, "vendorsb@=1.0"), ("vendorsb@=1.1", True, None), ], ) def test_conflicts_with_packages_that_are_not_dependencies( spec_str, expected_raise, expected_spec, tmp_path: pathlib.Path, config ): """Tests that we cannot concretize two specs together, if one conflicts with the other, even though they don't have a dependency relation. """ manifest = tmp_path / "spack.yaml" manifest.write_text( f"""\ spack: specs: - {spec_str} - pkg-b concretizer: unify: true """ ) with ev.Environment(manifest.parent) as e: if expected_raise: with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): e.concretize() else: e.concretize() assert any(s.satisfies(expected_spec) for s in e.concrete_roots()) @pytest.mark.regression("39455") @pytest.mark.parametrize( "possible_mpi_spec,unify", [("mpich", False), ("mpich", True), ("zmpi", False), ("zmpi", True)] ) def test_requires_on_virtual_and_potential_providers( possible_mpi_spec, unify, tmp_path: pathlib.Path, config ): """Tests that in an environment we can add packages explicitly, even though they provide a virtual package, and we require the provider of the same virtual to be another package, if they are added explicitly by their name. """ manifest = tmp_path / "spack.yaml" manifest.write_text( f"""\ spack: specs: - {possible_mpi_spec} - mpich2 - mpileaks packages: mpi: require: mpich2 concretizer: unify: {unify} """ ) with ev.Environment(manifest.parent) as e: e.concretize() assert e.matching_spec(possible_mpi_spec) assert e.matching_spec("mpich2") mpileaks = e.matching_spec("mpileaks") assert mpileaks.satisfies("^mpich2") assert mpileaks["mpi"].satisfies("mpich2") assert not mpileaks.satisfies(f"^{possible_mpi_spec}") @pytest.mark.regression("39387") @pytest.mark.parametrize( "spec_str", ["mpileaks +opt", "mpileaks +opt ~shared", "mpileaks ~shared +opt"] ) def test_manifest_file_removal_works_if_spec_is_not_normalized(tmp_path: pathlib.Path, spec_str): """Tests that we can remove a spec from a manifest file even if its string representation is not normalized. """ manifest = tmp_path / "spack.yaml" manifest.write_text( f"""\ spack: specs: - {spec_str} """ ) s = spack.spec.Spec(spec_str) spack_yaml = EnvironmentManifestFile(tmp_path) # Doing a round trip str -> Spec -> str normalizes the representation spack_yaml.remove_user_spec(str(s)) spack_yaml.flush() assert spec_str not in manifest.read_text() @pytest.mark.regression("39387") @pytest.mark.parametrize( "duplicate_specs,expected_number", [ # Swap variants, versions, etc. add spaces (["foo +bar ~baz", "foo ~baz +bar"], 3), (["foo @1.0 ~baz %gcc", "foo ~baz @1.0%gcc"], 3), # Item 1 and 3 are exactly the same (["zlib +shared", "zlib +shared", "zlib +shared"], 4), ], ) def test_removing_spec_from_manifest_with_exact_duplicates( duplicate_specs, expected_number, tmp_path: pathlib.Path ): """Tests that we can remove exact duplicates from a manifest file. Note that we can't get in a state with duplicates using only CLI, but this might happen on user edited spack.yaml files. """ manifest = tmp_path / "spack.yaml" manifest.write_text( f"""\ spack: specs: [{", ".join(duplicate_specs)} , "zlib"] """ ) with ev.Environment(tmp_path) as env: assert len(env.user_specs) == expected_number env.remove(duplicate_specs[0]) env.write() assert "+shared" not in manifest.read_text() assert "zlib" in manifest.read_text() with ev.Environment(tmp_path) as env: assert len(env.user_specs) == 1 @pytest.mark.regression("35298") def test_variant_propagation_with_unify_false(tmp_path: pathlib.Path, config): """Spack distributes concretizations to different processes, when unify:false is selected and the number of roots is 2 or more. When that happens, the specs to be concretized need to be properly reconstructed on the worker process, if variant propagation was requested. """ manifest = tmp_path / "spack.yaml" manifest.write_text( """ spack: specs: - parent-foo ++foo - pkg-c concretizer: unify: false """ ) with ev.Environment(tmp_path) as env: env.concretize() root = env.matching_spec("parent-foo") for node in root.traverse(): assert node.satisfies("+foo") def test_env_with_include_defs(mutable_mock_env_path): """Test environment with included definitions file.""" env_path = mutable_mock_env_path env_path.mkdir() defs_file = env_path / "definitions.yaml" defs_file.write_text( """definitions: - core_specs: [libdwarf, libelf] - compilers: ['%gcc'] """ ) spack_yaml = env_path / ev.manifest_name spack_yaml.write_text( f"""spack: include: - {defs_file.as_uri()} definitions: - my_packages: [zlib] specs: - matrix: - [$core_specs] - [$compilers] - $my_packages """ ) e = ev.Environment(env_path) with e: e.concretize() def test_env_with_include_def_missing(mutable_mock_env_path): """Test environment with included definitions file that is missing a definition.""" env_path = mutable_mock_env_path env_path.mkdir() filename = "missing-def.yaml" defs_file = env_path / filename defs_file.write_text("definitions:\n- my_compilers: ['%gcc']\n") spack_yaml = env_path / ev.manifest_name spack_yaml.write_text( f"""spack: include: - {defs_file.as_uri()} specs: - matrix: - [$core_specs] - [$my_compilers] """ ) with pytest.raises(UndefinedReferenceError, match=r"which is not defined"): _ = ev.Environment(env_path) @pytest.mark.regression("41292") @pytest.mark.parametrize("unify", ["true", "false", "when_possible"]) def test_deconcretize_then_concretize_does_not_error(mutable_mock_env_path, unify): """Tests that, after having deconcretized a spec, we can reconcretize an environment which has 2 or more user specs mapping to the same concrete spec. """ mutable_mock_env_path.mkdir() spack_yaml = mutable_mock_env_path / ev.manifest_name spack_yaml.write_text( f"""spack: specs: # These two specs concretize to the same hash - pkg-c - pkg-c@1.0 # Spec used to trigger the bug - pkg-a concretizer: unify: {unify} """ ) e = ev.Environment(mutable_mock_env_path) # Initial state assert len(e.user_specs) == 3 assert len(e.concretized_roots) == 0 with e: e.concretize() assert len(e.user_specs) == 3 assert len(e.concretized_roots) == 3 assert all(x.new for x in e.concretized_roots) e.deconcretize_by_user_spec(spack.spec.Spec("pkg-a")) assert len(e.user_specs) == 3 assert len(e.concretized_roots) == 2 assert all(x.new for x in e.concretized_roots) e.concretize() assert len(e.user_specs) == 3 assert len(e.concretized_roots) == 3 assert all(x.new for x in e.concretized_roots) all_root_hashes = {x.dag_hash() for x in e.concrete_roots()} assert len(all_root_hashes) == 2 @pytest.mark.regression("44216") def test_root_version_weights_for_old_versions(mutable_mock_env_path): """Tests that, when we select two old versions of root specs that have the same version optimization penalty, both are considered. """ mutable_mock_env_path.mkdir() spack_yaml = mutable_mock_env_path / ev.manifest_name spack_yaml.write_text( """spack: specs: # allow any version, but the most recent - bowtie@:1.3 # allows only the third most recent, so penalty is 2 - gcc@1 concretizer: unify: true """ ) e = ev.Environment(mutable_mock_env_path) with e: e.concretize() bowtie = [x for x in e.concrete_roots() if x.name == "bowtie"][0] gcc = [x for x in e.concrete_roots() if x.name == "gcc"][0] assert bowtie.satisfies("@=1.3.0") assert gcc.satisfies("@=1.0") def test_env_view_on_empty_dir_is_fine(tmp_path: pathlib.Path, config, temporary_store): """Tests that creating a view pointing to an empty dir is not an error.""" view_dir = tmp_path / "view" view_dir.mkdir() env = ev.create_in_dir(tmp_path, with_view="view") env.add("mpileaks") env.concretize() env.install_all(fake=True) env.regenerate_views() assert view_dir.is_symlink() def test_env_view_on_non_empty_dir_errors(tmp_path: pathlib.Path, config, temporary_store): """Tests that creating a view pointing to a non-empty dir errors.""" view_dir = tmp_path / "view" view_dir.mkdir() (view_dir / "file").write_text("") env = ev.create_in_dir(tmp_path, with_view="view") env.add("mpileaks") env.concretize() env.install_all(fake=True) with pytest.raises(ev.SpackEnvironmentError, match="because it is a non-empty dir"): env.regenerate_views() @pytest.mark.parametrize( "matrix_line", [("^zmpi", "^mpich"), ("~shared", "+shared"), ("shared=False", "+shared-libs")] ) @pytest.mark.regression("40791") def test_stack_enforcement_is_strict(tmp_path: pathlib.Path, matrix_line, config): """Ensure that constraints in matrices are applied strictly after expansion, to avoid inconsistencies between abstract user specs and concrete specs. """ manifest = tmp_path / "spack.yaml" manifest.write_text( f"""\ spack: definitions: - packages: [libelf, mpileaks] - install: - matrix: - [$packages] - [{", ".join(item for item in matrix_line)}] specs: - $install concretizer: unify: false """ ) # Here we raise different exceptions depending on whether we solve serially or not with pytest.raises(Exception): with ev.Environment(tmp_path) as e: e.concretize() def test_only_roots_are_explicitly_installed(tmp_path: pathlib.Path, config, temporary_store): """When installing specific non-root specs from an environment, we continue to mark them as implicitly installed. What makes installs explicit is that they are root of the env.""" env = ev.create_in_dir(tmp_path) env.add("mpileaks") env.concretize() mpileaks = env.concrete_roots()[0] callpath = mpileaks["callpath"] env.install_specs([callpath], fake=True) assert callpath in temporary_store.db.query(explicit=False) env.install_specs([mpileaks], fake=True) assert temporary_store.db.query(explicit=True) == [mpileaks] def test_environment_from_name_or_dir(mutable_mock_env_path): test_env = ev.create("test") name_env = ev.environment_from_name_or_dir(test_env.name) assert name_env.name == test_env.name assert name_env.path == test_env.path dir_env = ev.environment_from_name_or_dir(test_env.path) assert dir_env.name == test_env.name assert dir_env.path == test_env.path nested_test_env = ev.create("group/test") nested_name_env = ev.environment_from_name_or_dir(nested_test_env.name) assert nested_name_env.name == nested_test_env.name assert nested_name_env.path == nested_test_env.path nested_dir_env = ev.environment_from_name_or_dir(nested_test_env.path) assert nested_dir_env.name == nested_test_env.name assert nested_dir_env.path == nested_test_env.path with pytest.raises(ev.SpackEnvironmentError, match="no such environment"): _ = ev.environment_from_name_or_dir("fake-env") def test_env_include_configs(mutable_mock_env_path): """check config and package values using new include schema""" env_path = mutable_mock_env_path env_path.mkdir() this_os = spack.platforms.host().default_os config_root = env_path / this_os config_root.mkdir() config_path = str(config_root / "config.yaml") with open(config_path, "w", encoding="utf-8") as f: f.write( """\ config: verify_ssl: False """ ) packages_path = str(env_path / "packages.yaml") with open(packages_path, "w", encoding="utf-8") as f: f.write( """\ packages: python: require: - spec: "@3.11:" """ ) spack_yaml = env_path / ev.manifest_name spack_yaml.write_text( f"""\ spack: include: - path: {config_path} optional: true - path: {packages_path} """ ) e = ev.Environment(env_path) with e.manifest.use_config(): assert not spack.config.get("config:verify_ssl") python_reqs = spack.config.get("packages")["python"]["require"] req_specs = set(x["spec"] for x in python_reqs) assert req_specs == set(["@3.11:"]) def test_using_multiple_compilers_on_a_node_is_discouraged(tmp_path: pathlib.Path, mutable_config): """Tests that when we specify % Spack tries to use that compiler for all the languages needed by that node. """ manifest = tmp_path / "spack.yaml" manifest.write_text( """\ spack: specs: - mpileaks%clang ^mpich%gcc concretizer: unify: true """ ) with ev.Environment(tmp_path) as e: e.concretize() mpileaks = e.concrete_roots()[0] assert not mpileaks.satisfies("%gcc") and mpileaks.satisfies("%clang") assert len(mpileaks.dependencies(virtuals=("c", "cxx"))) == 1 mpich = mpileaks["mpich"] assert mpich.satisfies("%gcc") and not mpich.satisfies("%clang") assert len(mpich.dependencies(virtuals=("c", "cxx"))) == 1 @pytest.mark.parametrize( ["spack_yaml", "expected", "not_expected"], [ # Define a toolchain in spack.yaml ( """\ spack: specs: - mpileaks %llvm-toolchain toolchains: llvm-toolchain: - spec: "%[virtuals=c] llvm" when: "%c" - spec: "%[virtuals=cxx] llvm" when: "%cxx" concretizer: unify: true """, ["%[virtuals=c] llvm", "^[virtuals=mpi] mpich"], ["%[virtuals=c] gcc"], ), # Use a toolchain in a default requirement ( """\ spack: specs: - mpileaks toolchains: llvm-toolchain: - spec: "%[virtuals=c] llvm" when: "%c" - spec: "%[virtuals=cxx] llvm" when: "%cxx" - spec: "%[virtuals=mpi] zmpi" when: "%mpi" packages: all: require: - "%llvm-toolchain" concretizer: unify: true """, ["%[virtuals=c] llvm", "%[virtuals=mpi] zmpi", "^callpath %[virtuals=c] llvm"], ["%[virtuals=c] gcc"], ), ], ) def test_toolchain_definitions_are_allowed( spack_yaml, expected, not_expected, tmp_path: pathlib.Path, mutable_config ): """Tests that we can use toolchain definitions in spack.yaml files.""" manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() mpileaks = e.concrete_roots()[0] for c in expected: assert mpileaks.satisfies(c) for c in not_expected: assert not mpileaks.satisfies(c) MIXED_TOOLCHAIN = """ - spec: "%[virtuals=c] llvm" when: "%c" - spec: "%[virtuals=cxx] llvm" when: "%cxx" - spec: "%[virtuals=fortran] gcc" when: "%fortran" - spec: "%[virtuals=mpi] mpich" when: "%mpi" """ @pytest.mark.parametrize("unify", ["true", "false", "when_possible"]) def test_single_toolchain_and_matrix(unify, tmp_path: pathlib.Path, mutable_config): """Tests that toolchains can be used with matrices in environments""" spack_yaml = f""" spack: specs: - matrix: - [mpileaks, dt-diamond-right] - ["%mixed-toolchain"] toolchains: mixed-toolchain: {MIXED_TOOLCHAIN} concretizer: unify: {unify} """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() roots = e.concrete_roots() expected = [ "%[when='%c' virtuals=c] llvm", "%[when='%cxx' virtuals=cxx] llvm", "%[when='%fortran' virtuals=fortran] gcc", "%[when='%mpi' virtuals=mpi] mpich", ] for c in expected: assert all(s.satisfies(c) for s in roots) not_expected = ["^zmpi", "%[virtuals=c] gcc"] for c in not_expected: assert all(not s.satisfies(c) for s in roots) GCC_ZMPI = """ - spec: "%[virtuals=c] gcc" when: "%c" - spec: "%[virtuals=cxx] gcc" when: "%cxx" - spec: "%[virtuals=fortran] gcc" when: "%fortran" - spec: "%[virtuals=mpi] zmpi" when: "%mpi" """ @pytest.mark.parametrize("unify", ["false", "when_possible"]) def test_toolchains_as_matrix_dimension(unify, tmp_path: pathlib.Path, mutable_config): """Tests expanding a matrix using different toolchains as the last dimension""" spack_yaml = f""" spack: specs: - matrix: - [mpileaks, dt-diamond-right] - ["%mixed-toolchain", "%gcc-zmpi"] toolchains: mixed-toolchain: {MIXED_TOOLCHAIN} gcc-zmpi: {GCC_ZMPI} concretizer: unify: {unify} """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() roots = e.concrete_roots() mpileaks_gcc = [s for s in roots if s.satisfies("mpileaks %[virtuals=c] gcc")][0] mpileaks_clang = [s for s in roots if s.satisfies("mpileaks %[virtuals=c] clang")][0] # GCC-MPICH toolchain assert not mpileaks_gcc.satisfies("%[virtuals=mpi] mpich") assert mpileaks_gcc.satisfies("%[virtuals=mpi] zmpi") # Mixed toolchain assert mpileaks_clang.satisfies("%[virtuals=mpi] mpich") assert not mpileaks_clang.satisfies("%[virtuals=mpi] zmpi") assert mpileaks_clang["mpich"].satisfies("%[virtuals=fortran] gcc") @pytest.mark.parametrize("unify", ["true", "false", "when_possible"]) @pytest.mark.parametrize("requirement_type", ["require", "prefer"]) def test_using_toolchain_as_requirement( unify, requirement_type, tmp_path: pathlib.Path, mutable_config ): """Tests using a toolchain as a default requirement in an environment""" spack_yaml = f""" spack: specs: - mpileaks - dt-diamond-right toolchains: mixed-toolchain: {MIXED_TOOLCHAIN} packages: all: {requirement_type}: - "%mixed-toolchain" concretizer: unify: {unify} """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() roots = e.concrete_roots() mpileaks = [s for s in roots if s.satisfies("mpileaks")][0] assert mpileaks.satisfies("%[virtuals=mpi] mpich") assert mpileaks.satisfies("^[virtuals=mpi] mpich") mpich = mpileaks["mpi"] assert mpich.satisfies("%[virtuals=c] llvm") assert mpich.satisfies("%[virtuals=cxx] llvm") assert mpich.satisfies("%[virtuals=fortran] gcc") @pytest.mark.parametrize("unify", ["false", "when_possible"]) def test_using_toolchain_as_preferences(unify, tmp_path: pathlib.Path, mutable_config): """Tests using a toolchain as a strong preference in an environment""" spack_yaml = f""" spack: specs: - dt-diamond-right %gcc-zmpi toolchains: mixed-toolchain: {MIXED_TOOLCHAIN} gcc-zmpi: {GCC_ZMPI} packages: all: prefer: - "%mixed-toolchain" concretizer: unify: {unify} """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() roots = e.concrete_roots() dt = [s for s in roots if s.satisfies("dt-diamond-right")][0] assert dt.satisfies("%[virtuals=c] gcc") @pytest.mark.parametrize("unify", ["true", "false", "when_possible"]) def test_mixing_toolchains_in_an_input_spec(unify, tmp_path: pathlib.Path, mutable_config): """Tests using a toolchain as a strong preference in an environment""" spack_yaml = f""" spack: specs: - mpileaks %mixed-toolchain ^libelf %gcc-zmpi toolchains: mixed-toolchain: {MIXED_TOOLCHAIN} gcc-zmpi: {GCC_ZMPI} concretizer: unify: {unify} """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() roots = e.concrete_roots() mpileaks = [s for s in roots if s.satisfies("mpileaks")][0] assert mpileaks.satisfies("%[virtuals=mpi] mpich") assert mpileaks.satisfies("^[virtuals=mpi] mpich") assert mpileaks.satisfies("%[virtuals=c] llvm") libelf = mpileaks["libelf"] assert libelf.satisfies("%[virtuals=c] gcc") # libelf only depends on c def test_reuse_environment_dependencies(tmp_path: pathlib.Path, mutable_config): """Tests reusing specs from a separate, and concrete, environment.""" base = tmp_path / "base" base.mkdir() # Concretize the first environment asking for a non-default spec. In this way we'll know # that reuse from the derived environment is not accidental. manifest_base = base / "spack.yaml" manifest_base.write_text( """ spack: specs: - pkg-a@1.0 packages: pkg-b: require: - "@0.9" """ ) with ev.Environment(base) as e: e.concretize() # We need the spack.lock for reuse in the derived environment e.write(regenerate=False) base_pkga = e.concrete_roots()[0] # Create a second environment, reuse from the previous one and check pkg-a is the same derived = tmp_path / "derived" derived.mkdir() manifest_derived = derived / "spack.yaml" manifest_derived.write_text( f""" spack: specs: - pkg-a concretizer: reuse: from: - type: environment path: {base} """ ) with ev.Environment(derived) as e: e.concretize() derived_pkga = e.concrete_roots()[0] assert base_pkga.dag_hash() == derived_pkga.dag_hash() @pytest.mark.parametrize( "spack_yaml", [ # Use a plain requirement for callpath """ spack: specs: - mpileaks %%c,cxx=gcc - mpileaks %%c,cxx=llvm packages: callpath: require: - "%c=gcc" concretizer: unify: false """, # Propagate a toolchain """ spack: specs: - mpileaks %%c,cxx=gcc - mpileaks %%llvm_toolchain toolchains: llvm_toolchain: - spec: "%c=llvm" when: "%c" - spec: "%cxx=llvm" when: "%cxx" packages: callpath: require: - "%c=gcc" concretizer: unify: false """, # Override callpath from input spec """ spack: specs: - mpileaks %%c,cxx=gcc ^callpath %c=gcc - mpileaks %%llvm_toolchain ^callpath %c=gcc toolchains: llvm_toolchain: - spec: "%c=llvm" when: "%c" - spec: "%cxx=llvm" when: "%cxx" concretizer: unify: false """, ], ) def test_dependency_propagation_in_environments(spack_yaml, tmp_path, mutable_config): """Tests that we can enforce compiler preferences using %% in environments.""" manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() roots = e.concrete_roots() mpileaks_gcc = [s for s in roots if s.satisfies("mpileaks %c=gcc")][0] for c in ("%[when=%c]c=gcc", "%[when=%cxx]cxx=gcc"): assert all(x.satisfies(c) for x in mpileaks_gcc.traverse() if x.name != "callpath") mpileaks_llvm = [s for s in roots if s.satisfies("mpileaks %c=llvm")][0] for c in ("%[when=%c]c=llvm", "%[when=%cxx]cxx=llvm"): assert all(x.satisfies(c) for x in mpileaks_llvm.traverse() if x.name != "callpath") assert mpileaks_gcc["callpath"].satisfies("%c=gcc") assert mpileaks_llvm["callpath"].satisfies("%c=gcc") @pytest.mark.parametrize( "spack_yaml,exception_nodes", [ # trilinos and its link/run subdag are compiled with clang, all other nodes use gcc ( """ spack: specs: - trilinos %%c,cxx=clang packages: c: prefer: - gcc cxx: prefer: - gcc """, set(), ), # callpath and its link/run subdag are compiled with clang, all other nodes use gcc ( """ spack: specs: - trilinos ^callpath %%c,cxx=clang packages: c: prefer: - gcc cxx: prefer: - gcc """, {"trilinos", "mpich", "py-numpy"}, ), # trilinos and its link/run subdag, with the exception of mpich, are compiled with clang. # All other nodes use gcc. ( """ spack: specs: - trilinos %%c,cxx=clang ^mpich %c=gcc packages: c: prefer: - gcc cxx: prefer: - gcc """, {"mpich"}, ), ( """ spack: specs: - trilinos %%c,cxx=clang packages: c: prefer: - gcc cxx: prefer: - gcc mpich: require: - "%c=gcc" """, {"mpich"}, ), ], ) def test_double_percent_semantics(spack_yaml, exception_nodes, tmp_path, mutable_config): """Tests semantics of %% in environments, when combined with other features. The test assumes clang is the propagated compiler, and gcc is the preferred compiler. """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() trilinos = e.concrete_roots()[0] runtime_nodes = [ x for x in trilinos.traverse(deptype=("link", "run")) if x.name not in exception_nodes ] remaining_nodes = [x for x in trilinos.traverse() if x not in runtime_nodes] for x in runtime_nodes: error_msg = f"\n{x.tree()} does not use clang while expected to" assert x.satisfies("%[when=%c]c=clang %[when=%cxx]cxx=clang"), error_msg for x in remaining_nodes: error_msg = f"\n{x.tree()} does not use gcc while expected to" assert x.satisfies("%[when=%c]c=gcc %[when=%cxx]cxx=gcc"), error_msg def test_cannot_use_double_percent_with_require(tmp_path, mutable_config): """Tests that %% cannot be used with a requirement on languages, since they'll conflict.""" # trilinos wants to use clang, but we require gcc, so Spack will error spack_yaml = """ spack: specs: - trilinos %%c,cxx=clang packages: c: require: - gcc cxx: require: - gcc """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: with pytest.raises(spack.solver.asp.UnsatisfiableSpecError, match="failed to concretize"): e.concretize() @pytest.mark.parametrize( "spack_yaml", [ # Specs with reuse on """ spack: specs: - trilinos - mpileaks concretizer: reuse: true """, # Package with conditional dependency """ spack: specs: - ascent+adios2 - fftw+mpi """, """ spack: specs: - ascent~adios2 - fftw~mpi """, """ spack: specs: - ascent+adios2 - fftw~mpi """, ], ) def test_static_analysis_in_environments(spack_yaml, tmp_path, mutable_config): """Tests that concretizations with and without static analysis produce the same results.""" manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() no_static_analysis = {x.dag_hash() for x in e.concrete_roots()} mutable_config.set("concretizer:static_analysis", True) with ev.Environment(tmp_path) as e: e.concretize() static_analysis = {x.dag_hash() for x in e.concrete_roots()} assert no_static_analysis == static_analysis @pytest.mark.regression("51606") def test_ids_when_using_toolchain_twice_in_a_spec(tmp_path, mutable_config): """Tests that using the same toolchain twice in a spec constructs different objects""" spack_yaml = """ spack: toolchains: llvmtc: - spec: "%c=llvm" when: "%c" - spec: "%cxx=llvm" when: "%cxx" gnu: - spec: "%c=gcc@10" when: "%c" - spec: "%cxx=gcc@10" when: "%cxx" # This is missing the conditional when= on purpose - spec: "%fortran=gcc@10" """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path): # We rely on this behavior when emitting facts for the solver toolchains = spack.config.CONFIG.get("toolchains", {}) s = spack.spec_parser.parse("mpileaks %gnu ^callpath %gnu", toolchains=toolchains)[0] assert id(s["gcc"]) != id(s["callpath"]["gcc"]) def test_installed_specs_disregards_deprecation(tmp_path, mutable_config): """Tests that installed specs disregard deprecation. This is to avoid weird ordering issues, where an old version that _is not_ declared in package.py is considered as _not_ deprecated, and is preferred to a newer version that is explicitly marked as deprecated. """ spack_yaml = """ spack: specs: - mpileaks packages: c: require: - gcc cxx: require: - gcc gcc:: externals: - spec: gcc@7.3.1 languages:='c,c++,fortran' prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ fortran: /path/bin/gfortran - spec: gcc@=12.4.0 languages:='c,c++,fortran' prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc cxx: /usr/bin/g++ fortran: /usr/bin/gfortran """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() mpileaks = e.concrete_roots()[0] for node in mpileaks.traverse(): if node.satisfies("%c"): assert node.satisfies("%c=gcc@12"), node.tree() assert not node.satisfies("%c=gcc@7"), node.tree() @pytest.fixture() def create_temporary_manifest(tmp_path): manifest_path = tmp_path / "spack.yaml" def _create(spack_yaml: str): manifest_path.write_text(spack_yaml) return EnvironmentManifestFile(tmp_path) return _create @pytest.mark.usefixtures("mutable_config") class TestEnvironmentGroups: """Tests for the environment "groups" feature""" def test_manifest_and_groups(self, create_temporary_manifest): """Tests a basic case of reading groups from a manifest file""" manifest = create_temporary_manifest( """ spack: specs: - mpileaks - group: compiler matrix: - [gcc@14] - group: apps needs: [compiler] specs: - matrix: - [mpileaks] - ["%gcc@14"] - mpich - libelf """ ) # Check manifest properties assert set(manifest.groups()) == {"default", "compiler", "apps"} assert manifest.user_specs(group="default") == manifest.user_specs() assert manifest.user_specs() == ["mpileaks", "libelf"] assert manifest.user_specs(group="compiler") == [{"matrix": [["gcc@14"]]}] assert manifest.user_specs(group="apps") == [ {"matrix": [["mpileaks"], ["%gcc@14"]]}, "mpich", ] assert manifest.needs(group="default") == () assert manifest.needs(group="compiler") == () assert manifest.needs(group="apps") == ("compiler",) # Check user specs within the environment e = ev.Environment(manifest.manifest_dir) assert e.user_specs.specs == [spack.spec.Spec("mpileaks"), spack.spec.Spec("libelf")] compiler_specs = e.user_specs_by(group="compiler") assert compiler_specs.name == "specs:compiler" assert compiler_specs.specs == [spack.spec.Spec("gcc@14")] apps_specs = e.user_specs_by(group="apps") assert apps_specs.name == "specs:apps" assert apps_specs.specs == [spack.spec.Spec("mpileaks %gcc@14"), spack.spec.Spec("mpich")] def test_cannot_define_group_twice(self, create_temporary_manifest): """Tests that defining the same group twice raises an error""" with pytest.raises(SpackEnvironmentConfigError, match="defined more than once"): create_temporary_manifest( """ spack: specs: - group: compiler matrix: - [gcc@14] - group: compiler matrix: - [llvm@20] """ ) def test_matrix_can_be_expanded_in_groups(self, create_temporary_manifest): """Tests that definitions can be expanded also for matrix groups""" manifest = create_temporary_manifest( """ spack: definitions: - compilers: ["%gcc", "%clang"] - desired_specs: ["mpileaks@2.1"] specs: - group: apps specs: - matrix: - [$desired_specs] - [$compilers] - mpich """ ) e = ev.Environment(manifest.manifest_dir) assert e.user_specs.specs == [] assert e.user_specs_by(group="apps").specs == [ spack.spec.Spec("mpileaks@2.1 %gcc"), spack.spec.Spec("mpileaks@2.1 %clang"), spack.spec.Spec("mpich"), ] def test_environment_without_groups_use_lockfile_v6(self, create_temporary_manifest): manifest = create_temporary_manifest( """ spack: specs: - mpileaks - pkg-a """ ) with ev.Environment(manifest.manifest_dir) as e: e.concretize() lockfile_data = e._to_lockfile_dict() assert lockfile_data["_meta"]["lockfile-version"] == 6 assert all("group" not in x for x in lockfile_data["roots"]) def test_independent_groups_concretization(self, create_temporary_manifest): """Tests that groups of specs without dependencies among them can be concretized correctly """ manifest = create_temporary_manifest( """ spack: specs: - mpileaks - group: compiler matrix: - [gcc@14] - libelf """ ) with ev.Environment(manifest.manifest_dir) as e: e.concretize() roots = e.concrete_roots() assert len(roots) == 3 default_specs = list(e.concretized_specs_by(group="default")) assert len(default_specs) == 2 compiler_specs = list(e.concretized_specs_by(group="compiler")) assert len(compiler_specs) == 1 def test_independent_group_dont_reuse(self, create_temporary_manifest): """Tests that there is no cross-groups reuse among groups of specs without dependencies.""" manifest = create_temporary_manifest( """ spack: specs: - mpileaks@2.2 - group: app matrix: - [mpileaks] """ ) with ev.Environment(manifest.manifest_dir) as e: e.concretize() _, default_mpileaks = list(e.concretized_specs_by(group="default"))[0] assert default_mpileaks.satisfies("@2.2") _, app_mpileaks = list(e.concretized_specs_by(group="app"))[0] assert app_mpileaks.satisfies("@2.3") def test_relying_on_a_dependency_group(self, create_temporary_manifest): """Tests that a group of specs that would not concretize without a dependency group works correctly. """ manifest = create_temporary_manifest( """ spack: specs: - group: app matrix: - [mpileaks] - ["%c,cxx=gcc@14"] """ ) # We have no gcc@14 configured, so this will raise an error with ev.Environment(manifest.manifest_dir) as e: with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): e.concretize() manifest = create_temporary_manifest( """ spack: specs: - group: compiler specs: - gcc@14 - group: mpileaks needs: [compiler] matrix: - [mpileaks] - ["%c,cxx=gcc@14"] """ ) # In this case gcc@14 is taken from the "needed" group with ev.Environment(manifest.manifest_dir) as e: e.concretize() _, gcc = next(iter(e.concretized_specs_by(group="compiler"))) assert gcc.satisfies("gcc@14") _, mpileaks = next(iter(e.concretized_specs_by(group="mpileaks"))) assert mpileaks["c"].dag_hash() == gcc.dag_hash() def test_manifest_can_contain_config_override(self, mutable_config, create_temporary_manifest): manifest = create_temporary_manifest( """ spack: concretizer: unify: False specs: - group: compiler override: concretizer: unify: True """ ) with ev.Environment(manifest.manifest_dir) as e: assert mutable_config.get_config("concretizer")["unify"] is False # Assert the internal scope works when used manually override = manifest.config_override(group="compiler") mutable_config.push_scope( override, priority=ConfigScopePriority.ENVIRONMENT_SPEC_GROUPS ) assert mutable_config.get_config("concretizer")["unify"] is True mutable_config.remove_scope(override.name) assert mutable_config.get_config("concretizer")["unify"] is False # Assert the context manager works too with e.config_override_for_group(group="compiler"): assert mutable_config.get_config("concretizer")["unify"] is True assert mutable_config.get_config("concretizer")["unify"] is False def test_overriding_concretization_properties_per_group(self, create_temporary_manifest): manifest = create_temporary_manifest( """ spack: concretizer: unify: True specs: - group: compiler specs: - gcc@14 - group: scalapacks needs: [compiler] matrix: - [netlib-scalapack] - ["%mpi=mpich", "%mpi=mpich2"] - ["%lapack=openblas-with-lapack", "%lapack=netlib-lapack"] override: concretizer: unify: False packages: c: prefer: [gcc@14] cxx: prefer: [gcc@14] fortran: prefer: [gcc@14] """ ) with ev.Environment(manifest.manifest_dir) as e: e.concretize() assert len(list(e.concretized_specs_by(group="compiler"))) == 1 gcc = next(x for _, x in e.concretized_specs_by(group="compiler")) assert gcc.satisfies("gcc@14") and not gcc.external assert gcc.satisfies("%c,cxx=gcc") gcc_hash = gcc.dag_hash() assert len(list(e.concretized_specs_by(group="scalapacks"))) == 4 scalapacks = [x for _, x in e.concretized_specs_by(group="scalapacks")] for node in traverse_nodes(scalapacks, deptype=("link", "run")): assert node.satisfies(f"%[when=c]c=gcc/{gcc_hash}") assert node.satisfies(f"%[when=cxx]cxx=gcc/{gcc_hash}") assert node.satisfies(f"%[when=fortran]fortran=gcc/{gcc_hash}") def test_missing_needs_group_gives_clear_error(self, create_temporary_manifest): """Tests that referencing a non-existent group in 'needs' gives a clear error message that includes the name of the blocked group and the missing dependency. """ manifest = create_temporary_manifest( """ spack: specs: - group: apps needs: [nonexistent] specs: - mpileaks """ ) with ev.Environment(manifest.manifest_dir) as e: with pytest.raises( ev.SpackEnvironmentConfigError, match=r"but 'nonexistent' is not a defined group" ): e.concretize() def test_cyclic_group_dependencies_give_clear_error(self, create_temporary_manifest): """Tests that cyclic group dependencies give a clear error message that mentions the groups involved in the cycle. """ manifest = create_temporary_manifest( """ spack: specs: - group: alpha needs: [beta] specs: - mpileaks - group: beta needs: [alpha] specs: - zlib """ ) with ev.Environment(manifest.manifest_dir) as e: with pytest.raises(ev.SpackEnvironmentConfigError, match=r"among groups: alpha, beta"): e.concretize() def test_from_lockfile_preserves_groups(self, tmp_path): """Tests that EnvironmentManifestFile.from_lockfile reconstructs groups correctly from a v7 lockfile that contains group information in its roots. """ lockfile_data = { "_meta": {"file-type": "spack-lockfile", "lockfile-version": 7, "specfile-version": 5}, "roots": [ {"hash": "aaa", "spec": "mpileaks", "group": "default"}, {"hash": "bbb", "spec": "libelf", "group": "default"}, {"hash": "ccc", "spec": "gcc@14", "group": "compilers"}, ], "concrete_specs": {}, } lockfile_path = tmp_path / "spack.lock" lockfile_path.write_text(json.dumps(lockfile_data)) manifest = EnvironmentManifestFile.from_lockfile(tmp_path) # The reconstructed manifest must have both groups assert set(manifest.groups()) == {"default", "compilers"} assert manifest.user_specs(group="default") == ["mpileaks", "libelf"] assert manifest.user_specs(group="compilers") == ["gcc@14"] def test_from_lockfile_without_groups_stays_default(self, tmp_path): """Tests that a lockfile without group info (v6 and earlier) reconstructs all specs into the default group only. """ lockfile_data = { "_meta": {"file-type": "spack-lockfile", "lockfile-version": 6, "specfile-version": 5}, "roots": [{"hash": "aaa", "spec": "mpileaks"}, {"hash": "bbb", "spec": "libelf"}], "concrete_specs": {}, } lockfile_path = tmp_path / "spack.lock" lockfile_path.write_text(json.dumps(lockfile_data)) manifest = EnvironmentManifestFile.from_lockfile(tmp_path) assert set(manifest.groups()) == {"default"} assert manifest.user_specs(group="default") == ["mpileaks", "libelf"] @pytest.mark.regression("51995") def test_mixed_compilers_and_libllvm(tmp_path, config): """Tests that we divide virtual nodes correctly among unification sets. This test concretizes a unified environment where one package uses gcc as a C++ compiler and depends on llvm as a provider of libllvm, while the other package uses llvm as a C++ compiler. """ spack_yaml = """ spack: specs: - paraview %cxx=llvm - mesa %cxx=gcc %libllvm=llvm packages: c: prefer: - gcc cxx: prefer: - gcc gcc:: externals: - spec: gcc@13.2.0 languages:='c,c++,fortran' prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ fortran: /path/bin/gfortran llvm:: externals: - spec: llvm@20.1.8+clang+flang+lld+lldb prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc cxx: /usr/bin/g++ fortran: /usr/bin/gfortran concretizer: unify: true """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() for x in e.concrete_roots(): if x.name == "mesa": mesa = x else: paraview = x assert paraview.satisfies("%cxx=llvm@20") assert paraview.satisfies(f"%{mesa}") assert mesa.satisfies("%cxx=gcc %libllvm=llvm") assert paraview["cxx"].dag_hash() == mesa["libllvm"].dag_hash() @pytest.mark.regression("51512") def test_unified_environment_with_mixed_compilers_and_fortran(tmp_path, config): """Tests that we can concretize a unified environment using two C/C++ compilers for the root specs and GCC for Fortran, where both roots depend on Fortran. """ spack_yaml = """ spack: specs: - mpich %c,cxx=llvm - openblas %c,fortran=gcc packages: gcc:: externals: - spec: gcc@13.2.0 languages:='c,c++,fortran' prefix: /path extra_attributes: compilers: c: /path/bin/gcc cxx: /path/bin/g++ fortran: /path/bin/gfortran llvm:: externals: - spec: llvm@20.1.8+clang~flang prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc cxx: /usr/bin/g++ fortran: /usr/bin/gfortran concretizer: unify: true """ manifest = tmp_path / "spack.yaml" manifest.write_text(spack_yaml) with ev.Environment(tmp_path) as e: e.concretize() for x in e.concrete_roots(): if x.name == "mpich": mpich = x else: openblas = x assert mpich.satisfies("%c,cxx=llvm") assert mpich.satisfies("%fortran=gcc") assert openblas.satisfies("%c,fortran=gcc") assert mpich["fortran"].dag_hash() == openblas["fortran"].dag_hash() ================================================ FILE: lib/spack/spack/test/environment/mutate.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform import pytest import spack.concretize import spack.config import spack.environment as ev import spack.spec from spack.main import SpackCommand pytestmark = [ pytest.mark.usefixtures("mutable_config", "mutable_mock_env_path", "mutable_mock_repo"), pytest.mark.not_on_windows("Envs unsupported on Windows"), ] # See lib/spack/spack/platforms/test.py for how targets are defined on the Test platform test_targets = ("m1", "aarch64") if platform.machine() == "arm64" else ("core2", "x86_64") change = SpackCommand("change") @pytest.mark.parametrize("dep", [True, False]) @pytest.mark.parametrize( "orig_constraint,mutated_constraint", [ ("@3.23.1", "@3.4.3"), ("cflags=-O3", "cflags='-O0 -g'"), ("os=debian6", "os=redhat6"), (f"target={test_targets[0]}", f"target={test_targets[1]}"), ("build_system=generic", "build_system=foo"), ( f"@3.4.3 cflags=-g os=debian6 target={test_targets[1]} build_system=generic", f"@3.23.1 cflags=-O3 os=redhat6 target={test_targets[0]} build_system=foo", ), ], ) def test_mutate_internals(dep, orig_constraint, mutated_constraint): """ Check that Environment.mutate and Spec.mutate work for several different constraint types. Includes check that environment.mutate rehashing gets the same answer as spec.mutate rehashing. """ ev.create("test") env = ev.read("test") spack.config.set("packages:cmake", {"require": orig_constraint}) root_name = "cmake-client" if dep else "cmake" env.add(root_name) env.concretize() root_spec = next(env.roots()).copy() cmake_spec = root_spec["cmake"] if dep else root_spec orig_cmake_spec = cmake_spec.copy() orig_hash = root_spec.dag_hash() for spec in env.all_specs_generator(): if spec.name == "cmake": assert spec.satisfies(orig_constraint) selector = spack.spec.Spec("cmake") mutator = spack.spec.Spec(mutated_constraint) env.mutate(selector=selector, mutator=mutator) cmake_spec.mutate(mutator) for spec in env.all_specs_generator(): if spec.name == "cmake": assert spec.satisfies(mutated_constraint) assert cmake_spec.satisfies(mutated_constraint) # Make sure that we're not changing variant types single/multi for name, variant in cmake_spec.variants.items(): assert variant.type == orig_cmake_spec.variants[name].type new_hash = next(env.roots()).dag_hash() assert new_hash != orig_hash assert root_spec.dag_hash() != orig_hash assert root_spec.dag_hash() == new_hash @pytest.mark.parametrize("constraint", ["foo", "foo.bar", "foo%cmake@1.0", "foo@1.1:", "foo/abc"]) def test_mutate_spec_invalid(constraint): spec = spack.concretize.concretize_one("cmake-client") with pytest.raises(spack.spec.SpecMutationError): spec.mutate(spack.spec.Spec(constraint)) def _test_mutate_from_cli(args, create=True): if create: ev.create("test") env = ev.read("test") if create: env.add("cmake-client%cmake@3.4.3") env.add("cmake-client%cmake@3.23.1") env.concretize() env.write() with env: change(*args) return list(env.roots()) def test_mutate_from_cli(): match_spec = "%cmake@3.4.3" constraint = "@3.0" args = ["--concrete", f"--match-spec={match_spec}", constraint] roots = _test_mutate_from_cli(args) assert any(r.satisfies(match_spec) for r in roots) for root in roots: if root.satisfies("match_spec"): assert root.satisfies(constraint) def test_mutate_from_cli_multiple(): match_spec = "%cmake@3.4.3" constraint1 = "@3.0" constraint2 = "build_system=foo" args = ["--concrete", f"--match-spec={match_spec}", constraint1, constraint2] roots = _test_mutate_from_cli(args) assert any(r.satisfies(match_spec) for r in roots) for root in roots: if root.satisfies("match_spec"): assert root.satisfies(constraint1) assert root.satisfies(constraint2) def test_mutate_from_cli_no_abstract(): match_spec = "cmake" constraint = "@3.0" args = ["--concrete", f"--match-spec={match_spec}", constraint] with pytest.raises(ValueError, match="Cannot change abstract spec"): _ = _test_mutate_from_cli(args) args = ["--concrete-only"] + args[1:] roots = _test_mutate_from_cli(args, create=False) for root in roots: assert root[match_spec].satisfies(constraint) def test_mutate_from_cli_all_no_match_spec(): constraint = "cmake-client@3.0" args = ["--concrete", "--all", constraint] roots = _test_mutate_from_cli(args) for root in roots: assert root.satisfies(constraint) ================================================ FILE: lib/spack/spack/test/environment_modifications.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import sys import pytest import spack.util.environment as environment from spack.paths import spack_root from spack.util.environment import ( AppendPath, EnvironmentModifications, PrependPath, RemovePath, SetEnv, UnsetEnv, filter_system_paths, is_system_path, ) datadir = os.path.join(spack_root, "lib", "spack", "spack", "test", "data") shell_extension = ".bat" if sys.platform == "win32" else ".sh" # Returns a list of paths, including system ones miscellaneous_unix_sys_paths = [ "/usr/include", "/usr/local/lib", "/usr/local", "/usr/local/include", "/usr/local/lib64", "/usr/local/../bin", "/lib", "/", "/usr", "/usr/", "/usr/bin", "/bin64", "/lib64", "/include", "/include/", ] miscellaneous_win_sys_paths = [ "C:\\Users", "C:\\", "C:\\Program Files", "C:\\Program Files (x86)", "C:\\ProgramData", ] miscellaneous_win_paths = ["C:\\dev\\spack_window"] miscellaneous_unix_paths = [ "/usr/local/Cellar/gcc/5.3.0/lib", "/usr/local/opt/some-package/lib", "/usr/opt/lib", "/opt/some-package/include", "/opt/some-package/local/..", ] def test_inspect_path(tmp_path: pathlib.Path): inspections = { "bin": ["PATH"], "man": ["MANPATH"], "share/man": ["MANPATH"], "share/aclocal": ["ACLOCAL_PATH"], "lib": ["LIBRARY_PATH", "LD_LIBRARY_PATH"], "lib64": ["LIBRARY_PATH", "LD_LIBRARY_PATH"], "include": ["CPATH"], "lib/pkgconfig": ["PKG_CONFIG_PATH"], "lib64/pkgconfig": ["PKG_CONFIG_PATH"], "share/pkgconfig": ["PKG_CONFIG_PATH"], "": ["CMAKE_PREFIX_PATH"], } (tmp_path / "bin").mkdir() (tmp_path / "lib").mkdir() (tmp_path / "include").mkdir() env = environment.inspect_path(str(tmp_path), inspections) names = [item.name for item in env] assert "PATH" in names assert "LIBRARY_PATH" in names assert "LD_LIBRARY_PATH" in names assert "CPATH" in names def test_exclude_paths_from_inspection(): inspections = { "lib": ["LIBRARY_PATH", "LD_LIBRARY_PATH"], "lib64": ["LIBRARY_PATH", "LD_LIBRARY_PATH"], "include": ["CPATH"], } env = environment.inspect_path("/usr", inspections, exclude=is_system_path) assert len(env) == 0 def make_path(*path): """Joins given components a,b,c... to form a valid absolute path for the current host OS being tested. Created path does not necessarily exist on system. """ abs_path_prefix = "C:\\" if sys.platform == "win32" else "/" return os.path.join(abs_path_prefix, *path) def make_pathlist(paths): """Makes a fake list of platform specific paths""" return os.pathsep.join( [make_path(*path) if isinstance(path, list) else make_path(path) for path in paths] ) @pytest.fixture def system_paths_for_os(): return miscellaneous_win_sys_paths if sys.platform == "win32" else miscellaneous_unix_sys_paths @pytest.fixture def non_system_paths_for_os(): return miscellaneous_win_paths if sys.platform == "win32" else miscellaneous_unix_paths @pytest.fixture() def prepare_environment_for_tests(working_env, system_paths_for_os): """Sets a few dummy variables in the current environment, that will be useful for the tests below. """ os.environ["UNSET_ME"] = "foo" os.environ["EMPTY_PATH_LIST"] = "" os.environ["PATH_LIST"] = make_pathlist([["path", "second"], ["path", "third"]]) os.environ["REMOVE_PATH_LIST"] = make_pathlist( [ ["a", "b"], ["duplicate"], ["a", "c"], ["remove", "this"], ["a", "d"], ["duplicate"], ["f", "g"], ] ) # grab arbitrary system path sys_path = system_paths_for_os[0] + os.pathsep os.environ["PATH_LIST_WITH_SYSTEM_PATHS"] = sys_path + os.environ["REMOVE_PATH_LIST"] os.environ["PATH_LIST_WITH_DUPLICATES"] = os.environ["REMOVE_PATH_LIST"] @pytest.fixture def env(prepare_environment_for_tests): """Returns an empty EnvironmentModifications object.""" return EnvironmentModifications() @pytest.fixture def files_to_be_sourced(): """Returns a list of files to be sourced""" return [ os.path.join(datadir, "sourceme_first" + shell_extension), os.path.join(datadir, "sourceme_second" + shell_extension), os.path.join(datadir, "sourceme_parameters" + shell_extension), os.path.join(datadir, "sourceme_unicode" + shell_extension), ] def test_set(env): """Tests setting values in the environment.""" # Here we are storing the commands to set a couple of variables env.set("A", "dummy value") env.set("B", 3) # ...and then we are executing them env.apply_modifications() assert "dummy value" == os.environ["A"] assert str(3) == os.environ["B"] def test_append_flags(env): """Tests appending to a value in the environment.""" # Store a couple of commands env.append_flags("APPEND_TO_ME", "flag1") env.append_flags("APPEND_TO_ME", "flag2") # ... execute the commands env.apply_modifications() assert "flag1 flag2" == os.environ["APPEND_TO_ME"] def test_unset(env): """Tests unsetting values in the environment.""" # Assert that the target variable is there and unset it assert "foo" == os.environ["UNSET_ME"] env.unset("UNSET_ME") env.apply_modifications() # Trying to retrieve is after deletion should cause a KeyError with pytest.raises(KeyError): os.environ["UNSET_ME"] def test_filter_system_paths(system_paths_for_os, non_system_paths_for_os): """Tests that the filtering of system paths works as expected.""" filtered = filter_system_paths(system_paths_for_os + non_system_paths_for_os) assert filtered == non_system_paths_for_os def test_set_path(env): """Tests setting paths in an environment variable.""" name = "A" elements = ["foo", "bar", "baz"] # Check setting paths with a specific separator env.set_path(name, elements, separator=os.pathsep) env.apply_modifications() expected = os.pathsep.join(elements) assert expected == os.environ[name] def test_path_manipulation(env): """Tests manipulating list of paths in the environment.""" env.prepend_path("PATH_LIST", make_path("path", "first")) env.append_path("PATH_LIST", make_path("path", "fourth")) env.append_path("PATH_LIST", make_path("path", "last")) env.remove_path("REMOVE_PATH_LIST", make_path("remove", "this")) env.remove_path("REMOVE_PATH_LIST", make_path("duplicate") + os.sep) env.prune_duplicate_paths("PATH_LIST_WITH_DUPLICATES") env.apply_modifications() assert os.environ["PATH_LIST"] == make_pathlist( [ ["path", "first"], ["path", "second"], ["path", "third"], ["path", "fourth"], ["path", "last"], ] ) assert os.environ["REMOVE_PATH_LIST"] == make_pathlist( [["a", "b"], ["a", "c"], ["a", "d"], ["f", "g"]] ) assert os.environ["PATH_LIST_WITH_DUPLICATES"].count(make_path("duplicate")) == 1 @pytest.mark.not_on_windows("Skip unix path tests on Windows") def test_unix_system_path_manipulation(env): """Tests manipulting paths that have special meaning as system paths on Unix""" env.deprioritize_system_paths("PATH_LIST_WITH_SYSTEM_PATHS") env.apply_modifications() assert not os.environ["PATH_LIST_WITH_SYSTEM_PATHS"].startswith( make_pathlist([["usr", "include" + os.pathsep]]) ) assert os.environ["PATH_LIST_WITH_SYSTEM_PATHS"].endswith(make_pathlist([["usr", "include"]])) @pytest.mark.skipif(sys.platform != "win32", reason="Skip Windows paths on not Windows") def test_windows_system_path_manipulation(env): """Tests manipulting paths that have special meaning as system paths on Windows""" env.deprioritize_system_paths("PATH_LIST_WITH_SYSTEM_PATHS") env.apply_modifications() assert not os.environ["PATH_LIST_WITH_SYSTEM_PATHS"].startswith( make_pathlist([["C:", "Users" + os.pathsep]]) ) assert os.environ["PATH_LIST_WITH_SYSTEM_PATHS"].endswith(make_pathlist([["C:", "Users"]])) def test_extend(env): """Tests that we can construct a list of environment modifications starting from another list. """ env.set("A", "dummy value") env.set("B", 3) copy_construct = EnvironmentModifications(env) assert len(copy_construct) == 2 for x, y in zip(env, copy_construct): assert x is y @pytest.mark.usefixtures("prepare_environment_for_tests") def test_source_files(files_to_be_sourced): """Tests the construction of a list of environment modifications that are the result of sourcing a file. """ env = EnvironmentModifications() for filename in files_to_be_sourced: if filename.endswith("sourceme_parameters" + shell_extension): env.extend(EnvironmentModifications.from_sourcing_file(filename, "intel64")) else: env.extend(EnvironmentModifications.from_sourcing_file(filename)) modifications = env.group_by_name() # This is sensitive to the user's environment; can include # spurious entries for things like PS1 # # TODO: figure out how to make a bit more robust. assert len(modifications) >= 5 # Set new variables assert len(modifications["NEW_VAR"]) == 1 assert isinstance(modifications["NEW_VAR"][0], SetEnv) assert modifications["NEW_VAR"][0].value == "new" assert len(modifications["FOO"]) == 1 assert isinstance(modifications["FOO"][0], SetEnv) assert modifications["FOO"][0].value == "intel64" # Unset variables assert len(modifications["EMPTY_PATH_LIST"]) == 1 assert isinstance(modifications["EMPTY_PATH_LIST"][0], UnsetEnv) # Modified variables assert len(modifications["UNSET_ME"]) == 1 assert isinstance(modifications["UNSET_ME"][0], SetEnv) assert modifications["UNSET_ME"][0].value == "overridden" assert len(modifications["PATH_LIST"]) == 3 assert isinstance(modifications["PATH_LIST"][0], RemovePath) assert modifications["PATH_LIST"][0].value == make_path("path", "third") assert isinstance(modifications["PATH_LIST"][1], AppendPath) assert modifications["PATH_LIST"][1].value == make_path("path", "fourth") assert isinstance(modifications["PATH_LIST"][2], PrependPath) assert modifications["PATH_LIST"][2].value == make_path("path", "first") @pytest.mark.regression("8345") def test_preserve_environment(prepare_environment_for_tests): # UNSET_ME is defined, and will be unset in the context manager, # NOT_SET is not in the environment and will be set within the # context manager, PATH_LIST is set and will be changed. with environment.preserve_environment("UNSET_ME", "NOT_SET", "PATH_LIST"): os.environ["NOT_SET"] = "a" assert os.environ["NOT_SET"] == "a" del os.environ["UNSET_ME"] assert "UNSET_ME" not in os.environ os.environ["PATH_LIST"] = "changed" assert "NOT_SET" not in os.environ assert os.environ["UNSET_ME"] == "foo" assert os.environ["PATH_LIST"] == make_pathlist([["path", "second"], ["path", "third"]]) @pytest.mark.parametrize( "files,expected,deleted", [ # Sets two variables ( (os.path.join(datadir, "sourceme_first" + shell_extension),), {"NEW_VAR": "new", "UNSET_ME": "overridden"}, [], ), # Check if we can set a variable to different values depending # on command line parameters ( (os.path.join(datadir, "sourceme_parameters" + shell_extension),), {"FOO": "default"}, [], ), ( ([os.path.join(datadir, "sourceme_parameters" + shell_extension), "intel64"],), {"FOO": "intel64"}, [], ), # Check unsetting variables ( (os.path.join(datadir, "sourceme_second" + shell_extension),), { "PATH_LIST": make_pathlist( [["path", "first"], ["path", "second"], ["path", "fourth"]] ) }, ["EMPTY_PATH_LIST"], ), # Check that order of sourcing matters ( ( os.path.join(datadir, "sourceme_unset" + shell_extension), os.path.join(datadir, "sourceme_first" + shell_extension), ), {"NEW_VAR": "new", "UNSET_ME": "overridden"}, [], ), ( ( os.path.join(datadir, "sourceme_first" + shell_extension), os.path.join(datadir, "sourceme_unset" + shell_extension), ), {"NEW_VAR": "new"}, ["UNSET_ME"], ), ], ) @pytest.mark.usefixtures("prepare_environment_for_tests") def test_environment_from_sourcing_files(files, expected, deleted): env = environment.environment_after_sourcing_files(*files) # Test that variables that have been modified are still there and contain # the expected output for name, value in expected.items(): assert name in env assert value in env[name] # Test that variables that have been unset are not there for name in deleted: assert name not in env def test_clear(env): env.set("A", "dummy value") assert len(env) > 0 env.clear() assert len(env) == 0 @pytest.mark.parametrize( "env,exclude,include", [ # Check we can exclude a literal ({"SHLVL": "1"}, ["SHLVL"], []), # Check include takes precedence ({"SHLVL": "1"}, ["SHLVL"], ["SHLVL"]), ], ) def test_sanitize_literals(env, exclude, include): after = environment.sanitize(env, exclude, include) # Check that all the included variables are there assert all(x in after for x in include) # Check that the excluded variables that are not # included are there exclude = list(set(exclude) - set(include)) assert all(x not in after for x in exclude) @pytest.mark.parametrize( "env,exclude,include,expected,deleted", [ # Check we can exclude using a regex ({"SHLVL": "1"}, ["SH.*"], [], [], ["SHLVL"]), # Check we can include using a regex ({"SHLVL": "1"}, ["SH.*"], ["SH.*"], ["SHLVL"], []), # Check regex to exclude Environment Modules related vars ( {"MODULES_LMALTNAME": "1", "MODULES_LMCONFLICT": "2"}, ["MODULES_(.*)"], [], [], ["MODULES_LMALTNAME", "MODULES_LMCONFLICT"], ), ( {"A_modquar": "1", "b_modquar": "2", "C_modshare": "3"}, [r"(\w*)_mod(quar|share)"], [], [], ["A_modquar", "b_modquar", "C_modshare"], ), ( {"__MODULES_LMTAG": "1", "__MODULES_LMPREREQ": "2"}, ["__MODULES_(.*)"], [], [], ["__MODULES_LMTAG", "__MODULES_LMPREREQ"], ), ], ) def test_sanitize_regex(env, exclude, include, expected, deleted): after = environment.sanitize(env, exclude, include) assert all(x in after for x in expected) assert all(x not in after for x in deleted) @pytest.mark.regression("12085") @pytest.mark.parametrize( "before,after,search_list", [ # Set environment variables ({}, {"FOO": "foo"}, [environment.SetEnv("FOO", "foo")]), # Unset environment variables ({"FOO": "foo"}, {}, [environment.UnsetEnv("FOO")]), # Append paths to an environment variable ( {"FOO_PATH": make_pathlist([["a", "path"]])}, {"FOO_PATH": make_pathlist([["a", "path"], ["b", "path"]])}, [environment.AppendPath("FOO_PATH", make_pathlist([["b", "path"]]))], ), ( {}, {"FOO_PATH": make_pathlist([["a", "path"], ["b", "path"]])}, [environment.AppendPath("FOO_PATH", make_pathlist([["a", "path"], ["b", "path"]]))], ), ( {"FOO_PATH": make_pathlist([["a", "path"], ["b", "path"]])}, {"FOO_PATH": make_pathlist([["b", "path"]])}, [environment.RemovePath("FOO_PATH", make_pathlist([["a", "path"]]))], ), ( {"FOO_PATH": make_pathlist([["a", "path"], ["b", "path"]])}, {"FOO_PATH": make_pathlist([["a", "path"], ["c", "path"]])}, [ environment.RemovePath("FOO_PATH", make_pathlist([["b", "path"]])), environment.AppendPath("FOO_PATH", make_pathlist([["c", "path"]])), ], ), ( {"FOO_PATH": make_pathlist([["a", "path"], ["b", "path"]])}, {"FOO_PATH": make_pathlist([["c", "path"], ["a", "path"]])}, [ environment.RemovePath("FOO_PATH", make_pathlist([["b", "path"]])), environment.PrependPath("FOO_PATH", make_pathlist([["c", "path"]])), ], ), # Modify two variables in the same environment ( {"FOO": "foo", "BAR": "bar"}, {"FOO": "baz", "BAR": "baz"}, [environment.SetEnv("FOO", "baz"), environment.SetEnv("BAR", "baz")], ), ], ) def test_from_environment_diff(before, after, search_list): mod = environment.EnvironmentModifications.from_environment_diff(before, after) for item in search_list: assert item in mod @pytest.mark.not_on_windows("Lmod not supported on Windows") @pytest.mark.regression("15775") def test_exclude_lmod_variables(): # Construct the list of environment modifications file = os.path.join(datadir, "sourceme_lmod.sh") env = EnvironmentModifications.from_sourcing_file(file) # Check that variables related to lmod are not in there modifications = env.group_by_name() assert not any(x.startswith("LMOD_") for x in modifications) @pytest.mark.regression("13504") def test_exclude_modules_variables(): # Construct the list of environment modifications file = os.path.join(datadir, "sourceme_modules" + shell_extension) env = EnvironmentModifications.from_sourcing_file(file) # Check that variables related to modules are not in there modifications = env.group_by_name() assert not any(x.startswith("MODULES_") for x in modifications) assert not any(x.startswith("__MODULES_") for x in modifications) assert not any(x.startswith("BASH_FUNC_ml") for x in modifications) assert not any(x.startswith("BASH_FUNC_module") for x in modifications) assert not any(x.startswith("BASH_FUNC__module_raw") for x in modifications) ================================================ FILE: lib/spack/spack/test/error_messages.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import os.path import re from contextlib import contextmanager from typing import Iterable, Optional import pytest import spack.vendor.archspec.cpu import spack.config import spack.error import spack.repo import spack.util.file_cache import spack.util.spack_yaml as syaml from spack.concretize import concretize_one from spack.main import SpackCommand solve = SpackCommand("solve") def update_packages_config(conf_str): conf = syaml.load_config(conf_str) spack.config.set("packages", conf["packages"], scope="concretize") _pkgx1 = ( "x1", """\ class X1(Package): version("1.2") version("1.1") depends_on("x2") depends_on("x3") """, ) _pkgx2 = ( "x2", """\ class X2(Package): version("2.1") version("2.0") depends_on("x4@4.1") """, ) _pkgx3 = ( "x3", """\ class X3(Package): version("3.5") version("3.4") depends_on("x4@4.0") """, ) _pkgx4 = ( "x4", """\ class X4(Package): version("4.1") version("4.0") """, ) _pkgy1 = ( "y1", """\ class Y1(Package): version("1.2") version("1.1") depends_on("y2+v1") depends_on("y3") """, ) _pkgy2 = ( "y2", """\ class Y2(Package): version("2.1") version("2.0") variant("v1", default=True) depends_on("y4@4.1", when="+v1") depends_on("y4") """, ) _pkgy3 = ( "y3", """\ class Y3(Package): version("3.5") version("3.4") depends_on("y4@4.0") """, ) _pkgy4 = ( "y4", """\ class Y4(Package): version("4.1") version("4.0") """, ) _pkgz1 = ( "z1", """\ class Z1(Package): version("1.2") version("1.1") variant("v1", default=True) depends_on("z2") depends_on("z3") depends_on("z3+v2", when="~v1") conflicts("+v1", when="@:1.1") """, ) _pkgz2 = ( "z2", """\ class Z2(Package): version("3.1") version("3.0") depends_on("z3@:2.0") """, ) _pkgz3 = ( "z3", """\ class Z3(Package): version("2.1") version("2.0") variant("v2", default=True, when="@2.1:") """, ) # Cluster of packages that includes requirements - goal is to "chain" # the requirements like other constraints. _pkgw4 = ( "w4", """\ class W4(Package): version("2.1") version("2.0") variant("v1", default=True) depends_on("w2") depends_on("w2@:2.0", when="@:2.0") depends_on("w3") depends_on("w3+v1", when="@2.0") """, ) _pkgw3 = ( "w3", """\ class W3(Package): version("2.1") version("2.0") variant("v1", default=True) requires("~v1", when="@2.1") depends_on("w1") """, ) _pkgw2 = ( "w2", """\ class W2(Package): version("2.1") version("2.0") variant("v1", default=True) depends_on("w1") """, ) _pkgw1 = ( "w1", """\ class W1(Package): version("2.1") version("2.0") variant("v1", default=True) """, ) # Like the W* packages, but encodes the config requirements constraints # into the packages to see if that improves the error from # test_errmsg_requirements_2 _pkgt4 = ( "t4", """\ class T4(Package): version("2.1") version("2.0") variant("v1", default=True) depends_on("t2") depends_on("t2@:2.0", when="@:2.0") depends_on("t3") depends_on("t3~v1", when="@2.0") """, ) _pkgt3 = ( "t3", """\ class T3(Package): version("2.1") version("2.0") variant("v1", default=True) requires("+v1", when="@2.1") depends_on("t1") """, ) _pkgt2 = ( "t2", """\ class T2(Package): version("2.1") version("2.0") variant("v1", default=True) requires("~v1", when="@:2.0") depends_on("t1") """, ) _pkgt1 = ( "t1", """\ class T1(Package): version("2.1") version("2.0") variant("v1", default=True) """, ) all_pkgs = [ _pkgx1, _pkgx2, _pkgx3, _pkgx4, _pkgy1, _pkgy2, _pkgy3, _pkgy4, _pkgz1, _pkgz2, _pkgz3, _pkgw1, _pkgw2, _pkgw3, _pkgw4, _pkgt1, _pkgt2, _pkgt3, _pkgt4, ] def _add_import(pkg_def): return ( """\ from spack.package import * from spack.package import Package """ + pkg_def ) all_pkgs = list((x, _add_import(y)) for (x, y) in all_pkgs) _repo_name_id = 0 def create_test_repo(tmp_path, pkg_name_content_tuples): global _repo_name_id repo_name = f"testrepo{str(_repo_name_id)}" repo_path = tmp_path / "spack_repo" / repo_name os.makedirs(repo_path) with open(repo_path / "__init__.py", "w", encoding="utf-8"): pass repo_yaml = os.path.join(repo_path, "repo.yaml") with open(str(repo_yaml), "w", encoding="utf-8") as f: f.write( f"""\ repo: namespace: {repo_name} api: v2.1 """ ) _repo_name_id += 1 packages_dir = repo_path / "packages" os.mkdir(packages_dir) with open(packages_dir / "__init__.py", "w", encoding="utf-8"): pass for pkg_name, pkg_str in pkg_name_content_tuples: pkg_dir = packages_dir / pkg_name os.mkdir(pkg_dir) pkg_file = pkg_dir / "package.py" with open(str(pkg_file), "w", encoding="utf-8") as f: f.write(pkg_str) repo_cache = spack.util.file_cache.FileCache(str(tmp_path / "cache")) return spack.repo.Repo(str(repo_path), cache=repo_cache) @pytest.fixture def _create_test_repo(tmp_path, mutable_config): yield create_test_repo(tmp_path, all_pkgs) @pytest.fixture def test_repo(_create_test_repo, monkeypatch, mock_stage): with spack.repo.use_repositories(_create_test_repo) as mock_repo_path: yield mock_repo_path @contextmanager def expect_failure_and_print(should_mention=None): got_an_error_as_expected = False err_msg = None try: yield except spack.error.UnsatisfiableSpecError as e: got_an_error_as_expected = True err_msg = str(e) if not got_an_error_as_expected: raise ValueError("A failure was supposed to occur in this context manager") elif not err_msg: raise ValueError("No error message for failed concretization") print(err_msg) check_error(err_msg, should_mention) def check_error(msg, should_mention: Optional[Iterable] = None): excludes = [ "failed to concretize .* for the following reasons:", "Cannot satisfy .*", "required because .* requested explicitly", "cannot satisfy a requirement for package .*", ] lines = msg.split("\n") should_mention = set(should_mention) if should_mention else set() should_mention_hits = set() remaining = [] for line in lines: for p in should_mention: if re.search(p, line): should_mention_hits.add(p) if any(re.search(p, line) for p in excludes): continue remaining.append(line) if not remaining: raise ValueError("The error message contains only generic statements") should_mention_misses = should_mention - should_mention_hits if should_mention_misses: raise ValueError(f"The error message did not contain: {sorted(should_mention_misses)}") def test_diamond_with_pkg_conflict1(concretize_scope, test_repo): concretize_one("x2") concretize_one("x3") concretize_one("x4") important_points = ["x2 depends on x4@4.1", "x3 depends on x4@4.0"] with expect_failure_and_print(should_mention=important_points): concretize_one("x1") def test_diamond_with_pkg_conflict2(concretize_scope, test_repo): important_points = [ r"y2 depends on y4@4.1 when \+v1", r"y1 depends on y2\+v1", r"y3 depends on y4@4.0", ] with expect_failure_and_print(should_mention=important_points): concretize_one("y1") @pytest.mark.xfail(reason="Not addressed yet") def test_version_range_null(concretize_scope, test_repo): with expect_failure_and_print(): concretize_one("x2@3:4") # This error message is hard to follow: neither z2 or z3 # are mentioned, so if this hierarchy had 10 other "OK" # packages, a user would be conducting a tedious manual # search @pytest.mark.xfail(reason="Not addressed yet") def test_null_variant_for_requested_version(concretize_scope, test_repo): r""" Z1_ (@:1.1 -> !v1) | \ Z2 | \ | \| Z3 (z1~v1 -> z3+v2) (z2 ^z3:2.0) (v2 only exists for @2.1:) """ concretize_one("z1") with expect_failure_and_print(should_mention=["z2"]): concretize_one("z1@1.1") def test_errmsg_requirements_1(concretize_scope, test_repo): # w4 has: depends_on("w3+v1", when="@2.0") # w3 has: requires("~v1", when="@2.1") important_points = [ r"w4 depends on w3\+v1 when @2.0", r"w4@:2.0 \^w3@2.1 requested explicitly", r"~v1 is a requirement for package w3 when @2.1", ] with expect_failure_and_print(should_mention=important_points): concretize_one("w4@:2.0 ^w3@2.1") def test_errmsg_requirements_cfg(concretize_scope, test_repo): conf_str = """\ packages: w2: require: - one_of: ["~v1"] when: "@2.0" """ update_packages_config(conf_str) important_points = [ r"~v1 is a requirement for package w2 when @2.0", r"w4 depends on w2@:2.0 when @:2.0", r"w4@2.0 \^w2\+v1 requested explicitly", ] # w4 has: depends_on("w2@:2.0", when="@:2.0") with expect_failure_and_print(should_mention=important_points): concretize_one("w4@2.0 ^w2+v1") # This reencodes prior test test_errmsg_requirements_cfg # in terms of package `requires`, def test_errmsg_requirements_directives(concretize_scope, test_repo): # t4 has: depends_on("t2@:2.0", when="@:2.0") # t2 has: requires("~v1", when="@:2.0") important_points = [ r"~v1 is a requirement for package t2 when @:2.0", r"t4 depends on t2@:2.0 when @:2.0", r"t4@:2.0 \^t2\+v1 requested explicitly", ] with expect_failure_and_print(should_mention=important_points): concretize_one("t4@:2.0 ^t2+v1") # Simulates a user error: package is specified as external with a version, # but a different version was required in config. def test_errmsg_requirements_external_mismatch(concretize_scope, test_repo): conf_str = """\ packages: t1: buildable: false externals: - spec: "t1@2.1" prefix: /a/path/that/doesnt/need/to/exist/ require: - spec: "t1@2.0" """ update_packages_config(conf_str) important_points = ["no externals satisfy the request"] with expect_failure_and_print(should_mention=important_points): concretize_one("t1") @pytest.mark.parametrize("section", ["prefer", "require"]) def test_warns_on_compiler_constraint_in_all(concretize_scope, mock_packages, section): """Compiler constraints under packages:all: are a footgun and should warn.""" update_packages_config(f"packages:\n all:\n {section}:\n - '%c=gcc'\n") with pytest.warns(UserWarning, match="packages: all:"): concretize_one("gmake") @pytest.mark.regression("52209") def test_unknown_concrete_target_in_input_spec(concretize_scope, test_repo): """Tests that an input spec with an unknown concrete target raises a clear error naming the bad target, rather than a confusing 'cannot satisfy constraint' solver error. """ spec_str = "x4 target=not-a-real-uarch" with pytest.raises(spack.error.SpackError) as exc_info: concretize_one(spec_str) check_error(str(exc_info.value), should_mention=[spec_str, "not a known target"]) @pytest.mark.regression("52209") def test_require_single_unknown_target_errors(concretize_scope, test_repo): """Tests that a single-option require with an unknown target raises a clear error.""" target_str = "target=not-a-real-uarch" update_packages_config( f"""\ packages: x4: require: {target_str} """ ) with pytest.raises(spack.error.SpackError) as exc_info: concretize_one("x4") check_error(str(exc_info.value), should_mention=[target_str, "unknown target"]) @pytest.mark.regression("52209") def test_require_all_unknown_targets_errors(concretize_scope, test_repo): """Tests that a group where every option has an unknown target also raises a clear error.""" update_packages_config( """\ packages: x4: require: - any_of: ["target=not-a-real-uarch", "target=also-fake"] """ ) with pytest.raises(spack.error.SpackError) as exc_info: concretize_one("x4") check_error( str(exc_info.value), should_mention=["target=not-a-real-uarch", "target=also-fake", "unknown target"], ) @pytest.mark.regression("52209") @pytest.mark.skipif( str(spack.vendor.archspec.cpu.host().family) != "x86_64", reason="test assumes x86_64 uarchs" ) def test_require_mixed_unknown_and_valid_target_warns(concretize_scope, test_repo): """Tests that a "require" group with at least one valid option just warns.""" update_packages_config( """\ packages: x4: require: - one_of: ["target=not-a-real-uarch", "target=x86_64"] """ ) with pytest.warns(UserWarning, match="not-a-real-uarch"): concretize_one("x4") @pytest.mark.regression("52209") def test_prefer_unknown_target_warns(concretize_scope, test_repo): """A preference with an unknown target has the @: fallback, so it only warns.""" update_packages_config( """\ packages: x4: prefer: ["target=not-a-real-uarch"] """ ) with pytest.warns(UserWarning, match="not-a-real-uarch"): concretize_one("x4") ================================================ FILE: lib/spack/spack/test/externals.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from typing import List import pytest from spack.vendor.archspec.cpu import TARGETS import spack.archspec import spack.traverse from spack.externals import ( DuplicateExternalError, ExternalDict, ExternalSpecError, ExternalSpecsParser, complete_architecture, complete_variants_and_architecture, ) pytestmark = pytest.mark.usefixtures("config", "mock_packages") @pytest.mark.parametrize( "externals_dict,expected_length,expected_queries", [ # Empty dictionary case ([], 0, {"gmake": 0}), # Single spec case ( [{"spec": "gmake@1.0", "prefix": "/path/to/gmake"}], 1, {"gmake": 1, "gmake@1.0": 1, "gmake@2.0": 0}, ), # Multiple specs case ( [ {"spec": "gmake@1.0", "prefix": "/path/to/gmake1"}, {"spec": "gmake@2.0", "prefix": "/path/to/gmake2"}, {"spec": "gcc@1.0", "prefix": "/path/to/gcc"}, ], 3, {"gmake": 2, "gmake@2": 1, "gcc": 1, "baz": 0}, ), # Case with modules and extra attributes ( [ { "spec": "gmake@1.0", "prefix": "/path/to/gmake", "modules": ["module1", "module2"], "extra_attributes": {"attr1": "value1"}, } ], 1, {"gmake": 1}, ), ], ) def test_basic_parsing(externals_dict, expected_length, expected_queries): """Tests parsing external specs, in some basic cases""" parser = ExternalSpecsParser(externals_dict) assert len(parser.all_specs()) == expected_length assert len(parser.specs_by_external_id) == expected_length for node in parser.all_specs(): assert node.concrete for query, expected in expected_queries.items(): assert len(parser.query(query)) == expected @pytest.mark.parametrize( "externals_dict,expected_triplet", [ ([{"spec": "gmake@1.0", "prefix": "/path/to/gmake1"}], ("test", "debian6", "aarch64")), ( [{"spec": "gmake@1.0 target=icelake", "prefix": "/path/to/gmake1"}], ("test", "debian6", "icelake"), ), ( [{"spec": "gmake@1.0 platform=linux target=icelake", "prefix": "/path/to/gmake1"}], ("linux", "debian6", "icelake"), ), ( [{"spec": "gmake@1.0 os=rhel8", "prefix": "/path/to/gmake1"}], ("test", "rhel8", "aarch64"), ), ], ) def test_external_specs_architecture_completion( externals_dict: List[ExternalDict], expected_triplet, monkeypatch ): """Tests the completion of external specs architectures when using the default behavior""" monkeypatch.setattr(spack.archspec, "HOST_TARGET_FAMILY", TARGETS["aarch64"]) parser = ExternalSpecsParser(externals_dict) expected_platform, expected_os, expected_target = expected_triplet for node in parser.all_specs(): assert node.architecture is not None assert node.architecture.platform == expected_platform assert node.architecture.os == expected_os assert node.target == expected_target def test_external_specs_parser_with_missing_packages(): """Tests the parsing of external specs when some packages are missing""" externals_dict: List[ExternalDict] = [ {"spec": "gmake@1.0", "prefix": "/path/to/gmake1"}, {"spec": "gmake@2.0", "prefix": "/path/to/gmake2"}, {"spec": "gcc@1.0", "prefix": "/path/to/gcc"}, # This package does not exist in the builtin_mock repository {"spec": "baz@1.0", "prefix": "/path/to/baz"}, ] external_specs = ExternalSpecsParser(externals_dict, allow_nonexisting=True).all_specs() assert len(external_specs) == 3 assert len([x for x in external_specs if x.satisfies("gmake")]) == 2 assert len([x for x in external_specs if x.satisfies("gcc")]) == 1 with pytest.raises(ExternalSpecError, match="Package 'baz' does not exist"): ExternalSpecsParser(externals_dict, allow_nonexisting=False) def test_externals_with_duplicate_id(): """Tests the parsing of external specs when some specs have the same id""" externals_dict: List[ExternalDict] = [ {"spec": "gmake@1.0", "prefix": "/path/to/gmake1", "id": "gmake"}, {"spec": "gmake@2.0", "prefix": "/path/to/gmake2", "id": "gmake"}, {"spec": "gcc@1.0", "prefix": "/path/to/gcc", "id": "gcc"}, ] with pytest.raises(DuplicateExternalError, match="cannot have the same external id"): ExternalSpecsParser(externals_dict) @pytest.mark.parametrize( "externals_dicts,expected,not_expected", [ # o ascent@0.9.2 # o adios2@2.7.1 # o bzip2@1.0.8 ( [ { "spec": "ascent@0.9.2+adios2+shared", "prefix": "/user/path", "id": "ascent", "dependencies": [{"id": "adios2", "deptypes": ["build", "link"]}], }, { "spec": "adios2@2.7.1+shared", "prefix": "/user/path", "id": "adios2", "dependencies": [{"id": "bzip2", "deptypes": ["build", "link"]}], }, {"spec": "bzip2@1.0.8+shared", "prefix": "/user/path", "id": "bzip2"}, ], { "ascent": ["%[deptypes=build,link] adios2@2.7.1"], "adios2": ["%[deptypes=build,link] bzip2@1.0.8"], }, {}, ), # o ascent@0.9.2 # |\ # | o adios2@2.7.1 # |/ # o bzip2@1.0.8 ( [ { "spec": "ascent@0.9.2+adios2+shared", "prefix": "/user/path", "id": "ascent", "dependencies": [ {"id": "adios2", "deptypes": "link"}, {"id": "bzip2", "deptypes": "run"}, ], }, { "spec": "adios2@2.7.1+shared", "prefix": "/user/path", "id": "adios2", "dependencies": [{"id": "bzip2", "deptypes": ["build", "link"]}], }, {"spec": "bzip2@1.0.8+shared", "prefix": "/user/path", "id": "bzip2"}, ], { "ascent": ["%[deptypes=link] adios2@2.7.1", "%[deptypes=run] bzip2@1.0.8"], "adios2": ["%[deptypes=build,link] bzip2@1.0.8"], }, { "ascent": [ "%[deptypes=build] adios2@2.7.1", "%[deptypes=run] adios2@2.7.1", "%[deptypes=build] bzip2@1.0.8", "%[deptypes=link] bzip2@1.0.8", ] }, ), # Same, but specifying dependencies by spec: instead of id: ( [ { "spec": "ascent@0.9.2+adios2+shared", "prefix": "/user/path", "dependencies": [ {"spec": "adios2", "deptypes": "link"}, {"spec": "bzip2", "deptypes": "run"}, ], }, { "spec": "adios2@2.7.1+shared", "prefix": "/user/path", "dependencies": [{"spec": "bzip2", "deptypes": ["build", "link"]}], }, {"spec": "bzip2@1.0.8+shared", "prefix": "/user/path"}, ], { "ascent": ["%[deptypes=link] adios2@2.7.1", "%[deptypes=run] bzip2@1.0.8"], "adios2": ["%[deptypes=build,link] bzip2@1.0.8"], }, { "ascent": [ "%[deptypes=build] adios2@2.7.1", "%[deptypes=run] adios2@2.7.1", "%[deptypes=build] bzip2@1.0.8", "%[deptypes=link] bzip2@1.0.8", ] }, ), # Inline specification for # o mpileaks@2.2 # | \ # | o callpath@1.0 # | / # o gcc@15.0.1 ( [ [ {"spec": "mpileaks@2.2 %gcc %callpath", "prefix": "/user/path"}, {"spec": "callpath@1.0", "prefix": "/user/path"}, {"spec": "gcc@15.0.1 languages=c,c++", "prefix": "/user/path"}, ], {"mpileaks": ["%[deptypes=build] gcc@15", "%[deptypes=build,link] callpath@1.0"]}, {"mpileaks": ["%[deptypes=link] gcc@15"]}, ] ), # CMake dependency should be inferred of `deptypes=build` # o cmake-client # | # o cmake@3.23.1 ( [ [ {"spec": "cmake-client@1.0 %cmake", "prefix": "/user/path"}, {"spec": "cmake@3.23.1", "prefix": "/user/path"}, ], {"cmake-client": ["%[deptypes=build] cmake"]}, {"cmake-client": ["%[deptypes=link] cmake", "%[deptypes=run] cmake"]}, ] ), ], ) def test_externals_with_dependencies(externals_dicts: List[ExternalDict], expected, not_expected): """Tests constructing externals with dependencies""" parser = ExternalSpecsParser(externals_dicts) for query_spec, expected_list in expected.items(): result = parser.query(query_spec) assert len(result) == 1 assert all(result[0].satisfies(c) for c in expected_list) for query_spec, not_expected_list in not_expected.items(): result = parser.query(query_spec) assert len(result) == 1 assert all(not result[0].satisfies(c) for c in not_expected_list) # Assert all nodes have the namespace set for node in spack.traverse.traverse_nodes(parser.all_specs()): assert node.namespace is not None @pytest.mark.parametrize( "externals_dicts,expected_length,not_expected", [ ([{"spec": "mpileaks", "prefix": "/user/path", "id": "mpileaks"}], 0, ["mpileaks"]), ([{"spec": "mpileaks@2:", "prefix": "/user/path", "id": "mpileaks"}], 0, ["mpileaks"]), ], ) def test_externals_without_concrete_version( externals_dicts: List[ExternalDict], expected_length, not_expected ): """Tests parsing externals, when some dicts are malformed and don't have a concrete version""" parser = ExternalSpecsParser(externals_dicts) result = parser.all_specs() assert len(result) == expected_length for c in not_expected: assert all(not s.satisfies(c) for s in result) @pytest.mark.parametrize( "externals_dict,completion_fn,expected,not_expected", [ ( [{"spec": "mpileaks@2.3", "prefix": "/user/path"}], complete_architecture, {"mpileaks": ["platform=test"]}, {"mpileaks": ["debug=*", "opt=*", "shared=*", "static=*"]}, ), ( [{"spec": "mpileaks@2.3", "prefix": "/user/path"}], complete_variants_and_architecture, {"mpileaks": ["platform=test", "~debug", "~opt", "+shared", "+static"]}, {"mpileaks": ["+debug", "+opt", "~shared", "~static"]}, ), ], ) def test_external_node_completion( externals_dict: List[ExternalDict], completion_fn, expected, not_expected ): """Tests the completion of external specs with different node completion""" parser = ExternalSpecsParser(externals_dict, complete_node=completion_fn) for query_spec, expected_list in expected.items(): result = parser.query(query_spec) assert len(result) == 1 for expected in expected_list: assert result[0].satisfies(expected) for query_spec, expected_list in not_expected.items(): result = parser.query(query_spec) assert len(result) == 1 for expected in expected_list: assert not result[0].satisfies(expected) # Assert all nodes have the namespace set for node in spack.traverse.traverse_nodes(parser.all_specs()): assert node.namespace is not None @pytest.mark.regression("52179") def test_external_spec_single_valued_variant_type_is_corrected(): """Tests that an external spec string including a single-valued variant is parsed correctly.""" externals_dict = [ {"spec": "dual-cmake-autotools@1.0 build_system=autotools", "prefix": "/usr/dual"} ] parser = ExternalSpecsParser(externals_dict, complete_node=complete_variants_and_architecture) specs = parser.all_specs() assert len(specs) == 1 spec = specs[0] # Single-valued variants return the value, not a tuple of values build_system_value = spec.variants["build_system"].value assert build_system_value == "autotools", ( f"Expected 'autotools' but got {build_system_value!r} " f"(type: {type(build_system_value).__name__})" ) @pytest.mark.regression("52179") def test_external_spec_multi_valued_variant_is_not_changed(): """Tests that multi-valued variants in external specs are preserved as they are, even if the definition in package.py says otherwise. """ # Package.py prescribes a single-valued variant in this case externals_dict = [{"spec": "variant-values@1.0 v=foo,bar", "prefix": "/usr/variant-values"}] parser = ExternalSpecsParser(externals_dict, complete_node=complete_variants_and_architecture) specs = parser.all_specs() assert len(specs) == 1 assert specs[0].variants["v"].value == ("bar", "foo") ================================================ FILE: lib/spack/spack/test/fetch_strategy.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from io import StringIO import pytest from spack import fetch_strategy def test_fetchstrategy_bad_url_scheme(): """Ensure that trying to make a fetch strategy from a URL with an unsupported scheme fails as expected.""" with pytest.raises(ValueError): fetcher = fetch_strategy.from_url_scheme("bogus-scheme://example.com/a/b/c") # noqa: F841 @pytest.mark.parametrize( "expected,total_bytes", [ (" 0.00 B", 0), (" 999.00 B", 999), (" 1.00 KB", 1000), (" 2.05 KB", 2048), (" 1.00 MB", 1e6), (" 12.30 MB", 1.23e7), (" 1.23 GB", 1.23e9), (" 999.99 GB", 9.9999e11), ("5000.00 GB", 5e12), ], ) def test_format_bytes(expected, total_bytes): assert fetch_strategy._format_bytes(total_bytes) == expected @pytest.mark.parametrize( "expected,total_bytes,elapsed", [ (" 0.0 B/s", 0, 0), # no time passed -- defaults to 1s. (" 0.0 B/s", 0, 1), (" 999.0 B/s", 999, 1), (" 1.0 KB/s", 1000, 1), (" 500.0 B/s", 1000, 2), (" 2.0 KB/s", 2048, 1), (" 1.0 MB/s", 1e6, 1), (" 500.0 KB/s", 1e6, 2), (" 12.3 MB/s", 1.23e7, 1), (" 1.2 GB/s", 1.23e9, 1), (" 999.9 GB/s", 9.999e11, 1), ("5000.0 GB/s", 5e12, 1), ], ) def test_format_speed(expected, total_bytes, elapsed): assert fetch_strategy._format_speed(total_bytes, elapsed) == expected def test_fetch_progress_unknown_size(): # time stamps in seconds, with 0.1s delta except 1.5 -> 1.55. time_stamps = iter([1.0, 1.5, 1.55, 2.0, 3.0, 5.0, 5.5, 5.5]) progress = fetch_strategy.FetchProgress(total_bytes=None, get_time=lambda: next(time_stamps)) assert progress.start_time == 1.0 out = StringIO() progress.advance(1000, out) assert progress.last_printed == 1.5 progress.advance(50, out) assert progress.last_printed == 1.5 # does not print, too early after last print progress.advance(2000, out) assert progress.last_printed == 2.0 progress.advance(3000, out) assert progress.last_printed == 3.0 progress.advance(4000, out) assert progress.last_printed == 5.0 progress.advance(4000, out) assert progress.last_printed == 5.5 progress.print(final=True, out=out) # finalize download outputs = [ "\r [ | ] 1.00 KB @ 2.0 KB/s", "\r [ / ] 3.05 KB @ 3.0 KB/s", "\r [ - ] 6.05 KB @ 3.0 KB/s", "\r [ \\ ] 10.05 KB @ 2.5 KB/s", # have to escape \ here but is aligned in output "\r [ | ] 14.05 KB @ 3.1 KB/s", "\r [100%] 14.05 KB @ 3.1 KB/s\n", # final print: no spinner; newline ] assert out.getvalue() == "".join(outputs) def test_fetch_progress_known_size(): time_stamps = iter([1.0, 1.5, 3.0, 4.0, 4.0]) progress = fetch_strategy.FetchProgress(total_bytes=6000, get_time=lambda: next(time_stamps)) out = StringIO() progress.advance(1000, out) # time 1.5 progress.advance(2000, out) # time 3.0 progress.advance(3000, out) # time 4.0 progress.print(final=True, out=out) outputs = [ "\r [ 17%] 1.00 KB @ 2.0 KB/s", "\r [ 50%] 3.00 KB @ 1.5 KB/s", "\r [100%] 6.00 KB @ 2.0 KB/s", "\r [100%] 6.00 KB @ 2.0 KB/s\n", # final print has newline ] assert out.getvalue() == "".join(outputs) def test_fetch_progress_disabled(): """When disabled, FetchProgress shouldn't print anything when advanced""" def get_time(): raise RuntimeError("Should not be called") progress = fetch_strategy.FetchProgress(enabled=False, get_time=get_time) out = StringIO() progress.advance(1000, out) progress.advance(2000, out) progress.print(final=True, out=out) assert progress.last_printed == 0 assert not out.getvalue() @pytest.mark.parametrize( "header,value,total_bytes", [ ("Content-Length", "1234", 1234), ("Content-Length", "0", 0), ("Content-Length", "-10", 0), ("Content-Length", "not a number", 0), ("Not-Content-Length", "1234", 0), ], ) def test_fetch_progress_from_headers(header, value, total_bytes): time_stamps = iter([1.0, 1.5, 3.0, 4.0, 4.0]) progress = fetch_strategy.FetchProgress.from_headers( {header: value}, get_time=lambda: next(time_stamps), enabled=True ) assert progress.total_bytes == total_bytes assert progress.enabled assert progress.start_time == 1.0 def test_fetch_progress_from_headers_disabled(): progress = fetch_strategy.FetchProgress.from_headers( {"Content-Length": "1234"}, get_time=lambda: 1.0, enabled=False ) assert not progress.enabled ================================================ FILE: lib/spack/spack/test/flag_handlers.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pytest import spack.build_environment import spack.concretize from spack.package import build_system_flags, env_flags, inject_flags @pytest.fixture() def temp_env(): old_env = os.environ.copy() yield os.environ.clear() os.environ.update(old_env) def add_o3_to_build_system_cflags(pkg, name, flags): build_system_flags = [] if name == "cflags": build_system_flags.append("-O3") return flags, None, build_system_flags @pytest.mark.usefixtures("config", "mock_packages") class TestFlagHandlers: def test_no_build_system_flags(self, temp_env): # Test that both autotools and cmake work getting no build_system flags s1 = spack.concretize.concretize_one("cmake-client") spack.build_environment.setup_package(s1.package, False) s2 = spack.concretize.concretize_one("patchelf") spack.build_environment.setup_package(s2.package, False) # Use cppflags as a canary assert "SPACK_CPPFLAGS" not in os.environ assert "CPPFLAGS" not in os.environ def test_unbound_method(self, temp_env): # Other tests test flag_handlers set as bound methods and functions. # This tests an unbound method in python2 (no change in python3). s = spack.concretize.concretize_one("mpileaks cppflags=-g") s.package.flag_handler = s.package.__class__.inject_flags spack.build_environment.setup_package(s.package, False) assert os.environ["SPACK_CPPFLAGS"] == "-g" assert "CPPFLAGS" not in os.environ def test_inject_flags(self, temp_env): s = spack.concretize.concretize_one("mpileaks cppflags=-g") s.package.flag_handler = inject_flags spack.build_environment.setup_package(s.package, False) assert os.environ["SPACK_CPPFLAGS"] == "-g" assert "CPPFLAGS" not in os.environ def test_env_flags(self, temp_env): s = spack.concretize.concretize_one("mpileaks cppflags=-g") s.package.flag_handler = env_flags spack.build_environment.setup_package(s.package, False) assert os.environ["CPPFLAGS"] == "-g" assert "SPACK_CPPFLAGS" not in os.environ def test_build_system_flags_cmake(self, temp_env): s = spack.concretize.concretize_one("cmake-client cppflags=-g") s.package.flag_handler = build_system_flags spack.build_environment.setup_package(s.package, False) assert "SPACK_CPPFLAGS" not in os.environ assert "CPPFLAGS" not in os.environ assert set(s.package.cmake_flag_args) == { "-DCMAKE_C_FLAGS=-g", "-DCMAKE_CXX_FLAGS=-g", "-DCMAKE_Fortran_FLAGS=-g", } def test_build_system_flags_autotools(self, temp_env): s = spack.concretize.concretize_one("patchelf cppflags=-g") s.package.flag_handler = build_system_flags spack.build_environment.setup_package(s.package, False) assert "SPACK_CPPFLAGS" not in os.environ assert "CPPFLAGS" not in os.environ assert "CPPFLAGS=-g" in s.package.configure_flag_args def test_build_system_flags_not_implemented(self, temp_env): """Test the command line flags method raises a NotImplementedError""" s = spack.concretize.concretize_one("mpileaks cppflags=-g") s.package.flag_handler = build_system_flags try: spack.build_environment.setup_package(s.package, False) assert False except NotImplementedError: assert True def test_add_build_system_flags_autotools(self, temp_env): s = spack.concretize.concretize_one("patchelf cppflags=-g") s.package.flag_handler = add_o3_to_build_system_cflags spack.build_environment.setup_package(s.package, False) assert "-g" in os.environ["SPACK_CPPFLAGS"] assert "CPPFLAGS" not in os.environ assert s.package.configure_flag_args == ["CFLAGS=-O3"] def test_add_build_system_flags_cmake(self, temp_env): s = spack.concretize.concretize_one("cmake-client cppflags=-g") s.package.flag_handler = add_o3_to_build_system_cflags spack.build_environment.setup_package(s.package, False) assert "-g" in os.environ["SPACK_CPPFLAGS"] assert "CPPFLAGS" not in os.environ assert s.package.cmake_flag_args == ["-DCMAKE_C_FLAGS=-O3"] def test_ld_flags_cmake(self, temp_env): s = spack.concretize.concretize_one("cmake-client ldflags=-mthreads") s.package.flag_handler = build_system_flags spack.build_environment.setup_package(s.package, False) assert "SPACK_LDFLAGS" not in os.environ assert "LDFLAGS" not in os.environ assert set(s.package.cmake_flag_args) == { "-DCMAKE_EXE_LINKER_FLAGS=-mthreads", "-DCMAKE_MODULE_LINKER_FLAGS=-mthreads", "-DCMAKE_SHARED_LINKER_FLAGS=-mthreads", } def test_ld_libs_cmake(self, temp_env): s = spack.concretize.concretize_one("cmake-client ldlibs=-lfoo") s.package.flag_handler = build_system_flags spack.build_environment.setup_package(s.package, False) assert "SPACK_LDLIBS" not in os.environ assert "LDLIBS" not in os.environ assert set(s.package.cmake_flag_args) == { "-DCMAKE_C_STANDARD_LIBRARIES=-lfoo", "-DCMAKE_CXX_STANDARD_LIBRARIES=-lfoo", "-DCMAKE_Fortran_STANDARD_LIBRARIES=-lfoo", } def test_flag_handler_no_modify_specs(self, temp_env): def test_flag_handler(self, name, flags): flags.append("-foo") return (flags, None, None) s = spack.concretize.concretize_one("cmake-client") s.package.flag_handler = test_flag_handler spack.build_environment.setup_package(s.package, False) assert not s.compiler_flags["cflags"] assert os.environ["SPACK_CFLAGS"] == "-foo" ================================================ FILE: lib/spack/spack/test/gcs_fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import spack.fetch_strategy import spack.stage def test_gcsfetchstrategy_downloaded(tmp_path: pathlib.Path): """Ensure fetch with archive file already downloaded is a noop.""" archive = tmp_path / "gcs.tar.gz" class Archived_GCSFS(spack.fetch_strategy.GCSFetchStrategy): @property def archive_file(self): return str(archive) fetcher = Archived_GCSFS(url="gs://example/gcs.tar.gz") with spack.stage.Stage(fetcher, path=str(tmp_path)): fetcher.fetch() ================================================ FILE: lib/spack/spack/test/git_fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import copy import os import pathlib import shutil import pytest import spack.concretize import spack.config import spack.error import spack.fetch_strategy import spack.package_base import spack.platforms import spack.repo import spack.util.git from spack.fetch_strategy import GitFetchStrategy from spack.llnl.util.filesystem import mkdirp, touch, working_dir from spack.package_base import PackageBase from spack.spec import Spec from spack.stage import Stage from spack.variant import SingleValuedVariant from spack.version import Version _mock_transport_error = "Mock HTTP transport error" min_opt_string = ".".join(map(str, spack.util.git.MIN_OPT_VERSION)) min_direct_commit = ".".join(map(str, spack.util.git.MIN_DIRECT_COMMIT_FETCH)) @pytest.fixture(params=[None, "1.8.5.2", "1.8.5.1", "1.7.10", "1.7.1", "1.7.0"]) def git_version(git, request, monkeypatch): """Tests GitFetchStrategy behavior for different git versions. GitFetchStrategy tries to optimize using features of newer git versions, but needs to work with older git versions. To ensure code paths for old versions still work, we fake it out here and make it use the backward-compatibility code paths with newer git versions. """ real_git_version = Version(spack.util.git.extract_git_version_str(git)) if request.param is None: # Don't patch; run with the real git_version method. yield real_git_version else: test_git_version = Version(request.param) if test_git_version > real_git_version: pytest.skip("Can't test clone logic for newer version of git.") # Patch the fetch strategy to think it's using a lower git version. # we use this to test what we'd need to do with older git versions # using a newer git installation. monkeypatch.setattr(spack.util.git, "extract_git_version_str", lambda _: request.param) yield test_git_version @pytest.fixture def mock_bad_git(mock_util_executable): """ Test GitFetchStrategy behavior with a bad git command for git >= 1.7.1 to trigger a SpackError. """ _, should_fail, registered_respones = mock_util_executable should_fail.extend(["clone", "fetch"]) registered_respones["--version"] = "1.7.1" def test_bad_git(tmp_path: pathlib.Path, mock_bad_git): """Trigger a SpackError when attempt a fetch with a bad git.""" testpath = str(tmp_path) with pytest.raises(spack.error.SpackError): fetcher = GitFetchStrategy(git="file:///not-a-real-git-repo") with Stage(fetcher, path=testpath): fetcher.fetch() @pytest.mark.parametrize("type_of_test", ["default", "branch", "tag", "commit"]) @pytest.mark.parametrize("secure", [True, False]) def test_fetch( git, type_of_test, secure, mock_git_repository, default_mock_concretization, mutable_mock_repo, git_version, monkeypatch, ): """Tries to: 1. Fetch the repo using a fetch strategy constructed with supplied args (they depend on type_of_test). 2. Check if the test_file is in the checked out repository. 3. Assert that the repository is at the revision supplied. 4. Add and remove some files, then reset the repo, and ensure it's all there again. """ # Retrieve the right test parameters t = mock_git_repository.checks[type_of_test] h = mock_git_repository.hash pkg_class = spack.repo.PATH.get_pkg_class("git-test") # This would fail using the default-no-per-version-git check but that # isn't included in this test monkeypatch.delattr(pkg_class, "git") # Construct the package under test s = default_mock_concretization("git-test") monkeypatch.setitem(s.package.versions, Version("git"), t.args) if type_of_test == "commit": s.variants["commit"] = SingleValuedVariant("commit", t.args["commit"]) # Enter the stage directory and check some properties with s.package.stage: with spack.config.override("config:verify_ssl", secure): s.package.do_stage() with working_dir(s.package.stage.source_path): assert h("HEAD") == h(t.revision) file_path = os.path.join(s.package.stage.source_path, t.file) assert os.path.isdir(s.package.stage.source_path) assert os.path.isfile(file_path) os.unlink(file_path) assert not os.path.isfile(file_path) untracked_file = "foobarbaz" touch(untracked_file) assert os.path.isfile(untracked_file) s.package.do_restage() assert not os.path.isfile(untracked_file) assert os.path.isdir(s.package.stage.source_path) assert os.path.isfile(file_path) assert h("HEAD") == h(t.revision) @pytest.mark.disable_clean_stage_check def test_fetch_pkg_attr_submodule_init( mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch, mock_stage ): """In this case the version() args do not contain a 'git' URL, so the fetcher must be assembled using the Package-level 'git' attribute. This test ensures that the submodules are properly initialized and the expected branch file is present. """ t = mock_git_repository.checks["default-no-per-version-git"] pkg_class = spack.repo.PATH.get_pkg_class("git-test") # For this test, the version args don't specify 'git' (which is # the majority of version specifications) monkeypatch.setattr(pkg_class, "git", mock_git_repository.url) # Construct the package under test s = default_mock_concretization("git-test") monkeypatch.setitem(s.package.versions, Version("git"), t.args) s.package.do_stage() collected_fnames = set() for root, dirs, files in os.walk(s.package.stage.source_path): collected_fnames.update(files) # The submodules generate files with the prefix "r0_file_" assert {"r0_file_0", "r0_file_1", t.file} < collected_fnames @pytest.mark.skipif( str(spack.platforms.host()) == "windows", reason=( "Git fails to clone because the src/dst paths" " are too long: the name of the staging directory" " for ad-hoc Git commit versions is longer than" " other staged sources" ), ) @pytest.mark.disable_clean_stage_check def test_adhoc_version_submodules( mock_git_repository, config, mutable_mock_repo, monkeypatch, mock_stage, override_git_repos_cache_path, ): t = mock_git_repository.checks["tag"] # Construct the package under test pkg_class = spack.repo.PATH.get_pkg_class("git-test") monkeypatch.setitem(pkg_class.versions, Version("git"), t.args) monkeypatch.setattr(pkg_class, "git", mock_git_repository.url, raising=False) spec = spack.concretize.concretize_one( Spec("git-test@{0}".format(mock_git_repository.unversioned_commit)) ) spec.package.do_stage() collected_fnames = set() for root, dirs, files in os.walk(spec.package.stage.source_path): collected_fnames.update(files) # The submodules generate files with the prefix "r0_file_" assert set(["r0_file_0", "r0_file_1"]) < collected_fnames @pytest.mark.parametrize("type_of_test", ["branch", "commit"]) def test_debug_fetch( mock_packages, type_of_test, mock_git_repository, default_mock_concretization, monkeypatch ): """Fetch the repo with debug enabled.""" # Retrieve the right test parameters t = mock_git_repository.checks[type_of_test] # Construct the package under test s = default_mock_concretization("git-test") monkeypatch.setitem(s.package.versions, Version("git"), t.args) # Fetch then ensure source path exists with s.package.stage: with spack.config.override("config:debug", True): s.package.do_fetch() assert os.path.isdir(s.package.stage.source_path) def test_git_extra_fetch(git, tmp_path: pathlib.Path): """Ensure a fetch after 'expanding' is effectively a no-op.""" testpath = str(tmp_path) fetcher = GitFetchStrategy(git="file:///not-a-real-git-repo") with Stage(fetcher, path=testpath) as stage: mkdirp(stage.source_path) fetcher.fetch() # Use fetcher to fetch for code coverage shutil.rmtree(stage.source_path) def test_needs_stage(git): """Trigger a NoStageError when attempt a fetch without a stage.""" with pytest.raises( spack.fetch_strategy.NoStageError, match=r"set_stage.*before calling fetch" ): fetcher = GitFetchStrategy(git="file:///not-a-real-git-repo") fetcher.fetch() @pytest.mark.parametrize("get_full_repo", [True, False]) @pytest.mark.parametrize("use_commit", [True, False]) def test_get_full_repo( get_full_repo, use_commit, git_version, mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch, ): """Ensure that we can clone a full repository.""" if git_version < Version(min_opt_string): pytest.skip("Not testing get_full_repo for older git {0}".format(git_version)) # newer git allows for direct commit fetching can_use_direct_commit = git_version >= Version(min_direct_commit) secure = True type_of_test = "tag-branch" t = mock_git_repository.checks[type_of_test] spec_string = "git-test" s = default_mock_concretization(spec_string) args = copy.copy(t.args) args["get_full_repo"] = get_full_repo monkeypatch.setitem(s.package.versions, Version("git"), args) if use_commit: git_exe = mock_git_repository.git_exe url = mock_git_repository.url commit = git_exe("ls-remote", url, t.revision, output=str).strip().split()[0] s.variants["commit"] = SingleValuedVariant("commit", commit) if can_use_direct_commit: path = mock_git_repository.path git_exe("-C", path, "config", "uploadpack.allowReachableSHA1InWant", "true") with s.package.stage: with spack.config.override("config:verify_ssl", secure): s.package.do_stage() with working_dir(s.package.stage.source_path): branches = mock_git_repository.git_exe("branch", "-a", output=str).splitlines() nbranches = len(branches) commits = mock_git_repository.git_exe( "log", "--graph", "--pretty=format:%h -%d %s (%ci) <%an>", "--abbrev-commit", output=str, ).splitlines() ncommits = len(commits) if get_full_repo: # default branch commit, plus checkout commit assert ncommits == 2, commits assert nbranches >= 5, branches else: assert ncommits == 1, commits if can_use_direct_commit: if use_commit: # only commit (detached state) assert nbranches == 1, branches else: # tag, commit (detached state) assert nbranches == 2, branches else: if use_commit: # default branch, tag, commit (detached state) # git does not have a rewind, avoid messing with git history by # accepting detachment assert nbranches == 3, branches else: # default branch plus tag assert nbranches == 2, branches @pytest.mark.disable_clean_stage_check @pytest.mark.parametrize("submodules", [True, False]) def test_gitsubmodule( submodules, mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch ): """ Test GitFetchStrategy behavior with submodules. This package has a `submodules` property which is always True: when a specific version also indicates to include submodules, this should not interfere; if the specific version explicitly requests that submodules *not* be initialized, this should override the Package-level request. """ type_of_test = "tag-branch" t = mock_git_repository.checks[type_of_test] # Construct the package under test s = default_mock_concretization("git-test") args = copy.copy(t.args) args["submodules"] = submodules monkeypatch.setitem(s.package.versions, Version("git"), args) s.package.do_stage() with working_dir(s.package.stage.source_path): for submodule_count in range(2): file_path = os.path.join( s.package.stage.source_path, "third_party/submodule{0}/r0_file_{0}".format(submodule_count), ) if submodules: assert os.path.isfile(file_path) else: assert not os.path.isfile(file_path) @pytest.mark.disable_clean_stage_check def test_gitsubmodules_callable( mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch ): """ Test GitFetchStrategy behavior with submodules selected after concretization """ def submodules_callback(package): assert isinstance(package, PackageBase) name = "third_party/submodule0" return [name] type_of_test = "tag-branch" t = mock_git_repository.checks[type_of_test] # Construct the package under test s = default_mock_concretization("git-test") args = copy.copy(t.args) args["submodules"] = submodules_callback monkeypatch.setitem(s.package.versions, Version("git"), args) s.package.do_stage() with working_dir(s.package.stage.source_path): file_path = os.path.join(s.package.stage.source_path, "third_party/submodule0/r0_file_0") assert os.path.isfile(file_path) file_path = os.path.join(s.package.stage.source_path, "third_party/submodule1/r0_file_1") assert not os.path.isfile(file_path) @pytest.mark.disable_clean_stage_check def test_gitsubmodules_delete( mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch ): """ Test GitFetchStrategy behavior with submodules_delete """ type_of_test = "tag-branch" t = mock_git_repository.checks[type_of_test] # Construct the package under test s = default_mock_concretization("git-test") args = copy.copy(t.args) args["submodules"] = True args["submodules_delete"] = ["third_party/submodule0", "third_party/submodule1"] monkeypatch.setitem(s.package.versions, Version("git"), args) s.package.do_stage() with working_dir(s.package.stage.source_path): file_path = os.path.join(s.package.stage.source_path, "third_party/submodule0") assert not os.path.isdir(file_path) file_path = os.path.join(s.package.stage.source_path, "third_party/submodule1") assert not os.path.isdir(file_path) @pytest.mark.disable_clean_stage_check def test_gitsubmodules_falsey( mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch ): """ Test GitFetchStrategy behavior when callable submodules returns Falsey """ def submodules_callback(package): assert isinstance(package, PackageBase) return False type_of_test = "tag-branch" t = mock_git_repository.checks[type_of_test] # Construct the package under test s = default_mock_concretization("git-test") args = copy.copy(t.args) args["submodules"] = submodules_callback monkeypatch.setitem(s.package.versions, Version("git"), args) s.package.do_stage() with working_dir(s.package.stage.source_path): file_path = os.path.join(s.package.stage.source_path, "third_party/submodule0/r0_file_0") assert not os.path.isfile(file_path) file_path = os.path.join(s.package.stage.source_path, "third_party/submodule1/r0_file_1") assert not os.path.isfile(file_path) @pytest.mark.disable_clean_stage_check def test_git_sparse_paths_partial_clone( mock_git_repository, git_version, default_mock_concretization, mutable_mock_repo, monkeypatch ): """ Test partial clone of repository when using git_sparse_paths property """ type_of_test = "many-directories" sparse_paths = ["dir0"] omitted_paths = ["dir1", "dir2"] t = mock_git_repository.checks[type_of_test] args = copy.copy(t.args) args["git_sparse_paths"] = sparse_paths s = default_mock_concretization("git-test") monkeypatch.setitem(s.package.versions, Version("git"), args) s.package.do_stage() with working_dir(s.package.stage.source_path): # top level directory files are cloned via sparse-checkout assert os.path.isfile("r0_file") for p in sparse_paths: assert os.path.isdir(p) if git_version < Version("2.26.0.0"): # older versions of git should fall back to a full clone for p in omitted_paths: assert os.path.isdir(p) else: for p in omitted_paths: assert not os.path.isdir(p) # fixture file is in the sparse-path expansion tree assert os.path.isfile(t.file) @pytest.mark.regression("50699") def test_git_sparse_path_have_unique_mirror_projections( git, mock_git_repository, mutable_mock_repo, monkeypatch, mutable_config ): """ Confirm two packages with different sparse paths but the same git commit have different mirror projections so tarfiles in the mirror are unique and don't get overwritten """ repo_path = mock_git_repository.path monkeypatch.setattr( spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) gold_commit = git("-C", repo_path, "rev-parse", "many_dirs", output=str).strip() s_a = spack.concretize.concretize_one(f"git-sparse-a commit={gold_commit}") s_b = spack.concretize.concretize_one(f"git-sparse-b commit={gold_commit}") assert s_a.package.stage[0].mirror_layout.path != s_b.package.stage[0].mirror_layout.path @pytest.mark.disable_clean_stage_check def test_commit_variant_clone( git, default_mock_concretization, mutable_mock_repo, mock_git_version_info, monkeypatch ): repo_path, filename, commits = mock_git_version_info test_commit = commits[-2] s = default_mock_concretization("git-test") args = {"git": pathlib.Path(repo_path).as_uri()} monkeypatch.setitem(s.package.versions, Version("git"), args) s.variants["commit"] = SingleValuedVariant("commit", test_commit) s.package.do_stage() with working_dir(s.package.stage.source_path): assert git("rev-parse", "HEAD", output=str, error=str).strip() == test_commit ================================================ FILE: lib/spack/spack/test/graph.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import spack.concretize import spack.graph def test_dynamic_dot_graph_mpileaks(default_mock_concretization): """Test dynamically graphing the mpileaks package.""" s = default_mock_concretization("mpileaks") stream = io.StringIO() spack.graph.graph_dot([s], out=stream) dot = stream.getvalue() nodes_to_check = ["mpileaks", "mpi", "callpath", "dyninst", "libdwarf", "libelf"] hashes, builder = {}, spack.graph.SimpleDAG() for name in nodes_to_check: current = s[name] current_hash = current.dag_hash() hashes[name] = current_hash node_options = builder.node_entry(current)[1] assert node_options in dot dependencies_to_check = [ ("dyninst", "libdwarf"), ("callpath", "dyninst"), ("mpileaks", "mpi"), ("libdwarf", "libelf"), ("callpath", "mpi"), ("mpileaks", "callpath"), ("dyninst", "libelf"), ] for parent, child in dependencies_to_check: assert ' "{0}" -> "{1}"\n'.format(hashes[parent], hashes[child]) in dot def test_ascii_graph_mpileaks(config, mock_packages, monkeypatch): monkeypatch.setattr(spack.graph.AsciiGraph, "_node_label", lambda self, node: node.name) s = spack.concretize.concretize_one("mpileaks") stream = io.StringIO() graph = spack.graph.AsciiGraph() graph.write(s, out=stream, color=False) graph_str = stream.getvalue() graph_str = "\n".join([line.rstrip() for line in graph_str.split("\n")]) assert ( graph_str == r"""o mpileaks |\ | |\ | | |\ | | | |\ | | | | o callpath | |_|_|/| |/| |_|/| | |/| |/| | | |/|/| | | | | o dyninst | | |_|/| | |/| |/| | | |/|/| | | | | |\ o | | | | | mpich |\| | | | | |\ \ \ \ \ \ | |_|/ / / / |/| | | | | | |/ / / / | | | | o libdwarf | |_|_|/| |/| |_|/| | |/| |/| | | |/|/ | | | o libelf | |_|/| |/| |/| | |/|/ | o | compiler-wrapper | / | o gcc-runtime |/ o gcc """ or graph_str == r"""o mpileaks |\ | |\ | | |\ | | | o callpath | |_|/| |/| |/| | |/|/| | | | o dyninst | | |/| | |/|/| | | | |\ o | | | | mpich |\| | | | | |/ / / |/| | | | | | o libdwarf | |_|/| |/| |/| | |/|/ | | o libelf | |/| |/|/ | o gcc-runtime |/ o gcc """ ) ================================================ FILE: lib/spack/spack/test/hg_fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.concretize import spack.config from spack.fetch_strategy import HgFetchStrategy from spack.llnl.util.filesystem import mkdirp, touch, working_dir from spack.stage import Stage from spack.util.executable import which from spack.version import Version # Test functionality covered is supported on Windows, but currently failing # and expected to be fixed pytestmark = [ pytest.mark.skipif(not which("hg"), reason="requires mercurial to be installed"), pytest.mark.not_on_windows("Failing on Win"), ] @pytest.mark.parametrize("type_of_test", ["default", "rev0"]) @pytest.mark.parametrize("secure", [True, False]) def test_fetch(type_of_test, secure, mock_hg_repository, config, mutable_mock_repo, monkeypatch): """Tries to: 1. Fetch the repo using a fetch strategy constructed with supplied args (they depend on type_of_test). 2. Check if the test_file is in the checked out repository. 3. Assert that the repository is at the revision supplied. 4. Add and remove some files, then reset the repo, and ensure it's all there again. """ # Retrieve the right test parameters t = mock_hg_repository.checks[type_of_test] h = mock_hg_repository.hash # Construct the package under test s = spack.concretize.concretize_one("hg-test") monkeypatch.setitem(s.package.versions, Version("hg"), t.args) # Enter the stage directory and check some properties with s.package.stage: with spack.config.override("config:verify_ssl", secure): s.package.do_stage() with working_dir(s.package.stage.source_path): assert h() == t.revision file_path = os.path.join(s.package.stage.source_path, t.file) assert os.path.isdir(s.package.stage.source_path) assert os.path.isfile(file_path) os.unlink(file_path) assert not os.path.isfile(file_path) untracked_file = "foobarbaz" touch(untracked_file) assert os.path.isfile(untracked_file) s.package.do_restage() assert not os.path.isfile(untracked_file) assert os.path.isdir(s.package.stage.source_path) assert os.path.isfile(file_path) assert h() == t.revision def test_hg_extra_fetch(tmp_path: pathlib.Path): """Ensure a fetch after expanding is effectively a no-op.""" testpath = str(tmp_path) fetcher = HgFetchStrategy(hg="file:///not-a-real-hg-repo") with Stage(fetcher, path=testpath) as stage: source_path = stage.source_path mkdirp(source_path) fetcher.fetch() ================================================ FILE: lib/spack/spack/test/hooks/absolutify_elf_sonames.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.llnl.util.filesystem as fs import spack.platforms from spack.hooks.absolutify_elf_sonames import SharedLibrariesVisitor, find_and_patch_sonames from spack.util.executable import Executable def skip_unless_linux(f): return pytest.mark.skipif( str(spack.platforms.real_host()) != "linux", reason="only tested on linux for now" )(f) class ExecutableIntercept: def __init__(self): self.calls = [] def __call__(self, *args, **kwargs): self.calls.append(args) @property def returncode(self): return 0 @pytest.mark.requires_executables("gcc") @skip_unless_linux def test_shared_libraries_visitor(tmp_path: pathlib.Path): """Integration test for soname rewriting""" gcc = Executable("gcc") # Create a directory structure like this: # ./no-soname.so # just a shared library without a soname # ./soname.so # a shared library with a soname # ./executable.so # an executable masquerading as a shared lib # ./libskipme.so # a shared library with a soname # ./mydir/parent_dir -> .. # a symlinked dir, causing a cycle # ./mydir/skip_symlink -> ../libskipme # a symlink to a library with fs.working_dir(str(tmp_path)): with open("hello.c", "w", encoding="utf-8") as f: f.write("int main(){return 0;}") gcc("hello.c", "-o", "no-soname.so", "--shared") gcc("hello.c", "-o", "soname.so", "--shared", "-Wl,-soname,example.so") gcc("hello.c", "-pie", "-o", "executable.so") gcc("hello.c", "-o", "libskipme.so", "-Wl,-soname,libskipme.so") os.mkdir("my_dir") os.symlink("..", os.path.join("my_dir", "parent_dir")) os.symlink(os.path.join("..", "libskipme.so"), os.path.join("my_dir", "skip_symlink")) # Visit the whole prefix, but exclude `skip_symlink` visitor = SharedLibrariesVisitor(exclude_list=["skip_symlink"]) fs.visit_directory_tree(str(tmp_path), visitor) relative_paths = visitor.get_shared_libraries_relative_paths() assert "no-soname.so" in relative_paths assert "soname.so" in relative_paths assert "executable.so" not in relative_paths assert "libskipme.so" not in relative_paths # Run the full hook of finding libs and setting sonames. patchelf = ExecutableIntercept() find_and_patch_sonames(str(tmp_path), ["skip_symlink"], patchelf) assert len(patchelf.calls) == 2 elf_1 = tmp_path / "no-soname.so" elf_2 = tmp_path / "soname.so" assert ("--set-soname", str(elf_1), str(elf_1)) in patchelf.calls assert ("--set-soname", str(elf_2), str(elf_2)) in patchelf.calls ================================================ FILE: lib/spack/spack/test/install.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import shutil import sys import pytest import spack.build_environment import spack.builder import spack.concretize import spack.config import spack.database import spack.error import spack.installer import spack.llnl.util.filesystem as fs import spack.mirrors.mirror import spack.mirrors.utils import spack.package_base import spack.patch import spack.repo import spack.store import spack.util.spack_json as sjson from spack import binary_distribution from spack.error import InstallError from spack.installer import PackageInstaller from spack.package_base import ( PackageBase, PackageStillNeededError, _spack_build_envfile, _spack_build_logfile, _spack_configure_argsfile, spack_times_log, ) from spack.spec import Spec def find_nothing(*args): raise spack.repo.UnknownPackageError("Repo package access is disabled for test") def test_install_and_uninstall(install_mockery, mock_fetch, monkeypatch): spec = spack.concretize.concretize_one("trivial-install-test-package") PackageInstaller([spec.package], explicit=True).install() assert spec.installed spec.package.do_uninstall() assert not spec.installed @pytest.mark.regression("11870") def test_uninstall_non_existing_package(install_mockery, mock_fetch, monkeypatch): """Ensure that we can uninstall a package that has been deleted from the repo""" spec = spack.concretize.concretize_one("trivial-install-test-package") PackageInstaller([spec.package], explicit=True).install() assert spec.installed # Mock deletion of the package spec._package = None monkeypatch.setattr(spack.repo.PATH, "get", find_nothing) with pytest.raises(spack.repo.UnknownPackageError): spec.package # Ensure we can uninstall it PackageBase.uninstall_by_spec(spec) assert not spec.installed def test_pkg_attributes(install_mockery, mock_fetch, monkeypatch): # Get a basic concrete spec for the dummy package. spec = spack.concretize.concretize_one("attributes-foo-app ^attributes-foo") assert spec.concrete pkg = spec.package PackageInstaller([pkg], explicit=True).install() foo = "attributes-foo" assert spec["bar"].prefix == spec[foo].prefix assert spec["baz"].prefix == spec[foo].prefix assert spec[foo].home == spec[foo].prefix assert spec["bar"].home == spec[foo].home assert spec["baz"].home == spec[foo].prefix.baz foo_headers = spec[foo].headers # assert foo_headers.basenames == ['foo.h'] assert foo_headers.directories == [spec[foo].home.include] bar_headers = spec["bar"].headers # assert bar_headers.basenames == ['bar.h'] assert bar_headers.directories == [spec["bar"].home.include] baz_headers = spec["baz"].headers # assert baz_headers.basenames == ['baz.h'] assert baz_headers.directories == [spec["baz"].home.include] lib_suffix = ".so" if sys.platform == "win32": lib_suffix = ".dll" elif sys.platform == "darwin": lib_suffix = ".dylib" foo_libs = spec[foo].libs assert foo_libs.basenames == ["libFoo" + lib_suffix] assert foo_libs.directories == [spec[foo].home.lib64] bar_libs = spec["bar"].libs assert bar_libs.basenames == ["libFooBar" + lib_suffix] assert bar_libs.directories == [spec["bar"].home.lib64] baz_libs = spec["baz"].libs assert baz_libs.basenames == ["libFooBaz" + lib_suffix] assert baz_libs.directories == [spec["baz"].home.lib] def mock_remove_prefix(*args): raise MockInstallError("Intentional error", "Mock remove_prefix method intentionally fails") class RemovePrefixChecker: def __init__(self, wrapped_rm_prefix): self.removed = False self.wrapped_rm_prefix = wrapped_rm_prefix def remove_prefix(self): self.removed = True self.wrapped_rm_prefix() def test_partial_install_delete_prefix_and_stage(install_mockery, mock_fetch, working_env): s = spack.concretize.concretize_one("canfail") s.package.succeed = False instance_rm_prefix = s.package.remove_prefix s.package.remove_prefix = mock_remove_prefix with pytest.raises(MockInstallError): PackageInstaller([s.package], explicit=True).install() assert os.path.isdir(s.package.prefix) rm_prefix_checker = RemovePrefixChecker(instance_rm_prefix) s.package.remove_prefix = rm_prefix_checker.remove_prefix # must clear failure markings for the package before re-installing it spack.store.STORE.failure_tracker.clear(s, True) s.package.succeed = True spack.builder._BUILDERS.clear() # the builder is cached with a copy of the pkg's __dict__. PackageInstaller([s.package], explicit=True, restage=True).install() assert rm_prefix_checker.removed assert s.package.spec.installed @pytest.mark.not_on_windows("Fails spuriously on Windows") @pytest.mark.disable_clean_stage_check def test_failing_overwrite_install_should_keep_previous_installation( mock_fetch, install_mockery, working_env ): """ Make sure that whenever `spack install --overwrite` fails, spack restores the original install prefix instead of cleaning it. """ # Do a successful install s = spack.concretize.concretize_one("canfail") s.package.succeed = True # Do a failing overwrite install PackageInstaller([s.package], explicit=True).install() s.package.succeed = False spack.builder._BUILDERS.clear() # the builder is cached with a copy of the pkg's __dict__. kwargs = {"overwrite": [s.dag_hash()]} with pytest.raises(Exception): PackageInstaller([s.package], explicit=True, **kwargs).install() assert s.package.spec.installed assert os.path.exists(s.prefix) def test_dont_add_patches_to_installed_package(install_mockery, mock_fetch, monkeypatch): dependency = spack.concretize.concretize_one("dependency-install") PackageInstaller([dependency.package], explicit=True).install() dependency_hash = dependency.dag_hash() dependent = spack.concretize.concretize_one("dependent-install ^/" + dependency_hash) monkeypatch.setitem( dependency.package.patches, "dependency-install", [spack.patch.UrlPatch(dependent.package, "file://fake.patch", sha256="unused-hash")], ) assert dependent["dependency-install"] == dependency def test_installed_dependency_request_conflicts(install_mockery, mock_fetch, mutable_mock_repo): dependency = spack.concretize.concretize_one("dependency-install") PackageInstaller([dependency.package], explicit=True).install() dependency_hash = dependency.dag_hash() dependent = Spec("conflicting-dependent ^/" + dependency_hash) with pytest.raises(spack.error.UnsatisfiableSpecError): spack.concretize.concretize_one(dependent) def test_install_times(install_mockery, mock_fetch, mutable_mock_repo): """Test install times added.""" spec = spack.concretize.concretize_one("dev-build-test-install-phases") PackageInstaller([spec.package], explicit=True).install() # Ensure dependency directory exists after the installation. install_times = os.path.join(spec.package.prefix, ".spack", spack_times_log) assert os.path.isfile(install_times) # Ensure the phases are included with open(install_times, "r", encoding="utf-8") as timefile: times = sjson.load(timefile.read()) # The order should be maintained phases = [x["name"] for x in times["phases"]] assert phases == ["stage", "one", "two", "three", "install", "post-install"] assert all(isinstance(x["seconds"], float) for x in times["phases"]) @pytest.fixture() def install_upstream(tmp_path_factory: pytest.TempPathFactory, gen_mock_layout, install_mockery): """Provides a function that installs a specified set of specs to an upstream database. The function returns a store which points to the upstream, as well as the upstream layout (for verifying that dependent installs are using the upstream installs). """ mock_db_root = str(tmp_path_factory.mktemp("mock_db_root")) upstream_layout = gen_mock_layout("a") prepared_db = spack.database.Database(mock_db_root, layout=upstream_layout) spack.config.CONFIG.push_scope( spack.config.InternalConfigScope( name="install-upstream-fixture", data={"upstreams": {"mock1": {"install_tree": prepared_db.root}}}, ) ) def _install_upstream(*specs): for spec_str in specs: prepared_db.add(spack.concretize.concretize_one(spec_str)) downstream_root = str(tmp_path_factory.mktemp("mock_downstream_db_root")) return downstream_root, upstream_layout return _install_upstream def test_installed_upstream_external(install_upstream, mock_fetch): """Check that when a dependency package is recorded as installed in an upstream database that it is not reinstalled. """ store_root, _ = install_upstream("externaltool") with spack.store.use_store(store_root): dependent = spack.concretize.concretize_one("externaltest") new_dependency = dependent["externaltool"] assert new_dependency.external assert new_dependency.prefix == os.path.sep + os.path.join("path", "to", "external_tool") PackageInstaller([dependent.package], explicit=True).install() assert not os.path.exists(new_dependency.prefix) assert os.path.exists(dependent.prefix) def test_installed_upstream(install_upstream, mock_fetch): """Check that when a dependency package is recorded as installed in an upstream database that it is not reinstalled. """ store_root, upstream_layout = install_upstream("dependency-install") with spack.store.use_store(store_root): dependency = spack.concretize.concretize_one("dependency-install") dependent = spack.concretize.concretize_one("dependent-install") new_dependency = dependent["dependency-install"] assert new_dependency.installed_upstream assert new_dependency.prefix == upstream_layout.path_for_spec(dependency) PackageInstaller([dependent.package], explicit=True).install() assert not os.path.exists(new_dependency.prefix) assert os.path.exists(dependent.prefix) @pytest.mark.disable_clean_stage_check def test_partial_install_keep_prefix(install_mockery, mock_fetch, monkeypatch, working_env): s = spack.concretize.concretize_one("canfail") s.package.succeed = False # If remove_prefix is called at any point in this test, that is an error monkeypatch.setattr(spack.package_base.PackageBase, "remove_prefix", mock_remove_prefix) with pytest.raises(spack.build_environment.ChildError): PackageInstaller([s.package], explicit=True, keep_prefix=True).install() assert os.path.exists(s.package.prefix) # must clear failure markings for the package before re-installing it spack.store.STORE.failure_tracker.clear(s, True) s.package.succeed = True spack.builder._BUILDERS.clear() # the builder is cached with a copy of the pkg's __dict__. PackageInstaller([s.package], explicit=True, keep_prefix=True).install() assert s.package.spec.installed def test_second_install_no_overwrite_first(install_mockery, mock_fetch, monkeypatch): s = spack.concretize.concretize_one("canfail") monkeypatch.setattr(spack.package_base.PackageBase, "remove_prefix", mock_remove_prefix) s.package.succeed = True PackageInstaller([s.package], explicit=True).install() assert s.package.spec.installed # If Package.install is called after this point, it will fail s.package.succeed = False PackageInstaller([s.package], explicit=True).install() def test_install_prefix_collision_fails(config, mock_fetch, mock_packages, tmp_path: pathlib.Path): """ Test that different specs with coinciding install prefixes will fail to install. """ projections = {"projections": {"all": "one-prefix-per-package-{name}"}} with spack.store.use_store(str(tmp_path), extra_data=projections): with spack.config.override("config:checksum", False): pkg_a = spack.concretize.concretize_one("libelf@0.8.13").package pkg_b = spack.concretize.concretize_one("libelf@0.8.12").package PackageInstaller([pkg_a], explicit=True, fake=True).install() with pytest.raises(InstallError, match="Install prefix collision"): PackageInstaller([pkg_b], explicit=True, fake=True).install() def test_store(install_mockery, mock_fetch): spec = spack.concretize.concretize_one("cmake-client") pkg = spec.package PackageInstaller([pkg], fake=True, explicit=True).install() @pytest.mark.disable_clean_stage_check def test_failing_build(install_mockery, mock_fetch): spec = spack.concretize.concretize_one("failing-build") pkg = spec.package with pytest.raises(spack.build_environment.ChildError, match="Expected failure"): PackageInstaller([pkg], explicit=True).install() class MockInstallError(spack.error.SpackError): pass def test_uninstall_by_spec_errors(mutable_database): """Test exceptional cases with the uninstall command.""" # Try to uninstall a spec that has not been installed spec = spack.concretize.concretize_one("dependent-install") with pytest.raises(InstallError, match="is not installed"): PackageBase.uninstall_by_spec(spec) # Try an unforced uninstall of a spec with dependencies rec = mutable_database.get_record("mpich") with pytest.raises(PackageStillNeededError, match="Cannot uninstall"): PackageBase.uninstall_by_spec(rec.spec) @pytest.mark.disable_clean_stage_check @pytest.mark.use_package_hash def test_nosource_pkg_install(install_mockery, mock_fetch, mock_packages, capfd, ensure_debug): """Test install phases with the nosource package.""" spec = spack.concretize.concretize_one("nosource") pkg = spec.package # Make sure install works even though there is no associated code. PackageInstaller([pkg], explicit=True).install() out = capfd.readouterr() assert "Installing dependency-install" in out[0] # Make sure a warning for missing code is issued assert "Missing a source id for nosource" in out[1] @pytest.mark.disable_clean_stage_check def test_nosource_bundle_pkg_install( install_mockery, mock_fetch, mock_packages, capfd, ensure_debug ): """Test install phases with the nosource-bundle package.""" spec = spack.concretize.concretize_one("nosource-bundle") pkg = spec.package # Make sure install works even though there is no associated code. PackageInstaller([pkg], explicit=True).install() out = capfd.readouterr() assert "Installing dependency-install" in out[0] # Make sure a warning for missing code is *not* issued assert "Missing a source id for nosource" not in out[1] def test_nosource_pkg_install_post_install(install_mockery, mock_fetch, mock_packages): """Test install phases with the nosource package with post-install.""" spec = spack.concretize.concretize_one("nosource-install") pkg = spec.package # Make sure both the install and post-install package methods work. PackageInstaller([pkg], explicit=True).install() # Ensure the file created in the package's `install` method exists. install_txt = os.path.join(spec.prefix, "install.txt") assert os.path.isfile(install_txt) # Ensure the file created in the package's `post-install` method exists. post_install_txt = os.path.join(spec.prefix, "post-install.txt") assert os.path.isfile(post_install_txt) def test_pkg_build_paths(install_mockery): # Get a basic concrete spec for the trivial install package. spec = spack.concretize.concretize_one("trivial-install-test-package") assert spec.package.log_path.endswith(_spack_build_logfile) assert spec.package.env_path.endswith(_spack_build_envfile) def test_pkg_install_paths(install_mockery): # Get a basic concrete spec for the trivial install package. spec = spack.concretize.concretize_one("trivial-install-test-package") log_path = os.path.join(spec.prefix, ".spack", _spack_build_logfile + ".gz") assert spec.package.install_log_path == log_path env_path = os.path.join(spec.prefix, ".spack", _spack_build_envfile) assert spec.package.install_env_path == env_path args_path = os.path.join(spec.prefix, ".spack", _spack_configure_argsfile) assert spec.package.install_configure_args_path == args_path # Backward compatibility checks log_dir = os.path.dirname(log_path) fs.mkdirp(log_dir) with fs.working_dir(log_dir): # Start with the older of the previous install log filenames older_log = "build.out" fs.touch(older_log) assert spec.package.install_log_path.endswith(older_log) # Now check the newer install log filename last_log = "build.txt" fs.rename(older_log, last_log) assert spec.package.install_log_path.endswith(last_log) # Check the old install environment file last_env = "build.env" fs.rename(last_log, last_env) assert spec.package.install_env_path.endswith(last_env) # Cleanup shutil.rmtree(log_dir) def test_log_install_without_build_files(install_mockery): """Test the installer log function when no build files are present.""" # Get a basic concrete spec for the trivial install package. spec = spack.concretize.concretize_one("trivial-install-test-package") # Attempt installing log without the build log file with pytest.raises(OSError, match="No such file or directory"): spack.installer.log(spec.package) def test_log_install_with_build_files(install_mockery, monkeypatch): """Test the installer's log function when have build files.""" config_log = "config.log" # Retain the original function for use in the monkey patch that is used # to raise an exception under the desired condition for test coverage. orig_install_fn = fs.install def _install(src, dest): orig_install_fn(src, dest) if src.endswith(config_log): raise Exception("Mock log install error") monkeypatch.setattr(fs, "install", _install) spec = spack.concretize.concretize_one("trivial-install-test-package") # Set up mock build files and try again to include archive failure log_path = spec.package.log_path log_dir = os.path.dirname(log_path) fs.mkdirp(log_dir) with fs.working_dir(log_dir): fs.touch(log_path) fs.touch(spec.package.env_path) fs.touch(spec.package.env_mods_path) fs.touch(spec.package.configure_args_path) install_path = os.path.dirname(spec.package.install_log_path) fs.mkdirp(install_path) source = spec.package.stage.source_path config = os.path.join(source, "config.log") fs.touchp(config) monkeypatch.setattr( type(spec.package), "archive_files", ["missing", "..", config], raising=False ) spack.installer.log(spec.package) assert os.path.exists(spec.package.install_log_path) assert os.path.exists(spec.package.install_env_path) assert os.path.exists(spec.package.install_configure_args_path) archive_dir = os.path.join(install_path, "archived-files") source_dir = os.path.dirname(source) rel_config = os.path.relpath(config, source_dir) assert os.path.exists(os.path.join(archive_dir, rel_config)) assert not os.path.exists(os.path.join(archive_dir, "missing")) expected_errs = ["OUTSIDE SOURCE PATH", "FAILED TO ARCHIVE"] # for '..' # for rel_config with open(os.path.join(archive_dir, "errors.txt"), "r", encoding="utf-8") as fd: for ln, expected in zip(fd, expected_errs): assert expected in ln # Cleanup shutil.rmtree(log_dir) def test_unconcretized_install(install_mockery, mock_fetch, mock_packages): """Test attempts to perform install phases with unconcretized spec.""" spec = Spec("trivial-install-test-package") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) with pytest.raises(ValueError, match="must have a concrete spec"): PackageInstaller([pkg_cls(spec)], explicit=True).install() with pytest.raises(ValueError, match="only patch concrete packages"): pkg_cls(spec).do_patch() def test_install_error(): try: msg = "test install error" long_msg = "this is the long version of test install error" raise InstallError(msg, long_msg=long_msg) except Exception as exc: assert exc.__class__.__name__ == "InstallError" assert exc.message == msg assert exc.long_message == long_msg @pytest.mark.disable_clean_stage_check def test_empty_install_sanity_check_prefix( monkeypatch, install_mockery, mock_fetch, mock_packages ): """Test empty install triggers sanity_check_prefix.""" spec = spack.concretize.concretize_one("failing-empty-install") with pytest.raises(spack.build_environment.ChildError, match="Nothing was installed"): PackageInstaller([spec.package], explicit=True).install() def test_install_from_binary_with_missing_patch_succeeds( temporary_store: spack.store.Store, mutable_config, tmp_path: pathlib.Path, mock_packages ): """If a patch is missing in the local package repository, but was present when building and pushing the package to a binary cache, installation from that binary cache shouldn't error out because of the missing patch.""" # Create a spec s with non-existing patches s = spack.concretize.concretize_one("trivial-install-test-package") patches = ["a" * 64] s_dict = s.to_dict() s_dict["spec"]["nodes"][0]["patches"] = patches s_dict["spec"]["nodes"][0]["parameters"]["patches"] = patches s = Spec.from_dict(s_dict) # Create an install dir for it os.makedirs(os.path.join(s.prefix, ".spack")) with open(os.path.join(s.prefix, ".spack", "spec.json"), "w", encoding="utf-8") as f: s.to_json(f) # And register it in the database temporary_store.db.add(s, explicit=True) # Push it to a binary cache mirror = spack.mirrors.mirror.Mirror.from_local_path(str(tmp_path / "my_build_cache")) with binary_distribution.make_uploader(mirror=mirror) as uploader: uploader.push_or_raise([s]) # Now re-install it. s.package.do_uninstall() assert not temporary_store.db.query_local_by_spec_hash(s.dag_hash()) # Source install: fails, we don't have the patch. with pytest.raises(spack.error.SpecError, match="Couldn't find patch for package"): PackageInstaller([s.package], explicit=True).install() # Binary install: succeeds, we don't need the patch. spack.mirrors.utils.add(mirror) PackageInstaller( [s.package], explicit=True, root_policy="cache_only", dependencies_policy="cache_only", unsigned=True, ).install() assert temporary_store.db.query_local_by_spec_hash(s.dag_hash()) ================================================ FILE: lib/spack/spack/test/installer.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import glob import os import pathlib import shutil import sys from typing import List, Optional, Union import py import pytest import spack.binary_distribution import spack.concretize import spack.database import spack.deptypes as dt import spack.error import spack.hooks import spack.installer as inst import spack.installer_dispatch import spack.llnl.util.filesystem as fs import spack.llnl.util.lock as ulk import spack.llnl.util.tty as tty import spack.package_base import spack.package_prefs as prefs import spack.repo import spack.report import spack.spec import spack.store import spack.util.lock as lk from spack.main import SpackCommand from spack.test.conftest import RepoBuilder def _mock_repo(root, namespace): """Create an empty repository at the specified root Args: root (str): path to the mock repository root namespace (str): mock repo's namespace """ repodir = py.path.local(root) if isinstance(root, str) else root repodir.ensure(spack.repo.packages_dir_name, dir=True) yaml = repodir.join("repo.yaml") yaml.write( f""" repo: namespace: {namespace} """ ) def _noop(*args, **kwargs): """Generic monkeypatch no-op routine.""" def _none(*args, **kwargs): """Generic monkeypatch function that always returns None.""" return None def _not_locked(installer, lock_type, pkg): """Generic monkeypatch function for _ensure_locked to return no lock""" tty.msg("{0} locked {1}".format(lock_type, pkg.spec.name)) return lock_type, None def _true(*args, **kwargs): """Generic monkeypatch function that always returns True.""" return True def create_build_task( pkg: spack.package_base.PackageBase, install_args: Optional[dict] = None ) -> inst.BuildTask: request = inst.BuildRequest(pkg, {} if install_args is None else install_args) return inst.BuildTask(pkg, request=request, status=inst.BuildStatus.QUEUED) def create_installer( specs: Union[List[str], List[spack.spec.Spec]], install_args: Optional[dict] = None ) -> inst.PackageInstaller: """Create an installer instance for a list of specs or package names that will be concretized.""" _specs = [spack.concretize.concretize_one(s) if isinstance(s, str) else s for s in specs] _install_args = {} if install_args is None else install_args return inst.PackageInstaller([spec.package for spec in _specs], **_install_args) @pytest.mark.parametrize( "sec,result", [(86400, "24h"), (3600, "1h"), (60, "1m"), (1.802, "1.80s"), (3723.456, "1h 2m 3.46s")], ) def test_hms(sec, result): assert inst._hms(sec) == result def test_get_dependent_ids(install_mockery, mock_packages): # Concretize the parent package, which handle dependency too spec = spack.concretize.concretize_one("pkg-a") assert spec.concrete pkg_id = inst.package_id(spec) # Grab the sole dependency of 'a', which is 'b' dep = spec.dependencies()[0] # Ensure the parent package is a dependent of the dependency package assert pkg_id in inst.get_dependent_ids(dep) def test_install_msg(monkeypatch): """Test results of call to install_msg based on debug level.""" name = "some-package" pid = 123456 install_msg = "Installing {0}".format(name) monkeypatch.setattr(tty, "_debug", 0) assert inst.install_msg(name, pid, None) == install_msg install_status = inst.InstallStatus(1) expected = "{0} [0/1]".format(install_msg) assert inst.install_msg(name, pid, install_status) == expected monkeypatch.setattr(tty, "_debug", 1) assert inst.install_msg(name, pid, None) == install_msg # Expect the PID to be added at debug level 2 monkeypatch.setattr(tty, "_debug", 2) expected = "{0}: {1}".format(pid, install_msg) assert inst.install_msg(name, pid, None) == expected def test_install_from_cache_errors(install_mockery): """Test to ensure cover install from cache errors.""" spec = spack.concretize.concretize_one("trivial-install-test-package") assert spec.concrete # Check with cache-only with pytest.raises( spack.error.InstallError, match="No binary found when cache-only was specified" ): inst.PackageInstaller( [spec.package], root_policy="cache_only", dependencies_policy="cache_only" ).install() assert not spec.package.installed_from_binary_cache # Check when don't expect to install only from binary cache assert not inst._install_from_cache(spec.package, explicit=True, unsigned=False) assert not spec.package.installed_from_binary_cache def test_install_from_cache_ok(install_mockery, monkeypatch): """Test to ensure cover _install_from_cache to the return.""" spec = spack.concretize.concretize_one("trivial-install-test-package") monkeypatch.setattr(inst, "_try_install_from_binary_cache", _true) monkeypatch.setattr(spack.hooks, "post_install", _noop) assert inst._install_from_cache(spec.package, explicit=True, unsigned=False) def test_process_external_package_module(install_mockery, monkeypatch, capfd): """Test to simply cover the external module message path.""" spec = spack.concretize.concretize_one("trivial-install-test-package") assert spec.concrete # Ensure take the external module path WITHOUT any changes to the database monkeypatch.setattr(spack.database.Database, "get_record", _none) spec.external_path = "/actual/external/path/not/checked" spec.external_modules = ["unchecked_module"] inst._process_external_package(spec.package, False) out = capfd.readouterr()[0] assert "has external module in {0}".format(spec.external_modules) in out def test_process_binary_cache_tarball_tar(install_mockery, monkeypatch, capfd): """Tests of _process_binary_cache_tarball with a tar file.""" def _spec(spec, unsigned=False, mirrors_for_spec=None): return spec # Skip binary distribution functionality since assume tested elsewhere monkeypatch.setattr(spack.binary_distribution, "download_tarball", _spec) monkeypatch.setattr(spack.binary_distribution, "extract_tarball", _noop) # Skip database updates monkeypatch.setattr(spack.database.Database, "add", _noop) spec = spack.concretize.concretize_one("pkg-a") assert inst._process_binary_cache_tarball(spec.package, explicit=False, unsigned=False) out = capfd.readouterr()[0] assert "Extracting pkg-a" in out assert "from binary cache" in out def test_try_install_from_binary_cache(install_mockery, mock_packages, monkeypatch): """Test return false when no match exists in the mirror""" spec = spack.concretize.concretize_one("mpich") result = inst._try_install_from_binary_cache(spec.package, False, False) assert not result def test_installer_repr(install_mockery): installer = create_installer(["trivial-install-test-package"]) irep = installer.__repr__() assert irep.startswith(installer.__class__.__name__) assert "installed=" in irep assert "failed=" in irep def test_installer_str(install_mockery): installer = create_installer(["trivial-install-test-package"]) istr = str(installer) assert "#tasks=0" in istr assert "installed (0)" in istr assert "failed (0)" in istr def test_installer_prune_built_build_deps(install_mockery, monkeypatch, repo_builder: RepoBuilder): r""" Ensure that build dependencies of installed deps are pruned from installer package queues. (a) / \ / \ (b) (c) <--- is installed already so we should \ / | \ prune (f) from this install since \ / | \ it is *only* needed to build (b) (d) (e) (f) Thus since (c) is already installed our build_pq dag should only include four packages. [(a), (b), (c), (d), (e)] """ def _mock_installed(self): return self.name == "pkg-c" # Mock the installed property to say that (c) is installed monkeypatch.setattr(spack.spec.Spec, "installed", property(_mock_installed)) # Create mock repository with packages (a), (b), (c), (d), and (e) repo_builder.add_package( "pkg-a", dependencies=[("pkg-b", "build", None), ("pkg-c", "build", None)] ) repo_builder.add_package("pkg-b", dependencies=[("pkg-d", "build", None)]) repo_builder.add_package( "pkg-c", dependencies=[("pkg-d", "build", None), ("pkg-e", "all", None), ("pkg-f", "build", None)], ) repo_builder.add_package("pkg-d") repo_builder.add_package("pkg-e") repo_builder.add_package("pkg-f") with spack.repo.use_repositories(repo_builder.root): installer = create_installer(["pkg-a"]) installer._init_queue() # Assert that (c) is not in the build_pq result = {task.pkg_id[:5] for _, task in installer.build_pq} expected = {"pkg-a", "pkg-b", "pkg-c", "pkg-d", "pkg-e"} assert result == expected def test_check_before_phase_error(install_mockery): s = spack.concretize.concretize_one("trivial-install-test-package") s.package.stop_before_phase = "beforephase" with pytest.raises(inst.BadInstallPhase) as exc_info: inst._check_last_phase(s.package) err = str(exc_info.value) assert "is not a valid phase" in err assert s.package.stop_before_phase in err def test_check_last_phase_error(install_mockery): s = spack.concretize.concretize_one("trivial-install-test-package") s.package.stop_before_phase = None s.package.last_phase = "badphase" with pytest.raises(inst.BadInstallPhase) as exc_info: inst._check_last_phase(s.package) err = str(exc_info.value) assert "is not a valid phase" in err assert s.package.last_phase in err def test_installer_ensure_ready_errors(install_mockery, monkeypatch): installer = create_installer(["trivial-install-test-package"]) spec = installer.build_requests[0].pkg.spec fmt = r"cannot be installed locally.*{0}" # Force an external package error path, modules = spec.external_path, spec.external_modules spec.external_path = "/actual/external/path/not/checked" spec.external_modules = ["unchecked_module"] msg = fmt.format("is external") with pytest.raises(inst.ExternalPackageError, match=msg): installer._ensure_install_ready(spec.package) # Force an upstream package error spec.external_path, spec.external_modules = path, modules monkeypatch.setattr(spack.spec.Spec, "installed_upstream", True) msg = fmt.format("is upstream") with pytest.raises(inst.UpstreamPackageError, match=msg): installer._ensure_install_ready(spec.package) # Force an install lock error, which should occur naturally since # we are calling an internal method prior to any lock-related setup monkeypatch.setattr(spack.spec.Spec, "installed_upstream", False) assert len(installer.locks) == 0 with pytest.raises(inst.InstallLockError, match=fmt.format("not locked")): installer._ensure_install_ready(spec.package) def test_ensure_locked_err(install_mockery, monkeypatch, tmp_path: pathlib.Path, capfd): """Test _ensure_locked when a non-lock exception is raised.""" mock_err_msg = "Mock exception error" def _raise(lock, timeout=None): raise RuntimeError(mock_err_msg) installer = create_installer(["trivial-install-test-package"]) spec = installer.build_requests[0].pkg.spec monkeypatch.setattr(ulk.Lock, "acquire_read", _raise) with fs.working_dir(str(tmp_path)): with pytest.raises(RuntimeError): installer._ensure_locked("read", spec.package) out = str(capfd.readouterr()[1]) assert "Failed to acquire a read lock" in out assert mock_err_msg in out def test_ensure_locked_have(install_mockery, tmp_path: pathlib.Path, capfd): """Test _ensure_locked when already have lock.""" installer = create_installer(["trivial-install-test-package"], {}) spec = installer.build_requests[0].pkg.spec pkg_id = inst.package_id(spec) with fs.working_dir(str(tmp_path)): # Test "downgrade" of a read lock (to a read lock) lock = lk.Lock("./test", default_timeout=1e-9, desc="test") lock_type = "read" tpl = (lock_type, lock) installer.locks[pkg_id] = tpl assert installer._ensure_locked(lock_type, spec.package) == tpl # Test "upgrade" of a read lock without read count to a write lock_type = "write" err = "Cannot upgrade lock" with pytest.raises(ulk.LockUpgradeError, match=err): installer._ensure_locked(lock_type, spec.package) out = str(capfd.readouterr()[1]) assert "Failed to upgrade to a write lock" in out assert "exception when releasing read lock" in out # Test "upgrade" of the read lock *with* read count to a write lock._reads = 1 tpl = (lock_type, lock) assert installer._ensure_locked(lock_type, spec.package) == tpl # Test "downgrade" of the write lock to a read lock lock_type = "read" tpl = (lock_type, lock) assert installer._ensure_locked(lock_type, spec.package) == tpl @pytest.mark.parametrize("lock_type,reads,writes", [("read", 1, 0), ("write", 0, 1)]) def test_ensure_locked_new_lock(install_mockery, tmp_path: pathlib.Path, lock_type, reads, writes): installer = create_installer(["pkg-a"], {}) spec = installer.build_requests[0].pkg.spec with fs.working_dir(str(tmp_path)): ltype, lock = installer._ensure_locked(lock_type, spec.package) assert ltype == lock_type assert lock is not None assert lock._reads == reads assert lock._writes == writes def test_ensure_locked_new_warn(install_mockery, monkeypatch, capfd): orig_pl = spack.database.SpecLocker.lock def _pl(db, spec, timeout): lock = orig_pl(db, spec, timeout) lock.default_timeout = 1e-9 if timeout is None else None return lock installer = create_installer(["pkg-a"], {}) spec = installer.build_requests[0].pkg.spec monkeypatch.setattr(spack.database.SpecLocker, "lock", _pl) lock_type = "read" ltype, lock = installer._ensure_locked(lock_type, spec.package) assert ltype == lock_type assert lock is not None out = str(capfd.readouterr()[1]) assert "Expected prefix lock timeout" in out def test_package_id_err(install_mockery): s = spack.spec.Spec("trivial-install-test-package") with pytest.raises(ValueError, match="spec is not concretized"): inst.package_id(s) def test_package_id_ok(install_mockery): spec = spack.concretize.concretize_one("trivial-install-test-package") assert spec.concrete assert spec.name in inst.package_id(spec) def test_fake_install(install_mockery): spec = spack.concretize.concretize_one("trivial-install-test-package") assert spec.concrete pkg = spec.package inst._do_fake_install(pkg) assert os.path.isdir(pkg.prefix.lib) def test_dump_packages_deps_ok(install_mockery, tmp_path: pathlib.Path, mock_packages): """Test happy path for dump_packages with dependencies.""" spec_name = "simple-inheritance" spec = spack.concretize.concretize_one(spec_name) inst.dump_packages(spec, str(tmp_path)) repo = mock_packages.repos[0] dest_pkg = repo.filename_for_package_name(spec_name) assert os.path.isfile(dest_pkg) def test_dump_packages_deps_errs(install_mockery, tmp_path: pathlib.Path, monkeypatch, capfd): """Test error paths for dump_packages with dependencies.""" orig_bpp = spack.store.STORE.layout.build_packages_path orig_dirname = spack.repo.Repo.dirname_for_package_name repo_err_msg = "Mock dirname_for_package_name" def bpp_path(spec): # Perform the original function source = orig_bpp(spec) # Mock the required directory structure for the repository _mock_repo(os.path.join(source, spec.namespace), spec.namespace) return source def _repoerr(repo, name): if name == "cmake": raise spack.repo.RepoError(repo_err_msg) else: return orig_dirname(repo, name) # Now mock the creation of the required directory structure to cover # the try-except block monkeypatch.setattr(spack.store.STORE.layout, "build_packages_path", bpp_path) spec = spack.concretize.concretize_one("simple-inheritance") path = str(tmp_path) # The call to install_tree will raise the exception since not mocking # creation of dependency package files within *install* directories. with pytest.raises(OSError, match=path if sys.platform != "win32" else ""): inst.dump_packages(spec, path) # Now try the error path, which requires the mock directory structure # above monkeypatch.setattr(spack.repo.Repo, "dirname_for_package_name", _repoerr) with pytest.raises(spack.repo.RepoError, match=repo_err_msg): inst.dump_packages(spec, path) out = str(capfd.readouterr()[1]) assert "Couldn't copy in provenance for cmake" in out def test_clear_failures_success(tmp_path: pathlib.Path): """Test the clear_failures happy path.""" failures = spack.database.FailureTracker(str(tmp_path), default_timeout=0.1) spec = spack.spec.Spec("pkg-a") spec._mark_concrete() # Set up a test prefix failure lock failures.mark(spec) assert failures.has_failed(spec) # Now clear failure tracking failures.clear_all() # Ensure there are no cached failure locks or failure marks assert len(failures.locker.locks) == 0 assert len(os.listdir(failures.dir)) == 0 # Ensure the core directory and failure lock file still exist assert os.path.isdir(failures.dir) # Locks on windows are a no-op if sys.platform != "win32": assert os.path.isfile(failures.locker.lock_path) @pytest.mark.not_on_windows("chmod does not prevent removal on Win") @pytest.mark.skipif(fs.getuid() == 0, reason="user is root") def test_clear_failures_errs(tmp_path: pathlib.Path, capfd): """Test the clear_failures exception paths.""" failures = spack.database.FailureTracker(str(tmp_path), default_timeout=0.1) spec = spack.spec.Spec("pkg-a") spec._mark_concrete() failures.mark(spec) # Make the file marker not writeable, so that clearing_failures fails failures.dir.chmod(0o000) # Clear failure tracking failures.clear_all() # Ensure expected warning generated out = str(capfd.readouterr()[1]) assert "Unable to remove failure" in out failures.dir.chmod(0o750) def test_combine_phase_logs(tmp_path: pathlib.Path): """Write temporary files, and assert that combine phase logs works to combine them into one file. We aren't currently using this function, but it's available when the logs are refactored to be written separately. """ log_files = ["configure-out.txt", "install-out.txt", "build-out.txt"] phase_log_files = [] # Create and write to dummy phase log files for log_file in log_files: phase_log_file = tmp_path / log_file with open(phase_log_file, "w", encoding="utf-8") as plf: plf.write("Output from %s\n" % log_file) phase_log_files.append(str(phase_log_file)) # This is the output log we will combine them into combined_log = tmp_path / "combined-out.txt" inst.combine_phase_logs(phase_log_files, str(combined_log)) with open(combined_log, "r", encoding="utf-8") as log_file: out = log_file.read() # Ensure each phase log file is represented for log_file in log_files: assert "Output from %s\n" % log_file in out def test_combine_phase_logs_does_not_care_about_encoding(tmp_path: pathlib.Path): # this is invalid utf-8 at a minimum data = b"\x00\xf4\xbf\x00\xbf\xbf" input = [str(tmp_path / "a"), str(tmp_path / "b")] output = str(tmp_path / "c") for path in input: with open(path, "wb") as f: f.write(data) inst.combine_phase_logs(input, output) with open(output, "rb") as f: assert f.read() == data * 2 def test_check_deps_status_install_failure(install_mockery): """Tests that checking the dependency status on a request to install 'a' fails, if we mark the dependency as failed. """ s = spack.concretize.concretize_one("pkg-a") for dep in s.traverse(root=False): spack.store.STORE.failure_tracker.mark(dep) installer = create_installer(["pkg-a"], {}) request = installer.build_requests[0] with pytest.raises(spack.error.InstallError, match="install failure"): installer._check_deps_status(request) def test_check_deps_status_write_locked(install_mockery, monkeypatch): installer = create_installer(["pkg-a"], {}) request = installer.build_requests[0] # Ensure the lock is not acquired monkeypatch.setattr(inst.PackageInstaller, "_ensure_locked", _not_locked) with pytest.raises(spack.error.InstallError, match="write locked by another"): installer._check_deps_status(request) def test_check_deps_status_external(install_mockery, monkeypatch): installer = create_installer(["pkg-a"], {}) request = installer.build_requests[0] # Mock the dependencies as external so assumed to be installed monkeypatch.setattr(spack.spec.Spec, "external", True) installer._check_deps_status(request) for dep in request.spec.traverse(root=False): assert inst.package_id(dep) in installer.installed def test_check_deps_status_upstream(install_mockery, monkeypatch): installer = create_installer(["pkg-a"], {}) request = installer.build_requests[0] # Mock the known dependencies as installed upstream monkeypatch.setattr(spack.spec.Spec, "installed_upstream", True) installer._check_deps_status(request) for dep in request.spec.traverse(root=False): assert inst.package_id(dep) in installer.installed def test_prepare_for_install_on_installed(install_mockery, monkeypatch): """Test of _prepare_for_install's early return for installed task path.""" installer = create_installer(["dependent-install"], {}) request = installer.build_requests[0] install_args = {"keep_prefix": True, "keep_stage": True, "restage": False} task = create_build_task(request.pkg, install_args) installer.installed.add(task.pkg_id) monkeypatch.setattr(inst.PackageInstaller, "_ensure_install_ready", _noop) installer._prepare_for_install(task) def test_installer_init_requests(install_mockery): """Test of installer initial requests.""" spec_name = "dependent-install" installer = create_installer([spec_name], {}) # There is only one explicit request in this case assert len(installer.build_requests) == 1 request = installer.build_requests[0] assert request.pkg.name == spec_name def false(*args, **kwargs): return False def test_rewire_task_no_tarball(monkeypatch, mock_packages): spec = spack.concretize.concretize_one("splice-t") dep = spack.concretize.concretize_one("splice-h+foo") out = spec.splice(dep) rewire_task = inst.RewireTask(out.package, inst.BuildRequest(out.package, {})) monkeypatch.setattr(inst, "_process_binary_cache_tarball", false) monkeypatch.setattr(spack.report.InstallRecord, "succeed", lambda x: None) assert rewire_task.complete() == inst.ExecuteResult.MISSING_BUILD_SPEC @pytest.mark.parametrize("transitive", [True, False]) def test_install_spliced(install_mockery, mock_fetch, monkeypatch, transitive): """Test installing a spliced spec""" spec = spack.concretize.concretize_one("splice-t") dep = spack.concretize.concretize_one("splice-h+foo") # Do the splice. out = spec.splice(dep, transitive) installer = create_installer([out], {"verbose": True, "fail_fast": True}) installer.install() for node in out.traverse(): assert node.installed assert node.build_spec.installed @pytest.mark.parametrize("transitive", [True, False]) def test_install_spliced_build_spec_installed(install_mockery, mock_fetch, transitive): """Test installing a spliced spec with the build spec already installed""" spec = spack.concretize.concretize_one("splice-t") dep = spack.concretize.concretize_one("splice-h+foo") # Do the splice. out = spec.splice(dep, transitive) inst.PackageInstaller([out.build_spec.package]).install() installer = create_installer([out], {"verbose": True, "fail_fast": True}) installer._init_queue() for _, task in installer.build_pq: assert isinstance(task, inst.RewireTask if task.pkg.spec.spliced else inst.BuildTask) installer.install() for node in out.traverse(): assert node.installed assert node.build_spec.installed # Unit tests should not be affected by the user's managed environments @pytest.mark.not_on_windows("lacking windows support for binary installs") @pytest.mark.parametrize("transitive", [True, False]) @pytest.mark.parametrize( "root_str", ["splice-t^splice-h~foo", "splice-h~foo", "splice-vt^splice-a"] ) def test_install_splice_root_from_binary( mutable_mock_env_path, install_mockery, mock_fetch, temporary_mirror, transitive, root_str ): """Test installing a spliced spec with the root available in binary cache""" # Test splicing and rewiring a spec with the same name, different hash. original_spec = spack.concretize.concretize_one(root_str) spec_to_splice = spack.concretize.concretize_one("splice-h+foo") inst.PackageInstaller([original_spec.package, spec_to_splice.package]).install() out = original_spec.splice(spec_to_splice, transitive) buildcache = SpackCommand("buildcache") buildcache( "push", "--unsigned", "--update-index", temporary_mirror, str(original_spec), str(spec_to_splice), ) uninstall = SpackCommand("uninstall") uninstall("-ay") inst.PackageInstaller([out.package], unsigned=True).install() assert len(spack.store.STORE.db.query()) == len(list(out.traverse())) class MockInstallStatus(inst.InstallStatus): def next_pkg(self, *args, **kwargs): pass def set_term_title(self, *args, **kwargs): pass def get_progress(self): return "1/1" class MockTermStatusLine(inst.TermStatusLine): def add(self, *args, **kwargs): pass def clear(self): pass def test_installing_task_use_cache(install_mockery, monkeypatch): installer = create_installer(["trivial-install-test-package"], {}) request = installer.build_requests[0] task = create_build_task(request.pkg) install_status = MockInstallStatus(1) term_status = MockTermStatusLine(True) monkeypatch.setattr(inst, "_install_from_cache", _true) installer.start_task(task, install_status, term_status) installer.complete_task(task, install_status) assert request.pkg_id in installer.installed def test_install_task_requeue_build_specs(install_mockery, monkeypatch): """Check that a missing build_spec spec is added by _complete_task.""" # This test also ensures coverage of most of the new # _requeue_with_build_spec_tasks method. def _missing(*args, **kwargs): return inst.ExecuteResult.MISSING_BUILD_SPEC # Set the configuration to ensure _requeue_with_build_spec_tasks actually # does something. installer = create_installer(["depb"], {}) installer._init_queue() request = installer.build_requests[0] task = create_build_task(request.pkg) # Drop one of the specs so its task is missing before _complete_task popped_task = installer._pop_ready_task() assert inst.package_id(popped_task.pkg.spec) not in installer.build_tasks monkeypatch.setattr(task, "complete", _missing) installer._complete_task(task, None) # Ensure the dropped task/spec was added back by _install_task assert inst.package_id(popped_task.pkg.spec) in installer.build_tasks def test_release_lock_write_n_exception(install_mockery, tmp_path: pathlib.Path, capfd): """Test _release_lock for supposed write lock with exception.""" installer = create_installer(["trivial-install-test-package"], {}) pkg_id = "test" with fs.working_dir(str(tmp_path)): lock = lk.Lock("./test", default_timeout=1e-9, desc="test") installer.locks[pkg_id] = ("write", lock) assert lock._writes == 0 installer._release_lock(pkg_id) out = str(capfd.readouterr()[1]) msg = "exception when releasing write lock for {0}".format(pkg_id) assert msg in out @pytest.mark.parametrize("installed", [True, False]) def test_push_task_skip_processed(install_mockery, installed): """Test to ensure skip re-queueing a processed package.""" installer = create_installer(["pkg-a"], {}) assert len(list(installer.build_tasks)) == 0 # Mark the package as installed OR failed task = create_build_task(installer.build_requests[0].pkg) if installed: installer.installed.add(task.pkg_id) else: installer.failed[task.pkg_id] = None installer._push_task(task) assert len(list(installer.build_tasks)) == 0 def test_requeue_task(install_mockery, capfd): """Test to ensure cover _requeue_task.""" installer = create_installer(["pkg-a"], {}) task = create_build_task(installer.build_requests[0].pkg) # temporarily set tty debug messages on so we can test output current_debug_level = tty.debug_level() tty.set_debug(1) installer._requeue_task(task, None) tty.set_debug(current_debug_level) ids = list(installer.build_tasks) assert len(ids) == 1 qtask = installer.build_tasks[ids[0]] assert qtask.status == inst.BuildStatus.INSTALLING assert qtask.sequence > task.sequence assert qtask.attempts == task.attempts + 1 out = capfd.readouterr()[1] assert "Installing pkg-a" in out assert " in progress by another process" in out def test_cleanup_all_tasks(install_mockery, monkeypatch): """Test to ensure cover _cleanup_all_tasks.""" def _mktask(pkg): return create_build_task(pkg) def _rmtask(installer, pkg_id): raise RuntimeError("Raise an exception to test except path") installer = create_installer(["pkg-a"], {}) spec = installer.build_requests[0].pkg.spec # Cover task removal happy path installer.build_tasks["pkg-a"] = _mktask(spec.package) installer._cleanup_all_tasks() assert len(installer.build_tasks) == 0 # Cover task removal exception path installer.build_tasks["pkg-a"] = _mktask(spec.package) monkeypatch.setattr(inst.PackageInstaller, "_remove_task", _rmtask) installer._cleanup_all_tasks() assert len(installer.build_tasks) == 1 def test_setup_install_dir_grp(install_mockery, monkeypatch, capfd): """Test _setup_install_dir's group change.""" mock_group = "mockgroup" mock_chgrp_msg = "Changing group for {0} to {1}" def _get_group(spec): return mock_group def _chgrp(path, group, follow_symlinks=True): tty.msg(mock_chgrp_msg.format(path, group)) monkeypatch.setattr(prefs, "get_package_group", _get_group) monkeypatch.setattr(fs, "chgrp", _chgrp) build_task = create_build_task( spack.concretize.concretize_one("trivial-install-test-package").package ) spec = build_task.request.pkg.spec fs.touchp(spec.prefix) metadatadir = spack.store.STORE.layout.metadata_path(spec) # Regex matching with Windows style paths typically fails # so we skip the match check here if sys.platform == "win32": metadatadir = None # Should fail with a "not a directory" error with pytest.raises(OSError, match=metadatadir): build_task._setup_install_dir(spec.package) out = str(capfd.readouterr()[0]) expected_msg = mock_chgrp_msg.format(spec.prefix, mock_group) assert expected_msg in out def test_cleanup_failed_err(install_mockery, tmp_path: pathlib.Path, monkeypatch, capfd): """Test _cleanup_failed exception path.""" msg = "Fake release_write exception" def _raise_except(lock): raise RuntimeError(msg) installer = create_installer(["trivial-install-test-package"], {}) monkeypatch.setattr(lk.Lock, "release_write", _raise_except) pkg_id = "test" with fs.working_dir(str(tmp_path)): lock = lk.Lock("./test", default_timeout=1e-9, desc="test") installer.failed[pkg_id] = lock installer._cleanup_failed(pkg_id) out = str(capfd.readouterr()[1]) assert "exception when removing failure tracking" in out assert msg in out def test_update_failed_no_dependent_task(install_mockery): """Test _update_failed with missing dependent build tasks.""" installer = create_installer(["dependent-install"], {}) spec = installer.build_requests[0].pkg.spec for dep in spec.traverse(root=False): task = create_build_task(dep.package) installer._update_failed(task, mark=False) assert installer.failed[task.pkg_id] is None def test_install_uninstalled_deps(install_mockery, monkeypatch, capfd): """Test install with uninstalled dependencies.""" installer = create_installer(["parallel-package-a"], {}) # Skip the actual installation and any status updates monkeypatch.setattr(inst.Task, "start", _noop) monkeypatch.setattr(inst.Task, "poll", _noop) monkeypatch.setattr(inst.Task, "complete", _noop) monkeypatch.setattr(inst.PackageInstaller, "_update_installed", _noop) monkeypatch.setattr(inst.PackageInstaller, "_update_failed", _noop) msg = "Cannot proceed with parallel-package-a" with pytest.raises(spack.error.InstallError, match=msg): installer.install() out = str(capfd.readouterr()) assert "Detected uninstalled dependencies for" in out def test_install_failed(install_mockery, monkeypatch, capfd): """Test install with failed install.""" installer = create_installer(["parallel-package-a"], {}) # Make sure the package is identified as failed monkeypatch.setattr(spack.database.FailureTracker, "has_failed", _true) with pytest.raises(spack.error.InstallError, match="request failed"): installer.install() out = str(capfd.readouterr()) assert installer.build_requests[0].pkg_id in out assert "failed to install" in out def test_install_failed_not_fast(install_mockery, monkeypatch, capfd): """Test install with failed install.""" installer = create_installer(["parallel-package-a"], {"fail_fast": False}) # Make sure the package is identified as failed monkeypatch.setattr(spack.database.FailureTracker, "has_failed", _true) with pytest.raises(spack.error.InstallError, match="request failed"): installer.install() out = str(capfd.readouterr()) assert "failed to install" in out assert "Skipping build of parallel-package-a" in out def _interrupt(installer, task, install_status, **kwargs): if task.pkg.name == "pkg-a": raise KeyboardInterrupt("mock keyboard interrupt for pkg-a") else: return installer._real_install_task(task, None) # installer.installed.add(task.pkg.name) def test_install_fail_on_interrupt(install_mockery, mock_fetch, monkeypatch): """Test ctrl-c interrupted install.""" spec_name = "pkg-a" err_msg = "mock keyboard interrupt for {0}".format(spec_name) installer = create_installer([spec_name], {"fake": True}) setattr(inst.PackageInstaller, "_real_install_task", inst.PackageInstaller._complete_task) # Raise a KeyboardInterrupt error to trigger early termination monkeypatch.setattr(inst.PackageInstaller, "_complete_task", _interrupt) with pytest.raises(KeyboardInterrupt, match=err_msg): installer.install() assert not any(i.startswith("pkg-a-") for i in installer.installed) assert any( i.startswith("pkg-b-") for i in installer.installed ) # ensure dependency of a is 'installed' class MyBuildException(Exception): pass _old_complete_task = None def _install_fail_my_build_exception(installer, task, install_status, **kwargs): if task.pkg.name == "pkg-a": raise MyBuildException("mock internal package build error for pkg-a") else: _old_complete_task(installer, task, install_status) def test_install_fail_single(install_mockery, mock_fetch, monkeypatch): """Test expected results for failure of single package.""" global _old_complete_task installer = create_installer(["pkg-a"], {"fake": True}) # Raise a KeyboardInterrupt error to trigger early termination _old_complete_task = inst.PackageInstaller._complete_task monkeypatch.setattr(inst.PackageInstaller, "_complete_task", _install_fail_my_build_exception) with pytest.raises(MyBuildException, match="mock internal package build error for pkg-a"): installer.install() # ensure dependency of a is 'installed' and a is not assert any(pkg_id.startswith("pkg-b-") for pkg_id in installer.installed) assert not any(pkg_id.startswith("pkg-a-") for pkg_id in installer.installed) def test_install_fail_multi(install_mockery, mock_fetch, monkeypatch): """Test expected results for failure of multiple packages.""" global _old_complete_task installer = create_installer(["pkg-a", "pkg-c"], {"fake": True}) # Raise a KeyboardInterrupt error to trigger early termination _old_complete_task = inst.PackageInstaller._complete_task monkeypatch.setattr(inst.PackageInstaller, "_complete_task", _install_fail_my_build_exception) with pytest.raises(spack.error.InstallError, match="Installation request failed"): installer.install() # ensure the the second spec installed but not the first assert any(pkg_id.startswith("pkg-c-") for pkg_id in installer.installed) assert not any(pkg_id.startswith("pkg-a-") for pkg_id in installer.installed) def test_install_fail_fast_on_detect(install_mockery, monkeypatch, capfd): """Test fail_fast install when an install failure is detected.""" a = spack.concretize.concretize_one("parallel-package-a") a_id = inst.package_id(a) b_id = inst.package_id(a["parallel-package-b"]) c_id = inst.package_id(a["parallel-package-c"]) installer = create_installer([a], {"fail_fast": True}) # Make sure all packages are identified as failed # This will prevent a and b from installing, which will cause the build of c to be skipped # and the active processes to be killed. monkeypatch.setattr(spack.database.FailureTracker, "has_failed", _true) installer.max_active_tasks = 2 with pytest.raises(spack.error.InstallError, match="after first install failure"): installer.install() assert b_id in installer.failed, "Expected b to be marked as failed" assert c_id in installer.failed, "Expected c to be marked as failed" assert a_id not in installer.installed, ( "Package a cannot install due to its dependencies failing" ) # check that b's active process got killed when c failed assert f"{b_id} failed to install" in capfd.readouterr().err def _test_install_fail_fast_on_except_patch(installer, **kwargs): """Helper for test_install_fail_fast_on_except.""" # This is a module-scope function and not a local function because it # needs to be pickleable. raise RuntimeError("mock patch failure") @pytest.mark.disable_clean_stage_check def test_install_fail_fast_on_except(install_mockery, monkeypatch, capfd): """Test fail_fast install when an install failure results from an error.""" installer = create_installer(["pkg-a"], {"fail_fast": True}) # Raise a non-KeyboardInterrupt exception to trigger fast failure. # # This will prevent b from installing, which will cause the build of a # to be skipped. monkeypatch.setattr( spack.package_base.PackageBase, "do_patch", _test_install_fail_fast_on_except_patch ) with pytest.raises(spack.error.InstallError, match="mock patch failure"): installer.install() out = str(capfd.readouterr()) assert "Skipping build of pkg-a" in out def test_install_lock_failures(install_mockery, monkeypatch, capfd): """Cover basic install lock failure handling in a single pass.""" # Note: this test relies on installing a package with no dependencies def _requeued(installer, task, install_status): tty.msg("requeued {0}".format(task.pkg.spec.name)) installer = create_installer(["pkg-c"], {}) # Ensure never acquire a lock monkeypatch.setattr(inst.PackageInstaller, "_ensure_locked", _not_locked) # Ensure don't continually requeue the task monkeypatch.setattr(inst.PackageInstaller, "_requeue_task", _requeued) with pytest.raises(spack.error.InstallError, match="request failed"): installer.install() out = capfd.readouterr()[0] expected = ["write locked", "read locked", "requeued"] for exp, ln in zip(expected, out.split("\n")): assert exp in ln def test_install_lock_installed_requeue(install_mockery, monkeypatch, capfd): """Cover basic install handling for installed package.""" # Note: this test relies on installing a package with no dependencies concrete_spec = spack.concretize.concretize_one("pkg-c") pkg_id = inst.package_id(concrete_spec) installer = create_installer([concrete_spec]) def _prep(installer, task): installer.installed.add(pkg_id) tty.msg(f"{pkg_id} is installed") # also do not allow the package to be locked again monkeypatch.setattr(inst.PackageInstaller, "_ensure_locked", _not_locked) def _requeued(installer, task, install_status): tty.msg(f"requeued {inst.package_id(task.pkg.spec)}") # Flag the package as installed monkeypatch.setattr(inst.PackageInstaller, "_prepare_for_install", _prep) # Ensure don't continually requeue the task monkeypatch.setattr(inst.PackageInstaller, "_requeue_task", _requeued) with pytest.raises(spack.error.InstallError, match="request failed"): installer.install() assert pkg_id not in installer.installed expected = ["is installed", "read locked", "requeued"] for exp, ln in zip(expected, capfd.readouterr().out.splitlines()): assert exp in ln def test_install_read_locked_requeue(install_mockery, monkeypatch, capfd): """Cover basic read lock handling for uninstalled package with requeue.""" # Note: this test relies on installing a package with no dependencies orig_fn = inst.PackageInstaller._ensure_locked def _read(installer, lock_type, pkg): tty.msg("{0}->read locked {1}".format(lock_type, pkg.spec.name)) return orig_fn(installer, "read", pkg) def _prep(installer, task): tty.msg("preparing {0}".format(task.pkg.spec.name)) assert task.pkg.spec.name not in installer.installed def _requeued(installer, task, install_status): tty.msg("requeued {0}".format(task.pkg.spec.name)) # Force a read lock monkeypatch.setattr(inst.PackageInstaller, "_ensure_locked", _read) # Flag the package as installed monkeypatch.setattr(inst.PackageInstaller, "_prepare_for_install", _prep) # Ensure don't continually requeue the task monkeypatch.setattr(inst.PackageInstaller, "_requeue_task", _requeued) installer = create_installer(["pkg-c"], {}) with pytest.raises(spack.error.InstallError, match="request failed"): installer.install() assert "b" not in installer.installed out = capfd.readouterr()[0] expected = ["write->read locked", "preparing", "requeued"] for exp, ln in zip(expected, out.split("\n")): assert exp in ln def test_install_skip_patch(install_mockery, mock_fetch): """Test the path skip_patch install path.""" # Note: this test relies on installing a package with no dependencies installer = create_installer(["pkg-c"], {"fake": False, "skip_patch": True}) installer.install() assert inst.package_id(installer.build_requests[0].pkg.spec) in installer.installed def test_install_implicit(install_mockery, mock_fetch): """Test the path skip_patch install path.""" spec_name = "trivial-install-test-package" installer = create_installer([spec_name], {"fake": False}) pkg = installer.build_requests[0].pkg assert not create_build_task(pkg, {"explicit": []}).explicit assert create_build_task(pkg, {"explicit": [pkg.spec.dag_hash()]}).explicit assert not create_build_task(pkg).explicit # Install that wipes the prefix directory def wipe_prefix(pkg, install_args): shutil.rmtree(pkg.prefix, ignore_errors=True) fs.mkdirp(pkg.prefix) raise Exception("Some fatal install error") def fail(*args, **kwargs): assert False def test_overwrite_install_backup_success(monkeypatch, temporary_store, config, mock_packages): """ When doing an overwrite install that fails, Spack should restore the backup of the original prefix, and leave the original spec marked installed. """ # Get a build task. TODO: Refactor this to avoid calling internal methods. # This task relies on installing something with no dependencies installer = create_installer(["pkg-c"]) installer._init_queue() task = installer._pop_task() install_status = MockInstallStatus(1) term_status = MockTermStatusLine(True) # Make sure the install prefix exists with some trivial file installed_file = os.path.join(task.pkg.prefix, "some_file") fs.touchp(installed_file) monkeypatch.setattr(inst, "build_process", wipe_prefix) # Make sure the package is not marked uninstalled monkeypatch.setattr(spack.store.STORE.db, "remove", fail) # Make sure that the installer does an overwrite install monkeypatch.setattr(task, "_install_action", inst.InstallAction.OVERWRITE) # Installation should throw the installation exception, not the backup # failure. installer.start_task(task, install_status, term_status) with pytest.raises(Exception, match="Some fatal install error"): installer.complete_task(task, install_status) # Check that the original file is back. assert os.path.exists(installed_file) # Install that removes the backup directory, which is at the same level as # the prefix, starting with .backup def remove_backup(pkg, install_args): backup_glob = os.path.join(os.path.dirname(os.path.normpath(pkg.prefix)), ".backup*") for backup in glob.iglob(backup_glob): shutil.rmtree(backup) raise Exception("Some fatal install error") def test_overwrite_install_backup_failure(monkeypatch, temporary_store, config, mock_packages): """ When doing an overwrite install that fails, Spack should try to recover the original prefix. If that fails, the spec is lost, and it should be removed from the database. """ # Get a build task. TODO: refactor this to avoid calling internal methods installer = create_installer(["pkg-c"]) installer._init_queue() task = installer._pop_task() install_status = MockInstallStatus(1) term_status = MockTermStatusLine(True) # Make sure the install prefix exists installed_file = os.path.join(task.pkg.prefix, "some_file") fs.touchp(installed_file) monkeypatch.setattr(inst, "build_process", remove_backup) # Make sure that the installer does an overwrite install monkeypatch.setattr(task, "_install_action", inst.InstallAction.OVERWRITE) # Make sure that `remove` was called on the database after an unsuccessful # attempt to restore the backup. # This error is raised while handling the original install error installer.start_task(task, install_status, term_status) with pytest.raises(Exception, match="No such spec in database"): installer.complete_task(task, install_status) def test_term_status_line(): # Smoke test for TermStatusLine; to actually test output it would be great # to pass a StringIO instance, but we use tty.msg() internally which does not # accept that. `with log_output(buf)` doesn't really work because it trims output # and we actually want to test for escape sequences etc. x = inst.TermStatusLine(enabled=True) x.add("pkg-a") x.add("pkg-b") x.clear() @pytest.mark.parametrize("explicit", [True, False]) def test_single_external_implicit_install(install_mockery, explicit): pkg = "trivial-install-test-package" s = spack.concretize.concretize_one(pkg) s.external_path = "/usr" args = {"explicit": [s.dag_hash()] if explicit else []} create_installer([s], args).install() assert spack.store.STORE.db.get_record(pkg).explicit == explicit def test_overwrite_install_does_install_build_deps(install_mockery, mock_fetch): """When overwrite installing something from sources, build deps should be installed.""" s = spack.concretize.concretize_one("dtrun3") create_installer([s]).install() # Verify there is a pure build dep edge = s.edges_to_dependencies(name="dtbuild3").pop() assert edge.depflag == dt.BUILD build_dep = edge.spec # Uninstall the build dep build_dep.package.do_uninstall() # Overwrite install the root dtrun3 create_installer([s], {"overwrite": [s.dag_hash()]}).install() # Verify that the build dep was also installed. assert build_dep.installed @pytest.mark.parametrize("run_tests", [True, False]) def test_print_install_test_log_skipped(install_mockery, mock_packages, capfd, run_tests): """Confirm printing of install log skipped if not run/no failures.""" name = "trivial-install-test-package" s = spack.concretize.concretize_one(name) pkg = s.package pkg.run_tests = run_tests inst.print_install_test_log(pkg) out = capfd.readouterr()[0] assert out == "" def test_print_install_test_log_failures( tmp_path: pathlib.Path, install_mockery, mock_packages, ensure_debug, capfd ): """Confirm expected outputs when there are test failures.""" name = "trivial-install-test-package" s = spack.concretize.concretize_one(name) pkg = s.package # Missing test log is an error pkg.run_tests = True pkg.tester.test_log_file = str(tmp_path / "test-log.txt") pkg.tester.add_failure(AssertionError("test"), "test-failure") inst.print_install_test_log(pkg) err = capfd.readouterr()[1] assert "no test log file" in err # Having test log results in path being output fs.touch(pkg.tester.test_log_file) inst.print_install_test_log(pkg) out = capfd.readouterr()[0] assert "See test results at" in out def test_fallback_to_old_installer_for_splicing(monkeypatch, mock_packages, mutable_config): """Test that the old installer is used for spliced specs (unsupported in the new installer)""" mutable_config.set("config:installer", "new") spec = spack.concretize.concretize_one("splice-t") dep = spack.concretize.concretize_one("splice-h+foo") out = spec.splice(dep) assert isinstance( spack.installer_dispatch.create_installer([out.package]), inst.PackageInstaller ) ================================================ FILE: lib/spack/spack/test/installer_build_graph.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for BuildGraph class in new_installer""" import sys from typing import Dict, List, Tuple, Union import pytest if sys.platform == "win32": pytest.skip("Skipping new installer tests on Windows", allow_module_level=True) import spack.deptypes as dt import spack.error import spack.traverse from spack.new_installer import BuildGraph from spack.spec import Spec from spack.store import Store def create_dag( nodes: List[str], edges: List[Tuple[str, str, Union[dt.DepType, Tuple[dt.DepType, ...]]]] ) -> Dict[str, Spec]: """ Create a DAG of concrete specs, as a mapping from package name to Spec. Arguments: nodes: list of unique package names edges: list of tuples (parent, child, deptype) """ specs = {name: Spec(name) for name in nodes} for parent, child, deptypes in edges: depflag = deptypes if isinstance(deptypes, dt.DepFlag) else dt.canonicalize(deptypes) specs[parent].add_dependency_edge(specs[child], depflag=depflag, virtuals=()) # Mark all specs as concrete for spec in specs.values(): spec._mark_concrete() return specs def install_spec_in_db(spec: Spec, store: Store): """Helper to install a spec in the database for testing.""" prefix = store.layout.path_for_spec(spec) spec.set_prefix(prefix) # Use the layout to create a proper installation directory structure store.layout.create_install_directory(spec) store.db.add(spec, explicit=False) @pytest.fixture def mock_specs(): """Create a set of mock specs for testing. DAG structure: root -> dep1 -> dep2 root -> dep3 """ return create_dag( nodes=["root", "dep1", "dep2", "dep3"], edges=[ ("root", "dep1", ("build", "link")), ("root", "dep3", ("build", "link")), ("dep1", "dep2", ("build", "link")), ], ) @pytest.fixture def diamond_dag(): """Create a diamond-shaped DAG to test shared dependencies. DAG structure: root -> dep1 -> shared root -> dep2 -> shared """ return create_dag( nodes=["root", "dep1", "dep2", "shared"], edges=[ ("root", "dep1", ("build", "link")), ("root", "dep2", ("build", "link")), ("dep1", "shared", ("build", "link")), ("dep2", "shared", ("build", "link")), ], ) @pytest.fixture def specs_with_build_deps(): """Create specs with different dependency types for testing build dep filtering. DAG structure: root -> link_dep (link only) root -> build_dep (build only) root -> all_dep (build, link, run) """ return create_dag( nodes=["root", "link_dep", "build_dep", "all_dep"], edges=[ ("root", "link_dep", "link"), ("root", "build_dep", "build"), ("root", "all_dep", ("build", "link", "run")), ], ) @pytest.fixture def complex_pruning_dag(): """Create a complex DAG for testing re-parenting logic. DAG structure: parent1 -> middle -> child1 parent2 -> middle -> child2 When 'middle' is installed and pruned, both parent1 and parent2 should become direct parents of both child1 and child2 (full Cartesian product). """ return create_dag( nodes=["parent1", "parent2", "middle", "child1", "child2"], edges=[ ("parent1", "middle", ("build", "link")), ("parent2", "middle", ("build", "link")), ("middle", "child1", ("build", "link")), ("middle", "child2", ("build", "link")), ], ) class TestBuildGraph: """Tests for the BuildGraph class.""" def test_basic_graph_construction(self, mock_specs: Dict[str, Spec], temporary_store: Store): """Test basic graph construction with all specs to be installed.""" graph = BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) # Root should be in roots set assert mock_specs["root"].dag_hash() in graph.roots # All uninstalled specs should be in nodes assert len(graph.nodes) == 4 # root, dep1, dep2, dep3 # Root should have 2 children (dep1, dep3) assert len(graph.parent_to_child[mock_specs["root"].dag_hash()]) == 2 def test_install_package_only_mode(self, mock_specs: Dict[str, Spec], temporary_store: Store): """Test that install_package=False removes root specs from graph.""" graph = BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=False, # Only install dependencies install_deps=True, database=temporary_store.db, ) # Root should NOT be in nodes when install_package=False assert mock_specs["root"].dag_hash() not in graph.nodes # But its dependencies should be assert mock_specs["dep1"].dag_hash() in graph.nodes def test_install_deps_false_with_uninstalled_deps( self, mock_specs: Dict[str, Spec], temporary_store: Store ): """Test that install_deps=False raises error when dependencies are not installed.""" # Should raise error because dependencies are not installed with pytest.raises( spack.error.InstallError, match="package only mode.*dependency.*not installed" ): BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=False, # Don't install dependencies database=temporary_store.db, ) def test_multiple_roots(self, mock_specs: Dict[str, Spec], temporary_store: Store): """Test graph construction with multiple root specs.""" graph = BuildGraph( specs=[mock_specs["root"], mock_specs["dep1"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) # Both should be in roots assert mock_specs["root"].dag_hash() in graph.roots assert mock_specs["dep1"].dag_hash() in graph.roots def test_parent_child_mappings(self, mock_specs: Dict[str, Spec], temporary_store: Store): """Test that parent-child mappings are correctly constructed.""" spec_root = mock_specs["root"] graph = BuildGraph( specs=[spec_root], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) # Verify parent_to_child and child_to_parent are inverse mappings for parent, children in graph.parent_to_child.items(): for child in children: assert child in graph.child_to_parent assert parent in graph.child_to_parent[child] def test_diamond_dag_with_shared_dependency( self, diamond_dag: Dict[str, Spec], temporary_store: Store ): """Test graph construction with a diamond DAG where a dependency has multiple parents.""" graph = BuildGraph( specs=[diamond_dag["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) # Shared dependency should have two parents shared_hash = diamond_dag["shared"].dag_hash() assert len(graph.child_to_parent[shared_hash]) == 2 # Both dep1 and dep2 should be parents of shared assert diamond_dag["dep1"].dag_hash() in graph.child_to_parent[shared_hash] assert diamond_dag["dep2"].dag_hash() in graph.child_to_parent[shared_hash] def test_pruning_installed_specs(self, mock_specs: Dict[str, Spec], temporary_store: Store): """Test that installed specs are correctly pruned from the graph.""" # Install dep2 in the database dep2 = mock_specs["dep2"] install_spec_in_db(dep2, temporary_store) graph = BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) # dep2 should be pruned since it's installed assert dep2.dag_hash() not in graph.nodes # But dep1 (its parent) should still be in the graph assert mock_specs["dep1"].dag_hash() in graph.nodes # And dep1 should have no children (since dep2 was pruned) assert len(graph.parent_to_child[mock_specs["dep1"].dag_hash()]) == 0 def test_pruning_with_shared_dependency_partially_installed( self, diamond_dag: Dict[str, Spec], temporary_store: Store ): """Test that pruning a shared dependency correctly updates all parents.""" # Install the shared dependency shared = diamond_dag["shared"] install_spec_in_db(shared, temporary_store) graph = BuildGraph( specs=[diamond_dag["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) # Shared should be pruned assert shared.dag_hash() not in graph.nodes # Both dep1 and dep2 should have no children assert len(graph.parent_to_child[diamond_dag["dep1"].dag_hash()]) == 0 assert len(graph.parent_to_child[diamond_dag["dep2"].dag_hash()]) == 0 def test_overwrite_set_prevents_pruning( self, mock_specs: Dict[str, Spec], temporary_store: Store ): """Test that specs in overwrite_set are not pruned even if installed.""" # Install dep2 in the database dep2 = mock_specs["dep2"] install_spec_in_db(dep2, temporary_store) # Create graph with dep2 in the overwrite set graph = BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, overwrite_set={dep2.dag_hash()}, ) # dep2 should NOT be pruned since it's in overwrite_set assert dep2.dag_hash() in graph.nodes # dep1 should still have dep2 as a child assert dep2.dag_hash() in graph.parent_to_child[mock_specs["dep1"].dag_hash()] # dep2 should have dep1 as a parent assert mock_specs["dep1"].dag_hash() in graph.child_to_parent[dep2.dag_hash()] def test_installed_root_excludes_build_deps_even_when_requested( self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store ): """Test that installed root specs never include build deps, even with include_build_deps=True.""" root = specs_with_build_deps["root"] install_spec_in_db(root, temporary_store) graph = BuildGraph( specs=[root], root_policy="auto", dependencies_policy="auto", include_build_deps=True, # Should be ignored for installed root install_package=True, install_deps=True, database=temporary_store.db, ) # build_dep should NOT be in the graph (installed root never needs build deps) assert specs_with_build_deps["build_dep"].dag_hash() not in graph.nodes # link_dep and all_dep should be in the graph (link/run deps) assert specs_with_build_deps["link_dep"].dag_hash() in graph.nodes assert specs_with_build_deps["all_dep"].dag_hash() in graph.nodes def test_cache_only_excludes_build_deps( self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store ): """Test that cache_only policy excludes build deps when include_build_deps=False.""" specs = [specs_with_build_deps["root"]] graph = BuildGraph( specs=specs, root_policy="cache_only", dependencies_policy="auto", include_build_deps=False, # exclude build deps when possible install_package=True, install_deps=True, database=temporary_store.db, ) assert specs_with_build_deps["build_dep"].dag_hash() not in graph.nodes assert specs_with_build_deps["link_dep"].dag_hash() in graph.nodes assert specs_with_build_deps["all_dep"].dag_hash() in graph.nodes # Verify that the entire graph has a prefix assigned, which avoids that the subprocess has # to obtain a read lock on the database. for s in spack.traverse.traverse_nodes(specs): assert s._prefix is not None def test_cache_only_includes_build_deps_when_requested( self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store ): """Test that cache_only policy includes build deps when include_build_deps=True.""" graph = BuildGraph( specs=[specs_with_build_deps["root"]], root_policy="cache_only", dependencies_policy="cache_only", include_build_deps=True, install_package=True, install_deps=True, database=temporary_store.db, ) # All dependencies should be in the graph, including build_dep assert specs_with_build_deps["build_dep"].dag_hash() in graph.nodes assert specs_with_build_deps["link_dep"].dag_hash() in graph.nodes assert specs_with_build_deps["all_dep"].dag_hash() in graph.nodes def test_install_deps_false_with_all_deps_installed( self, mock_specs: Dict[str, Spec], temporary_store: Store ): """Test successful package-only install when all dependencies are already installed.""" # Install all dependencies for dep_name in ["dep1", "dep2", "dep3"]: install_spec_in_db(mock_specs[dep_name], temporary_store) # Should succeed since all dependencies are installed graph = BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=False, database=temporary_store.db, ) # Only the root should be in the graph assert len(graph.nodes) == 1 assert mock_specs["root"].dag_hash() in graph.nodes # Root should have no children (all deps pruned) assert len(graph.parent_to_child.get(mock_specs["root"].dag_hash(), [])) == 0 def test_pruning_creates_cartesian_product_of_connections( self, complex_pruning_dag: Dict[str, Spec], temporary_store: Store ): """Test that pruning creates full Cartesian product of parent-child connections. When a node with multiple parents and multiple children is pruned, all parents should be connected to all children (parents x children). DAG structure: parent1 -> middle -> child1 parent2 -> middle -> child2 After pruning 'middle': parent1 -> child1 parent1 -> child2 parent2 -> child1 parent2 -> child2 """ # Install the middle node middle = complex_pruning_dag["middle"] install_spec_in_db(middle, temporary_store) # Use parent1 as the root to build the graph graph = BuildGraph( specs=[complex_pruning_dag["parent1"], complex_pruning_dag["parent2"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) parent1_hash = complex_pruning_dag["parent1"].dag_hash() parent2_hash = complex_pruning_dag["parent2"].dag_hash() middle_hash = middle.dag_hash() child1_hash = complex_pruning_dag["child1"].dag_hash() child2_hash = complex_pruning_dag["child2"].dag_hash() # middle should be pruned since it's installed assert middle_hash not in graph.nodes # All other nodes should be in the graph assert parent1_hash in graph.nodes assert parent2_hash in graph.nodes assert child1_hash in graph.nodes assert child2_hash in graph.nodes # Verify full Cartesian product: each parent should be connected to each child # parent1 -> child1, child2 assert child1_hash in graph.parent_to_child[parent1_hash] assert child2_hash in graph.parent_to_child[parent1_hash] # parent2 -> child1, child2 assert child1_hash in graph.parent_to_child[parent2_hash] assert child2_hash in graph.parent_to_child[parent2_hash] # Verify reverse mapping: each child should have both parents # child1 <- parent1, parent2 assert parent1_hash in graph.child_to_parent[child1_hash] assert parent2_hash in graph.child_to_parent[child1_hash] # child2 <- parent1, parent2 assert parent1_hash in graph.child_to_parent[child2_hash] assert parent2_hash in graph.child_to_parent[child2_hash] # middle should not appear in any parent-child relationships assert middle_hash not in graph.parent_to_child assert middle_hash not in graph.child_to_parent def test_empty_graph_all_specs_installed( self, mock_specs: Dict[str, Spec], temporary_store: Store ): """Test that the graph is empty when all specs are already installed.""" # Install all specs in the DAG for spec_name in ["root", "dep1", "dep2", "dep3"]: install_spec_in_db(mock_specs[spec_name], temporary_store) graph = BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) # All nodes should be pruned, resulting in an empty graph assert len(graph.nodes) == 0 assert len(graph.parent_to_child) == 0 assert len(graph.child_to_parent) == 0 def test_empty_graph_install_package_false_all_deps_installed( self, mock_specs: Dict[str, Spec], temporary_store: Store ): """Test empty graph when install_package=False and all dependencies are installed.""" # Install all dependencies (but not the root) for dep_name in ["dep1", "dep2", "dep3"]: install_spec_in_db(mock_specs[dep_name], temporary_store) graph = BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=False, # Don't install the root install_deps=True, database=temporary_store.db, ) # Root is pruned because install_package=False # Dependencies are pruned because they're installed # Result: empty graph assert len(graph.nodes) == 0 assert len(graph.parent_to_child) == 0 assert len(graph.child_to_parent) == 0 def test_pruning_leaf_node(self, mock_specs: Dict[str, Spec], temporary_store: Store): """Test that pruning a leaf node (no children) works correctly. This ensures the pruning logic handles the boundary condition where a node has no children to re-wire. """ # Install dep2, which is a leaf node (no children) dep2 = mock_specs["dep2"] install_spec_in_db(dep2, temporary_store) graph = BuildGraph( specs=[mock_specs["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=True, install_deps=True, database=temporary_store.db, ) dep2_hash = dep2.dag_hash() dep1_hash = mock_specs["dep1"].dag_hash() # dep2 should be pruned assert dep2_hash not in graph.nodes # dep1 (parent of dep2) should have no children now assert len(graph.parent_to_child[dep1_hash]) == 0 # dep2 should not appear in any mappings assert dep2_hash not in graph.parent_to_child assert dep2_hash not in graph.child_to_parent def test_pruning_root_node_with_install_package_false( self, mock_specs: Dict[str, Spec], temporary_store: Store ): """Test that pruning a root node (no parents in the context) works correctly. When install_package=False, root nodes are marked for pruning. This ensures the pruning logic handles the boundary condition where a node has no parents. """ graph = BuildGraph( specs=[mock_specs["dep1"]], root_policy="auto", dependencies_policy="auto", include_build_deps=False, install_package=False, # Prune the root install_deps=True, database=temporary_store.db, ) dep1_hash = mock_specs["dep1"].dag_hash() dep2_hash = mock_specs["dep2"].dag_hash() # dep1 should be pruned (it's the root and install_package=False) assert dep1_hash not in graph.nodes # dep2 (child of dep1) should still be in the graph assert dep2_hash in graph.nodes # dep2 should have no parents now (its only parent was pruned) assert not graph.child_to_parent.get(dep2_hash) # dep1 should not appear in any mappings assert dep1_hash not in graph.parent_to_child assert dep1_hash not in graph.child_to_parent @pytest.fixture def specs_with_test_deps(): """Create specs with test-typed dependencies. DAG structure: root -> dep (link) + test_dep (test) dep -> dep_test_dep (test) """ return create_dag( nodes=["root", "dep", "test_dep", "dep_test_dep"], edges=[ ("root", "dep", ("build", "link")), ("root", "test_dep", "test"), ("dep", "dep_test_dep", "test"), ], ) class TestBuildGraphTestDeps: """Tests for BuildGraph handling of TEST-typed dependencies.""" def test_tests_false_excludes_test_deps( self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store ): """Test that tests=False excludes TEST-typed dependencies.""" graph = BuildGraph( specs=[specs_with_test_deps["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=True, install_package=True, install_deps=True, database=temporary_store.db, tests=False, ) assert specs_with_test_deps["dep"].dag_hash() in graph.nodes assert specs_with_test_deps["test_dep"].dag_hash() not in graph.nodes assert specs_with_test_deps["dep_test_dep"].dag_hash() not in graph.nodes def test_tests_root_includes_test_deps_for_root( self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store ): """Test that tests=[root_name] includes test deps only for the root package.""" graph = BuildGraph( specs=[specs_with_test_deps["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=True, install_package=True, install_deps=True, database=temporary_store.db, tests=["root"], ) assert specs_with_test_deps["dep"].dag_hash() in graph.nodes assert specs_with_test_deps["test_dep"].dag_hash() in graph.nodes # dep's test dep is NOT included because tests=["root"] only applies to "root" assert specs_with_test_deps["dep_test_dep"].dag_hash() not in graph.nodes def test_tests_all_includes_test_deps_for_all( self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store ): """Test that tests=True includes TEST-typed deps for all packages.""" graph = BuildGraph( specs=[specs_with_test_deps["root"]], root_policy="auto", dependencies_policy="auto", include_build_deps=True, install_package=True, install_deps=True, database=temporary_store.db, tests=True, ) assert specs_with_test_deps["dep"].dag_hash() in graph.nodes assert specs_with_test_deps["test_dep"].dag_hash() in graph.nodes assert specs_with_test_deps["dep_test_dep"].dag_hash() in graph.nodes def test_mark_explicit_spec_excludes_build_only_deps( self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store ): """An installed-implicit spec in explicit_set should only traverse link/run deps, not build-only deps.""" root = specs_with_build_deps["root"] install_spec_in_db(root, temporary_store) assert temporary_store.db._data[root.dag_hash()].explicit is False graph = BuildGraph( specs=[root], root_policy="auto", dependencies_policy="auto", include_build_deps=True, install_package=True, install_deps=True, database=temporary_store.db, explicit_set={root.dag_hash()}, ) # root should be in graph (not pruned) because it needs to be marked explicit. assert root.dag_hash() in graph.nodes # build-only dep should NOT be pulled in since root is already installed. assert specs_with_build_deps["build_dep"].dag_hash() not in graph.nodes ================================================ FILE: lib/spack/spack/test/installer_tui.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for the BuildStatus terminal UI in new_installer.py""" import sys import pytest if sys.platform == "win32": pytest.skip("No Windows support", allow_module_level=True) import io import os from multiprocessing import Pipe from typing import List, Optional, Tuple import spack.new_installer as inst from spack.new_installer import BuildStatus, StdinReader class MockConnection: """Mock multiprocessing.Connection for testing""" def fileno(self): return -1 class MockSpec: """Minimal mock for spack.spec.Spec""" def __init__( self, name: str, version: str = "1.0", external: bool = False, prefix: Optional[str] = None ) -> None: self.name = name self.version = version self.external = external self.prefix = prefix or f"/fake/prefix/{name}" self._hash = name # Simple hash based on name def dag_hash(self, length: Optional[int] = None) -> str: if length: return self._hash[:length] return self._hash class SimpleTextIOWrapper(io.TextIOWrapper): """TextIOWrapper around a BytesIO buffer for testing of stdout behavior""" def __init__(self, tty: bool) -> None: self._buffer = io.BytesIO() self._tty = tty super().__init__(self._buffer, encoding="utf-8", line_buffering=True) def isatty(self) -> bool: return self._tty def getvalue(self) -> str: self.flush() return self._buffer.getvalue().decode("utf-8") def clear(self): self.flush() self._buffer.truncate(0) self._buffer.seek(0) def create_build_status( is_tty: bool = True, terminal_cols: int = 80, terminal_rows: int = 24, total: int = 0, verbose: bool = False, filter_padding: bool = False, color: Optional[bool] = None, ) -> Tuple[BuildStatus, List[float], SimpleTextIOWrapper]: """Helper function to create BuildStatus with mocked dependencies""" fake_stdout = SimpleTextIOWrapper(tty=is_tty) # Easy way to set the current time in tests before running UI updates time_values = [0.0] def mock_get_time(): return time_values[-1] def mock_get_terminal_size(): return os.terminal_size((terminal_cols, terminal_rows)) status = BuildStatus( total=total, stdout=fake_stdout, get_terminal_size=mock_get_terminal_size, get_time=mock_get_time, is_tty=is_tty, verbose=verbose, filter_padding=filter_padding, color=color, ) return status, time_values, fake_stdout def add_mock_builds(status: BuildStatus, count: int) -> List[MockSpec]: """Helper function to add builds to a BuildStatus instance""" specs = [MockSpec(f"pkg{i}", f"{i}.0") for i in range(count)] for spec in specs: status.add_build(spec, explicit=True, control_w_conn=MockConnection()) # type: ignore return specs class TestBasicStateManagement: """Test basic state management operations""" def test_on_resize(self): """Test that on_resize sets terminal_size_changed and update() fetches lazily""" sizes = [os.terminal_size((80, 24))] fake_stdout = SimpleTextIOWrapper(tty=True) status = BuildStatus( total=0, stdout=fake_stdout, get_terminal_size=lambda: sizes[-1], is_tty=True ) # terminal_size_changed is True from __init__; terminal_size is placeholder assert status.terminal_size_changed is True # After on_resize the flag stays set and dirty is True sizes.append(os.terminal_size((120, 40))) status.on_resize() assert status.terminal_size_changed is True assert status.dirty is True # The actual size is fetched lazily on the first update() status.update() assert status.terminal_size == os.terminal_size((120, 40)) assert status.terminal_size_changed is False def test_add_build(self): """Test that add_build adds builds correctly""" status, _, _ = create_build_status(total=2) spec1 = MockSpec("pkg1", "1.0") spec2 = MockSpec("pkg2", "2.0") status.add_build(spec1, explicit=True, control_w_conn=MockConnection()) assert len(status.builds) == 1 assert spec1.dag_hash() in status.builds assert status.builds[spec1.dag_hash()].name == "pkg1" assert status.builds[spec1.dag_hash()].explicit is True assert status.dirty is True status.add_build(spec2, explicit=False, control_w_conn=MockConnection()) assert len(status.builds) == 2 assert spec2.dag_hash() in status.builds assert status.builds[spec2.dag_hash()].explicit is False def test_update_state_transitions(self): """Test that update_state transitions states properly""" status, fake_time, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() # Update to 'building' state status.update_state(build_id, "building") assert status.builds[build_id].state == "building" assert status.builds[build_id].progress_percent is None assert status.completed == 0 # Update to 'finished' state status.update_state(build_id, "finished") assert status.builds[build_id].state == "finished" assert status.completed == 1 assert status.builds[build_id].finished_time == fake_time[0] + inst.CLEANUP_TIMEOUT def test_update_state_failed(self): """Test that failed state increments completed counter""" status, fake_time, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() status.update_state(build_id, "failed") assert status.builds[build_id].state == "failed" assert status.completed == 1 assert status.builds[build_id].finished_time == fake_time[0] + inst.CLEANUP_TIMEOUT def test_parse_log_summary(self, tmp_path): """Test that parse_log_summary parses the build log and stores the summary.""" status, _, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() # Create a fake log file with an error log_file = tmp_path / "build.log" log_file.write_text("error: something went wrong\n") status.builds[build_id].log_path = str(log_file) status.parse_log_summary(build_id) assert status.builds[build_id].log_summary is not None assert "error" in status.builds[build_id].log_summary.lower() def test_parse_log_summary_no_log_path(self): """Test that parse_log_summary is a no-op when log_path is not set.""" status, _, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() status.parse_log_summary(build_id) assert status.builds[build_id].log_summary is None def test_parse_log_summary_missing_file(self, tmp_path): """Test that parse_log_summary is a no-op when log file doesn't exist.""" status, _, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() status.builds[build_id].log_path = str(tmp_path / "nonexistent.log") status.parse_log_summary(build_id) assert status.builds[build_id].log_summary is None def test_update_progress(self): """Test that update_progress updates percentages""" status, _, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() # Update progress status.update_progress(build_id, 50, 100) assert status.builds[build_id].progress_percent == 50 assert status.dirty is True # Same percentage shouldn't mark dirty again status.dirty = False status.update_progress(build_id, 50, 100) assert status.dirty is False # Different percentage should mark dirty status.update_progress(build_id, 75, 100) assert status.builds[build_id].progress_percent == 75 assert status.dirty is True def test_completion_counter(self): """Test that completion counter increments correctly""" status, _, _ = create_build_status(total=3) specs = add_mock_builds(status, 3) assert status.completed == 0 status.update_state(specs[0].dag_hash(), "finished") assert status.completed == 1 status.update_state(specs[1].dag_hash(), "failed") assert status.completed == 2 status.update_state(specs[2].dag_hash(), "finished") assert status.completed == 3 class TestOutputRendering: """Test output rendering for TTY and non-TTY modes""" def test_non_tty_output(self): """Test that non-TTY mode prints simple state changes""" status, _, fake_stdout = create_build_status(is_tty=False) spec = MockSpec("mypackage", "1.0") status.add_build(spec, explicit=True, control_w_conn=MockConnection()) build_id = spec.dag_hash() status.update_state(build_id, "finished") output = fake_stdout.getvalue() assert "[+]" in output assert "mypackage" in output assert "1.0" in output assert "/fake/prefix/mypackage" in output # prefix is shown for finished builds # Non-TTY output should not contain ANSI escape codes assert "\033[" not in output def test_tty_output_contains_ansi(self): """Test that TTY mode produces ANSI codes""" status, _, fake_stdout = create_build_status() add_mock_builds(status, 1) # Call update to render status.update() output = fake_stdout.getvalue() # Should contain ANSI escape sequences assert "\033[" in output # Should contain progress header assert "Progress:" in output def test_no_output_when_not_dirty(self): """Test that update() skips rendering when not dirty""" status, _, fake_stdout = create_build_status() add_mock_builds(status, 1) status.update() # Clear stdout and mark not dirty fake_stdout.clear() status.dirty = False # Update should not produce output status.update() assert fake_stdout.getvalue() == "" def test_update_throttling(self): """Test that update() throttles redraws""" status, fake_time, fake_stdout = create_build_status() add_mock_builds(status, 1) # First update at time 0 fake_time[0] = 0.0 status.update() first_output = fake_stdout.getvalue() assert first_output != "" # Mark dirty and try to update immediately fake_stdout.clear() status.dirty = True fake_time[0] = 0.01 # Very small time advance # Should be throttled (next_update not reached) status.update() assert fake_stdout.getvalue() == "" # Advance time past throttle and try again fake_time[0] = 1.0 status.update() assert fake_stdout.getvalue() != "" def test_cursor_movement_vs_newlines(self): """Test that finished builds get newlines, active builds get cursor movements""" status, fake_time, fake_stdout = create_build_status(total=5) specs = add_mock_builds(status, 3) # First update renders 3 active builds fake_time[0] = 0.0 status.update() output1 = fake_stdout.getvalue() # Count newlines (\n) and cursor movements (\033[1B\r = move down 1 line) newlines1 = output1.count("\n") cursor_moves1 = output1.count("\033[1B\r") # Initially all lines should be newlines (nothing in history yet) assert newlines1 > 0 assert cursor_moves1 == 0 # Now finish 2 builds and add 2 more fake_stdout.clear() fake_time[0] = inst.CLEANUP_TIMEOUT + 0.1 status.update_state(specs[0].dag_hash(), "finished") status.update_state(specs[1].dag_hash(), "finished") spec4 = MockSpec("pkg3", "3.0") spec5 = MockSpec("pkg4", "4.0") status.add_build(spec4, explicit=True, control_w_conn=MockConnection()) status.add_build(spec5, explicit=True, control_w_conn=MockConnection()) # Second update: finished builds persist (newlines), active area updates (cursor moves) status.update() output2 = fake_stdout.getvalue() newlines2 = output2.count("\n") cursor_moves2 = output2.count("\033[1B\r") # Should have newlines for the 2 finished builds persisted to history # and cursor movements for the active area (header + 3 active builds) assert newlines2 > 0, "Should have newlines for finished builds" assert cursor_moves2 > 0, "Should have cursor movements for active area" # Finished builds should be printed with newlines assert "pkg0" in output2 assert "pkg1" in output2 class TestTimeBasedBehavior: """Test time-based behaviors like spinner and cleanup""" def test_spinner_updates(self): """Test that spinner advances over time""" status, fake_time, _ = create_build_status() add_mock_builds(status, 1) # Initial spinner index initial_index = status.spinner_index # Advance time past spinner interval fake_time[0] = inst.SPINNER_INTERVAL + 0.01 status.update() # Spinner should have advanced assert status.spinner_index == (initial_index + 1) % len(status.spinner_chars) def test_finished_package_cleanup(self): """Test that finished packages are cleaned up after timeout""" status, fake_time, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() # Mark as finished fake_time[0] = 0.0 status.update_state(build_id, "finished") # Build should still be in active builds assert build_id in status.builds assert len(status.finished_builds) == 0 # Advance time past cleanup timeout fake_time[0] = inst.CLEANUP_TIMEOUT + 0.01 status.update() # Build should now be moved to finished_builds and removed from active assert build_id not in status.builds # Note: finished_builds is cleared after rendering, so check it happened via side effects assert status.dirty or build_id not in status.builds def test_failed_packages_not_cleaned_up(self): """Test that failed packages stay in active builds""" status, fake_time, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() # Mark as failed fake_time[0] = 0.0 status.update_state(build_id, "failed") # Advance time past cleanup timeout fake_time[0] = inst.CLEANUP_TIMEOUT + 0.01 status.update() # Failed build should remain in active builds assert build_id in status.builds class TestSearchAndFilter: """Test search mode and filtering""" def test_enter_search_mode(self): """Test that enter_search enables search mode""" status, _, _ = create_build_status() assert status.search_mode is False status.enter_search() assert status.search_mode is True assert status.dirty is True def test_search_input_printable(self): """Test that printable characters are added to search term""" status, _, _ = create_build_status() status.enter_search() status.search_input("a") assert status.search_term == "a" status.search_input("b") assert status.search_term == "ab" status.search_input("c") assert status.search_term == "abc" def test_search_input_backspace(self): """Test that backspace removes characters""" status, _, _ = create_build_status() status.enter_search() status.search_input("a") status.search_input("b") status.search_input("c") assert status.search_term == "abc" status.search_input("\x7f") # Backspace assert status.search_term == "ab" status.search_input("\b") # Alternative backspace assert status.search_term == "a" def test_search_input_escape(self): """Test that escape exits search mode""" status, _, _ = create_build_status() status.enter_search() status.search_input("test") status.search_input("\x1b") # Escape assert status.search_mode is False assert status.search_term == "" def test_is_displayed_filters_by_name(self): """Test that _is_displayed filters by package name""" status, _, _ = create_build_status(total=3) spec1 = MockSpec("package-foo", "1.0") spec2 = MockSpec("package-bar", "1.0") spec3 = MockSpec("other", "1.0") status.add_build(spec1, explicit=True, control_w_conn=MockConnection()) status.add_build(spec2, explicit=True, control_w_conn=MockConnection()) status.add_build(spec3, explicit=True, control_w_conn=MockConnection()) build1 = status.builds[spec1.dag_hash()] build2 = status.builds[spec2.dag_hash()] build3 = status.builds[spec3.dag_hash()] # No search term: all displayed status.search_term = "" assert status._is_displayed(build1) assert status._is_displayed(build2) assert status._is_displayed(build3) # Search for "package" status.search_term = "package" assert status._is_displayed(build1) assert status._is_displayed(build2) assert not status._is_displayed(build3) # Search for "foo" status.search_term = "foo" assert status._is_displayed(build1) assert not status._is_displayed(build2) assert not status._is_displayed(build3) def test_is_displayed_filters_by_hash(self): """Test that _is_displayed filters by hash prefix""" status, _, _ = create_build_status(total=2) spec1 = MockSpec("pkg1", "1.0") spec1._hash = "abc123" spec2 = MockSpec("pkg2", "1.0") spec2._hash = "def456" status.add_build(spec1, explicit=True, control_w_conn=MockConnection()) status.add_build(spec2, explicit=True, control_w_conn=MockConnection()) build1 = status.builds[spec1.dag_hash()] build2 = status.builds[spec2.dag_hash()] # Search by hash prefix status.search_term = "abc" assert status._is_displayed(build1) assert not status._is_displayed(build2) status.search_term = "def" assert not status._is_displayed(build1) assert status._is_displayed(build2) class TestNavigation: """Test navigation between builds""" def test_get_next_basic(self): """Test basic next/previous navigation""" status, _, _ = create_build_status(total=3) specs = add_mock_builds(status, 3) # Get first build first_id = status._get_next(1) assert first_id == specs[0].dag_hash() # Set tracked and get next status.tracked_build_id = first_id next_id = status._get_next(1) assert next_id == specs[1].dag_hash() # Get next again status.tracked_build_id = next_id next_id = status._get_next(1) assert next_id == specs[2].dag_hash() # Wrap around status.tracked_build_id = next_id next_id = status._get_next(1) assert next_id == specs[0].dag_hash() def test_get_next_previous(self): """Test backward navigation""" status, _, _ = create_build_status(total=3) specs = add_mock_builds(status, 3) # Start at second build status.tracked_build_id = specs[1].dag_hash() # Go backward prev_id = status._get_next(-1) assert prev_id == specs[0].dag_hash() # Go backward again (wrap around) status.tracked_build_id = prev_id prev_id = status._get_next(-1) assert prev_id == specs[2].dag_hash() def test_get_next_with_filter(self): """Test navigation respects search filter""" status, _, _ = create_build_status(total=4) specs = [ MockSpec("package-a", "1.0"), MockSpec("package-b", "1.0"), MockSpec("other-c", "1.0"), MockSpec("package-d", "1.0"), ] for spec in specs: status.add_build(spec, explicit=True, control_w_conn=MockConnection()) # Filter to only "package-*" status.search_term = "package" # Should only navigate through matching builds first_id = status._get_next(1) assert first_id and first_id == specs[0].dag_hash() status.tracked_build_id = first_id next_id = status._get_next(1) assert next_id and next_id == specs[1].dag_hash() status.tracked_build_id = next_id next_id = status._get_next(1) # Should skip "other-c" and go to "package-d" assert next_id and next_id == specs[3].dag_hash() def test_get_next_skips_finished(self): """Test that navigation skips finished builds""" status, _, _ = create_build_status(total=3) specs = add_mock_builds(status, 3) # Mark middle build as finished status.update_state(specs[1].dag_hash(), "finished") # Navigate from first status.tracked_build_id = specs[0].dag_hash() next_id = status._get_next(1) # Should skip finished build and go to third assert next_id == specs[2].dag_hash() def test_get_next_no_matching(self): """Test that _get_next returns None when no builds match""" status, _, _ = create_build_status(total=2) specs = add_mock_builds(status, 2) # Mark both as finished for spec in specs: status.update_state(spec.dag_hash(), "finished") # Should return None since no unfinished builds result = status._get_next(1) assert result is None def test_get_next_fallback_when_tracked_filtered_out(self): """Test that _get_next falls back correctly when tracked build no longer matches filter""" status, _, _ = create_build_status(total=3) specs = [ MockSpec("package-a", "1.0"), MockSpec("package-b", "1.0"), MockSpec("other-c", "1.0"), ] for spec in specs: status.add_build(spec, explicit=True, control_w_conn=MockConnection()) # Start tracking "other-c" status.tracked_build_id = specs[2].dag_hash() # Now apply a filter that excludes the tracked build status.search_term = "package" # _get_next should fall back to first matching build (forward) next_id = status._get_next(1) assert next_id == specs[0].dag_hash() # Test backward direction, should fall back to last matching build status.tracked_build_id = specs[2].dag_hash() # Reset to filtered-out build prev_id = status._get_next(-1) assert prev_id == specs[1].dag_hash() class TestTerminalSizes: """Test behavior with different terminal sizes""" def test_small_terminal_truncation(self): """Test that output is truncated for small terminals""" status, _, fake_stdout = create_build_status(total=10, terminal_cols=80, terminal_rows=10) # Add more builds than can fit on screen add_mock_builds(status, 10) status.update() output = fake_stdout.getvalue() # Should contain "more..." message indicating truncation assert "more..." in output def test_large_terminal_no_truncation(self): """Test that all builds shown on large terminal""" status, _, fake_stdout = create_build_status(total=3, terminal_cols=120) add_mock_builds(status, 3) status.update() output = fake_stdout.getvalue() # Should not contain truncation message assert "more..." not in output # Should contain all package names for i in range(3): assert f"pkg{i}" in output def test_narrow_terminal_short_header(self): """Test that narrow terminals get shortened header""" status, _, fake_stdout = create_build_status(total=1, terminal_cols=40) add_mock_builds(status, 1) status.update() output = fake_stdout.getvalue() # Should not contain the full header with hints assert "filter" not in output # But should contain progress assert "Progress:" in output class TestBuildInfo: """Test the BuildInfo dataclass""" def test_build_info_creation(self): """Test that BuildInfo is created correctly""" spec = MockSpec("mypackage", "1.0") build_info = inst.BuildInfo(spec, explicit=True, control_w_conn=MockConnection()) assert build_info.name == "mypackage" assert build_info.version == "1.0" assert build_info.explicit is True assert build_info.external is False assert build_info.state == "starting" assert build_info.finished_time is None assert build_info.progress_percent is None def test_build_info_external_package(self): """Test BuildInfo for external package""" spec = MockSpec("external-pkg", "1.0", external=True) build_info = inst.BuildInfo(spec, explicit=False, control_w_conn=MockConnection()) assert build_info.external is True class TestLogFollowing: """Test log following and print_logs functionality""" def test_print_logs_when_following(self): """Test that logs are printed when following a specific build""" status, _, fake_stdout = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() # Switch to log-following mode status.overview_mode = False status.tracked_build_id = build_id # Send some log data log_data = b"Building package...\nRunning tests...\n" status.print_logs(build_id, log_data) # Check that logs were echoed to stdout assert fake_stdout._buffer.getvalue() == log_data def test_print_logs_discarded_when_in_overview_mode(self): """Test that logs are discarded when in overview mode""" status, _, fake_stdout = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() # Stay in overview mode assert status.overview_mode is True # Try to print logs log_data = b"Should not be printed\n" status.print_logs(build_id, log_data) # Nothing should be printed assert fake_stdout.getvalue() == "" def test_print_logs_discarded_when_not_tracked(self): """Test that logs from non-tracked builds are discarded""" status, _, fake_stdout = create_build_status(total=2) spec1, spec2 = add_mock_builds(status, 2) # Switch to log-following mode for spec1 status.overview_mode = False status.tracked_build_id = spec1.dag_hash() # Try to print logs from spec2 (not tracked) log_data = b"Logs from pkg2\n" status.print_logs(spec2.dag_hash(), log_data) # Nothing should be printed since we're tracking pkg1, not pkg2 assert fake_stdout.getvalue() == "" def test_can_navigate_to_failed_build(self): """Test that navigating to a failed build shows log summary and path""" status, _, fake_stdout = create_build_status(total=3) specs = add_mock_builds(status, 3) # Mark the middle build as failed and set log info status.update_state(specs[1].dag_hash(), "failed") build_info = status.builds[specs[1].dag_hash()] build_info.log_summary = "Error: something went wrong\n" build_info.log_path = "/tmp/spack/pkg1.log" # Navigate from pkg0 to next -- should land on failed pkg1 status.tracked_build_id = specs[0].dag_hash() next_id = status._get_next(1) assert next_id == specs[1].dag_hash() # Actually navigate to it status.next(1) output = fake_stdout.getvalue() assert "Log summary of pkg1" in output assert "Error: something went wrong" in output assert "/tmp/spack/pkg1.log" in output def test_navigation_skips_finished_build(self): """Test that navigation skips successfully finished builds""" status, _, _ = create_build_status(total=3) specs = add_mock_builds(status, 3) # Mark the middle build as finished (successful) status.update_state(specs[1].dag_hash(), "finished") # Try to get next build, should skip the finished one status.tracked_build_id = specs[0].dag_hash() next_id = status._get_next(1) assert next_id == specs[2].dag_hash() class TestNavigationIntegration: """Test the next() method and navigation between builds""" def test_next_switches_from_overview_to_logs(self): """Test that next() switches from overview mode to log-following mode""" status, _, fake_stdout = create_build_status(total=2) specs = add_mock_builds(status, 2) # Start in overview mode assert status.overview_mode is True assert status.tracked_build_id == "" # Call next() to start following first build status.next() # Should have switched to log-following mode assert status.overview_mode is False assert status.tracked_build_id == specs[0].dag_hash() # Should have printed "Following logs" message output = fake_stdout.getvalue() assert "Following logs of" in output assert "pkg0" in output def test_next_cycles_through_builds(self): """Test that next() cycles through multiple builds""" status, _, fake_stdout = create_build_status(total=3) specs = add_mock_builds(status, 3) # Start following first build status.next() assert status.tracked_build_id == specs[0].dag_hash() fake_stdout.clear() # Navigate to next status.next(1) assert status.tracked_build_id == specs[1].dag_hash() assert "pkg1" in fake_stdout.getvalue() fake_stdout.clear() # Navigate to next (third build) status.next(1) assert status.tracked_build_id == specs[2].dag_hash() assert "pkg2" in fake_stdout.getvalue() fake_stdout.clear() # Navigate to next (should wrap to first) status.next(1) assert status.tracked_build_id == specs[0].dag_hash() assert "pkg0" in fake_stdout.getvalue() def test_next_backward_navigation(self): """Test that next(-1) navigates backward""" status, _, _ = create_build_status(total=3) specs = add_mock_builds(status, 3) # Start at first build status.next() assert status.tracked_build_id == specs[0].dag_hash() # Go backward (should wrap to last) status.next(-1) assert status.tracked_build_id == specs[2].dag_hash() # Go backward again status.next(-1) assert status.tracked_build_id == specs[1].dag_hash() def test_next_does_nothing_when_no_builds(self): """Test that next() does nothing when no unfinished builds exist""" status, _, _ = create_build_status(total=1) (spec,) = add_mock_builds(status, 1) # Mark as finished status.update_state(spec.dag_hash(), "finished") # Try to navigate initial_mode = status.overview_mode initial_tracked = status.tracked_build_id status.next() # Nothing should change assert status.overview_mode == initial_mode assert status.tracked_build_id == initial_tracked def test_next_does_nothing_when_same_build(self): """Test that next() doesn't re-print when already on the same build""" status, _, fake_stdout = create_build_status(total=1) (spec,) = add_mock_builds(status, 1) # Start following status.next() assert status.tracked_build_id == spec.dag_hash() # Clear output fake_stdout.clear() # Try to navigate to "next" (which is the same build) status.next() # Should not print anything assert fake_stdout.getvalue() == "" class TestToggle: """Test toggle() method for switching between overview and log-following modes""" def test_toggle_from_overview_calls_next(self): """Test that toggle() from overview mode calls next()""" status, _, fake_stdout = create_build_status(total=2) add_mock_builds(status, 2) # Start in overview mode assert status.overview_mode is True # Toggle should call next() status.toggle() # Should now be following logs assert status.overview_mode is False assert status.tracked_build_id != "" assert "Following logs of" in fake_stdout.getvalue() def test_toggle_from_logs_returns_to_overview(self): """Test that toggle() from log-following mode returns to overview""" status, _, _ = create_build_status(total=2) add_mock_builds(status, 2) # Switch to log-following mode first status.next() assert status.overview_mode is False tracked_id = status.tracked_build_id assert tracked_id != "" # Set some search state to verify cleanup status.search_term = "test" status.search_mode = True status.active_area_rows = 5 # Toggle back to overview status.toggle() # Should be back in overview mode with cleaned state assert status.overview_mode is True assert status.tracked_build_id == "" assert status.search_term == "" assert status.search_mode is False assert status.active_area_rows == 0 assert status.dirty is True def test_update_state_finished_triggers_toggle_when_tracking(self): """Test that finishing a tracked build triggers toggle back to overview""" status, _, _ = create_build_status(total=2) specs = add_mock_builds(status, 2) # Start tracking first build status.next() assert status.overview_mode is False assert status.tracked_build_id == specs[0].dag_hash() # Mark the tracked build as finished status.update_state(specs[0].dag_hash(), "finished") # Should have toggled back to overview mode assert status.overview_mode is True assert status.tracked_build_id == "" def test_partial_line_newline_on_toggle_and_next(self): """Ensure newline is inserted before mode transitions when log doesn't end with newline.""" status, _, fake_stdout = create_build_status(total=2) specs = add_mock_builds(status, 2) build_a, build_b = specs[0].dag_hash(), specs[1].dag_hash() # Follow a build, toggle back and forth between logs and overview mode, and receive logs # that may or may not end with newlines. status.next() status.print_logs(build_a, b"checking for foo...") status.toggle() status.next() status.print_logs(build_a, b"checking for bar... yes\n") status.next(1) status.print_logs(build_b, b"checking for baz...") status.next(-1) written = fake_stdout.getvalue() # There shouldn't be any double newlines: assert "\n\n" not in written # All partial and newline-terminated logs should be present with appropriate newlines: assert "checking for foo...\n" in written assert "checking for bar... yes\n" in written assert "checking for baz...\n" in written @pytest.mark.parametrize("filter_padding", [True, False]) def test_print_logs_filters_padding(self, filter_padding): """print_logs strips path-padding placeholders before writing to stdout.""" status, _, fake_stdout = create_build_status(filter_padding=filter_padding) (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() log_output = b"--with-foo=/base/__spack_path_placeholder__/__spack_path_placeholder__/bin" # track the build and print logs with the relevant path. status.overview_mode = False status.tracked_build_id = build_id status.print_logs(build_id, log_output) written = fake_stdout._buffer.getvalue() if filter_padding: assert written == b"--with-foo=/base/[padded-to-59-chars]/bin" else: assert written == log_output @pytest.mark.parametrize("filter_padding", [True, False]) def test_prefix_padding_filter_in_status(self, filter_padding): """Test that prefix in status indicator applies padding filter.""" padded_prefix = "/base/__spack_path_placeholder__/__spack_path_placeholder__/mypackage" status, _, fake_stdout = create_build_status(is_tty=False, filter_padding=filter_padding) spec = MockSpec("mypackage", "1.0", prefix=padded_prefix) status.add_build(spec, explicit=True, control_w_conn=MockConnection()) build_id = spec.dag_hash() status.update_state(build_id, "finished") output = fake_stdout.getvalue() common = f"[+] {spec.dag_hash(7)} {spec.name}@{spec.version}" if filter_padding: assert output == f"{common} /base/[padded-to-59-chars]/mypackage\n" else: assert output == f"{common} {padded_prefix}\n" class TestSearchFilteringIntegration: """Test search mode with display filtering""" def test_search_mode_filters_displayed_builds(self): """Test that search mode actually filters what's displayed""" status, _, fake_stdout = create_build_status(total=4) specs = [ MockSpec("package-foo", "1.0"), MockSpec("package-bar", "2.0"), MockSpec("other-thing", "3.0"), MockSpec("package-baz", "4.0"), ] for spec in specs: status.add_build(spec, explicit=True, control_w_conn=MockConnection()) # Enter search mode and search for "package" status.enter_search() assert status.search_mode is True for character in "package": status.search_input(character) assert status.search_term == "package" # Update to render status.update() output = fake_stdout.getvalue() # Should contain filtered builds assert "package-foo" in output assert "package-bar" in output assert "package-baz" in output # Should not contain the filtered-out build assert "other-thing" not in output # Should show filter prompt assert "filter>" in output assert status.search_term in output def test_search_mode_with_navigation(self): """Test that navigation respects search filter""" status, _, _ = create_build_status(total=4) specs = [ MockSpec("package-a", "1.0"), MockSpec("other-b", "2.0"), MockSpec("package-c", "3.0"), MockSpec("other-d", "4.0"), ] for spec in specs: status.add_build(spec, explicit=True, control_w_conn=MockConnection()) # Set search term to filter for "package" status.search_term = "package" # Start navigating, should only go through "package-a" and "package-c" status.next() assert status.tracked_build_id == specs[0].dag_hash() # package-a status.next(1) # Should skip other-b and go to package-c assert status.tracked_build_id == specs[2].dag_hash() # package-c status.next(1) # Should wrap around to package-a assert status.tracked_build_id == specs[0].dag_hash() # package-a def test_search_input_enter_navigates_to_next(self): """Test that pressing enter in search mode navigates to next match""" status, _, _ = create_build_status(total=3) specs = add_mock_builds(status, 3) # Enter search mode status.enter_search() for character in "pkg": status.search_input(character) # Press enter (should navigate to first match) status.search_input("\r") # Should have started following first matching build assert status.overview_mode is False assert status.tracked_build_id == specs[0].dag_hash() def test_clearing_search_shows_all_builds(self): """Test that clearing search term shows all builds again""" status, _, fake_stdout = create_build_status(total=3) specs = [ MockSpec("package-a", "1.0"), MockSpec("other-b", "2.0"), MockSpec("package-c", "3.0"), ] for spec in specs: status.add_build(spec, explicit=True, control_w_conn=MockConnection()) # Enter search and type something status.enter_search() status.search_input("p") status.search_input("a") status.search_input("c") assert status.search_term == "pac" # Clear it with backspace status.search_input("\x7f") # backspace status.search_input("\x7f") # backspace status.search_input("\x7f") # backspace assert status.search_term == "" # Update to render status.update() output = fake_stdout.getvalue() # All builds should be visible now assert "package-a" in output assert "other-b" in output assert "package-c" in output class TestEdgeCases: """Test edge cases and error conditions""" def test_empty_build_list(self): """Test update with no builds""" status, _, fake_stdout = create_build_status(total=0) status.update() output = fake_stdout.getvalue() # Should render header but no builds assert "Progress:" in output assert "0/0" in output def test_no_header_with_finalize(self): """Test that we don't print a header with finalize=True""" status, _, fake_stdout = create_build_status(total=2, color=False) spec_a, spec_b = add_mock_builds(status, 2) status.update_state(spec_a.dag_hash(), "finished") status.update_state(spec_b.dag_hash(), "failed") status.update(finalize=True) output = fake_stdout.getvalue() # Should not contain header assert "Progress:" not in output # Should contain final status lines for both builds assert f"[+] {spec_a.dag_hash(7)} {spec_a.name}@{spec_a.version}" in output assert f"[x] {spec_b.dag_hash(7)} {spec_b.name}@{spec_b.version}" in output def test_all_builds_finished(self): """Test when all builds are finished""" status, fake_time, _ = create_build_status(total=2) specs = add_mock_builds(status, 2) # Mark all as finished for spec in specs: status.update_state(spec.dag_hash(), "finished") # Advance time and update fake_time[0] = inst.CLEANUP_TIMEOUT + 0.01 status.update() # All should be cleaned up assert len(status.builds) == 0 assert status.completed == 2 def test_update_progress_rounds_correctly(self): """Test that progress percentage rounding works""" status, _, _ = create_build_status() (spec,) = add_mock_builds(status, 1) build_id = spec.dag_hash() # Test rounding status.update_progress(build_id, 1, 3) assert status.builds[build_id].progress_percent == 33 # int(100/3) status.update_progress(build_id, 2, 3) assert status.builds[build_id].progress_percent == 66 # int(200/3) status.update_progress(build_id, 3, 3) assert status.builds[build_id].progress_percent == 100 class TestBuildStatusVerbose: """Tests for verbose non-TTY log tracking in BuildStatus.""" def test_verbose_tracks_first_build(self): """First add_build() in verbose non-TTY mode sets tracked_build_id and enables echoing.""" bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4) spec = MockSpec("trivial-install-test-package", "1.0") r_conn, w_conn = Pipe(duplex=False) with r_conn, w_conn: bs.add_build(spec, explicit=True, control_w_conn=w_conn) assert bs.tracked_build_id == spec.dag_hash() written = os.read(r_conn.fileno(), 1) assert written == b"1" def test_verbose_does_not_track_when_already_tracking(self): """Second add_build() while already tracking does not switch tracking.""" bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4) spec1 = MockSpec("pkg1", "1.0") spec2 = MockSpec("pkg2", "1.0") r1, w1 = Pipe(duplex=False) r2, w2 = Pipe(duplex=False) with r1, w1, r2, w2: bs.add_build(spec1, explicit=True, control_w_conn=w1) first_tracked = bs.tracked_build_id bs.add_build(spec2, explicit=False, control_w_conn=w2) assert bs.tracked_build_id == first_tracked assert bs.tracked_build_id == spec1.dag_hash() # Second build should not have received b"1" assert not r2.poll(), "Second build should not be enabled" def test_verbose_switches_on_finish(self): """After the tracked build finishes, tracked_build_id is cleared.""" bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4) spec = MockSpec("trivial-install-test-package", "1.0") r_conn, w_conn = Pipe(duplex=False) with r_conn, w_conn: bs.add_build(spec, explicit=True, control_w_conn=w_conn) assert bs.tracked_build_id == spec.dag_hash() bs.update_state(spec.dag_hash(), "finished") assert bs.tracked_build_id == "" def test_verbose_print_logs_tracked(self): """print_logs() for the tracked build writes to stdout.""" bs, _, stdout = create_build_status(is_tty=False, verbose=True, total=1) spec = MockSpec("trivial-install-test-package", "1.0") r_conn, w_conn = Pipe(duplex=False) with r_conn, w_conn: bs.add_build(spec, explicit=True, control_w_conn=w_conn) bs.print_logs(spec.dag_hash(), b"hello log\n") stdout.flush() assert stdout.buffer.getvalue() == b"hello log\n" def test_verbose_print_logs_untracked(self): """print_logs() for an untracked build discards data.""" bs, _, stdout = create_build_status(is_tty=False, verbose=True, total=2) spec1 = MockSpec("pkg1", "1.0") spec2 = MockSpec("pkg2", "1.0") r1, w1 = Pipe(duplex=False) with r1, w1: bs.add_build(spec1, explicit=True, control_w_conn=w1) bs.add_build(spec2, explicit=False, control_w_conn=None) # Only spec1 is tracked; spec2 logs should be discarded bs.print_logs(spec2.dag_hash(), b"ignored\n") stdout.flush() assert stdout.buffer.getvalue() == b"" def test_verbose_tty_no_effect(self): """In TTY mode, add_build() does not set tracked_build_id automatically.""" bs, _, _ = create_build_status(is_tty=True, verbose=True, total=4) spec = MockSpec("trivial-install-test-package", "1.0") r_conn, w_conn = Pipe(duplex=False) with r_conn, w_conn: bs.add_build(spec, explicit=True, control_w_conn=w_conn) assert bs.tracked_build_id == "" class TestBuildStatusColor: """Tests that BuildStatus respects the explicit color=True/False parameter.""" def test_non_tty_finished_color_true_emits_green(self): """color=True in non-TTY mode: finished line has per-component ANSI colors.""" spec = MockSpec("pkg", "1.0") status, _, stdout = create_build_status(is_tty=False, total=1, color=True) status.add_build(spec, explicit=True) status.update_state(spec.dag_hash(), "finished") # green indicator, reset, dark-gray hash assert stdout.getvalue().startswith("\033[32m[+]\033[0m \033[0;90m") def test_non_tty_failed_color_true_emits_red(self): """color=True in non-TTY mode: failed line has per-component ANSI colors.""" spec = MockSpec("pkg", "1.0") status, _, stdout = create_build_status(is_tty=False, total=1, color=True) status.add_build(spec, explicit=True) status.update_state(spec.dag_hash(), "failed") # red indicator, reset, dark-gray hash assert stdout.getvalue().startswith("\033[31m[x]\033[0m \033[0;90m") def test_non_tty_finished_color_false_no_ansi(self): """color=False in non-TTY mode: finished line has no ANSI escape codes.""" spec = MockSpec("pkg", "1.0") status, _, stdout = create_build_status(is_tty=False, total=1, color=False) status.add_build(spec, explicit=True) status.update_state(spec.dag_hash(), "finished") assert "\033[" not in stdout.getvalue() class TestTargetJobs: """Test set_jobs and its effect on the header.""" def test_set_jobs_marks_dirty(self): """set_jobs with a new value should update target_jobs and mark dirty.""" status, _, _ = create_build_status() status.dirty = False status.set_jobs(3, 2) assert status.actual_jobs == 3 assert status.target_jobs == 2 assert status.dirty is True status.set_jobs(2, 2) assert status.actual_jobs == 2 assert status.target_jobs == 2 def test_set_jobs_same_value_no_dirty(self): """set_jobs with the same value should not mark dirty.""" status, _, _ = create_build_status() status.set_jobs(5, 5) status.dirty = False status.set_jobs(5, 5) assert status.dirty is False def test_header_shows_target_jobs(self): """The rendered header should contain the target_jobs count and the word 'jobs'.""" status, _, fake_stdout = create_build_status(total=1) add_mock_builds(status, 1) status.set_jobs(4, 4) status.update() output = fake_stdout.getvalue() assert "4" in output assert "jobs" in output def test_header_shows_arrow_when_pending(self): """When actual != target, the header should show 'actual=>target jobs'.""" status, _, fake_stdout = create_build_status(total=1) add_mock_builds(status, 1) status.set_jobs(4, 2) status.update() output = fake_stdout.getvalue() assert "4=>2" in output class TestHeadlessMode: """Test that headless mode suppresses terminal output.""" def test_update_suppressed_when_headless(self): """update() should not write anything when headless is True.""" status, time_values, stdout = create_build_status(is_tty=True, total=1) add_mock_builds(status, 1) status.headless = True time_values.append(10.0) status.update() assert stdout.getvalue() == "" def test_print_logs_suppressed_when_headless(self): """print_logs() should discard data when headless is True.""" status, _, stdout = create_build_status(is_tty=True, total=1) specs = add_mock_builds(status, 1) status.tracked_build_id = specs[0].dag_hash() status.headless = True status.print_logs(specs[0].dag_hash(), b"hello world\n") assert stdout.getvalue() == "" def test_update_state_non_tty_suppressed_when_headless(self): """update_state() non-TTY output should be suppressed when headless.""" status, _, stdout = create_build_status(is_tty=False, total=1) spec = MockSpec("pkg", "1.0") status.add_build(spec, explicit=True) status.headless = True stdout.clear() status.update_state(spec.dag_hash(), "finished") assert stdout.getvalue() == "" def test_update_works_after_headless_cleared(self): """update() should work normally once headless is cleared.""" status, time_values, stdout = create_build_status(is_tty=True, total=1, color=False) add_mock_builds(status, 1) status.headless = True time_values.append(10.0) status.update() assert stdout.getvalue() == "" # Clear headless and verify output resumes status.headless = False status.dirty = True status.update() assert "[/] pkg0 pkg0@0.0 starting" in stdout.getvalue() class TestStdinReader: def test_basic_ascii(self): r, w = os.pipe() try: reader = StdinReader(r) os.write(w, b"abc") assert reader.read() == "abc" finally: os.close(r) os.close(w) def test_ansi_stripping(self): r, w = os.pipe() try: reader = StdinReader(r) os.write(w, b"hello\x1b[Aworld\x1b[B!") assert reader.read() == "helloworld!" finally: os.close(r) os.close(w) def test_multibyte_utf8(self): r, w = os.pipe() try: reader = StdinReader(r) encoded = "é".encode("utf-8") # 0xc3 0xa9 os.write(w, encoded[:1]) # First read: incomplete char, decoder buffers it result1 = reader.read() os.write(w, encoded[1:]) result2 = reader.read() assert result1 + result2 == "é" finally: os.close(r) os.close(w) def test_oserror_returns_empty(self): r, w = os.pipe() os.close(w) os.close(r) reader = StdinReader(r) assert reader.read() == "" ================================================ FILE: lib/spack/spack/test/jobserver.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import sys import pytest if sys.platform == "win32": pytest.skip("Jobserver tests are not supported on Windows", allow_module_level=True) import fcntl import os import pathlib import stat from spack.new_installer import ( JobServer, create_jobserver_fifo, get_jobserver_config, open_existing_jobserver_fifo, ) from spack.spec import Spec class TestGetJobserverConfig: """Test parsing of MAKEFLAGS for jobserver configuration.""" def test_empty_makeflags(self): """Empty MAKEFLAGS should return None.""" assert get_jobserver_config("") is None def test_no_jobserver_flag(self): """MAKEFLAGS without jobserver flag should return None.""" assert get_jobserver_config(" -j4 --silent") is None def test_fifo_format_new(self): """Parse new FIFO format""" assert get_jobserver_config(" -j4 --jobserver-auth=fifo:/tmp/my_fifo") == "/tmp/my_fifo" def test_pipe_format_new(self): """Parse new pipe format""" assert get_jobserver_config(" -j4 --jobserver-auth=3,4") == (3, 4) def test_pipe_format_old(self): """Parse old pipe format (on old versions of gmake this was not publicized)""" assert get_jobserver_config(" -j4 --jobserver-fds=5,6") == (5, 6) def test_multiple_flags_last_wins(self): """When multiple jobserver flags exist, last one wins.""" makeflags = " --jobserver-fds=3,4 --jobserver-auth=fifo:/tmp/fifo --jobserver-auth=7,8" assert get_jobserver_config(makeflags) == (7, 8) def test_invalid_format(self): assert get_jobserver_config(" --jobserver-auth=3") is None assert get_jobserver_config(" --jobserver-auth=a,b") is None assert get_jobserver_config(" --jobserver-auth=3,b") is None assert get_jobserver_config(" --jobserver-auth=3,4,5") is None assert get_jobserver_config(" --jobserver-auth=") is None class TestCreateJobserverFifo: """Test FIFO creation for jobserver.""" def test_creates_fifo(self): """Should create a FIFO with correct properties.""" r, w, path = create_jobserver_fifo(4) try: assert os.path.exists(path) assert stat.S_ISFIFO(os.stat(path).st_mode) assert (os.stat(path).st_mode & 0o777) == 0o600 assert fcntl.fcntl(r, fcntl.F_GETFD) != -1 assert fcntl.fcntl(w, fcntl.F_GETFD) != -1 assert fcntl.fcntl(r, fcntl.F_GETFL) & os.O_NONBLOCK finally: os.close(r) os.close(w) os.unlink(path) os.rmdir(os.path.dirname(path)) def test_writes_correct_tokens(self): """Should write num_jobs - 1 tokens.""" r, w, path = create_jobserver_fifo(5) try: assert os.read(r, 10) == b"++++" # 4 tokens for 5 jobs finally: os.close(r) os.close(w) os.unlink(path) os.rmdir(os.path.dirname(path)) def test_single_job_no_tokens(self): """Single job should write 0 tokens.""" r, w, path = create_jobserver_fifo(1) try: with pytest.raises(BlockingIOError): os.read(r, 10) # No tokens for 1 job finally: os.close(r) os.close(w) os.unlink(path) os.rmdir(os.path.dirname(path)) class TestOpenExistingJobserverFifo: """Test opening existing jobserver FIFOs.""" def test_opens_existing_fifo(self, tmp_path: pathlib.Path): """Should successfully open an existing FIFO.""" fifo_path = str(tmp_path / "test_fifo") os.mkfifo(fifo_path, 0o600) result = open_existing_jobserver_fifo(fifo_path) assert result is not None r, w = result assert fcntl.fcntl(r, fcntl.F_GETFD) != -1 assert fcntl.fcntl(w, fcntl.F_GETFD) != -1 assert fcntl.fcntl(r, fcntl.F_GETFL) & os.O_NONBLOCK os.close(r) os.close(w) def test_returns_none_for_missing_fifo(self, tmp_path: pathlib.Path): """Should return None if FIFO doesn't exist.""" result = open_existing_jobserver_fifo(str(tmp_path / "nonexistent_fifo")) assert result is None #: Constant that's larger than the number of jobs used in tests. ALL_TOKENS = 100 class TestJobServer: """Test JobServer class functionality.""" def test_creates_new_jobserver(self): """Should create a new FIFO-based jobserver when none exists.""" js = JobServer(4) try: assert js.created is True assert js.fifo_path is not None assert os.path.exists(js.fifo_path) assert js.tokens_acquired == 0 assert fcntl.fcntl(js.r, fcntl.F_GETFD) != -1 assert fcntl.fcntl(js.w, fcntl.F_GETFD) != -1 finally: js.close() def test_attaches_to_existing_fifo(self): """Should attach to existing FIFO jobserver from environment.""" js1 = JobServer(4) assert js1.fifo_path try: fifo_config = get_jobserver_config(f" -j4 --jobserver-auth=fifo:{js1.fifo_path}") assert fifo_config == js1.fifo_path result = open_existing_jobserver_fifo(js1.fifo_path) assert result is not None r, w = result os.close(r) os.close(w) finally: js1.close() def test_acquire_tokens(self): """Should acquire tokens from jobserver.""" js = JobServer(5) try: assert js.acquire(2) == 2 assert js.tokens_acquired == 2 assert js.acquire(2) == 2 assert js.tokens_acquired == 4 assert js.acquire(2) == 0 assert js.tokens_acquired == 4 finally: js.close() def test_release_tokens(self): """Should release tokens back to jobserver.""" js = JobServer(5) try: assert js.acquire(2) == 2 assert js.tokens_acquired == 2 js.release() assert js.tokens_acquired == 1 assert js.acquire(1) == 1 assert js.tokens_acquired == 2 finally: js.close() def test_release_without_tokens_is_noop(self): """Releasing without acquired tokens should be a no-op.""" js = JobServer(4) try: assert js.tokens_acquired == 0 js.release() assert js.tokens_acquired == 0 finally: js.close() def test_makeflags_fifo_gmake_44(self): """Should return FIFO format for gmake >= 4.4.""" js = JobServer(8) try: flags = js.makeflags(Spec("gmake@=4.4")) assert flags == f" -j8 --jobserver-auth=fifo:{js.fifo_path}" finally: js.close() def test_makeflags_pipe_gmake_40(self): """Should return pipe format for gmake 4.0-4.3.""" js = JobServer(8) try: flags = js.makeflags(Spec("gmake@=4.0")) assert flags == f" -j8 --jobserver-auth={js.r},{js.w}" finally: js.close() def test_makeflags_old_format_gmake_3(self): """Should return old --jobserver-fds format for gmake < 4.0.""" js = JobServer(8) try: flags = js.makeflags(Spec("gmake@=3.9")) assert flags == f" -j8 --jobserver-fds={js.r},{js.w}" finally: js.close() def test_makeflags_no_gmake(self): """Should return FIFO format when no gmake (modern default).""" js = JobServer(6) try: flags = js.makeflags(None) assert flags == f" -j6 --jobserver-auth=fifo:{js.fifo_path}" finally: js.close() def test_close_removes_created_fifo(self): """Should remove FIFO and directory if created by this instance.""" js = JobServer(4) fifo_path = js.fifo_path assert fifo_path and os.path.exists(fifo_path) js.close() assert not os.path.exists(os.path.dirname(fifo_path)) def test_file_descriptors_are_inheritable(self): """Should set file descriptors as inheritable for child processes.""" js = JobServer(4) try: assert os.get_inheritable(js.r) assert os.get_inheritable(js.w) finally: js.close() def test_connection_objects_exist(self): """Should create Connection objects for fd inheritance.""" js = JobServer(4) try: assert js.r_conn is not None and js.r_conn.fileno() == js.r assert js.w_conn is not None and js.w_conn.fileno() == js.w finally: js.close() def test_close_warns_when_spack_holds_tokens(self): """Should warn when Spack closes the jobserver while still holding acquired tokens.""" js = JobServer(4) js.acquire(1) # Spack acquires a token without releasing it with pytest.warns(UserWarning, match="Spack failed to release jobserver tokens"): js.close() def test_close_warns_when_subprocess_holds_tokens(self): """Should warn when a subprocess acquired a token but never released it.""" js1 = JobServer(4) os.read(js1.r, 1) # A subprocess acquires a token without releasing it with pytest.warns(UserWarning, match="1 jobserver token was not released"): js1.close() js2 = JobServer(4) os.read(js2.r, 2) # A subprocess acquires two tokens without releasing them with pytest.warns(UserWarning, match="2 jobserver tokens were not released"): js2.close() def test_has_target_parallelism(self): """has_target_parallelism() should be True initially.""" js = JobServer(4) try: assert js.has_target_parallelism() is True js.target_jobs = js.num_jobs - 1 assert js.has_target_parallelism() is False finally: js.close() def test_increase_parallelism_not_created(self): """increase_parallelism() should be a no-op when not self.created.""" # Simulate an externally attached jobserver by patching created after construction. js = JobServer(3) try: original_num = js.num_jobs original_target = js.target_jobs js.created = False js.increase_parallelism() assert js.num_jobs == original_num assert js.target_jobs == original_target js.decrease_parallelism() assert js.num_jobs == original_num assert js.target_jobs == original_target finally: js.created = True # restore so close() works js.close() def test_increase_parallelism(self): """increase_parallelism() should increment num_jobs and target_jobs and add a token.""" js = JobServer(3) try: original_num = js.num_jobs original_target = js.target_jobs js.increase_parallelism() assert js.num_jobs == original_num + 1 assert js.target_jobs == original_target + 1 # Verify the "js.num_jobs - 1 tokens in the pipe" invariant. assert js.acquire(ALL_TOKENS) + 1 == js.num_jobs finally: js.close() def test_decrease_parallelism_at_floor(self): """decrease_parallelism() should not go below target_jobs == 1.""" js = JobServer(1) try: # target_jobs starts at 1 assert js.target_jobs == 1 js.decrease_parallelism() assert js.target_jobs == 1 finally: js.close() def test_decrease_parallelism_token_available(self): """When pipe has tokens, decrease_parallelism discards one immediately.""" js = JobServer(3) try: # 3-job server starts with 2 tokens in the pipe. original_num = js.num_jobs js.decrease_parallelism() assert js.target_jobs == original_num - 1 assert js.num_jobs == original_num - 1 assert js.acquire(ALL_TOKENS) + 1 == js.num_jobs finally: js.close() def test_decrease_parallelism_no_token_available(self): """When all tokens are held, decrease_parallelism defers the discard.""" js = JobServer(3) try: # Drain the pipe so no tokens are available for immediate discard. assert js.acquire(ALL_TOKENS) == js.num_jobs - 1 original_num = js.num_jobs js.decrease_parallelism() # target_jobs decremented but num_jobs unchanged (no token to discard yet). assert js.target_jobs == original_num - 1 assert js.num_jobs == original_num finally: js.close() def test_maybe_discard_tokens_noop_at_target(self): """maybe_discard_tokens() should be a no-op when num_jobs == target_jobs.""" js = JobServer(3) try: original_num = js.num_jobs js.maybe_discard_tokens() # to_discard == 0 assert js.num_jobs == original_num finally: js.close() def test_maybe_discard_tokens_discards_when_available(self): """maybe_discard_tokens() should consume tokens from the pipe.""" js = JobServer(4) try: # Manually set target lower to create a discard requirement. js.target_jobs = js.num_jobs - 2 original_num = js.num_jobs js.maybe_discard_tokens() assert js.num_jobs < original_num finally: js.close() def test_maybe_discard_tokens_noop_on_blocking(self): """maybe_discard_tokens() should not raise when pipe is empty.""" js = JobServer(3) try: # Drain all tokens from the pipe (simulates subprocesses holding them). assert js.acquire(ALL_TOKENS) == js.num_jobs - 1 original_num = js.num_jobs # Artificially lower target so a discard is requested, but pipe is empty. js.target_jobs = js.num_jobs - 1 js.maybe_discard_tokens() # Should not raise; num_jobs unchanged. assert js.num_jobs == original_num finally: js.close() def test_release_discards_token_when_target_below_num(self): """release() should discard a token (not return it) when target_jobs < num_jobs.""" js = JobServer(4) try: # Acquire a token. assert js.acquire(1) == 1 assert js.tokens_acquired == 1 # Manually lower target to simulate a pending decrease. js.target_jobs = js.num_jobs - 1 original_num = js.num_jobs # Drain the free tokens from the pipe so we can count them after. drained = os.read(js.r, ALL_TOKENS) # Release should discard the token (decrement num_jobs) instead of writing to pipe. js.release() assert js.tokens_acquired == 0 assert js.num_jobs == original_num - 1 # Pipe should remain empty (nothing written back). with pytest.raises(BlockingIOError): os.read(js.r, 1) finally: # Restore drained tokens so close() can clean up cleanly. os.write(js.w, drained) js.close() ================================================ FILE: lib/spack/spack/test/link_paths.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import re import sys import pytest import spack.compilers.libraries import spack.paths from spack.compilers.libraries import parse_non_system_link_dirs drive = "" if sys.platform == "win32": match = re.search(r"[A-Za-z]:", spack.paths.test_path) if match: drive = match.group() root = drive + os.sep #: directory with sample compiler data datadir = os.path.join(spack.paths.test_path, "data", "compiler_verbose_output") @pytest.fixture(autouse=True) def allow_nonexistent_paths(monkeypatch): # Allow nonexistent paths to be detected as part of the output # for testing purposes. monkeypatch.setattr(spack.compilers.libraries, "filter_non_existing_dirs", lambda x: x) def check_link_paths(filename, paths): with open(os.path.join(datadir, filename), encoding="utf-8") as file: output = file.read() detected_paths = parse_non_system_link_dirs(output) actual = detected_paths expected = paths missing_paths = list(x for x in expected if x not in actual) assert not missing_paths extra_paths = list(x for x in actual if x not in expected) assert not extra_paths assert actual == expected def test_icc16_link_paths(): prefix = os.path.join(root, "usr", "tce", "packages") check_link_paths( "icc-16.0.3.txt", [ os.path.join( prefix, "intel", "intel-16.0.3", "compilers_and_libraries_2016.3.210", "linux", "compiler", "lib", "intel64_lin", ), os.path.join( prefix, "gcc", "gcc-4.9.3", "lib64", "gcc", "x86_64-unknown-linux-gnu", "4.9.3" ), os.path.join(prefix, "gcc", "gcc-4.9.3", "lib64"), ], ) def test_gcc7_link_paths(): check_link_paths("gcc-7.3.1.txt", []) def test_clang4_link_paths(): check_link_paths("clang-4.0.1.txt", []) def test_xl_link_paths(): check_link_paths( "xl-13.1.5.txt", [ os.path.join(root, "opt", "ibm", "xlsmp", "4.1.5", "lib"), os.path.join(root, "opt", "ibm", "xlmass", "8.1.5", "lib"), os.path.join(root, "opt", "ibm", "xlC", "13.1.5", "lib"), ], ) def test_cce_link_paths(): gcc = os.path.join(root, "opt", "gcc") cray = os.path.join(root, "opt", "cray") check_link_paths( "cce-8.6.5.txt", [ os.path.join(gcc, "6.1.0", "snos", "lib64"), os.path.join(cray, "dmapp", "default", "lib64"), os.path.join(cray, "pe", "mpt", "7.7.0", "gni", "mpich-cray", "8.6", "lib"), os.path.join(cray, "pe", "libsci", "17.12.1", "CRAY", "8.6", "x86_64", "lib"), os.path.join(cray, "rca", "2.2.16-6.0.5.0_15.34__g5e09e6d.ari", "lib64"), os.path.join(cray, "pe", "pmi", "5.0.13", "lib64"), os.path.join(cray, "xpmem", "2.2.4-6.0.5.0_4.8__g35d5e73.ari", "lib64"), os.path.join(cray, "dmapp", "7.1.1-6.0.5.0_49.8__g1125556.ari", "lib64"), os.path.join(cray, "ugni", "6.0.14-6.0.5.0_16.9__g19583bb.ari", "lib64"), os.path.join(cray, "udreg", "2.3.2-6.0.5.0_13.12__ga14955a.ari", "lib64"), os.path.join(cray, "alps", "6.5.28-6.0.5.0_18.6__g13a91b6.ari", "lib64"), os.path.join(cray, "pe", "atp", "2.1.1", "libApp"), os.path.join(cray, "pe", "cce", "8.6.5", "cce", "x86_64", "lib"), os.path.join(cray, "wlm_detect", "1.3.2-6.0.5.0_3.1__g388ccd5.ari", "lib64"), os.path.join(gcc, "6.1.0", "snos", "lib", "gcc", "x86_64-suse-linux", "6.1.0"), os.path.join( cray, "pe", "cce", "8.6.5", "binutils", "x86_64", "x86_64-unknown-linux-gnu", "lib" ), ], ) def test_clang_apple_ld_link_paths(): check_link_paths( "clang-9.0.0-apple-ld.txt", [ os.path.join( root, "Applications", "Xcode.app", "Contents", "Developer", "Platforms", "MacOSX.platform", "Developer", "SDKs", "MacOSX10.13.sdk", "usr", "lib", ) ], ) def test_nag_mixed_gcc_gnu_ld_link_paths(): # This is a test of a mixed NAG/GCC toolchain, i.e. 'cxx' is set to g++ and # is used for the rpath detection. The reference compiler output is a # result of # '/path/to/gcc/bin/g++ -Wl,-v ./main.c'. prefix = os.path.join( root, "scratch", "local1", "spack", "opt", "spack", "gcc-6.3.0-haswell", "gcc-6.5.0-4sdjgrs", ) check_link_paths( "collect2-6.3.0-gnu-ld.txt", [ os.path.join(prefix, "lib", "gcc", "x86_64-pc-linux-gnu", "6.5.0"), os.path.join(prefix, "lib64"), os.path.join(prefix, "lib"), ], ) def test_nag_link_paths(): # This is a test of a NAG-only toolchain, i.e. 'cc' and 'cxx' are empty, # and therefore 'fc' is used for the rpath detection). The reference # compiler output is a result of # 'nagfor -Wc=/path/to/gcc/bin/gcc -Wl,-v ./main.c'. prefix = os.path.join( root, "scratch", "local1", "spack", "opt", "spack", "gcc-6.3.0-haswell", "gcc-6.5.0-4sdjgrs", ) check_link_paths( "nag-6.2-gcc-6.5.0.txt", [ os.path.join(prefix, "lib", "gcc", "x86_64-pc-linux-gnu", "6.5.0"), os.path.join(prefix, "lib64"), os.path.join(prefix, "lib"), ], ) def test_obscure_parsing_rules(): paths = [ os.path.join(root, "first", "path"), os.path.join(root, "second", "path"), os.path.join(root, "third", "path"), ] # TODO: add a comment explaining why this happens if sys.platform == "win32": paths.remove(os.path.join(root, "second", "path")) check_link_paths("obscure-parsing-rules.txt", paths) ================================================ FILE: lib/spack/spack/test/llnl/llnl_string.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.llnl.string @pytest.mark.parametrize( "arguments,expected", [ ((0, "thing"), "0 things"), ((1, "thing"), "1 thing"), ((2, "thing"), "2 things"), ((1, "thing", "wombats"), "1 thing"), ((2, "thing", "wombats"), "2 wombats"), ((2, "thing", "wombats", False), "wombats"), ], ) def test_plural(arguments, expected): assert spack.llnl.string.plural(*arguments) == expected @pytest.mark.parametrize( "arguments,expected", [((["one", "two"],), ["'one'", "'two'"]), ((["one", "two"], "^"), ["^one^", "^two^"])], ) def test_quote(arguments, expected): assert spack.llnl.string.quote(*arguments) == expected @pytest.mark.parametrize( "input,expected_and,expected_or", [ (["foo"], "foo", "foo"), (["foo", "bar"], "foo and bar", "foo or bar"), (["foo", "bar", "baz"], "foo, bar, and baz", "foo, bar, or baz"), ], ) def test_comma_and_or(input, expected_and, expected_or): assert spack.llnl.string.comma_and(input) == expected_and assert spack.llnl.string.comma_or(input) == expected_or ================================================ FILE: lib/spack/spack/test/llnl/url.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for spack.llnl.url functions""" import itertools import pytest import spack.llnl.url @pytest.fixture(params=spack.llnl.url.ALLOWED_ARCHIVE_TYPES) def archive_and_expected(request): archive_name = ".".join(["Foo", request.param]) return archive_name, request.param def test_get_extension(archive_and_expected): """Tests that we can predict correctly known extensions for simple cases.""" archive, expected = archive_and_expected result = spack.llnl.url.extension_from_path(archive) assert result == expected def test_get_bad_extension(): """Tests that a bad extension returns None""" result = spack.llnl.url.extension_from_path("Foo.cxx") assert result is None @pytest.mark.parametrize( "url,expected", [ # No suffix ("rgb-1.0.6", "rgb-1.0.6"), # Misleading prefix ("jpegsrc.v9b", "jpegsrc.v9b"), ("turbolinux702", "turbolinux702"), ("converge_install_2.3.16", "converge_install_2.3.16"), # Download type - code, source ("cistem-1.0.0-beta-source-code", "cistem-1.0.0-beta"), # Download type - src ("apache-ant-1.9.7-src", "apache-ant-1.9.7"), ("go1.7.4.src", "go1.7.4"), # Download type - source ("bowtie2-2.2.5-source", "bowtie2-2.2.5"), ("grib_api-1.17.0-Source", "grib_api-1.17.0"), # Download type - full ("julia-0.4.3-full", "julia-0.4.3"), # Download type - bin ("apache-maven-3.3.9-bin", "apache-maven-3.3.9"), # Download type - binary ("Jmol-14.8.0-binary", "Jmol-14.8.0"), # Download type - gem ("rubysl-date-2.0.9.gem", "rubysl-date-2.0.9"), # Download type - tar ("gromacs-4.6.1-tar", "gromacs-4.6.1"), # Download type - sh ("Miniconda2-4.3.11-Linux-x86_64.sh", "Miniconda2-4.3.11"), # Download version - release ("v1.0.4-release", "v1.0.4"), # Download version - stable ("libevent-2.0.21-stable", "libevent-2.0.21"), # Download version - final ("2.6.7-final", "2.6.7"), # Download version - rel ("v1.9.5.1rel", "v1.9.5.1"), # Download version - orig ("dash_0.5.5.1.orig", "dash_0.5.5.1"), # Download version - plus ("ncbi-blast-2.6.0+-src", "ncbi-blast-2.6.0"), # License ("cppad-20170114.gpl", "cppad-20170114"), # Arch ("pcraster-4.1.0_x86-64", "pcraster-4.1.0"), ("dislin-11.0.linux.i586_64", "dislin-11.0"), ("PAGIT.V1.01.64bit", "PAGIT.V1.01"), # OS - linux ("astyle_2.04_linux", "astyle_2.04"), # OS - unix ("install-tl-unx", "install-tl"), # OS - macos ("astyle_1.23_macosx", "astyle_1.23"), ("haxe-2.08-osx", "haxe-2.08"), # PyPI - wheel ("wheel-1.2.3-py3-none-any", "wheel-1.2.3"), ("wheel-1.2.3-py2.py3-none-any", "wheel-1.2.3"), ("wheel-1.2.3-cp38-abi3-macosx_10_12_x86_64", "wheel-1.2.3"), ("entrypoints-0.2.2-py2.py3-none-any", "entrypoints-0.2.2"), ( "numpy-1.12.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel." "macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64", "numpy-1.12.0", ), # Combinations of multiple patterns - bin, release ("rocketmq-all-4.5.2-bin-release", "rocketmq-all-4.5.2"), # Combinations of multiple patterns - all ("p7zip_9.04_src_all", "p7zip_9.04"), # Combinations of multiple patterns - run ("cuda_8.0.44_linux.run", "cuda_8.0.44"), # Combinations of multiple patterns - file ("ack-2.14-single-file", "ack-2.14"), # Combinations of multiple patterns - jar ("antlr-3.4-complete.jar", "antlr-3.4"), # Combinations of multiple patterns - oss ("tbb44_20160128oss_src_0", "tbb44_20160128"), # Combinations of multiple patterns - darwin ("ghc-7.0.4-x86_64-apple-darwin", "ghc-7.0.4"), ("ghc-7.0.4-i386-apple-darwin", "ghc-7.0.4"), # Combinations of multiple patterns - centos ("sratoolkit.2.8.2-1-centos_linux64", "sratoolkit.2.8.2-1"), # Combinations of multiple patterns - arch ( "VizGlow_v2.2alpha17-R21November2016-Linux-x86_64-Install", "VizGlow_v2.2alpha17-R21November2016", ), ("jdk-8u92-linux-x64", "jdk-8u92"), ("cuda_6.5.14_linux_64.run", "cuda_6.5.14"), ("Mathematica_12.0.0_LINUX.sh", "Mathematica_12.0.0"), ("trf407b.linux64", "trf407b"), # Combinations of multiple patterns - with ("mafft-7.221-with-extensions-src", "mafft-7.221"), ("spark-2.0.0-bin-without-hadoop", "spark-2.0.0"), ("conduit-v0.3.0-src-with-blt", "conduit-v0.3.0"), # Combinations of multiple patterns - rock ("bitlib-23-2.src.rock", "bitlib-23-2"), # Combinations of multiple patterns - public ("dakota-6.3-public.src", "dakota-6.3"), # Combinations of multiple patterns - universal ("synergy-1.3.6p2-MacOSX-Universal", "synergy-1.3.6p2"), # Combinations of multiple patterns - dynamic ("snptest_v2.5.2_linux_x86_64_dynamic", "snptest_v2.5.2"), # Combinations of multiple patterns - other ("alglib-3.11.0.cpp.gpl", "alglib-3.11.0"), ("hpcviewer-2019.08-linux.gtk.x86_64", "hpcviewer-2019.08"), ("apache-mxnet-src-1.3.0-incubating", "apache-mxnet-src-1.3.0"), ], ) def test_url_strip_version_suffixes(url, expected): stripped = spack.llnl.url.strip_version_suffixes(url) assert stripped == expected def test_strip_compression_extension(archive_and_expected): archive, extension = archive_and_expected stripped = spack.llnl.url.strip_compression_extension(archive) if extension == "zip": assert stripped == "Foo.zip" stripped = spack.llnl.url.strip_compression_extension(archive, "zip") assert stripped == "Foo" elif extension == "whl": assert stripped == "Foo.whl" elif ( extension.lower() == "tar" or extension in spack.llnl.url.CONTRACTION_MAP or extension in [ ".".join(ext) for ext in itertools.product( spack.llnl.url.PREFIX_EXTENSIONS, spack.llnl.url.EXTENSIONS ) ] ): assert stripped == "Foo.tar" or stripped == "Foo.TAR" else: assert stripped == "Foo" def test_allowed_archive(archive_and_expected): archive, _ = archive_and_expected assert spack.llnl.url.allowed_archive(archive) ================================================ FILE: lib/spack/spack/test/llnl/util/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/test/llnl/util/argparsewriter.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for ``llnl/util/argparsewriter.py`` These tests are fairly minimal, and ArgparseWriter is more extensively tested in ``cmd/commands.py``. """ import pytest import spack.llnl.util.argparsewriter as aw import spack.main parser = spack.main.make_argument_parser() spack.main.add_all_commands(parser) def test_format_not_overridden(): with pytest.raises(TypeError): aw.ArgparseWriter("spack") ================================================ FILE: lib/spack/spack/test/llnl/util/file_list.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import fnmatch import os import sys import pytest import spack.paths from spack.llnl.util.filesystem import HeaderList, LibraryList, find_headers, find_libraries @pytest.fixture() def library_list(): """Returns an instance of LibraryList.""" # Test all valid extensions: ['.a', '.dylib', '.so'] libs = ( [ "/dir1/liblapack.a", "/dir2/libpython3.6.dylib", # name may contain periods "/dir1/libblas.a", "/dir3/libz.so", "libmpi.so.20.10.1", # shared object libraries may be versioned ] if sys.platform != "win32" else [ "/dir1/liblapack.lib", "/dir2/libpython3.6.dll", "/dir1/libblas.lib", "/dir3/libz.dll", "libmpi.dll.20.10.1", ] ) return LibraryList(libs) @pytest.fixture() def header_list(): """Returns an instance of header list""" # Test all valid extensions: ['.h', '.hpp', '.hh', '.cuh'] headers = [ "/dir1/Python.h", "/dir2/date.time.h", "/dir1/pyconfig.hpp", "/dir3/core.hh", "pymem.cuh", ] h = HeaderList(headers) h.add_macro("-DBOOST_LIB_NAME=boost_regex") h.add_macro("-DBOOST_DYN_LINK") return h # TODO: Remove below when spack.llnl.util.filesystem.find_libraries becomes spec aware plat_static_ext = "lib" if sys.platform == "win32" else "a" plat_shared_ext = "dll" if sys.platform == "win32" else "so" plat_apple_shared_ext = "dylib" class TestLibraryList: def test_repr(self, library_list): x = eval(repr(library_list)) assert library_list == x def test_joined_and_str(self, library_list): s1 = library_list.joined() expected = " ".join( [ "/dir1/liblapack.%s" % plat_static_ext, "/dir2/libpython3.6.%s" % (plat_apple_shared_ext if sys.platform != "win32" else "dll"), "/dir1/libblas.%s" % plat_static_ext, "/dir3/libz.%s" % plat_shared_ext, "libmpi.%s.20.10.1" % plat_shared_ext, ] ) assert s1 == expected s2 = str(library_list) assert s1 == s2 s3 = library_list.joined(";") expected = ";".join( [ "/dir1/liblapack.%s" % plat_static_ext, "/dir2/libpython3.6.%s" % (plat_apple_shared_ext if sys.platform != "win32" else "dll"), "/dir1/libblas.%s" % plat_static_ext, "/dir3/libz.%s" % plat_shared_ext, "libmpi.%s.20.10.1" % plat_shared_ext, ] ) assert s3 == expected def test_flags(self, library_list): search_flags = library_list.search_flags assert "-L/dir1" in search_flags assert "-L/dir2" in search_flags assert "-L/dir3" in search_flags assert isinstance(search_flags, str) assert search_flags == "-L/dir1 -L/dir2 -L/dir3" link_flags = library_list.link_flags assert "-llapack" in link_flags assert "-lpython3.6" in link_flags assert "-lblas" in link_flags assert "-lz" in link_flags assert "-lmpi" in link_flags assert isinstance(link_flags, str) assert link_flags == "-llapack -lpython3.6 -lblas -lz -lmpi" ld_flags = library_list.ld_flags assert isinstance(ld_flags, str) assert ld_flags == search_flags + " " + link_flags def test_paths_manipulation(self, library_list): names = library_list.names assert names == ["lapack", "python3.6", "blas", "z", "mpi"] directories = library_list.directories assert directories == ["/dir1", "/dir2", "/dir3"] def test_get_item(self, library_list): a = library_list[0] assert a == "/dir1/liblapack.%s" % plat_static_ext b = library_list[:] assert type(b) is type(library_list) assert library_list == b assert library_list is not b def test_add(self, library_list): pylist = [ "/dir1/liblapack.%s" % plat_static_ext, # removed from the final list "/dir2/libmpi.%s" % plat_shared_ext, "/dir4/libnew.%s" % plat_static_ext, ] another = LibraryList(pylist) both = library_list + another assert len(both) == 7 # Invariant assert both == both + both # Always produce an instance of LibraryList assert type(library_list + pylist) is type(library_list) assert type(pylist + library_list) is type(library_list) class TestHeaderList: def test_repr(self, header_list): x = eval(repr(header_list)) assert header_list == x def test_joined_and_str(self, header_list): s1 = header_list.joined() expected = " ".join( [ "/dir1/Python.h", "/dir2/date.time.h", "/dir1/pyconfig.hpp", "/dir3/core.hh", "pymem.cuh", ] ) assert s1 == expected s2 = str(header_list) assert s1 == s2 s3 = header_list.joined(";") expected = ";".join( [ "/dir1/Python.h", "/dir2/date.time.h", "/dir1/pyconfig.hpp", "/dir3/core.hh", "pymem.cuh", ] ) assert s3 == expected def test_flags(self, header_list): include_flags = header_list.include_flags assert "-I/dir1" in include_flags assert "-I/dir2" in include_flags assert "-I/dir3" in include_flags assert isinstance(include_flags, str) assert include_flags == "-I/dir1 -I/dir2 -I/dir3" macros = header_list.macro_definitions assert "-DBOOST_LIB_NAME=boost_regex" in macros assert "-DBOOST_DYN_LINK" in macros assert isinstance(macros, str) assert macros == "-DBOOST_LIB_NAME=boost_regex -DBOOST_DYN_LINK" cpp_flags = header_list.cpp_flags assert isinstance(cpp_flags, str) assert cpp_flags == include_flags + " " + macros def test_paths_manipulation(self, header_list): names = header_list.names assert names == ["Python", "date.time", "pyconfig", "core", "pymem"] directories = header_list.directories assert directories == ["/dir1", "/dir2", "/dir3"] def test_get_item(self, header_list): a = header_list[0] assert a == "/dir1/Python.h" b = header_list[:] assert type(b) is type(header_list) assert header_list == b assert header_list is not b def test_add(self, header_list): pylist = [ "/dir1/Python.h", # removed from the final list "/dir2/pyconfig.hpp", "/dir4/date.time.h", ] another = HeaderList(pylist) h = header_list + another assert len(h) == 7 # Invariant : l == l + l assert h == h + h # Always produce an instance of HeaderList assert type(header_list + pylist) is type(header_list) assert type(pylist + header_list) is type(header_list) #: Directory where the data for the test below is stored search_dir = os.path.join(spack.paths.test_path, "data", "directory_search") @pytest.mark.parametrize( "lib_list,kwargs", [ (["liba"], {"shared": True, "recursive": True}), (["liba"], {"shared": False, "recursive": True}), (["libc", "liba"], {"shared": True, "recursive": True}), (["liba", "libc"], {"shared": False, "recursive": True}), (["libc", "libb", "liba"], {"shared": True, "recursive": True}), (["liba", "libb", "libc"], {"shared": False, "recursive": True}), ], ) def test_library_type_search(lib_list, kwargs): results = find_libraries(lib_list, search_dir, **kwargs) assert len(results) != 0 for result in results: lib_type_ext = plat_shared_ext if not kwargs["shared"]: lib_type_ext = plat_static_ext assert result.endswith(lib_type_ext) or ( kwargs["shared"] and result.endswith(plat_apple_shared_ext) ) @pytest.mark.parametrize( "search_fn,search_list,root,kwargs", [ (find_libraries, "liba", search_dir, {"recursive": True}), (find_libraries, ["liba"], search_dir, {"recursive": True}), (find_libraries, "libb", search_dir, {"recursive": True}), (find_libraries, ["libc"], search_dir, {"recursive": True}), (find_libraries, ["libc", "liba"], search_dir, {"recursive": True}), (find_libraries, ["liba", "libc"], search_dir, {"recursive": True}), (find_libraries, ["libc", "libb", "liba"], search_dir, {"recursive": True}), (find_libraries, ["liba", "libc"], search_dir, {"recursive": True}), ( find_libraries, ["libc", "libb", "liba"], search_dir, {"recursive": True, "shared": False}, ), (find_headers, "a", search_dir, {"recursive": True}), (find_headers, ["a"], search_dir, {"recursive": True}), (find_headers, "b", search_dir, {"recursive": True}), (find_headers, ["c"], search_dir, {"recursive": True}), (find_headers, ["c", "a"], search_dir, {"recursive": True}), (find_headers, ["a", "c"], search_dir, {"recursive": True}), (find_headers, ["c", "b", "a"], search_dir, {"recursive": True}), (find_headers, ["a", "c"], search_dir, {"recursive": True}), (find_libraries, ["liba", "libd"], os.path.join(search_dir, "b"), {"recursive": False}), (find_headers, ["b", "d"], os.path.join(search_dir, "b"), {"recursive": False}), ], ) def test_searching_order(search_fn, search_list, root, kwargs): # Test search result = search_fn(search_list, root, **kwargs) # The tests are set-up so that something is always found assert len(result) != 0 # Now reverse the result and start discarding things # as soon as you have matches. In the end the list should # be emptied. rlist = list(reversed(result)) # At this point make sure the search list is a sequence if isinstance(search_list, str): search_list = [search_list] # Discard entries in the order they appear in search list for x in search_list: try: while fnmatch.fnmatch(rlist[-1], x) or x in rlist[-1]: rlist.pop() except IndexError: # List is empty pass # List should be empty here assert len(rlist) == 0 ================================================ FILE: lib/spack/spack/test/llnl/util/filesystem.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for ``llnl/util/filesystem.py``""" import filecmp import os import pathlib import shutil import stat import sys from contextlib import contextmanager import pytest import spack.llnl.util.filesystem as fs import spack.paths @pytest.fixture() def stage(tmp_path_factory: pytest.TempPathFactory): """Creates a stage with the directory structure for the tests.""" s = tmp_path_factory.mktemp("filesystem_test") with fs.working_dir(s): # Create source file hierarchy fs.touchp("source/1") fs.touchp("source/a/b/2") fs.touchp("source/a/b/3") fs.touchp("source/c/4") fs.touchp("source/c/d/5") fs.touchp("source/c/d/6") fs.touchp("source/c/d/e/7") fs.touchp("source/g/h/i/8") fs.touchp("source/g/h/i/9") fs.touchp("source/g/i/j/10") # Create symlinks fs.symlink(os.path.abspath("source/1"), "source/2") fs.symlink("b/2", "source/a/b2") fs.symlink("a/b", "source/f") # Create destination directory fs.mkdirp("dest") yield s class TestCopy: """Tests for ``filesystem.copy``""" def test_file_dest(self, stage): """Test using a filename as the destination.""" with fs.working_dir(str(stage)): fs.copy("source/1", "dest/1") assert os.path.exists("dest/1") def test_dir_dest(self, stage): """Test using a directory as the destination.""" with fs.working_dir(str(stage)): fs.copy("source/1", "dest") assert os.path.exists("dest/1") def test_glob_src(self, stage): """Test using a glob as the source.""" with fs.working_dir(str(stage)): fs.copy("source/a/*/*", "dest") assert os.path.exists("dest/2") assert os.path.exists("dest/3") def test_non_existing_src(self, stage): """Test using a non-existing source.""" with fs.working_dir(str(stage)): with pytest.raises(OSError, match="No such file or directory"): fs.copy("source/none", "dest") def test_multiple_src_file_dest(self, stage): """Test a glob that matches multiple source files and a dest that is not a directory.""" with fs.working_dir(str(stage)): match = ".* matches multiple files but .* is not a directory" with pytest.raises(ValueError, match=match): fs.copy("source/a/*/*", "dest/1") def check_added_exe_permissions(src, dst): src_mode = os.stat(src).st_mode dst_mode = os.stat(dst).st_mode for perm in [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH]: if src_mode & perm: assert dst_mode & perm class TestInstall: """Tests for ``filesystem.install``""" def test_file_dest(self, stage): """Test using a filename as the destination.""" with fs.working_dir(str(stage)): fs.install("source/1", "dest/1") assert os.path.exists("dest/1") check_added_exe_permissions("source/1", "dest/1") def test_dir_dest(self, stage): """Test using a directory as the destination.""" with fs.working_dir(str(stage)): fs.install("source/1", "dest") assert os.path.exists("dest/1") check_added_exe_permissions("source/1", "dest/1") def test_glob_src(self, stage): """Test using a glob as the source.""" with fs.working_dir(str(stage)): fs.install("source/a/*/*", "dest") assert os.path.exists("dest/2") assert os.path.exists("dest/3") check_added_exe_permissions("source/a/b/2", "dest/2") check_added_exe_permissions("source/a/b/3", "dest/3") def test_non_existing_src(self, stage): """Test using a non-existing source.""" with fs.working_dir(str(stage)): with pytest.raises(OSError, match="No such file or directory"): fs.install("source/none", "dest") def test_multiple_src_file_dest(self, stage): """Test a glob that matches multiple source files and a dest that is not a directory.""" with fs.working_dir(str(stage)): match = ".* matches multiple files but .* is not a directory" with pytest.raises(ValueError, match=match): fs.install("source/a/*/*", "dest/1") class TestCopyTree: """Tests for ``filesystem.copy_tree``""" def test_existing_dir(self, stage): """Test copying to an existing directory.""" with fs.working_dir(str(stage)): fs.copy_tree("source", "dest") assert os.path.exists("dest/a/b/2") def test_non_existing_dir(self, stage): """Test copying to a non-existing directory.""" with fs.working_dir(str(stage)): fs.copy_tree("source", "dest/sub/directory") assert os.path.exists("dest/sub/directory/a/b/2") def test_symlinks_true(self, stage): """Test copying with symlink preservation.""" with fs.working_dir(str(stage)): fs.copy_tree("source", "dest", symlinks=True) assert os.path.exists("dest/2") assert fs.islink("dest/2") assert os.path.exists("dest/a/b2") with fs.working_dir("dest/a"): assert os.path.exists(fs.readlink("b2")) assert os.path.realpath("dest/f/2") == os.path.abspath("dest/a/b/2") assert os.path.realpath("dest/2") == os.path.abspath("dest/1") def test_symlinks_true_ignore(self, stage): """Test copying when specifying relative paths that should be ignored""" with fs.working_dir(str(stage)): ignore = lambda p: p in [os.path.join("c", "d", "e"), "a"] fs.copy_tree("source", "dest", symlinks=True, ignore=ignore) assert not os.path.exists("dest/a") assert os.path.exists("dest/c/d") assert not os.path.exists("dest/c/d/e") def test_symlinks_false(self, stage): """Test copying without symlink preservation.""" with fs.working_dir(str(stage)): fs.copy_tree("source", "dest", symlinks=False) assert os.path.exists("dest/2") if sys.platform != "win32": assert not os.path.islink("dest/2") def test_glob_src(self, stage): """Test using a glob as the source.""" with fs.working_dir(str(stage)): fs.copy_tree("source/g/*", "dest") assert os.path.exists("dest/i/8") assert os.path.exists("dest/i/9") assert os.path.exists("dest/j/10") def test_non_existing_src(self, stage): """Test using a non-existing source.""" with fs.working_dir(str(stage)): with pytest.raises(OSError, match="No such file or directory"): fs.copy_tree("source/none", "dest") def test_parent_dir(self, stage): """Test source as a parent directory of destination.""" with fs.working_dir(str(stage)): match = "Cannot copy ancestor directory" with pytest.raises(ValueError, match=match): fs.copy_tree("source", "source/sub/directory") class TestInstallTree: """Tests for ``filesystem.install_tree``""" def test_existing_dir(self, stage): """Test installing to an existing directory.""" with fs.working_dir(str(stage)): fs.install_tree("source", "dest") assert os.path.exists("dest/a/b/2") check_added_exe_permissions("source/a/b/2", "dest/a/b/2") def test_non_existing_dir(self, stage): """Test installing to a non-existing directory.""" with fs.working_dir(str(stage)): fs.install_tree("source", "dest/sub/directory") assert os.path.exists("dest/sub/directory/a/b/2") check_added_exe_permissions("source/a/b/2", "dest/sub/directory/a/b/2") def test_symlinks_true(self, stage): """Test installing with symlink preservation.""" with fs.working_dir(str(stage)): fs.install_tree("source", "dest", symlinks=True) assert os.path.exists("dest/2") if sys.platform != "win32": assert os.path.islink("dest/2") check_added_exe_permissions("source/2", "dest/2") def test_symlinks_false(self, stage): """Test installing without symlink preservation.""" with fs.working_dir(str(stage)): fs.install_tree("source", "dest", symlinks=False) assert os.path.exists("dest/2") if sys.platform != "win32": assert not os.path.islink("dest/2") check_added_exe_permissions("source/2", "dest/2") @pytest.mark.not_on_windows("Broken symlinks not allowed on Windows") def test_allow_broken_symlinks(self, stage): """Test installing with a broken symlink.""" with fs.working_dir(str(stage)): fs.symlink("nonexistant.txt", "source/broken") fs.install_tree("source", "dest", symlinks=True) assert os.path.islink("dest/broken") assert not os.path.exists(fs.readlink("dest/broken")) def test_glob_src(self, stage): """Test using a glob as the source.""" with fs.working_dir(str(stage)): fs.install_tree("source/g/*", "dest") assert os.path.exists("dest/i/8") assert os.path.exists("dest/i/9") assert os.path.exists("dest/j/10") check_added_exe_permissions("source/g/h/i/8", "dest/i/8") check_added_exe_permissions("source/g/h/i/9", "dest/i/9") check_added_exe_permissions("source/g/i/j/10", "dest/j/10") def test_non_existing_src(self, stage): """Test using a non-existing source.""" with fs.working_dir(str(stage)): with pytest.raises(OSError, match="No such file or directory"): fs.install_tree("source/none", "dest") def test_parent_dir(self, stage): """Test source as a parent directory of destination.""" with fs.working_dir(str(stage)): match = "Cannot copy ancestor directory" with pytest.raises(ValueError, match=match): fs.install_tree("source", "source/sub/directory") def test_paths_containing_libs(dirs_with_libfiles): lib_to_dirs, all_dirs = dirs_with_libfiles assert set(fs.paths_containing_libs(all_dirs, ["libgfortran"])) == set( lib_to_dirs["libgfortran"] ) assert set(fs.paths_containing_libs(all_dirs, ["libirc"])) == set(lib_to_dirs["libirc"]) def test_move_transaction_commit(tmp_path: pathlib.Path): lib_dir = tmp_path / "lib" lib_dir.mkdir() fake_library = lib_dir / "libfoo.so" fake_library.write_text("Just some fake content.") with fs.replace_directory_transaction(str(lib_dir)) as backup: assert os.path.isdir(backup) lib_dir.mkdir() fake_library = lib_dir / "libfoo.so" fake_library.write_text("Other content.") assert not os.path.lexists(backup) with open(str(lib_dir / "libfoo.so"), "r", encoding="utf-8") as f: assert "Other content." == f.read() def test_move_transaction_rollback(tmp_path: pathlib.Path): lib_dir = tmp_path / "lib" lib_dir.mkdir() fake_library = lib_dir / "libfoo.so" fake_library.write_text("Initial content.") try: with fs.replace_directory_transaction(str(lib_dir)) as backup: assert os.path.isdir(backup) lib_dir.mkdir() fake_library = lib_dir / "libfoo.so" fake_library.write_text("New content.") raise RuntimeError("") except RuntimeError: pass assert not os.path.lexists(backup) with open(str(lib_dir / "libfoo.so"), "r", encoding="utf-8") as f: assert "Initial content." == f.read() @pytest.mark.regression("10601") @pytest.mark.regression("10603") def test_recursive_search_of_headers_from_prefix(installation_dir_with_headers): # Try to inspect recursively from and ensure we don't get # subdirectories of the '/include' path prefix = str(installation_dir_with_headers) header_list = fs.find_all_headers(prefix) include_dirs = header_list.directories if sys.platform == "win32": header_list = [header.replace("/", "\\") for header in header_list] include_dirs = [dir.replace("/", "\\") for dir in include_dirs] # Check that the header files we expect are all listed assert os.path.join(prefix, "include", "ex3.h") in header_list assert os.path.join(prefix, "include", "boost", "ex3.h") in header_list assert os.path.join(prefix, "path", "to", "ex1.h") in header_list assert os.path.join(prefix, "path", "to", "subdir", "ex2.h") in header_list # Check that when computing directories we exclude /include/boost assert os.path.join(prefix, "include") in include_dirs assert os.path.join(prefix, "include", "boost") not in include_dirs assert os.path.join(prefix, "path", "to") in include_dirs assert os.path.join(prefix, "path", "to", "subdir") in include_dirs if sys.platform == "win32": dir_list = [ (["C:/pfx/include/foo.h", "C:/pfx/include/subdir/foo.h"], ["C:/pfx/include"]), (["C:/pfx/include/foo.h", "C:/pfx/subdir/foo.h"], ["C:/pfx/include", "C:/pfx/subdir"]), ( ["C:/pfx/include/subdir/foo.h", "C:/pfx/subdir/foo.h"], ["C:/pfx/include", "C:/pfx/subdir"], ), ] else: dir_list = [ (["/pfx/include/foo.h", "/pfx/include/subdir/foo.h"], ["/pfx/include"]), (["/pfx/include/foo.h", "/pfx/subdir/foo.h"], ["/pfx/include", "/pfx/subdir"]), (["/pfx/include/subdir/foo.h", "/pfx/subdir/foo.h"], ["/pfx/include", "/pfx/subdir"]), ] @pytest.mark.parametrize("list_of_headers,expected_directories", dir_list) def test_computation_of_header_directories(list_of_headers, expected_directories): hl = fs.HeaderList(list_of_headers) assert hl.directories == expected_directories def test_headers_directory_setter(): if sys.platform == "win32": root = r"C:\pfx\include\subdir" else: root = "/pfx/include/subdir" hl = fs.HeaderList([root + "/foo.h", root + "/bar.h"]) # Set directories using a list hl.directories = [root] assert hl.directories == [root] # If it's a single directory it's fine to not wrap it into a list # when setting the property hl.directories = root assert hl.directories == [root] # Paths are normalized, so it doesn't matter how many backslashes etc. # are present in the original directory being used if sys.platform == "win32": # TODO: Test with \\'s hl.directories = "C:/pfx/include//subdir" else: hl.directories = "/pfx/include//subdir/" assert hl.directories == [root] # Setting an empty list is allowed and returns an empty list hl.directories = [] assert hl.directories == [] # Setting directories to None also returns an empty list hl.directories = None assert hl.directories == [] if sys.platform == "win32": # TODO: Test \\s paths = [ (r"C:\user\root", None, (["C:\\", r"C:\user", r"C:\user\root"], "", [])), (r"C:\user\root", "C:\\", ([], "C:\\", [r"C:\user", r"C:\user\root"])), (r"C:\user\root", r"user", (["C:\\"], r"C:\user", [r"C:\user\root"])), (r"C:\user\root", r"root", (["C:\\", r"C:\user"], r"C:\user\root", [])), (r"relative\path", None, ([r"relative", r"relative\path"], "", [])), (r"relative\path", r"relative", ([], r"relative", [r"relative\path"])), (r"relative\path", r"path", ([r"relative"], r"relative\path", [])), ] else: paths = [ ("/tmp/user/root", None, (["/tmp", "/tmp/user", "/tmp/user/root"], "", [])), ("/tmp/user/root", "tmp", ([], "/tmp", ["/tmp/user", "/tmp/user/root"])), ("/tmp/user/root", "user", (["/tmp"], "/tmp/user", ["/tmp/user/root"])), ("/tmp/user/root", "root", (["/tmp", "/tmp/user"], "/tmp/user/root", [])), ("relative/path", None, (["relative", "relative/path"], "", [])), ("relative/path", "relative", ([], "relative", ["relative/path"])), ("relative/path", "path", (["relative"], "relative/path", [])), ] @pytest.mark.parametrize("path,entry,expected", paths) def test_partition_path(path, entry, expected): assert fs.partition_path(path, entry) == expected if sys.platform == "win32": path_list = [ ("", []), (r".\some\sub\dir", [r".\some", r".\some\sub", r".\some\sub\dir"]), (r"another\sub\dir", [r"another", r"another\sub", r"another\sub\dir"]), ] else: path_list = [ ("", []), ("/tmp/user/dir", ["/tmp", "/tmp/user", "/tmp/user/dir"]), ("./some/sub/dir", ["./some", "./some/sub", "./some/sub/dir"]), ("another/sub/dir", ["another", "another/sub", "another/sub/dir"]), ] @pytest.mark.parametrize("path,expected", path_list) def test_prefixes(path, expected): assert fs.prefixes(path) == expected @pytest.mark.regression("7358") @pytest.mark.parametrize( "regex,replacement,filename,keyword_args", [ (r"\", "", "x86_cpuid_info.c", {}), (r"CDIR", "CURRENT_DIRECTORY", "selfextract.bsx", {"stop_at": "__ARCHIVE_BELOW__"}), ], ) def test_filter_files_with_different_encodings( regex, replacement, filename, tmp_path: pathlib.Path, keyword_args ): # All files given as input to this test must satisfy the pre-requisite # that the 'replacement' string is not present in the file initially and # that there's at least one match for the regex original_file = os.path.join(spack.paths.test_path, "data", "filter_file", filename) target_file = os.path.join(str(tmp_path), filename) shutil.copy(original_file, target_file) # This should not raise exceptions fs.filter_file(regex, replacement, target_file, **keyword_args) # Check the strings have been replaced with open(target_file, mode="r", encoding="utf-8", errors="surrogateescape") as f: assert replacement in f.read() @pytest.mark.not_on_windows("chgrp isn't used on Windows") def test_chgrp_dont_set_group_if_already_set(tmp_path: pathlib.Path, monkeypatch): with fs.working_dir(str(tmp_path)): os.mkdir("test-dir_chgrp_dont_set_group_if_already_set") def _fail(*args, **kwargs): raise Exception("chrgrp should not be called") class FakeStat(object): def __init__(self, gid): self.st_gid = gid original_stat = os.stat def _stat(*args, **kwargs): path = args[0] if path == "test-dir_chgrp_dont_set_group_if_already_set": return FakeStat(gid=1001) else: # Monkeypatching stat can interfere with post-test cleanup, so for # paths that aren't part of the test, we want the original behavior # of stat return original_stat(*args, **kwargs) monkeypatch.setattr(os, "chown", _fail) monkeypatch.setattr(os, "lchown", _fail) monkeypatch.setattr(os, "stat", _stat) with fs.working_dir(str(tmp_path)): with pytest.raises(Exception): fs.chgrp("test-dir_chgrp_dont_set_group_if_already_set", 1002) fs.chgrp("test-dir_chgrp_dont_set_group_if_already_set", 1001) def test_filter_files_multiple(tmp_path: pathlib.Path): # All files given as input to this test must satisfy the pre-requisite # that the 'replacement' string is not present in the file initially and # that there's at least one match for the regex original_file = os.path.join(spack.paths.test_path, "data", "filter_file", "x86_cpuid_info.c") target_file = os.path.join(str(tmp_path), "x86_cpuid_info.c") shutil.copy(original_file, target_file) # This should not raise exceptions fs.filter_file(r"\", "", target_file) fs.filter_file(r"\", "", target_file) fs.filter_file(r"\", "", target_file) # Check the strings have been replaced with open(target_file, mode="r", encoding="utf-8", errors="surrogateescape") as f: assert "" not in f.read() assert "" not in f.read() assert "" not in f.read() def test_filter_files_start_stop(tmp_path: pathlib.Path): original_file = os.path.join(spack.paths.test_path, "data", "filter_file", "start_stop.txt") target_file = os.path.join(str(tmp_path), "start_stop.txt") shutil.copy(original_file, target_file) # None of the following should happen: # - filtering starts after A is found in the file: fs.filter_file("A", "X", target_file, string=True, start_at="B") # - filtering starts exactly when B is found: fs.filter_file("B", "X", target_file, string=True, start_at="B") # - filtering stops before D is found: fs.filter_file("D", "X", target_file, string=True, stop_at="C") assert filecmp.cmp(original_file, target_file) # All of the following should happen: fs.filter_file("A", "X", target_file, string=True) fs.filter_file("B", "X", target_file, string=True, start_at="X", stop_at="C") fs.filter_file(r"C|D", "X", target_file, start_at="X", stop_at="E") with open(target_file, mode="r", encoding="utf-8") as f: assert all("X" == line.strip() for line in f.readlines()) # Each test input is a tuple of entries which prescribe # - the 'subdirs' to be created from tmp_path # - the 'files' in that directory # - what is to be removed @pytest.mark.parametrize( "files_or_dirs", [ # Remove a file over the two that are present [{"subdirs": None, "files": ["spack.lock", "spack.yaml"], "remove": ["spack.lock"]}], # Remove the entire directory where two files are stored [{"subdirs": "myenv", "files": ["spack.lock", "spack.yaml"], "remove": ["myenv"]}], # Combine a mix of directories and files [ {"subdirs": None, "files": ["spack.lock", "spack.yaml"], "remove": ["spack.lock"]}, {"subdirs": "myenv", "files": ["spack.lock", "spack.yaml"], "remove": ["myenv"]}, ], # Multiple subdirectories, remove root [ {"subdirs": "work/myenv1", "files": ["spack.lock", "spack.yaml"], "remove": []}, {"subdirs": "work/myenv2", "files": ["spack.lock", "spack.yaml"], "remove": ["work"]}, ], # Multiple subdirectories, remove each one [ { "subdirs": "work/myenv1", "files": ["spack.lock", "spack.yaml"], "remove": ["work/myenv1"], }, { "subdirs": "work/myenv2", "files": ["spack.lock", "spack.yaml"], "remove": ["work/myenv2"], }, ], # Remove files with the same name in different directories [ { "subdirs": "work/myenv1", "files": ["spack.lock", "spack.yaml"], "remove": ["work/myenv1/spack.lock"], }, { "subdirs": "work/myenv2", "files": ["spack.lock", "spack.yaml"], "remove": ["work/myenv2/spack.lock"], }, ], # Remove first the directory, then a file within the directory [ { "subdirs": "myenv", "files": ["spack.lock", "spack.yaml"], "remove": ["myenv", "myenv/spack.lock"], } ], # Remove first a file within a directory, then the directory [ { "subdirs": "myenv", "files": ["spack.lock", "spack.yaml"], "remove": ["myenv/spack.lock", "myenv"], } ], ], ) @pytest.mark.regression("18441") def test_safe_remove(files_or_dirs, tmp_path: pathlib.Path): # Create a fake directory structure as prescribed by test input to_be_removed, to_be_checked = [], [] for entry in files_or_dirs: # Create relative dir subdirs = entry["subdirs"] if not subdirs: dir_path = tmp_path else: dir_path = tmp_path for subdir in subdirs.split("/"): dir_path = dir_path / subdir dir_path.mkdir(parents=True, exist_ok=True) # Create files in the directory files = entry["files"] for f in files: abspath = str(dir_path / f) to_be_checked.append(abspath) fs.touch(abspath) # List of things to be removed for r in entry["remove"]: to_be_removed.append(str(tmp_path / r)) # Assert that files are deleted in the context block, # mock a failure by raising an exception with pytest.raises(RuntimeError): with fs.safe_remove(*to_be_removed): for entry in to_be_removed: assert not os.path.exists(entry) raise RuntimeError("Mock a failure") # Assert that files are restored for entry in to_be_checked: assert os.path.exists(entry) @pytest.mark.regression("18441") def test_content_of_files_with_same_name(tmp_path: pathlib.Path): # Create two subdirectories containing a file with the same name, # differentiate the files by their content myenv1_dir = tmp_path / "myenv1" myenv1_dir.mkdir() file1 = myenv1_dir / "spack.lock" file1.write_text("file1") myenv2_dir = tmp_path / "myenv2" myenv2_dir.mkdir() file2 = myenv2_dir / "spack.lock" file2.write_text("file2") # Use 'safe_remove' to remove the two files with pytest.raises(RuntimeError): with fs.safe_remove(str(file1), str(file2)): raise RuntimeError("Mock a failure") # Check both files have been restored correctly # and have not been mixed assert file1.read_text().strip() == "file1" assert file2.read_text().strip() == "file2" def test_keep_modification_time(tmp_path: pathlib.Path): file1 = tmp_path / "file1" file1.touch() file2 = tmp_path / "file2" file2.touch() # Shift the modification time of the file 10 seconds back: stat1 = file1.stat() mtime1 = stat1.st_mtime - 10 os.utime(file1, (stat1.st_atime, mtime1)) with fs.keep_modification_time(str(file1), str(file2), "non-existing-file"): file1.write_text("file1") file2.unlink() # Assert that the modifications took place the modification time has not # changed; assert file1.read_text().strip() == "file1" assert not file2.exists() assert int(mtime1) == int(file1.stat().st_mtime) def test_temporary_dir_context_manager(): previous_dir = os.path.realpath(os.getcwd()) with fs.temporary_dir() as tmp_dir: assert previous_dir != os.path.realpath(os.getcwd()) assert os.path.realpath(str(tmp_dir)) == os.path.realpath(os.getcwd()) @pytest.mark.not_on_windows("No shebang on Windows") def test_is_nonsymlink_exe_with_shebang(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): # Create an executable with shebang. with open("executable_script", "wb") as f: f.write(b"#!/interpreter") os.chmod("executable_script", 0o100775) with open("executable_but_not_script", "wb") as f: f.write(b"#/not-a-shebang") os.chmod("executable_but_not_script", 0o100775) with open("not_executable_with_shebang", "wb") as f: f.write(b"#!/interpreter") os.chmod("not_executable_with_shebang", 0o100664) os.symlink("executable_script", "symlink_to_executable_script") assert fs.is_nonsymlink_exe_with_shebang("executable_script") assert not fs.is_nonsymlink_exe_with_shebang("executable_but_not_script") assert not fs.is_nonsymlink_exe_with_shebang("not_executable_with_shebang") assert not fs.is_nonsymlink_exe_with_shebang("symlink_to_executable_script") class RegisterVisitor(fs.BaseDirectoryVisitor): """A directory visitor that keeps track of all visited paths""" def __init__(self, root, follow_dirs=True, follow_symlink_dirs=True): self.files = [] self.dirs_before = [] self.symlinked_dirs_before = [] self.dirs_after = [] self.symlinked_dirs_after = [] self.root = root self.follow_dirs = follow_dirs self.follow_symlink_dirs = follow_symlink_dirs def check(self, root, rel_path, depth): # verify the (root, rel_path, depth) make sense. assert root == self.root and depth + 1 == len(rel_path.split(os.sep)) def visit_file(self, root, rel_path, depth): self.check(root, rel_path, depth) self.files.append(rel_path) def visit_symlinked_file(self, root, rel_path, depth): self.visit_file(root, rel_path, depth) def before_visit_dir(self, root, rel_path, depth): self.check(root, rel_path, depth) self.dirs_before.append(rel_path) return self.follow_dirs def before_visit_symlinked_dir(self, root, rel_path, depth): self.check(root, rel_path, depth) self.symlinked_dirs_before.append(rel_path) return self.follow_symlink_dirs def after_visit_dir(self, root, rel_path, depth): self.check(root, rel_path, depth) self.dirs_after.append(rel_path) def after_visit_symlinked_dir(self, root, rel_path, depth): self.check(root, rel_path, depth) self.symlinked_dirs_after.append(rel_path) @pytest.mark.not_on_windows("Requires symlinks") def test_visit_directory_tree_follow_all(noncyclical_dir_structure): root = str(noncyclical_dir_structure) visitor = RegisterVisitor(root, follow_dirs=True, follow_symlink_dirs=True) fs.visit_directory_tree(root, visitor) j = os.path.join assert visitor.files == [ j("a", "file_1"), j("a", "to_c", "dangling_link"), j("a", "to_c", "file_2"), j("a", "to_file_1"), j("b", "file_1"), j("b", "to_c", "dangling_link"), j("b", "to_c", "file_2"), j("b", "to_file_1"), j("c", "dangling_link"), j("c", "file_2"), j("file_3"), ] assert visitor.dirs_before == [j("a"), j("a", "d"), j("b", "d"), j("c")] assert visitor.dirs_after == [j("a", "d"), j("a"), j("b", "d"), j("c")] assert visitor.symlinked_dirs_before == [j("a", "to_c"), j("b"), j("b", "to_c")] assert visitor.symlinked_dirs_after == [j("a", "to_c"), j("b", "to_c"), j("b")] @pytest.mark.not_on_windows("Requires symlinks") def test_visit_directory_tree_follow_dirs(noncyclical_dir_structure): root = str(noncyclical_dir_structure) visitor = RegisterVisitor(root, follow_dirs=True, follow_symlink_dirs=False) fs.visit_directory_tree(root, visitor) j = os.path.join assert visitor.files == [ j("a", "file_1"), j("a", "to_file_1"), j("c", "dangling_link"), j("c", "file_2"), j("file_3"), ] assert visitor.dirs_before == [j("a"), j("a", "d"), j("c")] assert visitor.dirs_after == [j("a", "d"), j("a"), j("c")] assert visitor.symlinked_dirs_before == [j("a", "to_c"), j("b")] assert not visitor.symlinked_dirs_after @pytest.mark.not_on_windows("Requires symlinks") def test_visit_directory_tree_follow_none(noncyclical_dir_structure): root = str(noncyclical_dir_structure) visitor = RegisterVisitor(root, follow_dirs=False, follow_symlink_dirs=False) fs.visit_directory_tree(root, visitor) j = os.path.join assert visitor.files == [j("file_3")] assert visitor.dirs_before == [j("a"), j("c")] assert not visitor.dirs_after assert visitor.symlinked_dirs_before == [j("b")] assert not visitor.symlinked_dirs_after @pytest.mark.regression("29687") @pytest.mark.parametrize("initial_mode", [stat.S_IRUSR | stat.S_IXUSR, stat.S_IWGRP]) @pytest.mark.not_on_windows("Windows might change permissions") def test_remove_linked_tree_doesnt_change_file_permission(tmp_path: pathlib.Path, initial_mode): # Here we test that a failed call to remove_linked_tree, due to passing a file # as an argument instead of a directory, doesn't leave the file with different # permissions as a side effect of trying to handle the error. file_instead_of_dir = tmp_path / "foo" file_instead_of_dir.touch() file_instead_of_dir.chmod(initial_mode) initial_stat = os.stat(str(file_instead_of_dir)) fs.remove_linked_tree(str(file_instead_of_dir)) final_stat = os.stat(str(file_instead_of_dir)) assert final_stat == initial_stat def test_filesummary(tmp_path: pathlib.Path): p = str(tmp_path / "xyz") with open(p, "wb") as f: f.write(b"abcdefghijklmnopqrstuvwxyz") assert fs.filesummary(p, print_bytes=8) == (26, b"abcdefgh...stuvwxyz") assert fs.filesummary(p, print_bytes=13) == (26, b"abcdefghijklmnopqrstuvwxyz") assert fs.filesummary(p, print_bytes=100) == (26, b"abcdefghijklmnopqrstuvwxyz") @pytest.mark.parametrize("bfs_depth", [1, 2, 10]) def test_find_first_file(tmp_path: pathlib.Path, bfs_depth): # Create a structure: a/a/a/{file1,file2}, b/a, c/a, d/{a,file1} (tmp_path / "a" / "a" / "a").mkdir(parents=True) (tmp_path / "b" / "a").mkdir(parents=True) (tmp_path / "c" / "a").mkdir(parents=True) (tmp_path / "d" / "a").mkdir(parents=True) (tmp_path / "e").mkdir() fs.touch(str(tmp_path / "a" / "a" / "a" / "file1")) fs.touch(str(tmp_path / "a" / "a" / "a" / "file2")) fs.touch(str(tmp_path / "d" / "file1")) root = str(tmp_path) # Iterative deepening: should find low-depth file1. f1 = fs.find_first(root, "file*", bfs_depth=bfs_depth) assert f1 is not None and os.path.samefile(f1, os.path.join(root, "d", "file1")) assert fs.find_first(root, "nonexisting", bfs_depth=bfs_depth) is None f2 = fs.find_first(root, ["nonexisting", "file2"], bfs_depth=bfs_depth) assert f2 is not None and os.path.samefile(f2, os.path.join(root, "a", "a", "a", "file2")) # Should find first dir f3 = fs.find_first(root, "a", bfs_depth=bfs_depth) assert f3 is not None and os.path.samefile(f3, os.path.join(root, "a")) def test_rename_dest_exists(tmp_path: pathlib.Path): @contextmanager def setup_test_files(): a_dir = tmp_path / "a" a_dir.mkdir() a = a_dir / "file1" b = a_dir / "file2" fs.touchp(str(a)) fs.touchp(str(b)) with open(a, "w", encoding="utf-8") as oa, open(b, "w", encoding="utf-8") as ob: oa.write("I am A") ob.write("I am B") yield a, b shutil.rmtree(str(a_dir)) @contextmanager def setup_test_dirs(): d_dir = tmp_path / "d" d_dir.mkdir() a = d_dir / "a" b = d_dir / "b" fs.mkdirp(str(a)) fs.mkdirp(str(b)) yield a, b shutil.rmtree(str(d_dir)) # test standard behavior of rename # smoke test with setup_test_files() as files: a, b = files fs.rename(str(a), str(b)) assert os.path.exists(b) assert not os.path.exists(a) with open(b, "r", encoding="utf-8") as ob: content = ob.read() assert content == "I am A" # test relatitve paths # another sanity check/smoke test with setup_test_files() as files: a, b = files with fs.working_dir(str(tmp_path)): fs.rename(os.path.join("a", "file1"), os.path.join("a", "file2")) assert os.path.exists(b) assert not os.path.exists(a) with open(b, "r", encoding="utf-8") as ob: content = ob.read() assert content == "I am A" # Test rename symlinks to same file a_dir = tmp_path / "a" a_dir.mkdir() c = a_dir / "file1" a = a_dir / "link1" b = a_dir / "link2" fs.touchp(str(c)) fs.symlink(str(c), str(a)) fs.symlink(str(c), str(b)) fs.rename(str(a), str(b)) assert os.path.exists(b) assert not os.path.exists(a) assert os.path.realpath(str(b)) == str(c) shutil.rmtree(str(a_dir)) # test rename onto itself a_dir = tmp_path / "a" a_dir.mkdir() a = a_dir / "file1" b = a fs.touchp(str(a)) with open(a, "w", encoding="utf-8") as oa: oa.write("I am A") fs.rename(str(a), str(b)) # check a, or b, doesn't matter, same file assert os.path.exists(a) # ensure original file was not duplicated assert len(os.listdir(str(a_dir))) == 1 with open(a, "r", encoding="utf-8") as oa: assert oa.read() shutil.rmtree(str(a_dir)) # test rename onto symlink # to directory from symlink to directory # (this is something spack does when regenerating views) with setup_test_dirs() as dirs: a, b = dirs f_dir = tmp_path / "f" f_dir.mkdir() link1 = f_dir / "link1" link2 = f_dir / "link2" fs.symlink(str(a), str(link1)) fs.symlink(str(b), str(link2)) fs.rename(str(link1), str(link2)) assert os.path.exists(link2) assert os.path.realpath(str(link2)) == str(a) shutil.rmtree(str(f_dir)) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_windows_sfn(tmp_path: pathlib.Path): # first check some standard Windows locations # we know require sfn names # this is basically a smoke test # ensure spaces are replaced + path abbreviated assert fs.windows_sfn("C:\\Program Files (x86)") == "C:\\PROGRA~2" # ensure path without spaces is still properly shortened assert fs.windows_sfn("C:\\ProgramData") == "C:\\PROGRA~3" # test user created paths # ensure longer path with spaces is properly abbreviated a = tmp_path / "d" / "this is a test" / "a" / "still test" # ensure longer path is properly abbreviated b = tmp_path / "d" / "long_path_with_no_spaces" / "more_long_path" # ensure path not in need of abbreviation is properly roundtripped c = tmp_path / "d" / "this" / "is" / "short" # ensure paths that are the same in the first six letters # are incremented post tilde d = tmp_path / "d" / "longerpath1" e = tmp_path / "d" / "longerpath2" fs.mkdirp(str(a)) fs.mkdirp(str(b)) fs.mkdirp(str(c)) fs.mkdirp(str(d)) fs.mkdirp(str(e)) # check only for path of path we can control, # pytest prefix may or may not be mangled by windows_sfn # based on user/pytest config assert "d\\THISIS~1\\a\\STILLT~1" in fs.windows_sfn(str(a)) assert "d\\LONG_P~1\\MORE_L~1" in fs.windows_sfn(str(b)) assert "d\\this\\is\\short" in fs.windows_sfn(str(c)) assert "d\\LONGER~1" in fs.windows_sfn(str(d)) assert "d\\LONGER~2" in fs.windows_sfn(str(e)) shutil.rmtree(str(tmp_path / "d")) @pytest.fixture def dir_structure_with_things_to_find(tmp_path_factory: pytest.TempPathFactory): """ / dir_one/ file_one dir_two/ dir_three/ dir_four/ file_two file_three file_four """ tmp_main_dir = tmp_path_factory.mktemp("dir_structure_with_things_to_find") dir_one = tmp_main_dir / "dir_one" dir_one.mkdir() dir_two = tmp_main_dir / "dir_two" dir_two.mkdir() dir_three = tmp_main_dir / "dir_three" dir_three.mkdir() dir_four = dir_three / "dir_four" dir_four.mkdir() locations = {} file_one = dir_one / "file_one" file_one.touch() locations["file_one"] = str(file_one) file_two = dir_four / "file_two" file_two.touch() locations["file_two"] = str(file_two) file_three = dir_three / "file_three" file_three.touch() locations["file_three"] = str(file_three) file_four = tmp_main_dir / "file_four" file_four.touch() locations["file_four"] = str(file_four) return str(tmp_main_dir), locations def test_find_path_glob_matches(dir_structure_with_things_to_find): root, locations = dir_structure_with_things_to_find # both file name and path match assert ( fs.find(root, "file_two") == fs.find(root, "*/*/file_two") == fs.find(root, "dir_t*/*/*two") == [locations["file_two"]] ) # ensure that * does not match directory separators assert fs.find(root, "dir*file_two") == [] # ensure that file name matches after / are matched from the start of the file name assert fs.find(root, "*/ile_two") == [] # file name matches exist, but not with these paths assert fs.find(root, "dir_one/*/*two") == fs.find(root, "*/*/*/*/file_two") == [] def test_find_max_depth(dir_structure_with_things_to_find): root, locations = dir_structure_with_things_to_find # Make sure the paths we use to verify are absolute assert os.path.isabs(locations["file_one"]) assert set(fs.find(root, "file_*", max_depth=0)) == {locations["file_four"]} assert set(fs.find(root, "file_*", max_depth=1)) == { locations["file_one"], locations["file_three"], locations["file_four"], } assert set(fs.find(root, "file_two", max_depth=2)) == {locations["file_two"]} assert not set(fs.find(root, "file_two", max_depth=1)) assert set(fs.find(root, "file_two")) == {locations["file_two"]} assert set(fs.find(root, "file_*")) == set(locations.values()) def test_find_max_depth_relative(dir_structure_with_things_to_find): """find_max_depth should return absolute paths even if the provided path is relative.""" root, locations = dir_structure_with_things_to_find with fs.working_dir(root): assert set(fs.find(".", "file_*", max_depth=0)) == {locations["file_four"]} assert set(fs.find(".", "file_two", max_depth=2)) == {locations["file_two"]} @pytest.mark.parametrize("recursive,max_depth", [(False, -1), (False, 1)]) def test_max_depth_and_recursive_errors(tmp_path: pathlib.Path, recursive, max_depth): root = str(tmp_path) error_str = "cannot be set if recursive is False" with pytest.raises(ValueError, match=error_str): fs.find(root, ["some_file"], recursive=recursive, max_depth=max_depth) with pytest.raises(ValueError, match=error_str): fs.find_libraries(["some_lib"], root, recursive=recursive, max_depth=max_depth) @pytest.fixture(params=[True, False]) def complex_dir_structure(request, tmp_path_factory: pytest.TempPathFactory): """ "lx-dy" means "level x, directory y" "lx-fy" means "level x, file y" "lx-sy" means "level x, symlink y" / l1-d1/ l2-d1/ l3-d2/ l4-f1 l3-d4/ l4-f2 l3-s1 -> l1-d2 # points to directory above l2-d1 l3-s3 -> l1-d1 # cyclic link l1-d2/ l2-d2/ l3-f3 l2-f1 l2-s3 -> l2-d2 l1-s3 -> l3-d4 # a link that "skips" a directory level l1-s4 -> l2-s3 # a link to a link to a dir """ use_junctions = request.param if sys.platform == "win32" and not use_junctions and not fs._windows_can_symlink(): pytest.skip("This Windows instance is not configured with symlink support") elif sys.platform != "win32" and use_junctions: pytest.skip("Junctions are a Windows-only feature") tmp_complex_dir = tmp_path_factory.mktemp("complex_dir_structure") l1_d1 = tmp_complex_dir / "l1-d1" l1_d1.mkdir() l2_d1 = l1_d1 / "l2-d1" l2_d1.mkdir() l3_d2 = l2_d1 / "l3-d2" l3_d2.mkdir() l3_d4 = l2_d1 / "l3-d4" l3_d4.mkdir() l1_d2 = tmp_complex_dir / "l1-d2" l1_d2.mkdir() l2_d2 = l1_d2 / "l2-d2" l2_d2.mkdir() if sys.platform == "win32" and use_junctions: link_fn = fs._windows_create_junction else: link_fn = os.symlink link_fn(str(l1_d2), str(l2_d1 / "l3-s1")) link_fn(str(l1_d1), str(l2_d1 / "l3-s3")) link_fn(str(l3_d4), str(tmp_complex_dir / "l1-s3")) l2_s3 = l1_d2 / "l2-s3" link_fn(str(l2_d2), str(l2_s3)) link_fn(str(l2_s3), str(tmp_complex_dir / "l1-s4")) # Create files l4_f1 = l3_d2 / "l4-f1" l4_f1.touch() l4_f2 = l3_d4 / "l4-f2" l4_f2.touch() l2_f1 = l1_d2 / "l2-f1" l2_f1.touch() l3_f3 = l2_d2 / "l3-f3" l3_f3.touch() locations = { "l4-f1": str(l4_f1), "l4-f2-full": str(l4_f2), "l4-f2-link": str(tmp_complex_dir / "l1-s3" / "l4-f2"), "l2-f1": str(l2_f1), "l2-f1-link": str(tmp_complex_dir / "l1-d1" / "l2-d1" / "l3-s1" / "l2-f1"), "l3-f3-full": str(l3_f3), "l3-f3-link-l1": str(tmp_complex_dir / "l1-s4" / "l3-f3"), } return str(tmp_complex_dir), locations def test_find_max_depth_symlinks(complex_dir_structure): root, locations = complex_dir_structure root = pathlib.Path(root) assert set(fs.find(root, "l4-f1")) == {locations["l4-f1"]} assert set(fs.find(root / "l1-s3", "l4-f2", max_depth=0)) == {locations["l4-f2-link"]} assert set(fs.find(root / "l1-d1", "l2-f1")) == {locations["l2-f1-link"]} # File is accessible via symlink and subdir, the link path will be # searched first, and the directory will not be searched again when # it is encountered the second time (via not-link) in the traversal assert set(fs.find(root, "l4-f2")) == {locations["l4-f2-link"]} # File is accessible only via the dir, so the full file path should # be reported assert set(fs.find(root / "l1-d1", "l4-f2")) == {locations["l4-f2-full"]} # Check following links to links assert set(fs.find(root, "l3-f3")) == {locations["l3-f3-link-l1"]} def test_find_max_depth_multiple_and_repeated_entry_points(complex_dir_structure): root, locations = complex_dir_structure fst = str(pathlib.Path(root) / "l1-d1" / "l2-d1") snd = str(pathlib.Path(root) / "l1-d2") nonexistent = str(pathlib.Path(root) / "nonexistent") assert set(fs.find([fst, snd, fst, snd, nonexistent], ["l*-f*"], max_depth=1)) == { locations["l2-f1"], locations["l4-f1"], locations["l4-f2-full"], locations["l3-f3-full"], } def test_multiple_patterns(complex_dir_structure): root, _ = complex_dir_structure paths = fs.find(root, ["l2-f1", "l*-d*/l3-f3", "*-f*", "*/*-f*"]) # There shouldn't be duplicate results with multiple, overlapping patterns assert len(set(paths)) == len(paths) # All files should be found filenames = [os.path.basename(p) for p in paths] assert set(filenames) == {"l2-f1", "l3-f3", "l4-f1", "l4-f2"} # They are ordered by first matching pattern (this is a bit of an implementation detail, # and we could decide to change the exact order in the future) assert filenames[0] == "l2-f1" assert filenames[1] == "l3-f3" def test_find_input_types(tmp_path: pathlib.Path): """test that find only accepts sequences and instances of pathlib.Path and str for root, and only sequences and instances of str for patterns. In principle mypy catches these issues, but it is not enabled on all call-sites.""" (tmp_path / "file.txt").write_text("") assert ( fs.find(tmp_path, "file.txt") == fs.find(str(tmp_path), "file.txt") == fs.find([tmp_path, str(tmp_path)], "file.txt") == fs.find((tmp_path, str(tmp_path)), "file.txt") == fs.find(tmp_path, "file.txt") == fs.find(tmp_path, ["file.txt"]) == fs.find(tmp_path, ("file.txt",)) == [str(tmp_path / "file.txt")] ) with pytest.raises(TypeError): fs.find(tmp_path, pathlib.Path("file.txt")) # type: ignore with pytest.raises(TypeError): fs.find(1, "file.txt") # type: ignore def test_edit_in_place_through_temporary_file(tmp_path: pathlib.Path): (tmp_path / "example.txt").write_text("Hello") current_ino = os.stat(tmp_path / "example.txt").st_ino with fs.edit_in_place_through_temporary_file(str(tmp_path / "example.txt")) as temporary: os.unlink(temporary) with open(temporary, "w", encoding="utf-8") as f: f.write("World") assert (tmp_path / "example.txt").read_text() == "World" assert os.stat(tmp_path / "example.txt").st_ino == current_ino ================================================ FILE: lib/spack/spack/test/llnl/util/lang.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import re import sys from datetime import datetime, timedelta import pytest import spack.llnl.util.lang from spack.llnl.util.lang import ( Singleton, SingletonInstantiationError, dedupe, match_predicate, memoized, pretty_date, ) @pytest.fixture() def now(): return datetime.now() @pytest.fixture() def module_path(tmp_path: pathlib.Path): m = tmp_path / "foo.py" content = """ import os value = 1 path = os.path.join('/usr', 'bin') """ m.write_text(content) yield str(m) # Don't leave garbage in the module system if "foo" in sys.modules: del sys.modules["foo"] def test_pretty_date(): """Make sure pretty_date prints the right dates.""" now = datetime.now() just_now = now - timedelta(seconds=5) assert pretty_date(just_now, now) == "just now" seconds = now - timedelta(seconds=30) assert pretty_date(seconds, now) == "30 seconds ago" a_minute = now - timedelta(seconds=60) assert pretty_date(a_minute, now) == "a minute ago" minutes = now - timedelta(seconds=1800) assert pretty_date(minutes, now) == "30 minutes ago" an_hour = now - timedelta(hours=1) assert pretty_date(an_hour, now) == "an hour ago" hours = now - timedelta(hours=2) assert pretty_date(hours, now) == "2 hours ago" yesterday = now - timedelta(days=1) assert pretty_date(yesterday, now) == "yesterday" days = now - timedelta(days=3) assert pretty_date(days, now) == "3 days ago" a_week = now - timedelta(weeks=1) assert pretty_date(a_week, now) == "a week ago" weeks = now - timedelta(weeks=2) assert pretty_date(weeks, now) == "2 weeks ago" a_month = now - timedelta(days=30) assert pretty_date(a_month, now) == "a month ago" months = now - timedelta(days=60) assert pretty_date(months, now) == "2 months ago" a_year = now - timedelta(days=365) assert pretty_date(a_year, now) == "a year ago" years = now - timedelta(days=365 * 2) assert pretty_date(years, now) == "2 years ago" @pytest.mark.parametrize( "delta,pretty_string", [ (timedelta(days=1), "a day ago"), (timedelta(days=1), "yesterday"), (timedelta(days=1), "1 day ago"), (timedelta(weeks=1), "1 week ago"), (timedelta(weeks=3), "3 weeks ago"), (timedelta(days=30), "1 month ago"), (timedelta(days=730), "2 years ago"), ], ) def test_pretty_string_to_date_delta(now, delta, pretty_string): t1 = now - delta t2 = spack.llnl.util.lang.pretty_string_to_date(pretty_string, now) assert t1 == t2 @pytest.mark.parametrize( "format,pretty_string", [ ("%Y", "2018"), ("%Y-%m", "2015-03"), ("%Y-%m-%d", "2015-03-28"), ("%Y-%m-%d %H:%M", "2015-03-28 11:12"), ("%Y-%m-%d %H:%M:%S", "2015-03-28 23:34:45"), ], ) def test_pretty_string_to_date(format, pretty_string): t1 = datetime.strptime(pretty_string, format) t2 = spack.llnl.util.lang.pretty_string_to_date(pretty_string, now) assert t1 == t2 def test_pretty_seconds(): assert spack.llnl.util.lang.pretty_seconds(2.1) == "2.100s" assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000) == "2.100ms" assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000 / 1000) == "2.100us" assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000 / 1000 / 1000) == "2.100ns" assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000 / 1000 / 1000 / 10) == "0.210ns" def test_pretty_duration(): assert spack.llnl.util.lang.pretty_duration(0) == "0s" assert spack.llnl.util.lang.pretty_duration(45) == "45s" assert spack.llnl.util.lang.pretty_duration(60) == "1m00s" assert spack.llnl.util.lang.pretty_duration(125) == "2m05s" assert spack.llnl.util.lang.pretty_duration(3600) == "1h00m" assert spack.llnl.util.lang.pretty_duration(3661) == "1h01m" def test_match_predicate(): matcher = match_predicate(lambda x: True) assert matcher("foo") assert matcher("bar") assert matcher("baz") matcher = match_predicate(["foo", "bar"]) assert matcher("foo") assert matcher("bar") assert not matcher("baz") matcher = match_predicate(r"^(foo|bar)$") assert matcher("foo") assert matcher("bar") assert not matcher("baz") with pytest.raises(ValueError): matcher = match_predicate(object()) matcher("foo") def test_load_modules_from_file(module_path): # Check prerequisites assert "foo" not in sys.modules # Check that the module is loaded correctly from file foo = spack.llnl.util.lang.load_module_from_file("foo", module_path) assert "foo" in sys.modules assert foo.value == 1 assert foo.path == os.path.join("/usr", "bin") # Check that the module is not reloaded a second time on subsequent calls foo.value = 2 foo = spack.llnl.util.lang.load_module_from_file("foo", module_path) assert "foo" in sys.modules assert foo.value == 2 assert foo.path == os.path.join("/usr", "bin") def test_uniq(): assert [1, 2, 3] == spack.llnl.util.lang.uniq([1, 2, 3]) assert [1, 2, 3] == spack.llnl.util.lang.uniq([1, 1, 1, 1, 2, 2, 2, 3, 3]) assert [1, 2, 1] == spack.llnl.util.lang.uniq([1, 1, 1, 1, 2, 2, 2, 1, 1]) assert [] == spack.llnl.util.lang.uniq([]) def test_key_ordering(): """Ensure that key ordering works correctly.""" with pytest.raises(TypeError): @spack.llnl.util.lang.key_ordering class ClassThatHasNoCmpKeyMethod: # this will raise b/c it does not define _cmp_key pass @spack.llnl.util.lang.key_ordering class KeyComparable: def __init__(self, t): self.t = t def _cmp_key(self): return self.t a = KeyComparable((1, 2, 3)) a2 = KeyComparable((1, 2, 3)) b = KeyComparable((2, 3, 4)) b2 = KeyComparable((2, 3, 4)) assert a == a assert a == a2 assert a2 == a assert b == b assert b == b2 assert b2 == b assert a != b assert a < b assert b > a assert a <= b assert b >= a assert a <= a assert a <= a2 assert b >= b assert b >= b2 assert hash(a) != hash(b) assert hash(a) == hash(a) assert hash(a) == hash(a2) assert hash(b) == hash(b) assert hash(b) == hash(b2) @pytest.mark.parametrize("args, kwargs", [((1,), {}), ((), {"a": 3}), ((1,), {"a": 3})]) def test_memoized(args, kwargs): @memoized def f(*args, **kwargs): return "return-value" assert f(*args, **kwargs) == "return-value" assert f(*args, **kwargs) == "return-value" assert f.cache_info().hits == 1 @pytest.mark.parametrize("args, kwargs", [(([1],), {}), ((), {"a": [1]})]) def test_memoized_unhashable(args, kwargs): """Check that an exception is raised clearly""" @memoized def f(*args, **kwargs): return None with pytest.raises(TypeError, match="unhashable type:"): f(*args, **kwargs) def test_dedupe(): assert [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3] assert [x for x in dedupe([1, -2, 1, 3, 2], key=abs)] == [1, -2, 3] def test_grouped_exception(): h = spack.llnl.util.lang.GroupedExceptionHandler() def inner(): raise ValueError("wow!") with h.forward("inner method"): inner() with h.forward("top-level"): raise TypeError("ok") def test_grouped_exception_base_type(): h = spack.llnl.util.lang.GroupedExceptionHandler() with h.forward("catch-runtime-error", RuntimeError): raise NotImplementedError() with pytest.raises(NotImplementedError): with h.forward("catch-value-error", ValueError): raise NotImplementedError() message = h.grouped_message(with_tracebacks=False) assert "catch-runtime-error" in message assert "catch-value-error" not in message def test_class_level_constant_value(): """Tests that the Const descriptor does not allow overwriting the value from an instance""" class _SomeClass: CONST_VALUE = spack.llnl.util.lang.Const(10) with pytest.raises(TypeError, match="not support assignment"): _SomeClass().CONST_VALUE = 11 def test_deprecated_property(): """Tests the behavior of the DeprecatedProperty descriptor, which is can be used when deprecating an attribute. """ class _Deprecated(spack.llnl.util.lang.DeprecatedProperty): def factory(self, instance, owner): return 46 class _SomeClass: deprecated = _Deprecated("deprecated") # Default behavior is to just return the deprecated value s = _SomeClass() assert s.deprecated == 46 # When setting error_level to 1 the attribute warns _SomeClass.deprecated.error_lvl = 1 with pytest.warns(UserWarning): assert s.deprecated == 46 # When setting error_level to 2 an exception is raised _SomeClass.deprecated.error_lvl = 2 with pytest.raises(AttributeError): _ = s.deprecated def test_fnmatch_multiple(): named_patterns = {"a": "libf*o.so", "b": "libb*r.so"} regex = re.compile(spack.llnl.util.lang.fnmatch_translate_multiple(named_patterns)) a = regex.match("libfoo.so") assert a and a.group("a") == "libfoo.so" b = regex.match("libbar.so") assert b and b.group("b") == "libbar.so" assert not regex.match("libfoo.so.1") assert not regex.match("libbar.so.1") assert not regex.match("libfoo.solibbar.so") assert not regex.match("libbaz.so") def _attr_error_factory(): raise AttributeError("Could not make something") def test_singleton_instantiation_attr_failure(): """ If an AttributeError occurs during the instantiation of a Singleton object, we want to see that error. """ x = Singleton(_attr_error_factory) with pytest.raises(SingletonInstantiationError) as last_exception: x.something def follow_exceptions(e): while e: yield e e = e.__cause__ or e.__context__ assert any( "Could not make something" in str(e) for e in follow_exceptions(last_exception.value) ) class TestPriorityOrderedMapping: @pytest.mark.parametrize( "elements,expected", [ # Push out-of-order with explicit, and different, priorities ([("b", 2), ("a", 1), ("d", 4), ("c", 3)], ["a", "b", "c", "d"]), # Push in-order with priority=None ([("a", None), ("b", None), ("c", None), ("d", None)], ["a", "b", "c", "d"]), # Mix explicit and implicit priorities ([("b", 2), ("c", None), ("a", 1), ("d", None)], ["a", "b", "c", "d"]), ([("b", 10), ("c", None), ("a", -20), ("d", None)], ["a", "b", "c", "d"]), ([("b", 10), ("c", None), ("a", 20), ("d", None)], ["b", "c", "a", "d"]), # Adding the same key twice with different priorities ([("b", 10), ("c", None), ("a", 20), ("d", None), ("a", -20)], ["a", "b", "c", "d"]), # Adding the same key twice, no priorities ([("b", None), ("a", None), ("b", None)], ["a", "b"]), ], ) def test_iteration_order(self, elements, expected): """Tests that the iteration order respects priorities, no matter the insertion order.""" m = spack.llnl.util.lang.PriorityOrderedMapping() for key, priority in elements: m.add(key, value=None, priority=priority) assert list(m) == expected def test_reverse_iteration(self): """Tests that we can conveniently use reverse iteration""" m = spack.llnl.util.lang.PriorityOrderedMapping() for key, value in [("a", 1), ("b", 2), ("c", 3)]: m.add(key, value=value) assert list(m) == ["a", "b", "c"] assert list(reversed(m)) == ["c", "b", "a"] assert list(m.keys()) == ["a", "b", "c"] assert list(m.reversed_keys()) == ["c", "b", "a"] assert list(m.values()) == [1, 2, 3] assert list(m.reversed_values()) == [3, 2, 1] ================================================ FILE: lib/spack/spack/test/llnl/util/link_tree.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import sys import pytest import spack.llnl.util.filesystem from spack.llnl.util.filesystem import ( _windows_can_symlink, islink, mkdirp, readlink, symlink, touchp, visit_directory_tree, working_dir, ) from spack.llnl.util.link_tree import DestinationMergeVisitor, LinkTree, SourceMergeVisitor @pytest.fixture def stage(tmp_path: pathlib.Path): touchp(str(tmp_path / "source" / "1")) touchp(str(tmp_path / "source" / "a" / "b" / "2")) touchp(str(tmp_path / "source" / "a" / "b" / "3")) touchp(str(tmp_path / "source" / "c" / "4")) touchp(str(tmp_path / "source" / "c" / "d" / "5")) touchp(str(tmp_path / "source" / "c" / "d" / "6")) touchp(str(tmp_path / "source" / "c" / "d" / "e" / "7")) yield str(tmp_path) def check_file_link(filename: str, expected_target: str): assert os.path.isfile(filename) assert islink(filename) if sys.platform != "win32" or spack.llnl.util.filesystem._windows_can_symlink(): assert os.path.abspath(os.path.realpath(filename)) == os.path.abspath(expected_target) @pytest.mark.parametrize("run_as_root", [True, False] if sys.platform == "win32" else [False]) def test_merge_to_new_directory(stage: str, monkeypatch, run_as_root: bool): if sys.platform == "win32": if run_as_root and not _windows_can_symlink(): pytest.skip("Skipping portion of test which required dev-mode privileges.") monkeypatch.setattr( spack.llnl.util.filesystem, "_windows_can_symlink", lambda: run_as_root ) link_tree = LinkTree(os.path.join(stage, "source")) with working_dir(stage): link_tree.merge("dest") files = [ ("dest/1", "source/1"), ("dest/a/b/2", "source/a/b/2"), ("dest/a/b/3", "source/a/b/3"), ("dest/c/4", "source/c/4"), ("dest/c/d/5", "source/c/d/5"), ("dest/c/d/6", "source/c/d/6"), ("dest/c/d/e/7", "source/c/d/e/7"), ] for dest, source in files: check_file_link(dest, source) assert os.path.isabs(readlink(dest)) link_tree.unmerge("dest") assert not os.path.exists("dest") @pytest.mark.parametrize("run_as_root", [True, False] if sys.platform == "win32" else [False]) def test_merge_to_new_directory_relative(stage: str, monkeypatch, run_as_root: bool): if sys.platform == "win32": if run_as_root and not _windows_can_symlink(): pytest.skip("Skipping portion of test which required dev-mode privileges.") monkeypatch.setattr( spack.llnl.util.filesystem, "_windows_can_symlink", lambda: run_as_root ) link_tree = LinkTree(os.path.join(stage, "source")) with working_dir(stage): link_tree.merge("dest", relative=True) files = [ ("dest/1", "source/1"), ("dest/a/b/2", "source/a/b/2"), ("dest/a/b/3", "source/a/b/3"), ("dest/c/4", "source/c/4"), ("dest/c/d/5", "source/c/d/5"), ("dest/c/d/6", "source/c/d/6"), ("dest/c/d/e/7", "source/c/d/e/7"), ] for dest, source in files: check_file_link(dest, source) # Hard links/junctions are inherently absolute. if sys.platform != "win32" or run_as_root: assert not os.path.isabs(readlink(dest)) link_tree.unmerge("dest") assert not os.path.exists("dest") @pytest.mark.parametrize("run_as_root", [True, False] if sys.platform == "win32" else [False]) def test_merge_to_existing_directory(stage: str, monkeypatch, run_as_root): if sys.platform == "win32": if run_as_root and not _windows_can_symlink(): pytest.skip("Skipping portion of test which required dev-mode privileges.") monkeypatch.setattr( spack.llnl.util.filesystem, "_windows_can_symlink", lambda: run_as_root ) link_tree = LinkTree(os.path.join(stage, "source")) with working_dir(stage): touchp("dest/x") touchp("dest/a/b/y") link_tree.merge("dest") files = [ ("dest/1", "source/1"), ("dest/a/b/2", "source/a/b/2"), ("dest/a/b/3", "source/a/b/3"), ("dest/c/4", "source/c/4"), ("dest/c/d/5", "source/c/d/5"), ("dest/c/d/6", "source/c/d/6"), ("dest/c/d/e/7", "source/c/d/e/7"), ] for dest, source in files: check_file_link(dest, source) assert os.path.isfile("dest/x") assert os.path.isfile("dest/a/b/y") link_tree.unmerge("dest") assert os.path.isfile("dest/x") assert os.path.isfile("dest/a/b/y") for dest, _ in files: assert not os.path.isfile(dest) def test_merge_with_empty_directories(stage: str): link_tree = LinkTree(os.path.join(stage, "source")) with working_dir(stage): mkdirp("dest/f/g") mkdirp("dest/a/b/h") link_tree.merge("dest") link_tree.unmerge("dest") assert not os.path.exists("dest/1") assert not os.path.exists("dest/a/b/2") assert not os.path.exists("dest/a/b/3") assert not os.path.exists("dest/c/4") assert not os.path.exists("dest/c/d/5") assert not os.path.exists("dest/c/d/6") assert not os.path.exists("dest/c/d/e/7") assert os.path.isdir("dest/a/b/h") assert os.path.isdir("dest/f/g") def test_ignore(stage: str): link_tree = LinkTree(os.path.join(stage, "source")) with working_dir(stage): touchp("source/.spec") touchp("dest/.spec") link_tree.merge("dest", ignore=lambda x: x == ".spec") link_tree.unmerge("dest", ignore=lambda x: x == ".spec") assert not os.path.exists("dest/1") assert not os.path.exists("dest/a") assert not os.path.exists("dest/c") assert os.path.isfile("source/.spec") assert os.path.isfile("dest/.spec") def test_source_merge_visitor_does_not_follow_symlinked_dirs_at_depth(tmp_path: pathlib.Path): """Given an dir structure like this:: . `-- a |-- b | |-- c | | |-- d | | | `-- file | | `-- symlink_d -> d | `-- symlink_c -> c `-- symlink_b -> b The SoureMergeVisitor will expand symlinked dirs to directories, but only to fixed depth, to avoid exponential explosion. In our current defaults, symlink_b will be expanded, but symlink_c and symlink_d will not. """ j = os.path.join with working_dir(str(tmp_path)): os.mkdir(j("a")) os.mkdir(j("a", "b")) os.mkdir(j("a", "b", "c")) os.mkdir(j("a", "b", "c", "d")) symlink(j("b"), j("a", "symlink_b")) symlink(j("c"), j("a", "b", "symlink_c")) symlink(j("d"), j("a", "b", "c", "symlink_d")) with open(j("a", "b", "c", "d", "file"), "wb"): pass visitor = SourceMergeVisitor() visit_directory_tree(str(tmp_path), visitor) assert [p for p in visitor.files.keys()] == [ j("a", "b", "c", "d", "file"), j("a", "b", "c", "symlink_d"), # treated as a file, not expanded j("a", "b", "symlink_c"), # treated as a file, not expanded j("a", "symlink_b", "c", "d", "file"), # symlink_b was expanded j("a", "symlink_b", "c", "symlink_d"), # symlink_b was expanded j("a", "symlink_b", "symlink_c"), # symlink_b was expanded ] assert [p for p in visitor.directories.keys()] == [ j("a"), j("a", "b"), j("a", "b", "c"), j("a", "b", "c", "d"), j("a", "symlink_b"), j("a", "symlink_b", "c"), j("a", "symlink_b", "c", "d"), ] def test_source_merge_visitor_cant_be_cyclical(tmp_path: pathlib.Path): """Given an dir structure like this:: . |-- a | `-- symlink_b -> ../b | `-- symlink_symlink_b -> symlink_b `-- b `-- symlink_a -> ../a The SoureMergeVisitor will not expand `a/symlink_b`, `a/symlink_symlink_b` and `b/symlink_a` to avoid recursion. The general rule is: only expand symlinked dirs pointing deeper into the directory structure. """ j = os.path.join with working_dir(str(tmp_path)): os.mkdir(j("a")) os.mkdir(j("b")) symlink(j("..", "b"), j("a", "symlink_b")) symlink(j("symlink_b"), j("a", "symlink_b_b")) symlink(j("..", "a"), j("b", "symlink_a")) visitor = SourceMergeVisitor() visit_directory_tree(str(tmp_path), visitor) assert [p for p in visitor.files.keys()] == [ j("a", "symlink_b"), j("a", "symlink_b_b"), j("b", "symlink_a"), ] assert [p for p in visitor.directories.keys()] == [j("a"), j("b")] def test_destination_merge_visitor_always_errors_on_symlinked_dirs(tmp_path: pathlib.Path): """When merging prefixes into a non-empty destination folder, and this destination folder has a symlinked dir where the prefix has a dir, we should never merge any files there, but register a fatal error.""" j = os.path.join # Here example_a and example_b are symlinks. dst_path = tmp_path / "dst" dst_path.mkdir() with working_dir(str(dst_path)): os.mkdir("a") os.symlink("a", "example_a") os.symlink("a", "example_b") # Here example_a is a directory, and example_b is a (non-expanded) symlinked # directory. src_path = tmp_path / "src" src_path.mkdir() with working_dir(str(src_path)): os.mkdir("example_a") with open(j("example_a", "file"), "wb"): pass os.symlink("..", "example_b") visitor = SourceMergeVisitor() visit_directory_tree(str(src_path), visitor) visit_directory_tree(str(dst_path), DestinationMergeVisitor(visitor)) assert visitor.fatal_conflicts conflicts = [c.dst for c in visitor.fatal_conflicts] assert "example_a" in conflicts assert "example_b" in conflicts def test_destination_merge_visitor_file_dir_clashes(tmp_path: pathlib.Path): """Tests whether non-symlink file-dir and dir-file clashes as registered as fatal errors""" a_path = tmp_path / "a" a_path.mkdir() with working_dir(str(a_path)): os.mkdir("example") b_path = tmp_path / "b" b_path.mkdir() with working_dir(str(b_path)): with open("example", "wb"): pass a_to_b = SourceMergeVisitor() visit_directory_tree(str(a_path), a_to_b) visit_directory_tree(str(b_path), DestinationMergeVisitor(a_to_b)) assert a_to_b.fatal_conflicts assert a_to_b.fatal_conflicts[0].dst == "example" b_to_a = SourceMergeVisitor() visit_directory_tree(str(b_path), b_to_a) visit_directory_tree(str(a_path), DestinationMergeVisitor(b_to_a)) assert b_to_a.fatal_conflicts assert b_to_a.fatal_conflicts[0].dst == "example" @pytest.mark.parametrize("normalize", [True, False]) def test_source_merge_visitor_handles_same_file_gracefully( tmp_path: pathlib.Path, normalize: bool ): """Symlinked files/dirs from one prefix to the other are not file or fatal conflicts, they are resolved by taking the underlying file/dir, and this does not depend on the order prefixes are visited.""" def u(path: str) -> str: return path.upper() if normalize else path (tmp_path / "a").mkdir() (tmp_path / "a" / "file").write_bytes(b"hello") (tmp_path / "a" / "dir").mkdir() (tmp_path / "a" / "dir" / "foo").write_bytes(b"hello") (tmp_path / "b").mkdir() (tmp_path / "b" / u("file")).symlink_to(tmp_path / "a" / "file") (tmp_path / "b" / u("dir")).symlink_to(tmp_path / "a" / "dir") (tmp_path / "b" / "bar").write_bytes(b"hello") visitor_1 = SourceMergeVisitor(normalize_paths=normalize) visitor_1.set_projection(str(tmp_path / "view")) for p in ("a", "b"): visit_directory_tree(str(tmp_path / p), visitor_1) visitor_2 = SourceMergeVisitor(normalize_paths=normalize) visitor_2.set_projection(str(tmp_path / "view")) for p in ("b", "a"): visit_directory_tree(str(tmp_path / p), visitor_2) assert not visitor_1.file_conflicts and not visitor_2.file_conflicts assert not visitor_1.fatal_conflicts and not visitor_2.fatal_conflicts assert ( sorted(visitor_1.files.items()) == sorted(visitor_2.files.items()) == [ (str(tmp_path / "view" / "bar"), (str(tmp_path / "b"), "bar")), (str(tmp_path / "view" / "dir" / "foo"), (str(tmp_path / "a"), f"dir{os.sep}foo")), (str(tmp_path / "view" / "file"), (str(tmp_path / "a"), "file")), ] ) assert visitor_1.directories[str(tmp_path / "view" / "dir")] == (str(tmp_path / "a"), "dir") assert visitor_2.directories[str(tmp_path / "view" / "dir")] == (str(tmp_path / "a"), "dir") def test_source_merge_visitor_deals_with_dangling_symlinks(tmp_path: pathlib.Path): """When a file and a dangling symlink conflict, this should be handled like a file conflict.""" (tmp_path / "dir_a").mkdir() os.symlink("non-existent", str(tmp_path / "dir_a" / "file")) (tmp_path / "dir_b").mkdir() (tmp_path / "dir_b" / "file").write_bytes(b"data") visitor = SourceMergeVisitor() visitor.set_projection(str(tmp_path / "view")) visit_directory_tree(str(tmp_path / "dir_a"), visitor) visit_directory_tree(str(tmp_path / "dir_b"), visitor) # Check that a conflict was registered. assert len(visitor.file_conflicts) == 1 conflict = visitor.file_conflicts[0] assert conflict.src_a == str(tmp_path / "dir_a" / "file") assert conflict.src_b == str(tmp_path / "dir_b" / "file") assert conflict.dst == str(tmp_path / "view" / "file") # The first file encountered should be listed. assert visitor.files == {str(tmp_path / "view" / "file"): (str(tmp_path / "dir_a"), "file")} @pytest.mark.parametrize("normalize", [True, False]) def test_source_visitor_file_file(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a").mkdir() (tmp_path / "b").mkdir() (tmp_path / "a" / "file").write_bytes(b"") (tmp_path / "b" / "FILE").write_bytes(b"") v = SourceMergeVisitor(normalize_paths=normalize) for p in ("a", "b"): visit_directory_tree(str(tmp_path / p), v) if normalize: assert len(v.files) == 1 assert len(v.directories) == 0 assert "file" in v.files # first file wins assert len(v.file_conflicts) == 1 else: assert len(v.files) == 2 assert len(v.directories) == 0 assert "file" in v.files and "FILE" in v.files assert not v.fatal_conflicts assert not v.file_conflicts @pytest.mark.parametrize("normalize", [True, False]) def test_source_visitor_file_dir(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a").mkdir() (tmp_path / "a" / "file").write_bytes(b"") (tmp_path / "b").mkdir() (tmp_path / "b" / "FILE").mkdir() v1 = SourceMergeVisitor(normalize_paths=normalize) for p in ("a", "b"): visit_directory_tree(str(tmp_path / p), v1) v2 = SourceMergeVisitor(normalize_paths=normalize) for p in ("b", "a"): visit_directory_tree(str(tmp_path / p), v2) assert not v1.file_conflicts and not v2.file_conflicts if normalize: assert len(v1.fatal_conflicts) == len(v2.fatal_conflicts) == 1 else: assert len(v1.files) == len(v2.files) == 1 assert "file" in v1.files and "file" in v2.files assert len(v1.directories) == len(v2.directories) == 1 assert "FILE" in v1.directories and "FILE" in v2.directories assert not v1.fatal_conflicts and not v2.fatal_conflicts @pytest.mark.parametrize("normalize", [True, False]) def test_source_visitor_dir_dir(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a").mkdir() (tmp_path / "a" / "dir").mkdir() (tmp_path / "b").mkdir() (tmp_path / "b" / "DIR").mkdir() v = SourceMergeVisitor(normalize_paths=normalize) for p in ("a", "b"): visit_directory_tree(str(tmp_path / p), v) assert not v.files assert not v.fatal_conflicts assert not v.file_conflicts if normalize: assert len(v.directories) == 1 assert "dir" in v.directories else: assert len(v.directories) == 2 assert "DIR" in v.directories and "dir" in v.directories @pytest.mark.parametrize("normalize", [True, False]) def test_dst_visitor_file_file(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a").mkdir() (tmp_path / "b").mkdir() (tmp_path / "a" / "file").write_bytes(b"") (tmp_path / "b" / "FILE").write_bytes(b"") src = SourceMergeVisitor(normalize_paths=normalize) visit_directory_tree(str(tmp_path / "a"), src) visit_directory_tree(str(tmp_path / "b"), DestinationMergeVisitor(src)) assert len(src.files) == 1 assert len(src.directories) == 0 assert "file" in src.files assert not src.file_conflicts if normalize: assert len(src.fatal_conflicts) == 1 assert "FILE" in [c.dst for c in src.fatal_conflicts] else: assert not src.fatal_conflicts @pytest.mark.parametrize("normalize", [True, False]) def test_dst_visitor_file_dir(tmp_path: pathlib.Path, normalize: bool): (tmp_path / "a").mkdir() (tmp_path / "a" / "file").write_bytes(b"") (tmp_path / "b").mkdir() (tmp_path / "b" / "FILE").mkdir() src1 = SourceMergeVisitor(normalize_paths=normalize) visit_directory_tree(str(tmp_path / "a"), src1) visit_directory_tree(str(tmp_path / "b"), DestinationMergeVisitor(src1)) src2 = SourceMergeVisitor(normalize_paths=normalize) visit_directory_tree(str(tmp_path / "b"), src2) visit_directory_tree(str(tmp_path / "a"), DestinationMergeVisitor(src2)) assert len(src1.files) == 1 assert "file" in src1.files assert not src1.directories assert not src2.file_conflicts assert len(src2.directories) == 1 if normalize: assert len(src1.fatal_conflicts) == 1 assert "FILE" in [c.dst for c in src1.fatal_conflicts] assert not src2.files assert len(src2.fatal_conflicts) == 1 assert "file" in [c.dst for c in src2.fatal_conflicts] else: assert not src1.fatal_conflicts and not src2.fatal_conflicts assert not src1.file_conflicts and not src2.file_conflicts ================================================ FILE: lib/spack/spack/test/llnl/util/lock.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """These tests ensure that our lock works correctly. Run with pytest:: pytest lib/spack/spack/test/llnl/util/lock.py You can use this to test whether your shared filesystem properly supports POSIX reader-writer locking with byte ranges through fcntl. If you want to test on multiple filesystems, you can modify the ``locations`` list below. By default it looks like this:: locations = [ tempfile.gettempdir(), # standard tmp directory (potentially local) '/nfs/tmp2/%u', # NFS tmp mount '/p/lscratch*/%u' # Lustre scratch mount ] Add names and paths for your preferred filesystem mounts to test on them; the tests are parametrized to run on all the filesystems listed in this dict. """ import collections import errno import getpass import glob import multiprocessing import os import pathlib import shutil import socket import stat import sys import tempfile import traceback from contextlib import contextmanager from multiprocessing import Barrier, Process, Queue import pytest import spack.llnl.util.lock as lk from spack.llnl.util.filesystem import getuid, touch, working_dir if sys.platform != "win32": import fcntl pytestmark = pytest.mark.not_on_windows("does not run on windows") # # This test can be run with MPI. MPI is "enabled" if we can import # mpi4py and the number of total MPI processes is greater than 1. # Otherwise it just runs as a node-local test. # # NOTE: MPI mode is different from node-local mode in that node-local # mode will spawn its own test processes, while MPI mode assumes you've # run this script as a SPMD application. In MPI mode, no additional # processes are spawned, and you need to ensure that you mpirun the # script with enough processes for all the multiproc_test cases below. # # If you don't run with enough processes, tests that require more # processes than you currently have will be skipped. # mpi = False comm = None try: from mpi4py import MPI comm = MPI.COMM_WORLD if comm.size > 1: mpi = True except ImportError: pass #: This is a list of filesystem locations to test locks in. Paths are #: expanded so that %u is replaced with the current username. '~' is also #: legal and will be expanded to the user's home directory. #: #: Tests are skipped for directories that don't exist, so you'll need to #: update this with the locations of NFS, Lustre, and other mounts on your #: system. locations = [ tempfile.gettempdir(), os.path.join("/nfs/tmp2/", getpass.getuser()), os.path.join("/p/lscratch*/", getpass.getuser()), ] #: This is the longest a failed multiproc test will take. #: Barriers will time out and raise an exception after this interval. #: In MPI mode, barriers don't time out (they hang). See mpi_multiproc_test. barrier_timeout = 5 #: This is the lock timeout for expected failures. #: This may need to be higher for some filesystems. lock_fail_timeout = 0.1 def make_readable(*paths): # TODO: From os.chmod doc: # "Note Although Windows supports chmod(), you can only # set the file's read-only flag with it (via the stat.S_IWRITE and # stat.S_IREAD constants or a corresponding integer value). All other # bits are ignored." for path in paths: if sys.platform != "win32": mode = 0o555 if os.path.isdir(path) else 0o444 else: mode = stat.S_IREAD os.chmod(path, mode) def make_writable(*paths): for path in paths: if sys.platform != "win32": mode = 0o755 if os.path.isdir(path) else 0o744 else: mode = stat.S_IWRITE os.chmod(path, mode) @contextmanager def read_only(*paths): modes = [os.stat(p).st_mode for p in paths] make_readable(*paths) yield for path, mode in zip(paths, modes): os.chmod(path, mode) @pytest.fixture(scope="session", params=locations) def lock_test_directory(request): """This fixture causes tests to be executed for many different mounts. See the ``locations`` dict above for details. """ return request.param @pytest.fixture(scope="session") def lock_dir(lock_test_directory): parent = next( (p for p in glob.glob(lock_test_directory) if os.path.exists(p) and os.access(p, os.W_OK)), None, ) if not parent: # Skip filesystems that don't exist or aren't writable pytest.skip("requires filesystem: '%s'" % lock_test_directory) elif mpi and parent == tempfile.gettempdir(): # Skip local tmp test for MPI runs pytest.skip("skipping local tmp directory for MPI test.") tempdir = None if not mpi or comm.rank == 0: tempdir = tempfile.mkdtemp(dir=parent) if mpi: tempdir = comm.bcast(tempdir) yield tempdir if mpi: # rank 0 may get here before others, in which case it'll try to # remove the directory while other processes try to re-create the # lock. This will give errno 39: directory not empty. Use a # barrier to ensure everyone is done first. comm.barrier() if not mpi or comm.rank == 0: make_writable(tempdir) shutil.rmtree(tempdir) @pytest.fixture def private_lock_path(lock_dir): """In MPI mode, this is a private lock for each rank in a multiproc test. For other modes, it is the same as a shared lock. """ lock_file = os.path.join(lock_dir, "lockfile") if mpi: lock_file += ".%s" % comm.rank yield lock_file if os.path.exists(lock_file): make_writable(lock_dir, lock_file) os.unlink(lock_file) @pytest.fixture def lock_path(lock_dir): """This lock is shared among all processes in a multiproc test.""" lock_file = os.path.join(lock_dir, "lockfile") yield lock_file if os.path.exists(lock_file): make_writable(lock_dir, lock_file) os.unlink(lock_file) def test_poll_interval_generator(): interval_iter = iter(lk.Lock._poll_interval_generator(_wait_times=[1, 2, 3])) intervals = list(next(interval_iter) for i in range(100)) assert intervals == [1] * 20 + [2] * 40 + [3] * 40 def local_multiproc_test(*functions, **kwargs): """Order some processes using simple barrier synchronization.""" b = Barrier(len(functions), timeout=barrier_timeout) args = (b,) + tuple(kwargs.get("extra_args", ())) procs = [Process(target=f, args=args, name=f.__name__) for f in functions] for p in procs: p.start() for p in procs: p.join() assert all(p.exitcode == 0 for p in procs) def mpi_multiproc_test(*functions): """SPMD version of multiproc test. This needs to be run like so: srun spack test lock Each process executes its corresponding function. This is different from ``multiproc_test`` above, which spawns the processes. This will skip tests if there are too few processes to run them. """ procs = len(functions) if procs > comm.size: pytest.skip("requires at least %d MPI processes" % procs) comm.Barrier() # barrier before each MPI test include = comm.rank < len(functions) subcomm = comm.Split(include) class subcomm_barrier: """Stand-in for multiproc barrier for MPI-parallel jobs.""" def wait(self): subcomm.Barrier() if include: try: functions[subcomm.rank](subcomm_barrier()) except BaseException: # aborting is the best we can do for MPI tests without # hanging, since we're using MPI barriers. This will fail # early and it loses the nice pytest output, but at least it # gets use a stacktrace on the processes that failed. traceback.print_exc() comm.Abort() subcomm.Free() comm.Barrier() # barrier after each MPI test. #: ``multiproc_test()`` should be called by tests below. #: ``multiproc_test()`` will work for either MPI runs or for local runs. multiproc_test = mpi_multiproc_test if mpi else local_multiproc_test # # Process snippets below can be composed into tests. # class AcquireWrite: def __init__(self, lock_path, start=0, length=0): self.lock_path = lock_path self.start = start self.length = length @property def __name__(self): return self.__class__.__name__ def __call__(self, barrier): lock = lk.Lock(self.lock_path, start=self.start, length=self.length) lock.acquire_write() # grab exclusive lock barrier.wait() barrier.wait() # hold the lock until timeout in other procs. class AcquireRead: def __init__(self, lock_path, start=0, length=0): self.lock_path = lock_path self.start = start self.length = length @property def __name__(self): return self.__class__.__name__ def __call__(self, barrier): lock = lk.Lock(self.lock_path, start=self.start, length=self.length) lock.acquire_read() # grab shared lock barrier.wait() barrier.wait() # hold the lock until timeout in other procs. class TimeoutWrite: def __init__(self, lock_path, start=0, length=0): self.lock_path = lock_path self.start = start self.length = length @property def __name__(self): return self.__class__.__name__ def __call__(self, barrier): lock = lk.Lock(self.lock_path, start=self.start, length=self.length) barrier.wait() # wait for lock acquire in first process with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) barrier.wait() class TimeoutRead: def __init__(self, lock_path, start=0, length=0): self.lock_path = lock_path self.start = start self.length = length @property def __name__(self): return self.__class__.__name__ def __call__(self, barrier): lock = lk.Lock(self.lock_path, start=self.start, length=self.length) barrier.wait() # wait for lock acquire in first process with pytest.raises(lk.LockTimeoutError): lock.acquire_read(lock_fail_timeout) barrier.wait() # # Test that exclusive locks on other processes time out when an # exclusive lock is held. # def test_write_lock_timeout_on_write(lock_path): multiproc_test(AcquireWrite(lock_path), TimeoutWrite(lock_path)) def test_write_lock_timeout_on_write_2(lock_path): multiproc_test(AcquireWrite(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path)) def test_write_lock_timeout_on_write_3(lock_path): multiproc_test( AcquireWrite(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path), ) def test_write_lock_timeout_on_write_ranges(lock_path): multiproc_test(AcquireWrite(lock_path, 0, 1), TimeoutWrite(lock_path, 0, 1)) def test_write_lock_timeout_on_write_ranges_2(lock_path): multiproc_test( AcquireWrite(lock_path, 0, 64), AcquireWrite(lock_path, 65, 1), TimeoutWrite(lock_path, 0, 1), TimeoutWrite(lock_path, 63, 1), ) def test_write_lock_timeout_on_write_ranges_3(lock_path): multiproc_test( AcquireWrite(lock_path, 0, 1), AcquireWrite(lock_path, 1, 1), TimeoutWrite(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path), ) def test_write_lock_timeout_on_write_ranges_4(lock_path): multiproc_test( AcquireWrite(lock_path, 0, 1), AcquireWrite(lock_path, 1, 1), AcquireWrite(lock_path, 2, 456), AcquireWrite(lock_path, 500, 64), TimeoutWrite(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path), ) # # Test that shared locks on other processes time out when an # exclusive lock is held. # def test_read_lock_timeout_on_write(lock_path): multiproc_test(AcquireWrite(lock_path), TimeoutRead(lock_path)) def test_read_lock_timeout_on_write_2(lock_path): multiproc_test(AcquireWrite(lock_path), TimeoutRead(lock_path), TimeoutRead(lock_path)) def test_read_lock_timeout_on_write_3(lock_path): multiproc_test( AcquireWrite(lock_path), TimeoutRead(lock_path), TimeoutRead(lock_path), TimeoutRead(lock_path), ) def test_read_lock_timeout_on_write_ranges(lock_path): """small write lock, read whole file.""" multiproc_test(AcquireWrite(lock_path, 0, 1), TimeoutRead(lock_path)) def test_read_lock_timeout_on_write_ranges_2(lock_path): """small write lock, small read lock""" multiproc_test(AcquireWrite(lock_path, 0, 1), TimeoutRead(lock_path, 0, 1)) def test_read_lock_timeout_on_write_ranges_3(lock_path): """two write locks, overlapping read locks""" multiproc_test( AcquireWrite(lock_path, 0, 1), AcquireWrite(lock_path, 64, 128), TimeoutRead(lock_path, 0, 1), TimeoutRead(lock_path, 128, 256), ) # # Test that exclusive locks time out when shared locks are held. # def test_write_lock_timeout_on_read(lock_path): multiproc_test(AcquireRead(lock_path), TimeoutWrite(lock_path)) def test_write_lock_timeout_on_read_2(lock_path): multiproc_test(AcquireRead(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path)) def test_write_lock_timeout_on_read_3(lock_path): multiproc_test( AcquireRead(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path), ) def test_write_lock_timeout_on_read_ranges(lock_path): multiproc_test(AcquireRead(lock_path, 0, 1), TimeoutWrite(lock_path)) def test_write_lock_timeout_on_read_ranges_2(lock_path): multiproc_test(AcquireRead(lock_path, 0, 1), TimeoutWrite(lock_path, 0, 1)) def test_write_lock_timeout_on_read_ranges_3(lock_path): multiproc_test( AcquireRead(lock_path, 0, 1), AcquireRead(lock_path, 10, 1), TimeoutWrite(lock_path, 0, 1), TimeoutWrite(lock_path, 10, 1), ) def test_write_lock_timeout_on_read_ranges_4(lock_path): multiproc_test( AcquireRead(lock_path, 0, 64), TimeoutWrite(lock_path, 10, 1), TimeoutWrite(lock_path, 32, 1), ) def test_write_lock_timeout_on_read_ranges_5(lock_path): multiproc_test( AcquireRead(lock_path, 64, 128), TimeoutWrite(lock_path, 65, 1), TimeoutWrite(lock_path, 127, 1), TimeoutWrite(lock_path, 90, 10), ) # # Test that exclusive locks time while lots of shared locks are held. # def test_write_lock_timeout_with_multiple_readers_2_1(lock_path): multiproc_test(AcquireRead(lock_path), AcquireRead(lock_path), TimeoutWrite(lock_path)) def test_write_lock_timeout_with_multiple_readers_2_2(lock_path): multiproc_test( AcquireRead(lock_path), AcquireRead(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path), ) def test_write_lock_timeout_with_multiple_readers_3_1(lock_path): multiproc_test( AcquireRead(lock_path), AcquireRead(lock_path), AcquireRead(lock_path), TimeoutWrite(lock_path), ) def test_write_lock_timeout_with_multiple_readers_3_2(lock_path): multiproc_test( AcquireRead(lock_path), AcquireRead(lock_path), AcquireRead(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path), ) def test_write_lock_timeout_with_multiple_readers_2_1_ranges(lock_path): multiproc_test( AcquireRead(lock_path, 0, 10), AcquireRead(lock_path, 2, 10), TimeoutWrite(lock_path, 5, 5) ) def test_write_lock_timeout_with_multiple_readers_2_3_ranges(lock_path): multiproc_test( AcquireRead(lock_path, 0, 10), AcquireRead(lock_path, 5, 15), TimeoutWrite(lock_path, 0, 1), TimeoutWrite(lock_path, 11, 3), TimeoutWrite(lock_path, 7, 1), ) def test_write_lock_timeout_with_multiple_readers_3_1_ranges(lock_path): multiproc_test( AcquireRead(lock_path, 0, 5), AcquireRead(lock_path, 5, 5), AcquireRead(lock_path, 10, 5), TimeoutWrite(lock_path, 0, 15), ) def test_write_lock_timeout_with_multiple_readers_3_2_ranges(lock_path): multiproc_test( AcquireRead(lock_path, 0, 5), AcquireRead(lock_path, 5, 5), AcquireRead(lock_path, 10, 5), TimeoutWrite(lock_path, 3, 10), TimeoutWrite(lock_path, 5, 1), ) @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_read_lock_on_read_only_lockfile(lock_dir, lock_path): """read-only directory, read-only lockfile.""" touch(lock_path) with read_only(lock_path, lock_dir): lock = lk.Lock(lock_path) with lk.ReadTransaction(lock): pass with pytest.raises(lk.LockROFileError): with lk.WriteTransaction(lock): pass def test_read_lock_read_only_dir_writable_lockfile(lock_dir, lock_path): """read-only directory, writable lockfile.""" touch(lock_path) with read_only(lock_dir): lock = lk.Lock(lock_path) with lk.ReadTransaction(lock): pass with lk.WriteTransaction(lock): pass @pytest.mark.skipif(False if sys.platform == "win32" else getuid() == 0, reason="user is root") def test_read_lock_no_lockfile(lock_dir, lock_path): """read-only directory, no lockfile (so can't create).""" with read_only(lock_dir): lock = lk.Lock(lock_path) with pytest.raises(lk.CantCreateLockError): with lk.ReadTransaction(lock): pass with pytest.raises(lk.CantCreateLockError): with lk.WriteTransaction(lock): pass def test_upgrade_read_to_write(private_lock_path): """Test that a read lock can be upgraded to a write lock. Note that to upgrade a read lock to a write lock, you have the be the only holder of a read lock. Client code needs to coordinate that for shared locks. For this test, we use a private lock just to test that an upgrade is possible. """ # ensure lock file exists the first time, so we open it read-only # to begin with. touch(private_lock_path) lock = lk.Lock(private_lock_path) assert lock._reads == 0 assert lock._writes == 0 lock.acquire_read() assert lock._reads == 1 assert lock._writes == 0 assert lock._file_ref.fh.mode == "rb+" lock.acquire_write() assert lock._reads == 1 assert lock._writes == 1 assert lock._file_ref.fh.mode == "rb+" lock.release_write() assert lock._reads == 1 assert lock._writes == 0 assert lock._file_ref.fh.mode == "rb+" lock.release_read() assert lock._reads == 0 assert lock._writes == 0 assert not lock._file_ref.fh.closed # recycle the file handle for next lock def test_release_write_downgrades_to_shared(private_lock_path): """Releasing a write lock while a read lock is held must downgrade the POSIX lock from exclusive to shared, allowing other processes to acquire read locks.""" lock = lk.Lock(private_lock_path) lock.acquire_read() lock.acquire_write() lock.release_write() assert lock._reads == 1 assert lock._writes == 0 ctx = multiprocessing.get_context() q = ctx.Queue() # Another process must be able to acquire a shared read lock concurrently. p = ctx.Process(target=_child_try_acquire_read, args=(private_lock_path, q)) p.start() p.join() assert q.get() is True # But must not be able to acquire an exclusive write lock. p = ctx.Process(target=_child_try_acquire_write, args=(private_lock_path, q)) p.start() p.join() assert q.get() is False lock.release_read() assert lock._reads == 0 assert lock._writes == 0 @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_upgrade_read_to_write_fails_with_readonly_file(private_lock_path): """Test that read-only file can be read-locked but not write-locked.""" # ensure lock file exists the first time touch(private_lock_path) # open it read-only to begin with. with read_only(private_lock_path): lock = lk.Lock(private_lock_path) assert lock._reads == 0 assert lock._writes == 0 lock.acquire_read() assert lock._reads == 1 assert lock._writes == 0 assert lock._file_ref.fh.mode == "rb" # upgrade to write here with pytest.raises(lk.LockROFileError): lock.acquire_write() class ComplexAcquireAndRelease: def __init__(self, lock_path): self.lock_path = lock_path def p1(self, barrier): lock = lk.Lock(self.lock_path) lock.acquire_write() barrier.wait() # ---------------------------------------- 1 # others test timeout barrier.wait() # ---------------------------------------- 2 lock.release_write() # release and others acquire read barrier.wait() # ---------------------------------------- 3 with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) lock.acquire_read() barrier.wait() # ---------------------------------------- 4 lock.release_read() barrier.wait() # ---------------------------------------- 5 # p2 upgrades read to write barrier.wait() # ---------------------------------------- 6 with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) with pytest.raises(lk.LockTimeoutError): lock.acquire_read(lock_fail_timeout) barrier.wait() # ---------------------------------------- 7 # p2 releases write and read barrier.wait() # ---------------------------------------- 8 # p3 acquires read barrier.wait() # ---------------------------------------- 9 # p3 upgrades read to write barrier.wait() # ---------------------------------------- 10 with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) with pytest.raises(lk.LockTimeoutError): lock.acquire_read(lock_fail_timeout) barrier.wait() # ---------------------------------------- 11 # p3 releases locks barrier.wait() # ---------------------------------------- 12 lock.acquire_read() barrier.wait() # ---------------------------------------- 13 lock.release_read() def p2(self, barrier): lock = lk.Lock(self.lock_path) # p1 acquires write barrier.wait() # ---------------------------------------- 1 with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) with pytest.raises(lk.LockTimeoutError): lock.acquire_read(lock_fail_timeout) barrier.wait() # ---------------------------------------- 2 lock.acquire_read() barrier.wait() # ---------------------------------------- 3 # p1 tests shared read barrier.wait() # ---------------------------------------- 4 # others release reads barrier.wait() # ---------------------------------------- 5 lock.acquire_write() # upgrade read to write barrier.wait() # ---------------------------------------- 6 # others test timeout barrier.wait() # ---------------------------------------- 7 lock.release_write() # release read AND write (need both) lock.release_read() barrier.wait() # ---------------------------------------- 8 # p3 acquires read barrier.wait() # ---------------------------------------- 9 # p3 upgrades read to write barrier.wait() # ---------------------------------------- 10 with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) with pytest.raises(lk.LockTimeoutError): lock.acquire_read(lock_fail_timeout) barrier.wait() # ---------------------------------------- 11 # p3 releases locks barrier.wait() # ---------------------------------------- 12 lock.acquire_read() barrier.wait() # ---------------------------------------- 13 lock.release_read() def p3(self, barrier): lock = lk.Lock(self.lock_path) # p1 acquires write barrier.wait() # ---------------------------------------- 1 with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) with pytest.raises(lk.LockTimeoutError): lock.acquire_read(lock_fail_timeout) barrier.wait() # ---------------------------------------- 2 lock.acquire_read() barrier.wait() # ---------------------------------------- 3 # p1 tests shared read barrier.wait() # ---------------------------------------- 4 lock.release_read() barrier.wait() # ---------------------------------------- 5 # p2 upgrades read to write barrier.wait() # ---------------------------------------- 6 with pytest.raises(lk.LockTimeoutError): lock.acquire_write(lock_fail_timeout) with pytest.raises(lk.LockTimeoutError): lock.acquire_read(lock_fail_timeout) barrier.wait() # ---------------------------------------- 7 # p2 releases write & read barrier.wait() # ---------------------------------------- 8 lock.acquire_read() barrier.wait() # ---------------------------------------- 9 lock.acquire_write() barrier.wait() # ---------------------------------------- 10 # others test timeout barrier.wait() # ---------------------------------------- 11 lock.release_read() # release read AND write in opposite lock.release_write() # order from before on p2 barrier.wait() # ---------------------------------------- 12 lock.acquire_read() barrier.wait() # ---------------------------------------- 13 lock.release_read() # # Longer test case that ensures locks are reusable. Ordering is # enforced by barriers throughout -- steps are shown with numbers. # def test_complex_acquire_and_release_chain(lock_path): test_chain = ComplexAcquireAndRelease(lock_path) multiproc_test(test_chain.p1, test_chain.p2, test_chain.p3) class AssertLock(lk.Lock): """Test lock class that marks acquire/release events.""" def __init__(self, lock_path, vals): super().__init__(lock_path) self.vals = vals # assert hooks for subclasses assert_acquire_read = lambda self: None assert_acquire_write = lambda self: None assert_release_read = lambda self: None assert_release_write = lambda self: None def acquire_read(self, timeout=None): self.assert_acquire_read() result = super().acquire_read(timeout) self.vals["acquired_read"] = True return result def acquire_write(self, timeout=None): self.assert_acquire_write() result = super().acquire_write(timeout) self.vals["acquired_write"] = True return result def release_read(self, release_fn=None): self.assert_release_read() result = super().release_read(release_fn) self.vals["released_read"] = True return result def release_write(self, release_fn=None): self.assert_release_write() result = super().release_write(release_fn) self.vals["released_write"] = True return result @pytest.mark.parametrize( "transaction,type", [(lk.ReadTransaction, "read"), (lk.WriteTransaction, "write")] ) def test_transaction(lock_path, transaction, type): class MockLock(AssertLock): def assert_acquire_read(self): assert not vals["entered_fn"] assert not vals["exited_fn"] def assert_release_read(self): assert vals["entered_fn"] assert not vals["exited_fn"] def assert_acquire_write(self): assert not vals["entered_fn"] assert not vals["exited_fn"] def assert_release_write(self): assert vals["entered_fn"] assert not vals["exited_fn"] def enter_fn(): # assert enter_fn is called while lock is held assert vals["acquired_%s" % type] vals["entered_fn"] = True def exit_fn(t, v, tb): # assert exit_fn is called while lock is held assert not vals["released_%s" % type] vals["exited_fn"] = True vals["exception"] = t or v or tb vals = collections.defaultdict(lambda: False) lock = MockLock(lock_path, vals) with transaction(lock, acquire=enter_fn, release=exit_fn): assert vals["acquired_%s" % type] assert not vals["released_%s" % type] assert vals["entered_fn"] assert vals["exited_fn"] assert vals["acquired_%s" % type] assert vals["released_%s" % type] assert not vals["exception"] @pytest.mark.parametrize( "transaction,type", [(lk.ReadTransaction, "read"), (lk.WriteTransaction, "write")] ) def test_transaction_with_exception(lock_path, transaction, type): class MockLock(AssertLock): def assert_acquire_read(self): assert not vals["entered_fn"] assert not vals["exited_fn"] def assert_release_read(self): assert vals["entered_fn"] assert not vals["exited_fn"] def assert_acquire_write(self): assert not vals["entered_fn"] assert not vals["exited_fn"] def assert_release_write(self): assert vals["entered_fn"] assert not vals["exited_fn"] def enter_fn(): assert vals["acquired_%s" % type] vals["entered_fn"] = True def exit_fn(t, v, tb): assert not vals["released_%s" % type] vals["exited_fn"] = True vals["exception"] = t or v or tb return exit_result exit_result = False vals = collections.defaultdict(lambda: False) lock = MockLock(lock_path, vals) with pytest.raises(Exception): with transaction(lock, acquire=enter_fn, release=exit_fn): raise Exception() assert vals["entered_fn"] assert vals["exited_fn"] assert vals["exception"] # test suppression of exceptions from exit_fn exit_result = True vals.clear() # should not raise now. with transaction(lock, acquire=enter_fn, release=exit_fn): raise Exception() assert vals["entered_fn"] assert vals["exited_fn"] assert vals["exception"] def test_nested_write_transaction(lock_path): """Ensure that the outermost write transaction writes.""" def write(t, v, tb): vals["wrote"] = True vals = collections.defaultdict(lambda: False) lock = AssertLock(lock_path, vals) # write/write with lk.WriteTransaction(lock, release=write): assert not vals["wrote"] with lk.WriteTransaction(lock, release=write): assert not vals["wrote"] assert not vals["wrote"] assert vals["wrote"] # read/write vals.clear() with lk.ReadTransaction(lock): assert not vals["wrote"] with lk.WriteTransaction(lock, release=write): assert not vals["wrote"] assert vals["wrote"] # write/read/write vals.clear() with lk.WriteTransaction(lock, release=write): assert not vals["wrote"] with lk.ReadTransaction(lock): assert not vals["wrote"] with lk.WriteTransaction(lock, release=write): assert not vals["wrote"] assert not vals["wrote"] assert not vals["wrote"] assert vals["wrote"] # read/write/read/write vals.clear() with lk.ReadTransaction(lock): with lk.WriteTransaction(lock, release=write): assert not vals["wrote"] with lk.ReadTransaction(lock): assert not vals["wrote"] with lk.WriteTransaction(lock, release=write): assert not vals["wrote"] assert not vals["wrote"] assert not vals["wrote"] assert vals["wrote"] def test_nested_reads(lock_path): """Ensure that write transactions won't re-read data.""" def read(): vals["read"] += 1 vals = collections.defaultdict(lambda: 0) lock = AssertLock(lock_path, vals) # read/read vals.clear() assert vals["read"] == 0 with lk.ReadTransaction(lock, acquire=read): assert vals["read"] == 1 with lk.ReadTransaction(lock, acquire=read): assert vals["read"] == 1 # write/write vals.clear() assert vals["read"] == 0 with lk.WriteTransaction(lock, acquire=read): assert vals["read"] == 1 with lk.WriteTransaction(lock, acquire=read): assert vals["read"] == 1 # read/write vals.clear() assert vals["read"] == 0 with lk.ReadTransaction(lock, acquire=read): assert vals["read"] == 1 with lk.WriteTransaction(lock, acquire=read): assert vals["read"] == 1 # write/read/write vals.clear() assert vals["read"] == 0 with lk.WriteTransaction(lock, acquire=read): assert vals["read"] == 1 with lk.ReadTransaction(lock, acquire=read): assert vals["read"] == 1 with lk.WriteTransaction(lock, acquire=read): assert vals["read"] == 1 # read/write/read/write vals.clear() assert vals["read"] == 0 with lk.ReadTransaction(lock, acquire=read): assert vals["read"] == 1 with lk.WriteTransaction(lock, acquire=read): assert vals["read"] == 1 with lk.ReadTransaction(lock, acquire=read): assert vals["read"] == 1 with lk.WriteTransaction(lock, acquire=read): assert vals["read"] == 1 class LockDebugOutput: def __init__(self, lock_path): self.lock_path = lock_path self.host = socket.gethostname() def p1(self, barrier, q1, q2): # exchange pids p1_pid = os.getpid() q1.put(p1_pid) p2_pid = q2.get() # set up lock lock = lk.Lock(self.lock_path, debug=True) with lk.WriteTransaction(lock): # p1 takes write lock and writes pid/host to file barrier.wait() # ------------------------------------ 1 assert lock.pid == p1_pid assert lock.host == self.host # wait for p2 to verify contents of file barrier.wait() # ---------------------------------------- 2 # wait for p2 to take a write lock barrier.wait() # ---------------------------------------- 3 # verify pid/host info again with lk.ReadTransaction(lock): assert lock.old_pid == p1_pid assert lock.old_host == self.host assert lock.pid == p2_pid assert lock.host == self.host barrier.wait() # ---------------------------------------- 4 def p2(self, barrier, q1, q2): # exchange pids p2_pid = os.getpid() p1_pid = q1.get() q2.put(p2_pid) # set up lock lock = lk.Lock(self.lock_path, debug=True) # p1 takes write lock and writes pid/host to file barrier.wait() # ---------------------------------------- 1 # verify that p1 wrote information to lock file with lk.ReadTransaction(lock): assert lock.pid == p1_pid assert lock.host == self.host barrier.wait() # ---------------------------------------- 2 # take a write lock on the file and verify pid/host info with lk.WriteTransaction(lock): assert lock.old_pid == p1_pid assert lock.old_host == self.host assert lock.pid == p2_pid assert lock.host == self.host barrier.wait() # ------------------------------------ 3 # wait for p1 to verify pid/host info barrier.wait() # ---------------------------------------- 4 def test_lock_debug_output(lock_path): test_debug = LockDebugOutput(lock_path) q1, q2 = Queue(), Queue() local_multiproc_test(test_debug.p2, test_debug.p1, extra_args=(q1, q2)) def test_lock_with_no_parent_directory(tmp_path: pathlib.Path): """Make sure locks work even when their parent directory does not exist.""" with working_dir(str(tmp_path)): lock = lk.Lock("foo/bar/baz/lockfile") with lk.WriteTransaction(lock): pass def test_lock_in_current_directory(tmp_path: pathlib.Path): """Make sure locks work even when their parent directory does not exist.""" with working_dir(str(tmp_path)): # test we can create a lock in the current directory lock = lk.Lock("lockfile") for i in range(10): with lk.ReadTransaction(lock): pass with lk.WriteTransaction(lock): pass # and that we can do the same thing after it's already there lock = lk.Lock("lockfile") for i in range(10): with lk.ReadTransaction(lock): pass with lk.WriteTransaction(lock): pass def test_attempts_str(): assert lk._attempts_str(0, 0) == "" assert lk._attempts_str(0.12, 1) == "" assert lk._attempts_str(12.345, 2) == " after 12.345s and 2 attempts" def test_lock_str(): lock = lk.Lock("lockfile") lockstr = str(lock) assert "lockfile[0:0]" in lockstr assert "timeout=None" in lockstr assert "#reads=0, #writes=0" in lockstr def test_downgrade_write_okay(tmp_path: pathlib.Path): """Test the lock write-to-read downgrade operation.""" with working_dir(str(tmp_path)): lock = lk.Lock("lockfile") lock.acquire_write() lock.downgrade_write_to_read() assert lock._reads == 1 assert lock._writes == 0 lock.release_read() def test_downgrade_write_fails(tmp_path: pathlib.Path): """Test failing the lock write-to-read downgrade operation.""" with working_dir(str(tmp_path)): lock = lk.Lock("lockfile") lock.acquire_read() msg = "Cannot downgrade lock from write to read on file: lockfile" with pytest.raises(lk.LockDowngradeError, match=msg): lock.downgrade_write_to_read() lock.release_read() @pytest.mark.parametrize( "err_num,err_msg", [ (errno.EACCES, "Fake EACCES error"), (errno.EAGAIN, "Fake EAGAIN error"), (errno.ENOENT, "Fake ENOENT error"), ], ) def test_poll_lock_exception(tmp_path: pathlib.Path, monkeypatch, err_num, err_msg): """Test poll lock exception handling.""" def _lockf(fd, cmd, len, start, whence): raise OSError(err_num, err_msg) with working_dir(str(tmp_path)): lockfile = "lockfile" lock = lk.Lock(lockfile) lock.acquire_read() monkeypatch.setattr(fcntl, "lockf", _lockf) if err_num in [errno.EAGAIN, errno.EACCES]: assert not lock._poll_lock(fcntl.LOCK_EX) else: with pytest.raises(OSError, match=err_msg): lock._poll_lock(fcntl.LOCK_EX) monkeypatch.undo() lock.release_read() def test_upgrade_read_okay(tmp_path: pathlib.Path): """Test the lock read-to-write upgrade operation.""" with working_dir(str(tmp_path)): lock = lk.Lock("lockfile") lock.acquire_read() lock.upgrade_read_to_write() assert lock._reads == 0 assert lock._writes == 1 lock.release_write() def test_upgrade_read_fails(tmp_path: pathlib.Path): """Test failing the lock read-to-write upgrade operation.""" with working_dir(str(tmp_path)): lock = lk.Lock("lockfile") lock.acquire_write() msg = "Cannot upgrade lock from read to write on file: lockfile" with pytest.raises(lk.LockUpgradeError, match=msg): lock.upgrade_read_to_write() lock.release_write() @pytest.mark.parametrize("acquire", ["acquire_write", "acquire_read"]) def test_acquire_after_fork(tmp_path: pathlib.Path, acquire: str): """After fork, acquire_write/read must not silently succeed due to inherited counters.""" try: ctx = multiprocessing.get_context("fork") except ValueError: pytest.skip("fork start method not available on this platform") lockfile = str(tmp_path / "lockfile") lock = lk.Lock(lockfile) result = ctx.Queue() def child(): assert lock._writes == 1 # due to forking, but POSIX lock is NOT held by this process try: if acquire == "acquire_write": lock.acquire_write(lock_fail_timeout) elif acquire == "acquire_read": lock.acquire_read(lock_fail_timeout) else: assert False # should never get here result.put("no_error") except lk.LockTimeoutError: result.put("timed_out") lock.acquire_write() try: p = ctx.Process(target=child) p.start() p.join() assert result.get() == "timed_out" finally: lock.release_write() def _child_try_acquire_write(lock_path: str, result_queue): lock = lk.Lock(lock_path) result_queue.put(lock.try_acquire_write()) def _child_try_acquire_read(lock_path: str, result_queue): lock = lk.Lock(lock_path) result_queue.put(lock.try_acquire_read()) def test_try_acquire_read(tmp_path: pathlib.Path): """Test non-blocking try_acquire_read.""" lock = lk.Lock(str(tmp_path / "lockfile")) # Succeeds on unlocked lock assert lock.try_acquire_read() is True assert lock._reads == 1 # Succeeds again (nested) assert lock.try_acquire_read() is True assert lock._reads == 2 lock.release_read() lock.release_read() ctx = multiprocessing.get_context() # Fails when another process holds an exclusive write lock lock.acquire_write() try: q = ctx.Queue() p = ctx.Process(target=_child_try_acquire_read, args=(str(tmp_path / "lockfile"), q)) p.start() p.join() assert q.get() is False finally: lock.release_write() def test_try_acquire_write(tmp_path: pathlib.Path): """Test non-blocking try_acquire_write.""" lock = lk.Lock(str(tmp_path / "lockfile")) ctx = multiprocessing.get_context() # Succeeds on unlocked lock assert lock.try_acquire_write() is True assert lock._writes == 1 # Succeeds again (nested) assert lock.try_acquire_write() is True assert lock._writes == 2 lock.release_write() lock.release_write() # Fails when another process holds a write lock lock.acquire_write() try: q = ctx.Queue() p = ctx.Process(target=_child_try_acquire_write, args=(str(tmp_path / "lockfile"), q)) p.start() p.join() assert q.get() is False finally: lock.release_write() # Fails when another process holds a read lock lock.acquire_read() try: q = ctx.Queue() p = ctx.Process(target=_child_try_acquire_write, args=(str(tmp_path / "lockfile"), q)) p.start() p.join() assert q.get() is False finally: lock.release_read() def _child_fails_to_acquire_read(_lock: lk.Lock): try: _lock.acquire_read(timeout=1e-9) except lk.LockTimeoutError: return assert False, "Child process should not have been able to acquire read lock" def test_read_after_write_does_not_accidentally_downgrade(tmp_path: pathlib.Path): """Test that acquiring a read lock after a write lock does not accidentally downgrade the write lock, by having another process attempt to acquire a read lock.""" lock = lk.Lock(str(tmp_path / "lockfile")) lock.acquire_write() lock.acquire_read() # should not downgrade the write lock try: # No matter the start method, the child process shouldn't be able to acquire a read lock. p = multiprocessing.Process(target=_child_fails_to_acquire_read, args=(lock,)) p.start() p.join() assert p.exitcode == 0 finally: lock.release_read() lock.release_write() ================================================ FILE: lib/spack/spack/test/llnl/util/symlink.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for Windows symlink functionality.""" import os import pathlib import tempfile import pytest import spack.llnl.util.filesystem as fs def test_symlink_file(tmp_path: pathlib.Path): """Test the symlink functionality on all operating systems for a file""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) link_file = str(tmp_path / "link.txt") assert os.path.exists(link_file) is False fs.symlink(real_file, link_file) assert os.path.exists(link_file) assert fs.islink(link_file) def test_symlink_dir(tmp_path: pathlib.Path): """Test the symlink functionality on all operating systems for a directory""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) real_dir = os.path.join(test_dir, "real_dir") link_dir = os.path.join(test_dir, "link_dir") os.mkdir(real_dir) fs.symlink(real_dir, link_dir) assert os.path.exists(link_dir) assert fs.islink(link_dir) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_symlink_source_not_exists(tmp_path: pathlib.Path): """Test the symlink method for the case where a source path does not exist""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) real_dir = os.path.join(test_dir, "real_dir") link_dir = os.path.join(test_dir, "link_dir") with pytest.raises(fs.SymlinkError): fs._windows_symlink(real_dir, link_dir) def test_symlink_src_relative_to_link(tmp_path: pathlib.Path): """Test the symlink functionality where the source value exists relative to the link but not relative to the cwd""" with fs.working_dir(str(tmp_path)): subdir_1 = tmp_path / "a" subdir_2 = os.path.join(subdir_1, "b") link_dir = os.path.join(subdir_1, "c") os.mkdir(subdir_1) os.mkdir(subdir_2) fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=subdir_2) link_file = os.path.join(subdir_1, "link.txt") fs.symlink(f"b/{os.path.basename(real_file)}", f"a/{os.path.basename(link_file)}") assert os.path.exists(link_file) assert fs.islink(link_file) # Check dirs assert not os.path.lexists(link_dir) fs.symlink("b", "a/c") assert os.path.lexists(link_dir) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_symlink_src_not_relative_to_link(tmp_path: pathlib.Path): """Test the symlink functionality where the source value does not exist relative to the link and not relative to the cwd. NOTE that this symlink api call is EXPECTED to raise a fs.SymlinkError exception that we catch.""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) subdir_1 = os.path.join(test_dir, "a") subdir_2 = os.path.join(subdir_1, "b") link_dir = os.path.join(subdir_1, "c") os.mkdir(subdir_1) os.mkdir(subdir_2) fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=subdir_2) link_file = str(tmp_path / "link.txt") # Expected SymlinkError because source path does not exist relative to link path with pytest.raises(fs.SymlinkError): fs._windows_symlink( f"d/{os.path.basename(real_file)}", f"a/{os.path.basename(link_file)}" ) assert not os.path.exists(link_file) # Check dirs assert not os.path.lexists(link_dir) with pytest.raises(fs.SymlinkError): fs._windows_symlink("d", "a/c") assert not os.path.lexists(link_dir) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_symlink_link_already_exists(tmp_path: pathlib.Path): """Test the symlink method for the case where a link already exists""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) real_dir = os.path.join(test_dir, "real_dir") link_dir = os.path.join(test_dir, "link_dir") os.mkdir(real_dir) fs._windows_symlink(real_dir, link_dir) assert os.path.exists(link_dir) with pytest.raises(fs.SymlinkError): fs._windows_symlink(real_dir, link_dir) @pytest.mark.skipif(not fs._windows_can_symlink(), reason="Test requires elevated privileges") @pytest.mark.only_windows("Test is for Windows specific behavior") def test_symlink_win_file(tmp_path: pathlib.Path): """Check that symlink makes a symlink file when run with elevated permissions""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) link_file = str(tmp_path / "link.txt") fs.symlink(real_file, link_file) # Verify that all expected conditions are met assert os.path.exists(link_file) assert fs.islink(link_file) assert os.path.islink(link_file) assert not fs._windows_is_hardlink(link_file) assert not fs._windows_is_junction(link_file) @pytest.mark.skipif(not fs._windows_can_symlink(), reason="Test requires elevated privileges") @pytest.mark.only_windows("Test is for Windows specific behavior") def test_symlink_win_dir(tmp_path: pathlib.Path): """Check that symlink makes a symlink dir when run with elevated permissions""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) real_dir = os.path.join(test_dir, "real") link_dir = os.path.join(test_dir, "link") os.mkdir(real_dir) fs.symlink(real_dir, link_dir) # Verify that all expected conditions are met assert os.path.exists(link_dir) assert fs.islink(link_dir) assert os.path.islink(link_dir) assert not fs._windows_is_hardlink(link_dir) assert not fs._windows_is_junction(link_dir) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_windows_create_junction(tmp_path: pathlib.Path): """Test the _windows_create_junction method""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) junction_real_dir = os.path.join(test_dir, "real_dir") junction_link_dir = os.path.join(test_dir, "link_dir") os.mkdir(junction_real_dir) fs._windows_create_junction(junction_real_dir, junction_link_dir) # Verify that all expected conditions are met assert os.path.exists(junction_link_dir) assert fs._windows_is_junction(junction_link_dir) assert fs.islink(junction_link_dir) assert not os.path.islink(junction_link_dir) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_windows_create_hard_link(tmp_path: pathlib.Path): """Test the _windows_create_hard_link method""" with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) link_file = str(tmp_path / "link.txt") fs._windows_create_hard_link(real_file, link_file) # Verify that all expected conditions are met assert os.path.exists(link_file) assert fs._windows_is_hardlink(real_file) assert fs._windows_is_hardlink(link_file) assert fs.islink(link_file) assert not os.path.islink(link_file) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_windows_create_link_dir(tmp_path: pathlib.Path): """Test the functionality of the windows_create_link method with a directory which should result in making a junction. """ with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) real_dir = os.path.join(test_dir, "real") link_dir = os.path.join(test_dir, "link") os.mkdir(real_dir) fs._windows_create_link(real_dir, link_dir) # Verify that all expected conditions are met assert os.path.exists(link_dir) assert fs.islink(link_dir) assert not fs._windows_is_hardlink(link_dir) assert fs._windows_is_junction(link_dir) assert not os.path.islink(link_dir) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_windows_create_link_file(tmp_path: pathlib.Path): """Test the functionality of the windows_create_link method with a file which should result in the creation of a hard link. It also tests the functionality of the symlink islink infrastructure. """ with fs.working_dir(str(tmp_path)): test_dir = str(tmp_path) fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir) link_file = str(tmp_path / "link.txt") fs._windows_create_link(real_file, link_file) # Verify that all expected conditions are met assert fs._windows_is_hardlink(link_file) assert fs.islink(link_file) assert not fs._windows_is_junction(link_file) @pytest.mark.only_windows("Test is for Windows specific behavior") def test_windows_read_link(tmp_path: pathlib.Path): """Makes sure readlink can read the link source for hard links and junctions on windows.""" with fs.working_dir(str(tmp_path)): real_dir_1 = "real_dir_1" real_dir_2 = "real_dir_2" link_dir_1 = "link_dir_1" link_dir_2 = "link_dir_2" os.mkdir(real_dir_1) os.mkdir(real_dir_2) # Create a file and a directory _, real_file_1 = tempfile.mkstemp(prefix="real_1", suffix=".txt", dir=".") _, real_file_2 = tempfile.mkstemp(prefix="real_2", suffix=".txt", dir=".") link_file_1 = "link_1.txt" link_file_2 = "link_2.txt" # Make hard link/junction fs._windows_create_hard_link(real_file_1, link_file_1) fs._windows_create_hard_link(real_file_2, link_file_2) fs._windows_create_junction(real_dir_1, link_dir_1) fs._windows_create_junction(real_dir_2, link_dir_2) assert fs.readlink(link_file_1) == os.path.abspath(real_file_1) assert fs.readlink(link_file_2) == os.path.abspath(real_file_2) assert fs.readlink(link_dir_1) == os.path.abspath(real_dir_1) assert fs.readlink(link_dir_2) == os.path.abspath(real_dir_2) ================================================ FILE: lib/spack/spack/test/llnl/util/tty/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/test/llnl/util/tty/colify.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import sys import pytest from spack.llnl.util.tty.colify import colify, colify_table # table as 3 rows x 6 columns lorem_table = [ ["There", "are", "many", "variations", "of", "passages"], ["of", "Lorem", "Ipsum", "available", "but", "many"], ["have", "suffered", "alteration", "in", "some", "form"], ] # width of each column in above table lorem_table_col_starts = [0, 7, 17, 29, 41, 47] # table in a single list lorem_words = lorem_table[0] + lorem_table[1] + lorem_table[2] @pytest.mark.parametrize("console_cols", [10, 20, 40, 60, 80, 100, 120]) def test_fixed_column_table(console_cols, capfd): "ensure output is a fixed table regardless of size" colify_table(lorem_table, output=sys.stdout, console_cols=console_cols) output, _ = capfd.readouterr() # 3 rows assert output.strip().count("\n") == 2 # right spacing lines = output.strip().split("\n") for line in lines: assert [line[w - 1] for w in lorem_table_col_starts[1:]] == [" "] * 5 # same data stripped_lines = [re.sub(r"\s+", " ", line.strip()) for line in lines] assert stripped_lines == [" ".join(row) for row in lorem_table] @pytest.mark.parametrize( "console_cols,expected_rows,expected_cols", [ (10, 18, 1), (20, 18, 1), (40, 5, 4), (60, 3, 6), (80, 2, 9), (100, 2, 9), (120, 2, 9), (140, 1, 18), ], ) def test_variable_width_columns(console_cols, expected_rows, expected_cols, capfd): colify(lorem_words, tty=True, output=sys.stdout, console_cols=console_cols) output, _ = capfd.readouterr() print(output) # expected rows assert output.strip().count("\n") == expected_rows - 1 # right cols lines = output.strip().split("\n") assert all(len(re.split(r"\s+", line)) <= expected_cols for line in lines) # padding between columns rows = [re.split(r"\s+", line) for line in lines] cols = list(zip(*rows)) max_col_widths = [max(len(s) for s in col) for col in cols] col_start = 0 for w in max_col_widths: col_start += w + 2 # plus padding # verify that every column boundary is at max width + padding assert all( [ line[col_start - 1] == " " and line[col_start] != " " for line in lines if col_start < len(line) ] ) ================================================ FILE: lib/spack/spack/test/llnl/util/tty/color.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import textwrap import pytest import spack.llnl.util.tty.color as color from spack.llnl.util.tty.color import cescape, colorize, csub test_text = [ "@r{The quick brown fox jumps over the lazy yellow dog.", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt " "ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco " "laboris nisi ut aliquip ex ea commodo consequat.}", "@c{none, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034}", "none, @c{gfx1010}, gfx1011, @r{gfx1012}, gfx1013, @b{gfx1030}, gfx1031, gfx1032, gfx1033", "@c{none, 10, 100, 100a, 100f, 101, 101a, 101f, 103, 103a, 103f, 11, 12, 120, 120a, 120f}", "@c{none, 10, 100, 100a, 100f, 101, 101a, 101f, 103, 103a, 103f, 11, 12, 120}", "none, @c{10}, @b{100}, 100a, @r{100f}, 101, @g{101a}, 101f, @c{103}, 103a, 103f" "@g{build}, @c{link}, @r{run}", ] @pytest.mark.parametrize("cols", list(range(30, 101, 10))) @pytest.mark.parametrize("text", test_text) @pytest.mark.parametrize("indent", [0, 4, 8]) def test_color_wrap(cols, text, indent): colorized = color.colorize(text, color=True) # True to force color plain = color.csub(colorized) spaces = indent * " " color_wrapped = " ".join( color.cwrap(colorized, width=cols, initial_indent=spaces, subsequent_indent=spaces) ) plain_cwrapped = " ".join( color.cwrap(plain, width=cols, initial_indent=spaces, subsequent_indent=spaces) ) wrapped = " ".join( textwrap.wrap(plain, width=cols, initial_indent=spaces, subsequent_indent=spaces) ) # make sure the concatenated, non-indented wrapped version is the same as the # original, modulo any spaces consumed while wrapping. assert re.sub(r"\s+", " ", color_wrapped).lstrip() == re.sub(r"\s+", " ", colorized) # make sure we wrap the same as textwrap assert color.csub(color_wrapped) == wrapped assert plain_cwrapped == wrapped def test_cescape_at_sign_roundtrip(): """cescape followed by colorize should not double-escape '@' inside color blocks.""" raw = 'if spec.satisfies("@:25.1"):' colorized = colorize("@R{%s}" % cescape(raw), color=True) assert csub(colorized) == raw def test_cescape_multiple_at_signs_roundtrip(): """Multiple consecutive '@' characters should survive a cescape/colorize roundtrip.""" raw = "foo @@@@@bar" colorized = colorize("@R{%s}" % cescape(raw), color=True) assert csub(colorized) == raw def test_colorize_top_level_consecutive_escaped_ats(): """Consecutive @@ at the top level (outside braces) must each unescape independently.""" assert colorize("@@@@", color=False) == "@@" assert colorize("@@@@@@", color=False) == "@@@" ================================================ FILE: lib/spack/spack/test/llnl/util/tty/log.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import contextlib import pathlib import sys from types import ModuleType from typing import Optional import pytest import spack.llnl.util.tty.log as log from spack.llnl.util.filesystem import working_dir from spack.util.executable import Executable termios: Optional[ModuleType] = None try: import termios as term_mod termios = term_mod except ImportError: pass pytestmark = pytest.mark.not_on_windows("does not run on windows") @contextlib.contextmanager def nullcontext(): yield def test_log_python_output_with_echo(capfd, tmp_path: pathlib.Path): with working_dir(str(tmp_path)): with log.log_output("foo.txt", echo=True): print("logged") # foo.txt has output with open("foo.txt", encoding="utf-8") as f: assert f.read() == "logged\n" # output is also echoed. assert capfd.readouterr()[0] == "logged\n" def test_log_python_output_without_echo(capfd, tmp_path: pathlib.Path): with working_dir(str(tmp_path)): with log.log_output("foo.txt"): print("logged") # foo.txt has output with open("foo.txt", encoding="utf-8") as f: assert f.read() == "logged\n" # nothing on stdout or stderr assert capfd.readouterr()[0] == "" def test_log_python_output_with_invalid_utf8(capfd, tmp_path: pathlib.Path): tmp_file = str(tmp_path / "foo.txt") with log.log_output(tmp_file, echo=True): sys.stdout.buffer.write(b"\xc3helloworld\n") # we should be able to read this as valid utf-8 with open(tmp_file, "r", encoding="utf-8") as f: assert f.read() == "�helloworld\n" assert capfd.readouterr().out == "�helloworld\n" def test_log_python_output_and_echo_output(capfd, tmp_path: pathlib.Path): with working_dir(str(tmp_path)): # echo two lines with log.log_output("foo.txt") as logger: with logger.force_echo(): print("force echo") print("logged") # log file contains everything with open("foo.txt", encoding="utf-8") as f: assert f.read() == "force echo\nlogged\n" # only force-echo'd stuff is in output assert capfd.readouterr()[0] == "force echo\n" def test_log_output_with_control_codes(capfd, tmp_path: pathlib.Path): with working_dir(str(tmp_path)): with log.log_output("foo.txt"): # Print a sample of formatted GCC error output # Line obtained from the file generated by running gcc on a nonexistent file: # gcc -fdiagnostics-color=always ./test.cpp 2>test.log csi = "\x1b[" print( f"{csi}01m{csi}Kgcc:{csi}m{csi}K {csi}01;31m{csi}Kerror: {csi}m{csi}K./test.cpp:" ) with open("foo.txt", encoding="utf-8") as f: assert f.read() == "gcc: error: ./test.cpp:\n" def _log_filter_fn(string): return string.replace("foo", "bar") def test_log_output_with_filter(capfd, tmp_path: pathlib.Path): with working_dir(str(tmp_path)): with log.log_output("foo.txt", filter_fn=_log_filter_fn): print("foo blah") print("blah foo") print("foo foo") # foo.txt output is not filtered with open("foo.txt", encoding="utf-8") as f: assert f.read() == "foo blah\nblah foo\nfoo foo\n" # output is not echoed assert capfd.readouterr()[0] == "" # now try with echo with working_dir(str(tmp_path)): with log.log_output("foo.txt", echo=True, filter_fn=_log_filter_fn): print("foo blah") print("blah foo") print("foo foo") # foo.txt output is still not filtered with open("foo.txt", encoding="utf-8") as f: assert f.read() == "foo blah\nblah foo\nfoo foo\n" # echoed output is filtered. assert capfd.readouterr()[0] == "bar blah\nblah bar\nbar bar\n" def test_log_output_with_filter_and_append(capfd, tmp_path: pathlib.Path): with working_dir(str(tmp_path)): with log.log_output("foo.txt", filter_fn=_log_filter_fn): print("foo blah") print("blah foo") print("foo foo") with open("foo.txt", encoding="utf-8") as f: assert f.read() == "foo blah\nblah foo\nfoo foo\n" with log.log_output("foo.txt", filter_fn=_log_filter_fn, append=True): print("more foo more blah") with open("foo.txt", encoding="utf-8") as f: assert f.read() == "foo blah\nblah foo\nfoo foo\nmore foo more blah\n" def test_log_subproc_and_echo_output(capfd, tmp_path: pathlib.Path): python = Executable(sys.executable) with working_dir(str(tmp_path)): with log.log_output("foo.txt") as logger: with logger.force_echo(): python("-c", "print('echo')") print("logged") # Check log file content with open("foo.txt", encoding="utf-8") as f: assert f.read() == "echo\nlogged\n" # Check captured output (echoed content) # Note: 'logged' is not echoed because force_echo() scope ended assert capfd.readouterr()[0] == "echo\n" ================================================ FILE: lib/spack/spack/test/llnl/util/tty/tty.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pytest import spack.llnl.util.tty as tty def test_get_timestamp(monkeypatch): """Ensure the results of get_timestamp are reasonable.""" # Debug disabled should return an empty string monkeypatch.setattr(tty, "_debug", 0) assert not tty.get_timestamp(False), "Expected an empty string" # Debug disabled but force the timestamp should return a string assert tty.get_timestamp(True), "Expected a timestamp/non-empty string" pid_str = " {0}".format(os.getpid()) # Level 1 debugging should return a timestamp WITHOUT the pid monkeypatch.setattr(tty, "_debug", 1) out_str = tty.get_timestamp(False) assert out_str and pid_str not in out_str, "Expected no PID in results" # Level 2 debugging should also return a timestamp WITH the pid monkeypatch.setattr(tty, "_debug", 2) out_str = tty.get_timestamp(False) assert out_str and pid_str in out_str, "Expected PID in results" @pytest.mark.parametrize( "msg,enabled,trace,newline", [ ("", False, False, False), # Nothing is output (Exception(""), True, False, True), # Exception output ("trace", True, True, False), # stacktrace output ("newline", True, False, True), # newline in output ("no newline", True, False, False), # no newline output ], ) def test_msg(capfd, monkeypatch, enabled, msg, trace, newline): """Ensure the output from msg with options is appropriate.""" # temporarily use the parameterized settings monkeypatch.setattr(tty, "_msg_enabled", enabled) monkeypatch.setattr(tty, "_stacktrace", trace) expected = [msg if isinstance(msg, str) else "Exception: "] if newline: expected[0] = "{0}\n".format(expected[0]) if trace: expected.insert(0, ".py") tty.msg(msg, newline=newline) out = capfd.readouterr()[0] for msg in expected: assert msg in out @pytest.mark.parametrize( "msg,trace,wrap", [ (Exception(""), False, False), # Exception output ("trace", True, False), # stacktrace output ("wrap", False, True), # wrap in output ], ) def test_info(capfd, monkeypatch, msg, trace, wrap): """Ensure the output from info with options is appropriate.""" # temporarily use the parameterized settings monkeypatch.setattr(tty, "_stacktrace", trace) expected = [msg if isinstance(msg, str) else "Exception: "] if trace: expected.insert(0, ".py") extra = ( "This extra argument *should* make for a sufficiently long line" " that needs to be wrapped if the option is enabled." ) args = [msg, extra] num_newlines = 3 if wrap else 2 tty.info(*args, wrap=wrap, countback=3) out = capfd.readouterr()[0] for msg in expected: assert msg in out assert out.count("\n") == num_newlines ================================================ FILE: lib/spack/spack/test/main.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import os.path import pathlib import pytest import spack import spack.config import spack.environment as ev import spack.error import spack.llnl.util.filesystem as fs import spack.main import spack.paths import spack.platforms import spack.util.executable as exe import spack.util.git import spack.util.spack_yaml as syaml pytestmark = pytest.mark.not_on_windows( "Test functionality supported but tests are failing on Win" ) @pytest.fixture(autouse=True) def _clear_commit_cache(): spack.get_spack_commit.cache_clear() def test_version_git_nonsense_output(tmp_path: pathlib.Path, working_env, monkeypatch): git = tmp_path / "git" with open(git, "w", encoding="utf-8") as f: f.write( """#!/bin/sh echo --|not a hash|---- """ ) fs.set_executable(str(git)) monkeypatch.setattr(spack.util.git, "git", lambda: exe.which(str(git))) assert spack.spack_version == spack.get_version() def test_version_git_fails(tmp_path: pathlib.Path, working_env, monkeypatch): git = tmp_path / "git" with open(git, "w", encoding="utf-8") as f: f.write( """#!/bin/sh echo 26552533be04e83e66be2c28e0eb5011cb54e8fa exit 1 """ ) fs.set_executable(str(git)) monkeypatch.setattr(spack.util.git, "git", lambda: exe.which(str(git))) assert spack.spack_version == spack.get_version() def test_git_sha_output(tmp_path: pathlib.Path, working_env, monkeypatch): git = tmp_path / "git" sha = "26552533be04e83e66be2c28e0eb5011cb54e8fa" with open(git, "w", encoding="utf-8") as f: f.write( """#!/bin/sh echo {0} """.format(sha) ) fs.set_executable(str(git)) monkeypatch.setattr(spack.util.git, "git", lambda: exe.which(str(git))) expected = "{0} ({1})".format(spack.spack_version, sha) assert expected == spack.get_version() def test_get_version_no_repo(tmp_path: pathlib.Path, monkeypatch): monkeypatch.setattr(spack.paths, "prefix", str(tmp_path)) assert spack.spack_version == spack.get_version() def test_get_version_no_git(working_env, monkeypatch): monkeypatch.setattr(spack.util.git, "git", lambda: None) assert spack.spack_version == spack.get_version() def test_main_calls_get_version(capfd, working_env, monkeypatch): # act like git is not found in the PATH monkeypatch.setattr(spack.util.git, "git", lambda: None) # make sure we get a bare version (without commit) when this happens spack.main.main(["-V"]) out, err = capfd.readouterr() assert spack.spack_version == out.strip() def test_unrecognized_top_level_flag(): assert spack.main.main(["-o", "mirror", "list"]) != 0 def test_get_version_bad_git(tmp_path: pathlib.Path, working_env, monkeypatch): bad_git = str(tmp_path / "git") with open(bad_git, "w", encoding="utf-8") as f: f.write( """#!/bin/sh exit 1 """ ) fs.set_executable(bad_git) monkeypatch.setattr(spack.util.git, "git", lambda: exe.which(bad_git)) assert spack.spack_version == spack.get_version() def test_bad_command_line_scopes(tmp_path: pathlib.Path, config): cfg = spack.config.Configuration() file_path = tmp_path / "file_instead_of_dir" non_existing_path = tmp_path / "non_existing_dir" file_path.write_text("") with pytest.raises(spack.error.ConfigError): spack.main.add_command_line_scopes(cfg, [str(file_path)]) with pytest.raises(spack.error.ConfigError): spack.main.add_command_line_scopes(cfg, [str(non_existing_path)]) def test_add_command_line_scopes(tmp_path: pathlib.Path, mutable_config): config_yaml = str(tmp_path / "config.yaml") with open(config_yaml, "w", encoding="utf-8") as f: f.write( """\ config: verify_ssl: False dirty: False """ ) spack.main.add_command_line_scopes(mutable_config, [str(tmp_path)]) assert mutable_config.get("config:verify_ssl") is False assert mutable_config.get("config:dirty") is False def test_add_command_line_scope_env(tmp_path: pathlib.Path, mutable_mock_env_path): """Test whether --config-scope works, either by name or path.""" managed_env = ev.create("example").manifest_path with open(managed_env, "w", encoding="utf-8") as f: f.write( """\ spack: config: install_tree: root: /tmp/first """ ) with open(tmp_path / "spack.yaml", "w", encoding="utf-8") as f: f.write( """\ spack: config: install_tree: root: /tmp/second """ ) config = spack.config.Configuration() spack.main.add_command_line_scopes(config, ["example", str(tmp_path)]) assert len(config.scopes) == 2 assert config.get("config:install_tree:root") == "/tmp/second" config = spack.config.Configuration() spack.main.add_command_line_scopes(config, [str(tmp_path), "example"]) assert len(config.scopes) == 2 assert config.get("config:install_tree:root") == "/tmp/first" assert ev.active_environment() is None # shouldn't cause an environment to be activated def test_include_cfg(mock_low_high_config, write_config_file, tmp_path: pathlib.Path): cfg1_path = str(tmp_path / "include1.yaml") with open(cfg1_path, "w", encoding="utf-8") as f: f.write( """\ config: verify_ssl: False dirty: True packages: python: require: - spec: "@3.11:" """ ) def python_cfg(_spec): return f"""\ packages: python: require: - spec: {_spec} """ def write_python_cfg(_spec, _cfg_name): cfg_path = str(tmp_path / _cfg_name) with open(cfg_path, "w", encoding="utf-8") as f: f.write(python_cfg(_spec)) return cfg_path # This config will not be included cfg2_path = write_python_cfg("+shared", "include2.yaml") # The config will point to this using substitutable variables, # namely $os; we expect that Spack resolves these variables # into the actual path of the config this_os = spack.platforms.host().default_os cfg3_expanded_path = os.path.join(str(tmp_path), f"{this_os}", "include3.yaml") fs.mkdirp(os.path.dirname(cfg3_expanded_path)) with open(cfg3_expanded_path, "w", encoding="utf-8") as f: f.write(python_cfg("+ssl")) cfg3_abstract_path = os.path.join(str(tmp_path), "$os", "include3.yaml") # This will be included unconditionally cfg4_path = write_python_cfg("+tk", "include4.yaml") # This config will not exist, and the config will explicitly # allow this cfg5_path = os.path.join(str(tmp_path), "non-existent.yaml") include_entries = [ {"path": f"{cfg1_path}", "when": f'os == "{this_os}"'}, {"path": f"{cfg2_path}", "when": "False"}, {"path": cfg3_abstract_path}, cfg4_path, {"path": cfg5_path, "optional": True}, ] include_cfg = {"include": include_entries} filename = write_config_file("include", include_cfg, "low") assert not spack.config.get("config:dirty") spack.main.add_command_line_scopes(mock_low_high_config, [os.path.dirname(filename)]) assert spack.config.get("config:dirty") python_reqs = spack.config.get("packages")["python"]["require"] req_specs = set(x["spec"] for x in python_reqs) assert req_specs == set(["@3.11:", "+ssl", "+tk"]) def test_include_duplicate_source(mutable_config): """Check precedence when include.yaml files have the same path.""" include_yaml = "debug.yaml" include_list = {"include": [f"./{include_yaml}"]} system_filename = mutable_config.get_config_filename("system", "include") site_filename = mutable_config.get_config_filename("site", "include") def write_configs(include_path, debug_data): fs.mkdirp(os.path.dirname(include_path)) with open(include_path, "w", encoding="utf-8") as f: syaml.dump_config(include_list, f) debug_path = fs.join_path(os.path.dirname(include_path), include_yaml) with open(debug_path, "w", encoding="utf-8") as f: syaml.dump_config(debug_data, f) system_config = {"config": {"debug": False}} write_configs(system_filename, system_config) spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(system_filename)]) site_config = {"config": {"debug": True}} write_configs(site_filename, site_config) spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(site_filename)]) # Ensure takes the last value of the option pushed onto the stack assert mutable_config.get("config:debug") == site_config["config"]["debug"] def test_include_recurse_limit(tmp_path: pathlib.Path, mutable_config): """Ensure hit the recursion limit.""" include_yaml = "include.yaml" include_list = {"include": [f"./{include_yaml}"]} include_path = str(tmp_path / include_yaml) with open(include_path, "w", encoding="utf-8") as f: syaml.dump_config(include_list, f) with pytest.raises(spack.config.RecursiveIncludeError, match="recursion exceeded"): spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(include_path)]) # TODO: Fix this once recursive includes are processed in the expected order. @pytest.mark.parametrize("child,expected", [("b", True), ("c", False)]) def test_include_recurse_diamond(tmp_path: pathlib.Path, mutable_config, child, expected): """Demonstrate include parent's value overrides that of child in diamond include. Check that the value set by b or c overrides that set by d. """ configs_root = tmp_path / "configs" configs_root.mkdir() def write(path, contents): with open(path, "w", encoding="utf-8") as f: f.write(contents) def debug_contents(value): return f"config:\n debug: {value}\n" def include_contents(paths): indent = "\n - " values = indent.join([str(p) for p in paths]) return f"include:{indent}{values}" a_yaml = tmp_path / "a.yaml" b_yaml = configs_root / "b.yaml" c_yaml = configs_root / "c.yaml" d_yaml = configs_root / "d.yaml" debug_yaml = configs_root / "enable_debug.yaml" write(debug_yaml, debug_contents("true")) a_contents = f"""\ include: - {b_yaml} - {c_yaml} """ write(a_yaml, a_contents) write(d_yaml, debug_contents("false")) write(b_yaml, include_contents([debug_yaml, d_yaml] if child == "b" else [d_yaml])) write(c_yaml, include_contents([debug_yaml, d_yaml] if child == "c" else [d_yaml])) spack.main.add_command_line_scopes(mutable_config, [str(tmp_path)]) try: assert mutable_config.get("config:debug") is expected except AssertionError: pytest.xfail("recursive includes are not processed in the expected order") ================================================ FILE: lib/spack/spack/test/make_executable.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ Tests for Spack's built-in parallel make support. This just tests whether the right args are getting passed to make. """ import os import pathlib import pytest from spack.build_environment import MakeExecutable from spack.util.environment import path_put_first pytestmark = pytest.mark.not_on_windows("MakeExecutable not supported on Windows") @pytest.fixture(autouse=True) def make_executable(tmp_path: pathlib.Path, working_env): make_exe = tmp_path / "make" with open(make_exe, "w", encoding="utf-8") as f: f.write("#!/bin/sh\n") f.write('echo "$@"') os.chmod(make_exe, 0o700) path_put_first("PATH", [str(tmp_path)]) def test_make_normal(): make = MakeExecutable("make", jobs=8) assert make(output=str).strip() == "-j8" assert make("install", output=str).strip() == "-j8 install" def test_make_explicit(): make = MakeExecutable("make", jobs=8) assert make(parallel=True, output=str).strip() == "-j8" assert make("install", parallel=True, output=str).strip() == "-j8 install" def test_make_one_job(): make = MakeExecutable("make", jobs=1) assert make(output=str).strip() == "-j1" assert make("install", output=str).strip() == "-j1 install" def test_make_parallel_false(): make = MakeExecutable("make", jobs=8) assert make(parallel=False, output=str).strip() == "-j1" assert make("install", parallel=False, output=str).strip() == "-j1 install" def test_make_parallel_disabled(monkeypatch): make = MakeExecutable("make", jobs=8) monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "true") assert make(output=str).strip() == "-j1" assert make("install", output=str).strip() == "-j1 install" monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "1") assert make(output=str).strip() == "-j1" assert make("install", output=str).strip() == "-j1 install" # These don't disable (false and random string) monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "false") assert make(output=str).strip() == "-j8" assert make("install", output=str).strip() == "-j8 install" monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "foobar") assert make(output=str).strip() == "-j8" assert make("install", output=str).strip() == "-j8 install" def test_make_parallel_precedence(monkeypatch): make = MakeExecutable("make", jobs=8) # These should work monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "true") assert make(parallel=True, output=str).strip() == "-j1" assert make("install", parallel=True, output=str).strip() == "-j1 install" monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "1") assert make(parallel=True, output=str).strip() == "-j1" assert make("install", parallel=True, output=str).strip() == "-j1 install" # These don't disable (false and random string) monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "false") assert make(parallel=True, output=str).strip() == "-j8" assert make("install", parallel=True, output=str).strip() == "-j8 install" monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "foobar") assert make(parallel=True, output=str).strip() == "-j8" assert make("install", parallel=True, output=str).strip() == "-j8 install" def test_make_jobs_env(): make = MakeExecutable("make", jobs=8) dump_env = {} assert make(output=str, jobs_env="MAKE_PARALLELISM", _dump_env=dump_env).strip() == "-j8" assert dump_env["MAKE_PARALLELISM"] == "8" def test_make_jobserver(monkeypatch): make = MakeExecutable("make", jobs=8) monkeypatch.setenv("MAKEFLAGS", "--jobserver-auth=X,Y") assert make(output=str).strip() == "" assert make(parallel=False, output=str).strip() == "-j1" def test_make_jobserver_not_supported(monkeypatch): make = MakeExecutable("make", jobs=8, supports_jobserver=False) monkeypatch.setenv("MAKEFLAGS", "--jobserver-auth=X,Y") # Currently fallback on default job count, Maybe it should force -j1 ? assert make(output=str).strip() == "-j8" ================================================ FILE: lib/spack/spack/test/mirror.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp import os import pathlib import pytest import spack.caches import spack.cmd.mirror import spack.concretize import spack.config import spack.fetch_strategy import spack.mirrors.layout import spack.mirrors.mirror import spack.mirrors.utils import spack.patch import spack.stage import spack.util.spack_json as sjson import spack.util.url as url_util from spack.cmd.common.arguments import mirror_name_or_url from spack.llnl.util.filesystem import resolve_link_target_relative_to_the_link, working_dir from spack.spec import Spec from spack.util.executable import which from spack.util.spack_yaml import SpackYAMLError pytestmark = [pytest.mark.usefixtures("mutable_config", "mutable_mock_repo")] # paths in repos that shouldn't be in the mirror tarballs. exclude = [".hg", ".git", ".svn"] repos = {} def set_up_package(name, repository, url_attr): """Set up a mock package to be mirrored. Each package needs us to: 1. Set up a mock repo/archive to fetch from. 2. Point the package's version args at that repo. """ # Set up packages to point at mock repos. s = spack.concretize.concretize_one(name) repos[name] = repository # change the fetch args of the first (only) version. assert len(s.package.versions) == 1 v = next(iter(s.package.versions)) s.package.versions[v][url_attr] = repository.url def check_mirror(): with spack.stage.Stage("spack-mirror-test") as stage: mirror_root = os.path.join(stage.path, "test-mirror") # register mirror with spack config mirrors = {"spack-mirror-test": url_util.path_to_file_url(mirror_root)} with spack.config.override("mirrors", mirrors): with spack.config.override("config:checksum", False): specs = [spack.concretize.concretize_one(x) for x in repos] spack.cmd.mirror.create(mirror_root, specs) # Stage directory exists assert os.path.isdir(mirror_root) for spec in specs: fetcher = spec.package.fetcher per_package_ref = os.path.join(spec.name, "-".join([spec.name, str(spec.version)])) mirror_layout = spack.mirrors.layout.default_mirror_layout( fetcher, per_package_ref ) expected_path = os.path.join(mirror_root, mirror_layout.path) assert os.path.exists(expected_path) # Now try to fetch each package. for name, mock_repo in repos.items(): spec = spack.concretize.concretize_one(name) pkg = spec.package with spack.config.override("config:checksum", False): with pkg.stage: pkg.do_stage(mirror_only=True) # Compare the original repo with the expanded archive original_path = mock_repo.path if "svn" in name: # have to check out the svn repo to compare. original_path = os.path.join(mock_repo.path, "checked_out") svn = which("svn", required=True) svn("checkout", mock_repo.url, original_path) dcmp = filecmp.dircmp(original_path, pkg.stage.source_path) # make sure there are no new files in the expanded # tarball assert not dcmp.right_only # and that all original files are present. assert all(left in exclude for left in dcmp.left_only) def test_url_mirror(mock_archive): set_up_package("trivial-install-test-package", mock_archive, "url") check_mirror() repos.clear() def test_git_mirror(git, mock_git_repository): set_up_package("git-test", mock_git_repository, "git") check_mirror() repos.clear() def test_svn_mirror(mock_svn_repository): set_up_package("svn-test", mock_svn_repository, "svn") check_mirror() repos.clear() def test_hg_mirror(mock_hg_repository): set_up_package("hg-test", mock_hg_repository, "hg") check_mirror() repos.clear() def test_all_mirror(mock_git_repository, mock_svn_repository, mock_hg_repository, mock_archive): set_up_package("git-test", mock_git_repository, "git") set_up_package("svn-test", mock_svn_repository, "svn") set_up_package("hg-test", mock_hg_repository, "hg") set_up_package("trivial-install-test-package", mock_archive, "url") check_mirror() repos.clear() @pytest.mark.parametrize( "mirror", [ spack.mirrors.mirror.Mirror( {"fetch": "https://example.com/fetch", "push": "https://example.com/push"} ) ], ) def test_roundtrip_mirror(mirror: spack.mirrors.mirror.Mirror): mirror_yaml = mirror.to_yaml() assert spack.mirrors.mirror.Mirror.from_yaml(mirror_yaml) == mirror mirror_json = mirror.to_json() assert spack.mirrors.mirror.Mirror.from_json(mirror_json) == mirror @pytest.mark.parametrize( "invalid_yaml", ["playing_playlist: {{ action }} playlist {{ playlist_name }}"] ) def test_invalid_yaml_mirror(invalid_yaml): with pytest.raises(SpackYAMLError, match="error parsing YAML") as e: spack.mirrors.mirror.Mirror.from_yaml(invalid_yaml) assert invalid_yaml in str(e.value) @pytest.mark.parametrize("invalid_json, error_message", [("{13:", "Expecting property name")]) def test_invalid_json_mirror(invalid_json, error_message): with pytest.raises(sjson.SpackJSONError) as e: spack.mirrors.mirror.Mirror.from_json(invalid_json) exc_msg = str(e.value) assert exc_msg.startswith("error parsing JSON mirror:") assert error_message in exc_msg @pytest.mark.parametrize( "mirror_collection", [ spack.mirrors.mirror.MirrorCollection( mirrors={ "example-mirror": spack.mirrors.mirror.Mirror( "https://example.com/fetch", "https://example.com/push" ).to_dict() } ) ], ) def test_roundtrip_mirror_collection(mirror_collection): mirror_collection_yaml = mirror_collection.to_yaml() assert ( spack.mirrors.mirror.MirrorCollection.from_yaml(mirror_collection_yaml) == mirror_collection ) mirror_collection_json = mirror_collection.to_json() assert ( spack.mirrors.mirror.MirrorCollection.from_json(mirror_collection_json) == mirror_collection ) @pytest.mark.parametrize( "invalid_yaml", ["playing_playlist: {{ action }} playlist {{ playlist_name }}"] ) def test_invalid_yaml_mirror_collection(invalid_yaml): with pytest.raises(SpackYAMLError, match="error parsing YAML") as e: spack.mirrors.mirror.MirrorCollection.from_yaml(invalid_yaml) assert invalid_yaml in str(e.value) @pytest.mark.parametrize("invalid_json, error_message", [("{13:", "Expecting property name")]) def test_invalid_json_mirror_collection(invalid_json, error_message): with pytest.raises(sjson.SpackJSONError) as e: spack.mirrors.mirror.MirrorCollection.from_json(invalid_json) exc_msg = str(e.value) assert exc_msg.startswith("error parsing JSON mirror collection:") assert error_message in exc_msg def test_mirror_archive_paths_no_version(mock_packages, mock_archive): spec = spack.concretize.concretize_one( Spec("trivial-install-test-package@=nonexistingversion") ) fetcher = spack.fetch_strategy.URLFetchStrategy(url=mock_archive.url) spack.mirrors.layout.default_mirror_layout(fetcher, "per-package-ref", spec) def test_mirror_with_url_patches(mock_packages, monkeypatch): spec = spack.concretize.concretize_one("patch-several-dependencies") files_cached_in_mirror = set() def record_store(_class, fetcher, relative_dst, cosmetic_path=None): files_cached_in_mirror.add(os.path.basename(relative_dst)) def successful_fetch(_class): with open(_class.stage.save_filename, "w", encoding="utf-8"): pass def successful_expand(_class): expanded_path = os.path.join(_class.stage.path, spack.stage._source_path_subdir) os.mkdir(expanded_path) with open(os.path.join(expanded_path, "test.patch"), "w", encoding="utf-8"): pass def successful_apply(*args, **kwargs): pass def successful_make_alias(*args, **kwargs): pass with spack.stage.Stage("spack-mirror-test") as stage: mirror_root = os.path.join(stage.path, "test-mirror") monkeypatch.setattr(spack.fetch_strategy.URLFetchStrategy, "fetch", successful_fetch) monkeypatch.setattr(spack.fetch_strategy.URLFetchStrategy, "expand", successful_expand) monkeypatch.setattr(spack.patch, "apply_patch", successful_apply) monkeypatch.setattr(spack.caches.MirrorCache, "store", record_store) monkeypatch.setattr( spack.mirrors.layout.DefaultLayout, "make_alias", successful_make_alias ) with spack.config.override("config:checksum", False): spack.cmd.mirror.create(mirror_root, list(spec.traverse())) assert { "abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234", "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd.gz", }.issubset(files_cached_in_mirror) class MockFetcher: """Mock fetcher object which implements the necessary functionality for testing MirrorCache """ @staticmethod def archive(dst): with open(dst, "w", encoding="utf-8"): pass @pytest.mark.regression("14067") def test_mirror_layout_make_alias(tmp_path: pathlib.Path): """Confirm that the cosmetic symlink created in the mirror cache (which may be relative) targets the storage path correctly. """ alias = os.path.join("zlib", "zlib-1.2.11.tar.gz") path = os.path.join("_source-cache", "archive", "c3", "c3e5.tar.gz") cache = spack.caches.MirrorCache(root=str(tmp_path), skip_unstable_versions=False) layout = spack.mirrors.layout.DefaultLayout(alias, path) cache.store(MockFetcher(), layout.path) layout.make_alias(cache.root) link_target = resolve_link_target_relative_to_the_link(os.path.join(cache.root, layout.alias)) assert os.path.exists(link_target) assert os.path.normpath(link_target) == os.path.join(cache.root, layout.path) @pytest.mark.regression("31627") @pytest.mark.parametrize( "specs,expected_specs", [ (["pkg-a"], ["pkg-a@=1.0", "pkg-a@=2.0"]), (["pkg-a", "brillig"], ["pkg-a@=1.0", "pkg-a@=2.0", "brillig@=1.0.0", "brillig@=2.0.0"]), ], ) def test_get_all_versions(specs, expected_specs): specs = [Spec(s) for s in specs] output_list = spack.mirrors.utils.get_all_versions(specs) output_list = [str(x) for x in output_list] # Compare sets since order is not important assert set(output_list) == set(expected_specs) def test_update_1(): # No change m = spack.mirrors.mirror.Mirror("https://example.com") assert not m.update({"url": "https://example.com"}) assert m.to_dict() == "https://example.com" def test_update_2(): # Change URL, shouldn't expand to {"url": ...} dict. m = spack.mirrors.mirror.Mirror("https://example.com") assert m.update({"url": "https://example.org"}) assert m.to_dict() == "https://example.org" assert m.fetch_url == "https://example.org" assert m.push_url == "https://example.org" def test_update_3(): # Change fetch url, ensure minimal config m = spack.mirrors.mirror.Mirror("https://example.com") assert m.update({"url": "https://example.org"}, "fetch") assert m.to_dict() == {"url": "https://example.com", "fetch": "https://example.org"} assert m.fetch_url == "https://example.org" assert m.push_url == "https://example.com" def test_update_4(): # Change push url, ensure minimal config m = spack.mirrors.mirror.Mirror("https://example.com") assert m.update({"url": "https://example.org"}, "push") assert m.to_dict() == {"url": "https://example.com", "push": "https://example.org"} assert m.push_url == "https://example.org" assert m.fetch_url == "https://example.com" @pytest.mark.parametrize("direction", ["fetch", "push"]) def test_update_connection_params(direction, monkeypatch): """Test whether new connection params expand the mirror config to a dict.""" m = spack.mirrors.mirror.Mirror("https://example.com", "example") assert m.update( { "url": "http://example.org", "access_pair": ["username", "password"], "access_token": "token", "profile": "profile", "endpoint_url": "https://example.com", }, direction, ) assert m.to_dict() == { "url": "https://example.com", direction: { "url": "http://example.org", "access_pair": ["username", "password"], "access_token": "token", "profile": "profile", "endpoint_url": "https://example.com", }, } assert m.get_access_pair(direction) == ("username", "password") assert m.get_access_token(direction) == "token" assert m.get_profile(direction) == "profile" assert m.get_endpoint_url(direction) == "https://example.com" # Expand environment variables os.environ["_SPACK_TEST_PAIR_USERNAME"] = "expanded_username" os.environ["_SPACK_TEST_PAIR_PASSWORD"] = "expanded_password" os.environ["_SPACK_TEST_TOKEN"] = "expanded_token" assert m.update( { "access_pair": { "id_variable": "_SPACK_TEST_PAIR_USERNAME", "secret_variable": "_SPACK_TEST_PAIR_PASSWORD", } }, direction, ) assert m.to_dict() == { "url": "https://example.com", direction: { "url": "http://example.org", "access_pair": { "id_variable": "_SPACK_TEST_PAIR_USERNAME", "secret_variable": "_SPACK_TEST_PAIR_PASSWORD", }, "access_token": "token", "profile": "profile", "endpoint_url": "https://example.com", }, } assert m.get_access_pair(direction) == ("expanded_username", "expanded_password") assert m.update( { "access_pair": {"id": "username", "secret_variable": "_SPACK_TEST_PAIR_PASSWORD"}, "access_token_variable": "_SPACK_TEST_TOKEN", }, direction, ) assert m.to_dict() == { "url": "https://example.com", direction: { "url": "http://example.org", "access_pair": {"id": "username", "secret_variable": "_SPACK_TEST_PAIR_PASSWORD"}, "access_token_variable": "_SPACK_TEST_TOKEN", "profile": "profile", "endpoint_url": "https://example.com", }, } assert m.get_access_pair(direction) == ("username", "expanded_password") assert m.get_access_token(direction) == "expanded_token" def test_mirror_name_or_url_dir_parsing(tmp_path: pathlib.Path): curdir = tmp_path / "mirror" curdir.mkdir() with working_dir(curdir): assert mirror_name_or_url(".").fetch_url == curdir.as_uri() assert mirror_name_or_url("..").fetch_url == tmp_path.as_uri() ================================================ FILE: lib/spack/spack/test/module_parsing.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.util.module_cmd from spack.util.module_cmd import ( get_path_args_from_module_line, get_path_from_module_contents, module, path_from_modules, ) pytestmark = pytest.mark.not_on_windows("Tests fail on Windows") test_module_lines = [ "prepend-path LD_LIBRARY_PATH /path/to/lib", "setenv MOD_DIR /path/to", "setenv LDFLAGS -Wl,-rpath/path/to/lib", "setenv LDFLAGS -L/path/to/lib", "prepend-path PATH /path/to/bin", ] def test_module_function_change_env(tmp_path: pathlib.Path): environb = {b"TEST_MODULE_ENV_VAR": b"TEST_FAIL", b"NOT_AFFECTED": b"NOT_AFFECTED"} src_file = tmp_path / "src_me" src_file.write_text("export TEST_MODULE_ENV_VAR=TEST_SUCCESS\n") module("load", str(src_file), module_template=f". {src_file} 2>&1", environb=environb) assert environb[b"TEST_MODULE_ENV_VAR"] == b"TEST_SUCCESS" assert environb[b"NOT_AFFECTED"] == b"NOT_AFFECTED" def test_module_function_change_env_with_module_src_cmd(tmp_path: pathlib.Path): environb = { b"MODULESHOME": b"here", b"TEST_MODULE_ENV_VAR": b"TEST_FAIL", b"TEST_ANOTHER_MODULE_ENV_VAR": b"TEST_FAIL", b"NOT_AFFECTED": b"NOT_AFFECTED", } src_file = tmp_path / "src_me" src_file.write_text("export TEST_MODULE_ENV_VAR=TEST_SUCCESS\n") module_src_file = tmp_path / "src_me_too" module_src_file.write_text("export TEST_ANOTHER_MODULE_ENV_VAR=TEST_SUCCESS\n") module("load", str(src_file), module_template=f". {src_file} 2>&1", environb=environb) module( "load", str(src_file), module_template=f". {src_file} 2>&1", module_src_cmd=f". {module_src_file} 2>&1; ", environb=environb, ) assert environb[b"TEST_MODULE_ENV_VAR"] == b"TEST_SUCCESS" assert environb[b"TEST_ANOTHER_MODULE_ENV_VAR"] == b"TEST_SUCCESS" assert environb[b"NOT_AFFECTED"] == b"NOT_AFFECTED" def test_module_function_change_env_without_moduleshome_no_module_src_cmd(tmp_path: pathlib.Path): environb = { b"TEST_MODULE_ENV_VAR": b"TEST_FAIL", b"TEST_ANOTHER_MODULE_ENV_VAR": b"TEST_FAIL", b"NOT_AFFECTED": b"NOT_AFFECTED", } src_file = tmp_path / "src_me" src_file.write_text("export TEST_MODULE_ENV_VAR=TEST_SUCCESS\n") module_src_file = tmp_path / "src_me_too" module_src_file.write_text("export TEST_ANOTHER_MODULE_ENV_VAR=TEST_SUCCESS\n") module("load", str(src_file), module_template=f". {src_file} 2>&1", environb=environb) module( "load", str(src_file), module_template=f". {src_file} 2>&1", module_src_cmd=f". {module_src_file} 2>&1; ", environb=environb, ) assert environb[b"TEST_MODULE_ENV_VAR"] == b"TEST_SUCCESS" assert environb[b"TEST_ANOTHER_MODULE_ENV_VAR"] == b"TEST_FAIL" assert environb[b"NOT_AFFECTED"] == b"NOT_AFFECTED" def test_module_function_no_change(tmp_path: pathlib.Path): src_file = str(tmp_path / "src_me") with open(src_file, "w", encoding="utf-8") as f: f.write("echo TEST_MODULE_FUNCTION_PRINT") old_env = os.environ.copy() text = module("show", src_file, module_template=". {0} 2>&1".format(src_file)) assert text == "TEST_MODULE_FUNCTION_PRINT\n" assert os.environ == old_env def test_get_path_from_module_faked(monkeypatch): for line in test_module_lines: def fake_module(*args): return line monkeypatch.setattr(spack.util.module_cmd, "module", fake_module) path = path_from_modules(["mod"]) assert path == "/path/to" def test_get_path_from_module_contents(): # A line with "MODULEPATH" appears early on, and the test confirms that it # is not extracted as the package's path module_show_output = """ os.environ["MODULEPATH"] = "/path/to/modules1:/path/to/modules2"; ---------------------------------------------------------------------------- /root/cmake/3.9.2.lua: ---------------------------------------------------------------------------- help([[CMake Version 3.9.2 ]]) whatis("Name: CMake") whatis("Version: 3.9.2") whatis("Category: Tools") whatis("URL: https://cmake.org/") prepend_path("LD_LIBRARY_PATH","/bad/path") prepend_path("PATH","/path/to/cmake-3.9.2/bin:/other/bad/path") prepend_path("MANPATH","/path/to/cmake/cmake-3.9.2/share/man") prepend_path("LD_LIBRARY_PATH","/path/to/cmake-3.9.2/lib64") """ module_show_lines = module_show_output.split("\n") # PATH and LD_LIBRARY_PATH outvote MANPATH and the other PATH and # LD_LIBRARY_PATH entries assert ( get_path_from_module_contents(module_show_lines, "cmake-3.9.2") == "/path/to/cmake-3.9.2" ) def test_get_path_from_empty_module(): assert get_path_from_module_contents("", "test") is None def test_pkg_dir_from_module_name(): module_show_lines = ["setenv FOO_BAR_DIR /path/to/foo-bar"] assert get_path_from_module_contents(module_show_lines, "foo-bar") == "/path/to/foo-bar" assert get_path_from_module_contents(module_show_lines, "foo-bar/1.0") == "/path/to/foo-bar" def test_get_argument_from_module_line(): simple_lines = [ "prepend-path LD_LIBRARY_PATH /lib/path", "prepend-path LD_LIBRARY_PATH /lib/path", "prepend_path('PATH' , '/lib/path')", 'prepend_path( "PATH" , "/lib/path" )', 'prepend_path("PATH",' + "'/lib/path')", ] complex_lines = [ "prepend-path LD_LIBRARY_PATH /lib/path:/pkg/path", "prepend-path LD_LIBRARY_PATH /lib/path:/pkg/path", "prepend_path('PATH' , '/lib/path:/pkg/path')", 'prepend_path( "PATH" , "/lib/path:/pkg/path" )', 'prepend_path("PATH",' + "'/lib/path:/pkg/path')", ] bad_lines = ["prepend_path(PATH,/lib/path)", "prepend-path (LD_LIBRARY_PATH) /lib/path"] assert all(get_path_args_from_module_line(x) == ["/lib/path"] for x in simple_lines) assert all( get_path_args_from_module_line(x) == ["/lib/path", "/pkg/path"] for x in complex_lines ) for bl in bad_lines: with pytest.raises(ValueError): get_path_args_from_module_line(bl) # lmod is entirely unsupported on Windows def test_lmod_quote_parsing(): lines = ['setenv("SOME_PARTICULAR_DIR","-L/opt/cray/pe/mpich/8.1.4/gtl/lib")'] result = get_path_from_module_contents(lines, "some-module") assert "/opt/cray/pe/mpich/8.1.4/gtl" == result ================================================ FILE: lib/spack/spack/test/modules/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/test/modules/common.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pickle import stat import pytest import spack.cmd.modules import spack.concretize import spack.config import spack.error import spack.modules import spack.modules.common import spack.modules.tcl import spack.package_base import spack.package_prefs import spack.repo from spack.installer import PackageInstaller from spack.llnl.util.filesystem import readlink from spack.modules.common import UpstreamModuleIndex pytestmark = [ pytest.mark.not_on_windows("does not run on windows"), pytest.mark.usefixtures("mock_modules_root"), ] def test_update_dictionary_extending_list(): target = {"foo": {"a": 1, "b": 2, "d": 4}, "bar": [1, 2, 4], "baz": "foobar"} update = {"foo": {"c": 3}, "bar": [3], "baz": "foobaz", "newkey": {"d": 4}} spack.modules.common.update_dictionary_extending_lists(target, update) assert len(target) == 4 assert len(target["foo"]) == 4 assert len(target["bar"]) == 4 assert target["baz"] == "foobaz" @pytest.fixture() def mock_module_defaults(monkeypatch): def impl(*args): # No need to patch both types because neither override base monkeypatch.setattr( spack.modules.common.BaseConfiguration, "defaults", [arg for arg in args] ) return impl @pytest.fixture() def mock_package_perms(monkeypatch): perms = stat.S_IRGRP | stat.S_IWGRP monkeypatch.setattr(spack.package_prefs, "get_package_permissions", lambda spec: perms) yield perms def test_modules_written_with_proper_permissions( mock_module_filename, mock_package_perms, mock_packages, config ): spec = spack.concretize.concretize_one("mpileaks") # The code tested is common to all module types, but has to be tested from # one. Tcl picked at random generator = spack.modules.tcl.TclModulefileWriter(spec, "default") generator.write() assert mock_package_perms & os.stat(mock_module_filename).st_mode == mock_package_perms @pytest.mark.parametrize("module_type", ["tcl", "lmod"]) def test_modules_default_symlink( module_type, mock_packages, mock_module_filename, mock_module_defaults, config ): spec = spack.concretize.concretize_one("mpileaks@2.3") mock_module_defaults(spec.format("{name}{@version}"), True) generator_cls = spack.modules.module_types[module_type] generator = generator_cls(spec, "default") generator.write() link_path = os.path.join(os.path.dirname(mock_module_filename), "default") assert os.path.islink(link_path) assert readlink(link_path) == mock_module_filename generator.remove() assert not os.path.lexists(link_path) class MockDb: def __init__(self, db_ids, spec_hash_to_db): self.upstream_dbs = db_ids self.spec_hash_to_db = spec_hash_to_db def db_for_spec_hash(self, spec_hash): return self.spec_hash_to_db.get(spec_hash) class MockSpec: def __init__(self, unique_id): self.unique_id = unique_id def dag_hash(self): return self.unique_id def test_upstream_module_index(): s1 = MockSpec("spec-1") s2 = MockSpec("spec-2") s3 = MockSpec("spec-3") s4 = MockSpec("spec-4") tcl_module_index = """\ module_index: {0}: path: /path/to/a use_name: a """.format(s1.dag_hash()) module_indices = [{"tcl": spack.modules.common._read_module_index(tcl_module_index)}, {}] dbs = ["d0", "d1"] mock_db = MockDb(dbs, {s1.dag_hash(): "d0", s2.dag_hash(): "d1", s3.dag_hash(): "d0"}) upstream_index = UpstreamModuleIndex(mock_db, module_indices) m1 = upstream_index.upstream_module(s1, "tcl") assert m1.path == "/path/to/a" # No modules are defined for the DB associated with s2 assert not upstream_index.upstream_module(s2, "tcl") # Modules are defined for the index associated with s1, but none are # defined for the requested type assert not upstream_index.upstream_module(s1, "lmod") # A module is registered with a DB and the associated module index has # modules of the specified type defined, but not for the requested spec assert not upstream_index.upstream_module(s3, "tcl") # The spec isn't recorded as installed in any of the DBs with pytest.raises(spack.error.SpackError): upstream_index.upstream_module(s4, "tcl") def test_get_module_upstream(): s1 = MockSpec("spec-1") tcl_module_index = """\ module_index: {0}: path: /path/to/a use_name: a """.format(s1.dag_hash()) module_indices = [{}, {"tcl": spack.modules.common._read_module_index(tcl_module_index)}] dbs = ["d0", "d1"] mock_db = MockDb(dbs, {s1.dag_hash(): "d1"}) upstream_index = UpstreamModuleIndex(mock_db, module_indices) setattr(s1, "installed_upstream", True) try: old_index = spack.modules.common.upstream_module_index spack.modules.common.upstream_module_index = upstream_index m1_path = spack.modules.get_module("tcl", s1, True) assert m1_path == "/path/to/a" finally: spack.modules.common.upstream_module_index = old_index @pytest.mark.regression("14347") def test_load_installed_package_not_in_repo(install_mockery, mock_fetch, monkeypatch): """Test that installed packages that have been removed are still loadable""" spec = spack.concretize.concretize_one("trivial-install-test-package") PackageInstaller([spec.package], explicit=True).install() spack.modules.module_types["tcl"](spec, "default", True).write() def find_nothing(*args): raise spack.repo.UnknownPackageError("Repo package access is disabled for test") # Mock deletion of the package spec._package = None monkeypatch.setattr(spack.repo.PATH, "get", find_nothing) with pytest.raises(spack.repo.UnknownPackageError): spec.package module_path = spack.modules.get_module("tcl", spec, True) assert module_path spack.package_base.PackageBase.uninstall_by_spec(spec) @pytest.mark.regression("37649") def test_check_module_set_name(mutable_config): """Tests that modules set name are validated correctly and an error is reported if the name we require does not exist or is reserved by the configuration.""" # Minimal modules.yaml config. spack.config.set( "modules", { "prefix_inspections": {"./bin": ["PATH"]}, # module sets "first": {}, "second": {}, }, ) # Valid module set name spack.cmd.modules.check_module_set_name("first") # Invalid module set names msg = "Valid module set names are" with pytest.raises(spack.error.ConfigError, match=msg): spack.cmd.modules.check_module_set_name("prefix_inspections") with pytest.raises(spack.error.ConfigError, match=msg): spack.cmd.modules.check_module_set_name("third") @pytest.mark.parametrize("module_type", ["tcl", "lmod"]) def test_module_writers_are_pickleable(default_mock_concretization, module_type): s = default_mock_concretization("mpileaks") writer = spack.modules.module_types[module_type](s, "default") assert pickle.loads(pickle.dumps(writer)).spec == s ================================================ FILE: lib/spack/spack/test/modules/conftest.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.concretize import spack.modules.lmod import spack.modules.tcl import spack.spec @pytest.fixture() def modulefile_content(request): """Returns a function that generates the content of a module file as a list of lines.""" writer_cls = getattr(request.module, "writer_cls") def _impl(spec_like, module_set_name="default", explicit=True): if isinstance(spec_like, str): spec_like = spack.spec.Spec(spec_like) spec = spack.concretize.concretize_one(spec_like) generator = writer_cls(spec, module_set_name, explicit) generator.write(overwrite=True) written_module = pathlib.Path(generator.layout.filename) content = written_module.read_text(encoding="utf-8").splitlines() generator.remove() return content return _impl @pytest.fixture() def factory(request, mock_modules_root): """Given a spec string, returns an instance of the writer and the corresponding spec.""" writer_cls = getattr(request.module, "writer_cls") def _mock(spec_string, module_set_name="default", explicit=True): spec = spack.concretize.concretize_one(spec_string) return writer_cls(spec, module_set_name, explicit), spec return _mock @pytest.fixture() def mock_module_filename(monkeypatch, tmp_path: pathlib.Path): filename = tmp_path / "module" # Set for both module types so we can test both monkeypatch.setattr(spack.modules.lmod.LmodFileLayout, "filename", str(filename)) monkeypatch.setattr(spack.modules.tcl.TclFileLayout, "filename", str(filename)) yield str(filename) ================================================ FILE: lib/spack/spack/test/modules/lmod.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.vendor.archspec.cpu import spack.concretize import spack.config import spack.environment as ev import spack.main import spack.modules.common import spack.modules.lmod import spack.spec import spack.util.environment mpich_spec_string = "mpich@3.0.4" mpileaks_spec_string = "mpileaks" libdwarf_spec_string = "libdwarf arch=x64-linux" install = spack.main.SpackCommand("install") #: Class of the writer tested in this module writer_cls = spack.modules.lmod.LmodModulefileWriter pytestmark = [ pytest.mark.not_on_windows("does not run on windows"), pytest.mark.usefixtures("mock_modules_root"), ] @pytest.fixture(params=["clang@=15.0.0", "gcc@=10.2.1"]) def compiler(request): return request.param @pytest.fixture( params=[ ("mpich@3.0.4", ("mpi",), True, False), ("mpich@3.0.1", [], True, True), ("openblas@0.2.15", ("blas",), True, False), ("openblas-with-lapack@0.2.15", ("blas", "lapack"), True, False), ("mpileaks@2.3", ("mpi",), True, False), ("mpileaks@2.1", [], True, False), ("py-extension1@2.0", ("python",), False, True), ("python@3.8.0", ("python",), False, True), ] ) def provider(request): return request.param @pytest.mark.usefixtures("mutable_config", "mock_packages") class TestLmod: @pytest.mark.regression("37788") @pytest.mark.parametrize("modules_config", ["core_compilers", "core_compilers_at_equal"]) def test_layout_for_specs_compiled_with_core_compilers( self, modules_config, module_configuration, factory ): """Tests that specs compiled with core compilers are in the 'Core' folder. Also tests that we can use both ``compiler@version`` and ``compiler@=version`` to specify a core compiler. """ module_configuration(modules_config) module, spec = factory("libelf%clang@15.0.0") assert "Core" in module.layout.available_path_parts def test_file_layout(self, compiler, provider, factory, module_configuration): """Tests the layout of files in the hierarchy is the one expected.""" module_configuration("complex_hierarchy") spec_string, services, use_compiler, place_in_core = provider # Non-python specs add compiler factory_string = spec_string if use_compiler: factory_string += "%" + compiler module, spec = factory(factory_string) layout = module.layout # Check that the services provided are in the hierarchy for s in services: assert s in layout.conf.hierarchy_tokens # Check that the compiler part of the path has no hash and that it # is transformed to r"Core" if the compiler is listed among core # compilers # Check that specs listed as core_specs are transformed to "Core" # Check that specs with no hierarchy components are transformed to "Core" if "clang@=15.0.0" in factory_string or place_in_core: assert "Core" in layout.available_path_parts else: assert compiler.replace("@=", "/") in layout.available_path_parts # Check that the provider part instead has always an hash even if # hash has been disallowed in the configuration file path_parts = layout.available_path_parts service_part = spec_string.replace("@", "/") service_part = "-".join([service_part, layout.spec.dag_hash(length=7)]) if "mpi" in spec: # It's a user, not a provider, so create the provider string service_part = layout.spec["mpi"].format("{name}/{version}-{hash:7}") elif "python" in spec: # It's a user, not a provider, so create the provider string service_part = layout.spec["python"].format("{name}/{version}-{hash:7}") else: # Only relevant for providers, not users, of virtuals assert service_part in path_parts # Check that multi-providers have repetitions in path parts repetitions = len([x for x in path_parts if service_part == x]) if spec_string == "openblas-with-lapack@0.2.15": assert repetitions == 2 elif spec_string == "mpileaks@2.1": assert repetitions == 0 else: assert repetitions == 1 def test_compilers_provided_different_name( self, factory, module_configuration, compiler_factory ): with spack.config.override( "packages", {"llvm": {"externals": [compiler_factory(spec="llvm@3.3 +clang")]}} ): module_configuration("complex_hierarchy") module, spec = factory("intel-oneapi-compilers%clang@3.3") provides = module.conf.provides assert "compiler" in provides assert provides["compiler"] == spack.spec.Spec("intel-oneapi-compilers@=3.0") @pytest.mark.parametrize("language", ["c", "cxx", "fortran"]) def test_compiler_language_virtuals(self, factory, module_configuration, language): """Tests all compiler virtuals for hierarchical module placement.""" module_configuration("complex_hierarchy") module, spec = factory(f"single-language-virtual +{language} %{language}=gcc@=10.2.1") requires = module.conf.requires assert "gcc@=10.2.1" in requires["compiler"] def test_simple_case(self, modulefile_content, module_configuration): """Tests the generation of a simple Lua module file.""" module_configuration("autoload_direct") content = modulefile_content(mpich_spec_string) assert "-- -*- lua -*-" in content assert "whatis([[Name : mpich]])" in content assert "whatis([[Version : 3.0.4]])" in content assert 'family("mpi")' in content def test_autoload_direct(self, modulefile_content, module_configuration): """Tests the automatic loading of direct dependencies.""" module_configuration("autoload_direct") content = modulefile_content(mpileaks_spec_string) assert len([x for x in content if "depends_on(" in x]) == 3 def test_autoload_all(self, modulefile_content, module_configuration): """Tests the automatic loading of all dependencies.""" module_configuration("autoload_all") content = modulefile_content(mpileaks_spec_string) assert len([x for x in content if "depends_on(" in x]) == 6 def test_alter_environment(self, modulefile_content, module_configuration): """Tests modifications to run-time environment.""" module_configuration("alter_environment") content = modulefile_content("mpileaks platform=test target=x86_64") assert len([x for x in content if x.startswith('prepend_path("CMAKE_PREFIX_PATH"')]) == 0 assert len([x for x in content if 'setenv("FOO", "foo")' in x]) == 1 assert len([x for x in content if 'unsetenv("BAR")' in x]) == 1 content = modulefile_content("libdwarf platform=test target=core2") assert len([x for x in content if x.startswith('prepend_path("CMAKE_PREFIX_PATH"')]) == 0 assert len([x for x in content if 'setenv("FOO", "foo")' in x]) == 0 assert len([x for x in content if 'unsetenv("BAR")' in x]) == 0 def test_prepend_path_separator(self, modulefile_content, module_configuration): """Tests that we can use custom delimiters to manipulate path lists.""" module_configuration("module_path_separator") content = modulefile_content("module-path-separator") assert len([x for x in content if 'append_path("COLON", "foo", ":")' in x]) == 1 assert len([x for x in content if 'prepend_path("COLON", "foo", ":")' in x]) == 1 assert len([x for x in content if 'remove_path("COLON", "foo", ":")' in x]) == 1 assert len([x for x in content if 'append_path("SEMICOLON", "bar", ";")' in x]) == 1 assert len([x for x in content if 'prepend_path("SEMICOLON", "bar", ";")' in x]) == 1 assert len([x for x in content if 'remove_path("SEMICOLON", "bar", ";")' in x]) == 1 assert len([x for x in content if 'append_path("SPACE", "qux", " ")' in x]) == 1 assert len([x for x in content if 'remove_path("SPACE", "qux", " ")' in x]) == 1 @pytest.mark.regression("11355") def test_manpath_setup(self, modulefile_content, module_configuration): """Tests specific setup of MANPATH environment variable.""" module_configuration("autoload_direct") # no manpath set by module content = modulefile_content("mpileaks") assert len([x for x in content if 'append_path("MANPATH", "", ":")' in x]) == 0 # manpath set by module with prepend_path content = modulefile_content("module-manpath-prepend") assert ( len([x for x in content if 'prepend_path("MANPATH", "/path/to/man", ":")' in x]) == 1 ) assert ( len([x for x in content if 'prepend_path("MANPATH", "/path/to/share/man", ":")' in x]) == 1 ) assert len([x for x in content if 'append_path("MANPATH", "", ":")' in x]) == 1 # manpath set by module with append_path content = modulefile_content("module-manpath-append") assert len([x for x in content if 'append_path("MANPATH", "/path/to/man", ":")' in x]) == 1 assert len([x for x in content if 'append_path("MANPATH", "", ":")' in x]) == 1 # manpath set by module with setenv content = modulefile_content("module-manpath-setenv") assert len([x for x in content if 'setenv("MANPATH", "/path/to/man")' in x]) == 1 assert len([x for x in content if 'append_path("MANPATH", "", ":")' in x]) == 0 @pytest.mark.regression("29578") def test_setenv_raw_value(self, modulefile_content, module_configuration): """Tests that we can set environment variable value without formatting it.""" module_configuration("autoload_direct") content = modulefile_content("module-setenv-raw") assert len([x for x in content if 'setenv("FOO", "{{name}}, {name}, {{}}, {}")' in x]) == 1 @pytest.mark.skipif( str(spack.vendor.archspec.cpu.host().family) != "x86_64", reason="test data is specific for x86_64", ) def test_help_message(self, modulefile_content, module_configuration): """Tests the generation of module help message.""" module_configuration("autoload_direct") content = modulefile_content("mpileaks target=core2") help_msg = ( "help([[Name : mpileaks]])" "help([[Version: 2.3]])" "help([[Target : core2]])" "help()" "help([[Mpileaks is a mock package that passes audits]])" ) assert help_msg in "".join(content) content = modulefile_content("libdwarf target=core2") help_msg = ( "help([[Name : libdwarf]])" "help([[Version: 20130729]])" "help([[Target : core2]])" "depends_on(" ) assert help_msg in "".join(content) content = modulefile_content("module-long-help target=core2") help_msg = ( "help([[Name : module-long-help]])" "help([[Version: 1.0]])" "help([[Target : core2]])" "help()" "help([[Package to test long description message generated in modulefile." "Message too long is wrapped over multiple lines.]])" ) assert help_msg in "".join(content) def test_exclude(self, modulefile_content, module_configuration): """Tests excluding the generation of selected modules.""" module_configuration("exclude") content = modulefile_content(mpileaks_spec_string) assert len([x for x in content if "depends_on(" in x]) == 2 def test_no_hash(self, factory, module_configuration): """Makes sure that virtual providers (in the hierarchy) always include a hash. Make sure that the module file for the spec does not include a hash if hash_length is 0. """ module_configuration("no_hash") module, spec = factory(mpileaks_spec_string) path = module.layout.filename mpi_spec = spec["mpi"] mpi_element = "{0}/{1}-{2}/".format( mpi_spec.name, mpi_spec.version, mpi_spec.dag_hash(length=7) ) assert mpi_element in path mpileaks_spec = spec mpileaks_element = "{0}/{1}.lua".format(mpileaks_spec.name, mpileaks_spec.version) assert path.endswith(mpileaks_element) def test_no_core_compilers(self, factory, module_configuration): """Ensures that missing 'core_compilers' in the configuration file raises the right exception. """ # In this case we miss the entry completely module_configuration("missing_core_compilers") module, spec = factory(mpileaks_spec_string) with pytest.raises(spack.modules.lmod.CoreCompilersNotFoundError): module.write() # Here we have an empty list module_configuration("core_compilers_empty") module, spec = factory(mpileaks_spec_string) with pytest.raises(spack.modules.lmod.CoreCompilersNotFoundError): module.write() def test_conflicts(self, modulefile_content, module_configuration): """Tests adding conflicts to the module.""" # This configuration has no error, so check the conflicts directives # are there module_configuration("conflicts") content = modulefile_content("mpileaks") assert len([x for x in content if x.startswith("conflict")]) == 2 assert len([x for x in content if x == 'conflict("mpileaks")']) == 1 assert len([x for x in content if x == 'conflict("intel/14.0.1")']) == 1 def test_inconsistent_conflict_in_modules_yaml(self, modulefile_content, module_configuration): """Tests inconsistent conflict definition in `modules.yaml`.""" # This configuration is inconsistent, check an error is raised module_configuration("wrong_conflicts") with pytest.raises(spack.modules.common.ModulesError): modulefile_content("mpileaks") def test_override_template_in_package(self, modulefile_content, module_configuration): """Tests overriding a template from and attribute in the package.""" module_configuration("autoload_direct") content = modulefile_content("override-module-templates") assert "Override successful!" in content def test_override_template_in_modules_yaml( self, modulefile_content, module_configuration, host_architecture_str ): """Tests overriding a template from `modules.yaml`""" module_configuration("override_template") content = modulefile_content("override-module-templates") assert "Override even better!" in content content = modulefile_content(f"mpileaks target={host_architecture_str}") assert "Override even better!" in content def test_external_configure_args(self, factory): # If this package is detected as an external, its configure option line # in the module file starts with 'unknown' writer, spec = factory("externaltool") assert "unknown" in writer.context.configure_options def test_guess_core_compilers(self, factory, module_configuration, monkeypatch): """Check that we can guess core compilers.""" # In this case we miss the entry completely module_configuration("missing_core_compilers") # Our mock paths must be detected as system paths monkeypatch.setattr(spack.util.environment, "SYSTEM_DIRS", ["/path/bin"]) # We don't want to really write into user configuration # when running tests def no_op_set(*args, **kwargs): pass monkeypatch.setattr(spack.config, "set", no_op_set) # Assert we have core compilers now writer, _ = factory(mpileaks_spec_string) assert writer.conf.core_compilers @pytest.mark.parametrize( "spec_str", ["mpileaks target=nocona", "mpileaks target=core2", "mpileaks target=x86_64"] ) @pytest.mark.regression("13005") def test_only_generic_microarchitectures_in_root( self, spec_str, factory, module_configuration ): module_configuration("complex_hierarchy") writer, spec = factory(spec_str) assert str(spec.target.family) in writer.layout.arch_dirname if spec.target.family != spec.target: assert str(spec.target) not in writer.layout.arch_dirname def test_projections_specific(self, factory, module_configuration): """Tests reading the correct naming scheme.""" # This configuration has no error, so check the conflicts directives # are there module_configuration("projections") # Test we read the expected configuration for the naming scheme writer, _ = factory("mpileaks") expected = {"all": "{name}/v{version}", "mpileaks": "{name}-mpiprojection"} assert writer.conf.projections == expected projection = writer.spec.format(writer.conf.projections["mpileaks"]) assert projection in writer.layout.use_name def test_projections_all(self, factory, module_configuration): """Tests reading the correct naming scheme.""" # This configuration has no error, so check the conflicts directives # are there module_configuration("projections") # Test we read the expected configuration for the naming scheme writer, _ = factory("libelf") expected = {"all": "{name}/v{version}", "mpileaks": "{name}-mpiprojection"} assert writer.conf.projections == expected projection = writer.spec.format(writer.conf.projections["all"]) assert projection in writer.layout.use_name def test_modules_relative_to_view( self, tmp_path: pathlib.Path, modulefile_content, module_configuration, install_mockery, mock_fetch, ): with ev.create_in_dir(str(tmp_path), with_view=True) as e: module_configuration("with_view") install("--fake", "--add", "cmake") spec = spack.concretize.concretize_one("cmake") content = modulefile_content("cmake") expected = e.default_view.get_projection_for_spec(spec) # Rather than parse all lines, ensure all prefixes in the content # point to the right one assert any(expected in line for line in content) assert not any(spec.prefix in line for line in content) def test_modules_no_arch(self, factory, module_configuration): module_configuration("no_arch") module, spec = factory(mpileaks_spec_string) path = module.layout.filename assert str(spec.os) not in path def test_hide_implicits(self, module_configuration, temporary_store): """Tests the addition and removal of hide command in modulerc.""" module_configuration("hide_implicits") spec = spack.concretize.concretize_one("mpileaks@2.3") # mpileaks is defined as implicit, thus hide command should appear in modulerc writer = writer_cls(spec, "default", False) writer.write() assert os.path.exists(writer.layout.modulerc) with open(writer.layout.modulerc, encoding="utf-8") as f: content = [line.strip() for line in f.readlines()] hide_implicit_mpileaks = f'hide_version("{writer.layout.use_name}")' assert len([x for x in content if hide_implicit_mpileaks == x]) == 1 # The direct dependencies are all implicitly installed, and they should all be hidden, # except for mpich, which is provider for mpi, which is in the hierarchy, and therefore # can't be hidden. All other hidden modules should have a 7 character hash (the config # hash_length = 0 only applies to exposed modules). with open(writer.layout.filename, encoding="utf-8") as f: depends_statements = [line.strip() for line in f.readlines() if "depends_on" in line] for dep in spec.dependencies(deptype=("link", "run")): if dep.satisfies("mpi"): assert not any(dep.dag_hash(7) in line for line in depends_statements) else: assert any(dep.dag_hash(7) in line for line in depends_statements) # when mpileaks becomes explicit, its file name changes (hash_length = 0), meaning an # extra module file is created; the old one still exists and remains hidden. writer = writer_cls(spec, "default", True) writer.write() assert os.path.exists(writer.layout.modulerc) with open(writer.layout.modulerc, encoding="utf-8") as f: content = [line.strip() for line in f.readlines()] assert hide_implicit_mpileaks in content # old, implicit mpileaks is still hidden assert f'hide_version("{writer.layout.use_name}")' not in content # after removing both the implicit and explicit module, the modulerc file would be empty # and should be removed. writer_cls(spec, "default", False).remove() writer_cls(spec, "default", True).remove() assert not os.path.exists(writer.layout.modulerc) assert not os.path.exists(writer.layout.filename) # implicit module is removed writer = writer_cls(spec, "default", False) writer.write() assert os.path.exists(writer.layout.filename) assert os.path.exists(writer.layout.modulerc) writer.remove() assert not os.path.exists(writer.layout.modulerc) assert not os.path.exists(writer.layout.filename) # three versions of mpileaks are implicit writer = writer_cls(spec, "default", False) writer.write(overwrite=True) spec_alt1 = spack.concretize.concretize_one("mpileaks@2.2") spec_alt2 = spack.concretize.concretize_one("mpileaks@2.1") writer_alt1 = writer_cls(spec_alt1, "default", False) writer_alt1.write(overwrite=True) writer_alt2 = writer_cls(spec_alt2, "default", False) writer_alt2.write(overwrite=True) assert os.path.exists(writer.layout.modulerc) with open(writer.layout.modulerc, encoding="utf-8") as f: content = [line.strip() for line in f.readlines()] hide_cmd = f'hide_version("{writer.layout.use_name}")' hide_cmd_alt1 = f'hide_version("{writer_alt1.layout.use_name}")' hide_cmd_alt2 = f'hide_version("{writer_alt2.layout.use_name}")' assert len([x for x in content if hide_cmd == x]) == 1 assert len([x for x in content if hide_cmd_alt1 == x]) == 1 assert len([x for x in content if hide_cmd_alt2 == x]) == 1 # one version is removed writer_alt1.remove() assert os.path.exists(writer.layout.modulerc) with open(writer.layout.modulerc, encoding="utf-8") as f: content = [line.strip() for line in f.readlines()] assert len([x for x in content if hide_cmd == x]) == 1 assert len([x for x in content if hide_cmd_alt1 == x]) == 0 assert len([x for x in content if hide_cmd_alt2 == x]) == 1 ================================================ FILE: lib/spack/spack/test/modules/tcl.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pytest import spack.vendor.archspec.cpu import spack.concretize import spack.modules.common import spack.modules.tcl mpich_spec_string = "mpich@3.0.4" mpileaks_spec_string = "mpileaks" libdwarf_spec_string = "libdwarf target=x86_64" #: Class of the writer tested in this module writer_cls = spack.modules.tcl.TclModulefileWriter pytestmark = [ pytest.mark.not_on_windows("does not run on windows"), pytest.mark.usefixtures("mock_modules_root"), ] @pytest.mark.usefixtures("mutable_config", "mock_packages", "mock_module_filename") class TestTcl: def test_simple_case(self, modulefile_content, module_configuration): """Tests the generation of a simple Tcl module file.""" module_configuration("autoload_direct") content = modulefile_content(mpich_spec_string) assert "module-whatis {mpich @3.0.4}" in content def test_autoload_direct(self, modulefile_content, module_configuration): """Tests the automatic loading of direct dependencies.""" module_configuration("autoload_direct") content = modulefile_content(mpileaks_spec_string) assert ( len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 1 ) assert len([x for x in content if " proc depends-on {args} {" in x]) == 1 assert len([x for x in content if " module load {*}$args" in x]) == 1 # depends-on command defined once and used 3 times assert len([x for x in content if "depends-on " in x]) == 4 # dtbuild1 has # - 1 ('run',) dependency # - 1 ('build','link') dependency # - 1 ('build',) dependency # Just make sure the 'build' dependency is not there content = modulefile_content("dtbuild1") assert ( len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 1 ) assert len([x for x in content if " proc depends-on {args} {" in x]) == 1 assert len([x for x in content if " module load {*}$args" in x]) == 1 # depends-on command defined once and used twice assert len([x for x in content if "depends-on " in x]) == 3 # The configuration file sets the verbose keyword to False messages = [x for x in content if 'puts stderr "Autoloading' in x] assert len(messages) == 0 def test_autoload_all(self, modulefile_content, module_configuration): """Tests the automatic loading of all dependencies.""" module_configuration("autoload_all") content = modulefile_content(mpileaks_spec_string) assert ( len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 1 ) assert len([x for x in content if " proc depends-on {args} {" in x]) == 1 assert len([x for x in content if " module load {*}$args" in x]) == 1 # depends-on command defined once and used 6 times assert len([x for x in content if "depends-on " in x]) == 7 # dtbuild1 has # - 1 ('run',) dependency # - 1 ('build','link') dependency # - 1 ('build',) dependency # Just make sure the 'build' dependency is not there content = modulefile_content("dtbuild1") assert ( len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 1 ) assert len([x for x in content if " proc depends-on {args} {" in x]) == 1 assert len([x for x in content if " module load {*}$args" in x]) == 1 # depends-on command defined once and used twice assert len([x for x in content if "depends-on " in x]) == 3 def test_prerequisites_direct( self, modulefile_content, module_configuration, host_architecture_str ): """Tests asking direct dependencies as prerequisites.""" module_configuration("prerequisites_direct") content = modulefile_content(f"mpileaks target={host_architecture_str}") assert len([x for x in content if "prereq" in x]) == 3 def test_prerequisites_all( self, modulefile_content, module_configuration, host_architecture_str ): """Tests asking all dependencies as prerequisites.""" module_configuration("prerequisites_all") content = modulefile_content(f"mpileaks target={host_architecture_str}") assert len([x for x in content if "prereq" in x]) == 6 def test_alter_environment(self, modulefile_content, module_configuration): """Tests modifications to run-time environment.""" module_configuration("alter_environment") content = modulefile_content("mpileaks platform=test target=x86_64") assert len([x for x in content if x.startswith("prepend-path CMAKE_PREFIX_PATH")]) == 0 assert len([x for x in content if "setenv FOO {foo}" in x]) == 1 assert len([x for x in content if "setenv OMPI_MCA_mpi_leave_pinned {1}" in x]) == 1 assert len([x for x in content if "setenv OMPI_MCA_MPI_LEAVE_PINNED {1}" in x]) == 0 assert len([x for x in content if "unsetenv BAR" in x]) == 1 assert len([x for x in content if "setenv MPILEAKS_ROOT" in x]) == 1 content = modulefile_content("libdwarf platform=test target=core2") assert len([x for x in content if x.startswith("prepend-path CMAKE_PREFIX_PATH")]) == 0 assert len([x for x in content if "setenv FOO {foo}" in x]) == 0 assert len([x for x in content if "unsetenv BAR" in x]) == 0 assert len([x for x in content if "depends-on foo/bar" in x]) == 1 assert len([x for x in content if "setenv LIBDWARF_ROOT" in x]) == 1 def test_prepend_path_separator(self, modulefile_content, module_configuration): """Tests that we can use custom delimiters to manipulate path lists.""" module_configuration("module_path_separator") content = modulefile_content("module-path-separator") assert len([x for x in content if "append-path -d {:} COLON {foo}" in x]) == 1 assert len([x for x in content if "prepend-path -d {:} COLON {foo}" in x]) == 1 assert len([x for x in content if "remove-path -d {:} COLON {foo}" in x]) == 1 assert len([x for x in content if "append-path -d {;} SEMICOLON {bar}" in x]) == 1 assert len([x for x in content if "prepend-path -d {;} SEMICOLON {bar}" in x]) == 1 assert len([x for x in content if "remove-path -d {;} SEMICOLON {bar}" in x]) == 1 assert len([x for x in content if "append-path -d { } SPACE {qux}" in x]) == 1 assert len([x for x in content if "remove-path -d { } SPACE {qux}" in x]) == 1 @pytest.mark.regression("11355") def test_manpath_setup(self, modulefile_content, module_configuration): """Tests specific setup of MANPATH environment variable.""" module_configuration("autoload_direct") # no manpath set by module content = modulefile_content("mpileaks") assert len([x for x in content if "append-path MANPATH {}" in x]) == 0 # manpath set by module with prepend-path content = modulefile_content("module-manpath-prepend") assert len([x for x in content if "prepend-path -d {:} MANPATH {/path/to/man}" in x]) == 1 assert ( len([x for x in content if "prepend-path -d {:} MANPATH {/path/to/share/man}" in x]) == 1 ) assert len([x for x in content if "append-path MANPATH {}" in x]) == 1 # manpath set by module with append-path content = modulefile_content("module-manpath-append") assert len([x for x in content if "append-path -d {:} MANPATH {/path/to/man}" in x]) == 1 assert len([x for x in content if "append-path MANPATH {}" in x]) == 1 # manpath set by module with setenv content = modulefile_content("module-manpath-setenv") assert len([x for x in content if "setenv MANPATH {/path/to/man}" in x]) == 1 assert len([x for x in content if "append-path MANPATH {}" in x]) == 0 @pytest.mark.regression("29578") def test_setenv_raw_value(self, modulefile_content, module_configuration): """Tests that we can set environment variable value without formatting it.""" module_configuration("autoload_direct") content = modulefile_content("module-setenv-raw") assert len([x for x in content if "setenv FOO {{{name}}, {name}, {{}}, {}}" in x]) == 1 @pytest.mark.skipif( str(spack.vendor.archspec.cpu.host().family) != "x86_64", reason="test data is specific for x86_64", ) def test_help_message(self, modulefile_content, module_configuration): """Tests the generation of module help message.""" module_configuration("autoload_direct") content = modulefile_content("mpileaks target=core2") help_msg = ( "proc ModulesHelp { } {" " puts stderr {Name : mpileaks}" " puts stderr {Version: 2.3}" " puts stderr {Target : core2}" " puts stderr {}" " puts stderr {Mpileaks is a mock package that passes audits}" "}" ) assert help_msg in "".join(content) content = modulefile_content("libdwarf target=core2") help_msg = ( "proc ModulesHelp { } {" " puts stderr {Name : libdwarf}" " puts stderr {Version: 20130729}" " puts stderr {Target : core2}" "}" ) assert help_msg in "".join(content) content = modulefile_content("module-long-help target=core2") help_msg = ( "proc ModulesHelp { } {" " puts stderr {Name : module-long-help}" " puts stderr {Version: 1.0}" " puts stderr {Target : core2}" " puts stderr {}" " puts stderr {Package to test long description message generated in modulefile.}" " puts stderr {Message too long is wrapped over multiple lines.}" "}" ) assert help_msg in "".join(content) def test_exclude(self, modulefile_content, module_configuration, host_architecture_str): """Tests excluding the generation of selected modules.""" module_configuration("exclude") content = modulefile_content("mpileaks ^zmpi") # depends-on command defined once and used twice assert len([x for x in content if "depends-on " in x]) == 3 with pytest.raises(FileNotFoundError): modulefile_content(f"callpath target={host_architecture_str}") content = modulefile_content(f"zmpi target={host_architecture_str}") # depends-on command defined once and used twice assert len([x for x in content if "depends-on " in x]) == 3 def test_naming_scheme_compat(self, factory, module_configuration): """Tests backwards compatibility for naming_scheme key""" module_configuration("naming_scheme") # Test we read the expected configuration for the naming scheme writer, _ = factory("mpileaks") expected = {"all": "{name}/{version}-{compiler.name}"} assert writer.conf.projections == expected projection = writer.spec.format(writer.conf.projections["all"]) assert projection in writer.layout.use_name def test_projections_specific(self, factory, module_configuration): """Tests reading the correct naming scheme.""" # This configuration has no error, so check the conflicts directives # are there module_configuration("projections") # Test we read the expected configuration for the naming scheme writer, _ = factory("mpileaks") expected = {"all": "{name}/{version}-{compiler.name}", "mpileaks": "{name}-mpiprojection"} assert writer.conf.projections == expected projection = writer.spec.format(writer.conf.projections["mpileaks"]) assert projection in writer.layout.use_name def test_projections_all(self, factory, module_configuration): """Tests reading the correct naming scheme.""" # This configuration has no error, so check the conflicts directives # are there module_configuration("projections") # Test we read the expected configuration for the naming scheme writer, _ = factory("libelf") expected = {"all": "{name}/{version}-{compiler.name}", "mpileaks": "{name}-mpiprojection"} assert writer.conf.projections == expected projection = writer.spec.format(writer.conf.projections["all"]) assert projection in writer.layout.use_name def test_invalid_naming_scheme(self, factory, module_configuration): """Tests the evaluation of an invalid naming scheme.""" module_configuration("invalid_naming_scheme") # Test that having invalid tokens in the naming scheme raises # a RuntimeError writer, _ = factory("mpileaks") with pytest.raises(RuntimeError): writer.layout.use_name def test_invalid_token_in_env_name(self, factory, module_configuration): """Tests setting environment variables with an invalid name.""" module_configuration("invalid_token_in_env_var_name") writer, _ = factory("mpileaks") with pytest.raises(RuntimeError): writer.write() def test_conflicts(self, modulefile_content, module_configuration): """Tests adding conflicts to the module.""" # This configuration has no error, so check the conflicts directives # are there module_configuration("conflicts") content = modulefile_content("mpileaks") assert len([x for x in content if x.startswith("conflict")]) == 2 assert len([x for x in content if x == "conflict mpileaks"]) == 1 assert len([x for x in content if x == "conflict intel/14.0.1"]) == 1 def test_inconsistent_conflict_in_modules_yaml(self, modulefile_content, module_configuration): """Tests inconsistent conflict definition in `modules.yaml`.""" # This configuration is inconsistent, check an error is raised module_configuration("wrong_conflicts") with pytest.raises(spack.modules.common.ModulesError): modulefile_content("mpileaks") def test_module_index( self, module_configuration, factory, tmp_path_factory: pytest.TempPathFactory ): module_configuration("suffix") w1, s1 = factory("mpileaks") w2, s2 = factory("callpath") w3, s3 = factory("openblas") test_root = str(tmp_path_factory.mktemp("module-root")) spack.modules.common.generate_module_index(test_root, [w1, w2]) index = spack.modules.common.read_module_index(test_root) assert index[s1.dag_hash()].use_name == w1.layout.use_name assert index[s2.dag_hash()].path == w2.layout.filename spack.modules.common.generate_module_index(test_root, [w3]) index = spack.modules.common.read_module_index(test_root) assert len(index) == 3 assert index[s1.dag_hash()].use_name == w1.layout.use_name assert index[s2.dag_hash()].path == w2.layout.filename spack.modules.common.generate_module_index(test_root, [w3], overwrite=True) index = spack.modules.common.read_module_index(test_root) assert len(index) == 1 assert index[s3.dag_hash()].use_name == w3.layout.use_name def test_suffixes(self, module_configuration, factory): """Tests adding suffixes to module file name.""" module_configuration("suffix") writer, spec = factory("mpileaks+debug target=x86_64") assert "foo" in writer.layout.use_name assert "foo-foo" not in writer.layout.use_name writer, spec = factory("mpileaks~debug target=x86_64") assert "foo-bar" in writer.layout.use_name assert "baz" not in writer.layout.use_name writer, spec = factory("mpileaks~debug+opt target=x86_64") assert "baz-foo-bar" in writer.layout.use_name def test_suffixes_format(self, module_configuration, factory): """Tests adding suffixes as spec format string to module file name.""" module_configuration("suffix-format") writer, spec = factory("mpileaks +debug target=x86_64 ^mpich@3.0.4") assert "debug=True" in writer.layout.use_name assert "mpi=mpich-v3.0.4" in writer.layout.use_name def test_setup_environment(self, modulefile_content, module_configuration): """Tests the internal set-up of run-time environment.""" module_configuration("suffix") content = modulefile_content("mpileaks") assert len([x for x in content if "setenv FOOBAR" in x]) == 1 assert len([x for x in content if "setenv FOOBAR {mpileaks}" in x]) == 1 spec = spack.concretize.concretize_one("mpileaks") content = modulefile_content(spec["callpath"]) assert len([x for x in content if "setenv FOOBAR" in x]) == 1 assert len([x for x in content if "setenv FOOBAR {callpath}" in x]) == 1 def test_override_config(self, module_configuration, factory): """Tests overriding some sections of the configuration file.""" module_configuration("override_config") writer, spec = factory("mpileaks~opt target=x86_64") assert "mpich-static" in writer.layout.use_name assert "over" not in writer.layout.use_name assert "ridden" not in writer.layout.use_name writer, spec = factory("mpileaks+opt target=x86_64") assert "over-ridden" in writer.layout.use_name assert "mpich" not in writer.layout.use_name assert "static" not in writer.layout.use_name def test_override_template_in_package(self, modulefile_content, module_configuration): """Tests overriding a template from and attribute in the package.""" module_configuration("autoload_direct") content = modulefile_content("override-module-templates") assert "Override successful!" in content def test_override_template_in_modules_yaml( self, modulefile_content, module_configuration, host_architecture_str ): """Tests overriding a template from `modules.yaml`""" module_configuration("override_template") content = modulefile_content("override-module-templates") assert "Override even better!" in content content = modulefile_content(f"mpileaks target={host_architecture_str}") assert "Override even better!" in content def test_extend_context(self, modulefile_content, module_configuration): """Tests using a package defined context""" module_configuration("autoload_direct") content = modulefile_content("override-context-templates") assert 'puts stderr "sentence from package"' in content short_description = "module-whatis {This package updates the context for Tcl modulefiles.}" assert short_description in content @pytest.mark.regression("4400") @pytest.mark.db def test_hide_implicits_no_arg(self, module_configuration, mutable_database): module_configuration("exclude_implicits") # mpileaks has been installed explicitly when setting up # the tests database mpileaks_specs = mutable_database.query("mpileaks") for item in mpileaks_specs: writer = writer_cls(item, "default") assert not writer.conf.excluded # callpath is a dependency of mpileaks, and has been pulled # in implicitly callpath_specs = mutable_database.query("callpath") for item in callpath_specs: writer = writer_cls(item, "default") assert writer.conf.excluded @pytest.mark.regression("12105") def test_hide_implicits_with_arg(self, module_configuration): module_configuration("exclude_implicits") # mpileaks is defined as explicit with explicit argument set on writer mpileaks_spec = spack.concretize.concretize_one("mpileaks") writer = writer_cls(mpileaks_spec, "default", True) assert not writer.conf.excluded # callpath is defined as implicit with explicit argument set on writer callpath_spec = spack.concretize.concretize_one("callpath") writer = writer_cls(callpath_spec, "default", False) assert writer.conf.excluded @pytest.mark.regression("9624") def test_autoload_with_constraints(self, modulefile_content, module_configuration): """Tests the automatic loading of direct dependencies.""" module_configuration("autoload_with_constraints") # Test the mpileaks that should have the autoloaded dependencies content = modulefile_content("mpileaks ^mpich2") # depends-on command defined once and used 3 times assert len([x for x in content if "depends-on " in x]) == 4 # Test the mpileaks that should NOT have the autoloaded dependencies content = modulefile_content("mpileaks ^mpich") assert ( len([x for x in content if "if {![llength [info commands depends-on]]} {" in x]) == 0 ) assert len([x for x in content if " proc depends-on {args} {" in x]) == 0 assert len([x for x in content if " module load {*}$args" in x]) == 0 assert len([x for x in content if "depends-on " in x]) == 0 def test_modules_no_arch(self, factory, module_configuration): module_configuration("no_arch") module, spec = factory(mpileaks_spec_string) path = module.layout.filename assert str(spec.os) not in path def test_hide_implicits(self, module_configuration, temporary_store): """Tests the addition and removal of hide command in modulerc.""" module_configuration("hide_implicits") spec = spack.concretize.concretize_one("mpileaks@2.3") # mpileaks is defined as implicit, thus hide command should appear in modulerc writer = writer_cls(spec, "default", False) writer.write() assert os.path.exists(writer.layout.modulerc) with open(writer.layout.modulerc, encoding="utf-8") as f: content = [line.strip() for line in f.readlines()] hide_implicit_mpileaks = f"module-hide --soft --hidden-loaded {writer.layout.use_name}" assert len([x for x in content if hide_implicit_mpileaks == x]) == 1 # The direct dependencies are all implicit, and they should have depends-on with fixed # 7 character hash, even though the config is set to hash_length = 0. with open(writer.layout.filename, encoding="utf-8") as f: depends_statements = [line.strip() for line in f.readlines() if "depends-on" in line] for dep in spec.dependencies(deptype=("link", "run")): assert any(dep.dag_hash(7) in line for line in depends_statements) # when mpileaks becomes explicit, its file name changes (hash_length = 0), meaning an # extra module file is created; the old one still exists and remains hidden. writer = writer_cls(spec, "default", True) writer.write() assert os.path.exists(writer.layout.modulerc) with open(writer.layout.modulerc, encoding="utf-8") as f: content = [line.strip() for line in f.readlines()] assert hide_implicit_mpileaks in content # old, implicit mpileaks is still hidden assert f"module-hide --soft --hidden-loaded {writer.layout.use_name}" not in content # after removing both the implicit and explicit module, the modulerc file would be empty # and should be removed. writer_cls(spec, "default", False).remove() writer_cls(spec, "default", True).remove() assert not os.path.exists(writer.layout.modulerc) assert not os.path.exists(writer.layout.filename) # implicit module is removed writer = writer_cls(spec, "default", False) writer.write() assert os.path.exists(writer.layout.filename) assert os.path.exists(writer.layout.modulerc) writer.remove() assert not os.path.exists(writer.layout.modulerc) assert not os.path.exists(writer.layout.filename) # three versions of mpileaks are implicit writer = writer_cls(spec, "default", False) writer.write(overwrite=True) spec_alt1 = spack.concretize.concretize_one("mpileaks@2.2") spec_alt2 = spack.concretize.concretize_one("mpileaks@2.1") writer_alt1 = writer_cls(spec_alt1, "default", False) writer_alt1.write(overwrite=True) writer_alt2 = writer_cls(spec_alt2, "default", False) writer_alt2.write(overwrite=True) assert os.path.exists(writer.layout.modulerc) with open(writer.layout.modulerc, encoding="utf-8") as f: content = [line.strip() for line in f.readlines()] hide_cmd = f"module-hide --soft --hidden-loaded {writer.layout.use_name}" hide_cmd_alt1 = f"module-hide --soft --hidden-loaded {writer_alt1.layout.use_name}" hide_cmd_alt2 = f"module-hide --soft --hidden-loaded {writer_alt2.layout.use_name}" assert len([x for x in content if hide_cmd == x]) == 1 assert len([x for x in content if hide_cmd_alt1 == x]) == 1 assert len([x for x in content if hide_cmd_alt2 == x]) == 1 # one version is removed writer_alt1.remove() assert os.path.exists(writer.layout.modulerc) with open(writer.layout.modulerc, encoding="utf-8") as f: content = [line.strip() for line in f.readlines()] assert len([x for x in content if hide_cmd == x]) == 1 assert len([x for x in content if hide_cmd_alt1 == x]) == 0 assert len([x for x in content if hide_cmd_alt2 == x]) == 1 ================================================ FILE: lib/spack/spack/test/multimethod.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test for multi_method dispatch.""" import pytest import spack.concretize import spack.platforms from spack.multimethod import NoSuchMethodError pytestmark = [ pytest.mark.usefixtures("mock_packages", "config"), pytest.mark.not_on_windows("Not running on windows"), ] @pytest.fixture(scope="module", params=["multimethod", "multimethod-inheritor"]) def pkg_name(request): """Make tests run on both multimethod and multimethod-inheritor. This means we test all of our @when methods on a class that uses them directly, AND on a class that inherits them. """ return request.param def test_no_version_match(pkg_name): spec = spack.concretize.concretize_one(pkg_name + "@2.0") with pytest.raises(NoSuchMethodError): spec.package.no_version_2() @pytest.mark.parametrize( "constraint_str,method_name,expected_result", [ # Only one version match these constraints ("@1.0", "no_version_2", 1), ("@3.0", "no_version_2", 3), ("@4.0", "no_version_2", 4), # These constraints overlap, in which case the first match wins ("@2.0", "version_overlap", 1), ("@5.0", "version_overlap", 2), # These constraints are on the version of a virtual dependency ("^mpich@3.0.4", "mpi_version", 3), ("^mpich2@1.2", "mpi_version", 2), ("^mpich@1.0", "mpi_version", 1), # Undefined mpi versions ("^mpich@=0.4", "mpi_version", 1), ("^mpich@=1.4", "mpi_version", 1), # Constraints on compilers with a default ("%gcc", "has_a_default", "gcc"), ("%clang", "has_a_default", "clang"), ("%gcc@9", "has_a_default", "default"), # Constraints on dependencies ("^zmpi", "different_by_dep", "zmpi"), ("^mpich", "different_by_dep", "mpich"), # Constraints on virtual dependencies ("^mpich2", "different_by_virtual_dep", 2), ("^mpich@1.0", "different_by_virtual_dep", 1), # Multimethod with base classes ("@1", "base_method", "base_method"), # Boolean ("", "boolean_true_first", "True"), ("", "boolean_false_first", "True"), ], ) def test_multimethod_calls( pkg_name, constraint_str, method_name, expected_result, default_mock_concretization ): s = default_mock_concretization(f"{pkg_name}{constraint_str}") msg = f"Method {method_name} from {s} is giving a wrong result" assert getattr(s.package, method_name)() == expected_result, msg def test_target_match(pkg_name): platform = spack.platforms.host() targets = list(platform.targets.values()) for target in targets[:-1]: s = spack.concretize.concretize_one(pkg_name + " target=" + target.name) assert s.package.different_by_target() == target.name s = spack.concretize.concretize_one(pkg_name + " target=" + targets[-1].name) if len(targets) == 1: assert s.package.different_by_target() == targets[-1].name else: with pytest.raises(NoSuchMethodError): s.package.different_by_target() @pytest.mark.parametrize( "spec_str,method_name,expected_result", [ # This is overridden in the second case ("multimethod@3", "base_method", "multimethod"), ("multimethod-inheritor@3", "base_method", "multimethod-inheritor"), # Here we have a mix of inherited and overridden methods ("multimethod-inheritor@1.0", "inherited_and_overridden", "inheritor@1.0"), ("multimethod-inheritor@2.0", "inherited_and_overridden", "base@2.0"), ("multimethod@1.0", "inherited_and_overridden", "base@1.0"), ("multimethod@2.0", "inherited_and_overridden", "base@2.0"), # Diamond-like inheritance (even though the MRO linearize everything) ("multimethod-diamond@1.0", "diamond_inheritance", "base_package"), ("multimethod-base@=1.0", "diamond_inheritance", "base_package"), ("multimethod-diamond@2.0", "diamond_inheritance", "first_parent"), ("multimethod-inheritor@2.0", "diamond_inheritance", "first_parent"), ("multimethod-diamond@=3.0", "diamond_inheritance", "second_parent"), ("multimethod-diamond-parent@=3.0", "diamond_inheritance", "second_parent"), ("multimethod-diamond@4.0", "diamond_inheritance", "subclass"), ], ) def test_multimethod_calls_and_inheritance(spec_str, method_name, expected_result): s = spack.concretize.concretize_one(spec_str) assert getattr(s.package, method_name)() == expected_result ================================================ FILE: lib/spack/spack/test/namespace_trie.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.util.naming @pytest.fixture() def trie(): return spack.util.naming.NamespaceTrie() def test_add_single(trie): trie["foo"] = "bar" assert trie.is_prefix("foo") assert trie.has_value("foo") assert trie["foo"] == "bar" def test_add_multiple(trie): trie["foo.bar"] = "baz" assert not trie.has_value("foo") assert trie.is_prefix("foo") assert trie.is_prefix("foo.bar") assert trie.has_value("foo.bar") assert trie["foo.bar"] == "baz" assert not trie.is_prefix("foo.bar.baz") assert not trie.has_value("foo.bar.baz") def test_add_three(trie): # add a three-level namespace trie["foo.bar.baz"] = "quux" assert trie.is_prefix("foo") assert not trie.has_value("foo") assert trie.is_prefix("foo.bar") assert not trie.has_value("foo.bar") assert trie.is_prefix("foo.bar.baz") assert trie.has_value("foo.bar.baz") assert trie["foo.bar.baz"] == "quux" assert not trie.is_prefix("foo.bar.baz.quux") assert not trie.has_value("foo.bar.baz.quux") # Try to add a second element in a prefix namespace trie["foo.bar"] = "blah" assert trie.is_prefix("foo") assert not trie.has_value("foo") assert trie.is_prefix("foo.bar") assert trie.has_value("foo.bar") assert trie["foo.bar"] == "blah" assert trie.is_prefix("foo.bar.baz") assert trie.has_value("foo.bar.baz") assert trie["foo.bar.baz"] == "quux" assert not trie.is_prefix("foo.bar.baz.quux") assert not trie.has_value("foo.bar.baz.quux") def test_add_none_single(trie): trie["foo"] = None assert trie.is_prefix("foo") assert trie.has_value("foo") assert trie["foo"] is None assert not trie.is_prefix("foo.bar") assert not trie.has_value("foo.bar") def test_add_none_multiple(trie): trie["foo.bar"] = None assert trie.is_prefix("foo") assert not trie.has_value("foo") assert trie.is_prefix("foo.bar") assert trie.has_value("foo.bar") assert trie["foo.bar"] is None assert not trie.is_prefix("foo.bar.baz") assert not trie.has_value("foo.bar.baz") ================================================ FILE: lib/spack/spack/test/new_installer.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for the new_installer.py module""" import pathlib import sys import time import pytest if sys.platform == "win32": pytest.skip("No Windows support", allow_module_level=True) import spack.spec from spack.new_installer import ( OVERWRITE_GARBAGE_SUFFIX, JobServer, PackageInstaller, PrefixPivoter, _node_to_roots, schedule_builds, ) from spack.test.traverse import create_dag @pytest.fixture def existing_prefix(tmp_path: pathlib.Path) -> pathlib.Path: """Creates a standard existing prefix with content.""" prefix = tmp_path / "existing_prefix" prefix.mkdir() (prefix / "old_file").write_text("old content") return prefix class TestPrefixPivoter: """Tests for the PrefixPivoter class.""" def test_no_existing_prefix(self, tmp_path: pathlib.Path): """Test installation when prefix doesn't exist yet.""" prefix = tmp_path / "new_prefix" with PrefixPivoter(str(prefix)): prefix.mkdir() (prefix / "installed_file").write_text("content") assert prefix.exists() assert (prefix / "installed_file").read_text() == "content" def test_existing_prefix_success_cleans_up_old_prefix( self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): """Test that an existing prefix is moved aside, and cleaned up on success.""" with PrefixPivoter(str(existing_prefix)): assert not existing_prefix.exists() existing_prefix.mkdir() (existing_prefix / "new_file").write_text("new content") assert existing_prefix.exists() assert (existing_prefix / "new_file").exists() assert not (existing_prefix / "old_file").exists() # Only the existing_prefix directory should remain assert len(list(tmp_path.iterdir())) == 1 def test_existing_prefix_failure_restores_original_prefix( self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): """Test that the original prefix is restored when installation fails.""" with pytest.raises(RuntimeError, match="simulated failure"): with PrefixPivoter(str(existing_prefix), keep_prefix=False): existing_prefix.mkdir() (existing_prefix / "partial_file").write_text("partial") raise RuntimeError("simulated failure") assert existing_prefix.exists() assert (existing_prefix / "old_file").read_text() == "old content" assert not (existing_prefix / "partial_file").exists() # Only the original prefix should remain assert len(list(tmp_path.iterdir())) == 1 def test_existing_prefix_failure_no_partial_prefix_created( self, existing_prefix: pathlib.Path ): """Test restoration when failure occurs before the build creates the prefix dir.""" with pytest.raises(RuntimeError, match="early failure"): with PrefixPivoter(str(existing_prefix)): raise RuntimeError("early failure") assert existing_prefix.exists() assert (existing_prefix / "old_file").read_text() == "old content" def test_no_existing_prefix_success(self, tmp_path: pathlib.Path): """Test that a fresh install with no pre-existing prefix works fine.""" prefix = tmp_path / "new_prefix" with PrefixPivoter(str(prefix)): prefix.mkdir() (prefix / "installed_file").write_text("content") assert prefix.exists() # Only the new_prefix directory should remain assert len(list(tmp_path.iterdir())) == 1 def test_keep_prefix_true_with_existing_prefix_keeps_failed_install( self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): """Test that keep_prefix=True keeps the failed install and discards the backup.""" with pytest.raises(RuntimeError, match="simulated failure"): with PrefixPivoter(str(existing_prefix), keep_prefix=True): existing_prefix.mkdir() (existing_prefix / "partial_file").write_text("partial content") raise RuntimeError("simulated failure") # The failed prefix should be kept (not the original) assert existing_prefix.exists() assert (existing_prefix / "partial_file").exists() assert not (existing_prefix / "old_file").exists() # Backup should have been removed assert len(list(tmp_path.iterdir())) == 1 def test_keep_prefix_false_removes_failed_install(self, tmp_path: pathlib.Path): """Test that keep_prefix=False removes the failed installation (no pre-existing prefix).""" prefix = tmp_path / "new_prefix" with pytest.raises(RuntimeError, match="simulated failure"): with PrefixPivoter(str(prefix), keep_prefix=False): prefix.mkdir() (prefix / "partial_file").write_text("partial content") raise RuntimeError("simulated failure") # Failed prefix should be removed assert not prefix.exists() # Nothing should remain assert len(list(tmp_path.iterdir())) == 0 def test_keep_prefix_true_no_existing_prefix(self, tmp_path: pathlib.Path): """Test failure with keep_prefix=True when no prefix existed beforehand.""" prefix = tmp_path / "new_prefix" with pytest.raises(RuntimeError, match="simulated failure"): with PrefixPivoter(str(prefix), keep_prefix=True): prefix.mkdir() (prefix / "partial_file").write_text("partial content") raise RuntimeError("simulated failure") # The failed prefix should be kept assert prefix.exists() assert (prefix / "partial_file").exists() # No backup should exist assert len(list(tmp_path.iterdir())) == 1 def test_failure_no_prefix_created(self, tmp_path: pathlib.Path): """Test failure when the prefix directory was never created.""" prefix = tmp_path / "new_prefix" with pytest.raises(RuntimeError, match="simulated failure"): with PrefixPivoter(str(prefix), keep_prefix=False): # Do NOT create the prefix directory raise RuntimeError("simulated failure") # Prefix should not exist assert not prefix.exists() # Nothing should remain assert len(list(tmp_path.iterdir())) == 0 class FailingPrefixPivoter(PrefixPivoter): """Test subclass that can simulate filesystem failures.""" def __init__( self, prefix: str, keep_prefix: bool = False, fail_on_restore: bool = False, fail_on_move_garbage: bool = False, ): super().__init__(prefix, keep_prefix) self.fail_on_restore = fail_on_restore self.fail_on_move_garbage = fail_on_move_garbage self.restore_rename_count = 0 def _rename(self, src: str, dst: str) -> None: if ( self.fail_on_restore and self.tmp_prefix and src == self.tmp_prefix and dst == self.prefix ): self.restore_rename_count += 1 raise OSError("Simulated rename failure during restore") if self.fail_on_move_garbage and dst.endswith(OVERWRITE_GARBAGE_SUFFIX): raise OSError("Simulated rename failure moving to garbage") super()._rename(src, dst) class TestPrefixPivoterFailureRecovery: """Tests for edge cases and failure recovery in PrefixPivoter.""" def test_restore_failure_leaves_backup( self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): """Test that if restoration fails, the backup is not deleted.""" pivoter = FailingPrefixPivoter(str(existing_prefix), fail_on_restore=True) with pytest.raises(OSError, match="Simulated rename failure during restore"): with pivoter: existing_prefix.mkdir() (existing_prefix / "partial_file").write_text("partial") raise RuntimeError("simulated failure") assert pivoter.restore_rename_count > 0 # Backup directory should still exist (plus the failed prefix) assert len(list(tmp_path.iterdir())) == 2 def test_garbage_move_failure_leaves_backup( self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path ): """Test that if moving the failed install to garbage fails, the backup is preserved.""" pivoter = FailingPrefixPivoter(str(existing_prefix), fail_on_move_garbage=True) with pytest.raises(OSError, match="Simulated rename failure moving to garbage"): with pivoter: existing_prefix.mkdir() (existing_prefix / "partial_file").write_text("partial") raise RuntimeError("simulated failure") assert (existing_prefix / "partial_file").exists() # Backup directory, failed prefix, and empty garbage directory should exist assert len(list(tmp_path.iterdir())) == 3 class TestPackageInstallerConstructor: """Tests for PackageInstaller constructor, especially capacity initialization.""" def test_capacity_explicit_concurrent_packages(self, temporary_store, mock_packages): """Test that capacity is set correctly when concurrent_packages is explicitly provided.""" spec = spack.spec.Spec("trivial-install-test-package") spec._mark_concrete() assert PackageInstaller([spec.package], concurrent_packages=5).capacity == 5 assert PackageInstaller([spec.package], concurrent_packages=1).capacity == 1 def test_capacity_from_config_default_one( self, temporary_store, mock_packages, mutable_config ): """Test that config value of 0 is treated as unlimited.""" mutable_config.set("config:concurrent_packages", 0) spec = spack.spec.Spec("trivial-install-test-package") spec._mark_concrete() assert PackageInstaller([spec.package]).capacity == sys.maxsize def test_capacity_from_config_non_zero(self, temporary_store, mock_packages, mutable_config): """Test that non-0 config values are used as-is.""" mutable_config.set("config:concurrent_packages", 1) spec = spack.spec.Spec("trivial-install-test-package") spec._mark_concrete() assert PackageInstaller([spec.package]).capacity == 1 class _FakeBuildGraph: """Minimal stand-in for BuildGraph in schedule_builds unit tests. Provides the two interface points that schedule_builds calls: - .nodes (dict: dag_hash -> Spec) - .enqueue_parents(dag_hash, pending_builds) """ def __init__(self, specs): self.nodes = {spec.dag_hash(): spec for spec in specs} def enqueue_parents(self, dag_hash, pending_builds): """Remove dag_hash from nodes; no parents in these simple unit tests.""" self.nodes.pop(dag_hash, None) class TestScheduleBuilds: """Unit tests for the module-level schedule_builds() function.""" def _make_spec(self, name): """Return a minimal concrete spec suitable for locking and DB queries.""" spec = spack.spec.Spec(name) spec._mark_concrete() return spec def _mark_installed(self, spec, store): """Create the install directory structure and register the spec in the DB as installed.""" store.layout.create_install_directory(spec) store.db.add(spec, explicit=True) def test_not_installed_no_running_starts_build(self, temporary_store, mock_packages): """A fresh spec with no running builds is added to to_start.""" spec = self._make_spec("trivial-install-test-package") pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) jobserver = JobServer(num_jobs=2) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite=set(), overwrite_time=0.0, capacity=1, needs_jobserver_token=False, jobserver=jobserver, explicit=set(), ) assert not result.blocked assert len(result.to_start) == 1 assert result.to_start[0][0] == spec.dag_hash() assert not result.newly_installed assert not pending # removed from the pending list finally: for _, lock in result.to_start: lock.release_write() jobserver.close() def test_already_installed_yields_newly_installed(self, temporary_store, mock_packages): """A spec already in the DB is returned in newly_installed, not in to_start.""" spec = self._make_spec("trivial-install-test-package") self._mark_installed(spec, temporary_store) pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) jobserver = JobServer(num_jobs=2) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite=set(), overwrite_time=0.0, capacity=1, needs_jobserver_token=False, jobserver=jobserver, explicit=set(), ) assert not result.blocked assert not result.to_start assert len(result.newly_installed) == 1 assert result.newly_installed[0][0] == spec.dag_hash() assert not pending # removed from the pending list finally: for _, _, lock in result.newly_installed: lock.release_read() jobserver.close() def test_no_jobserver_token_returns_empty(self, temporary_store, mock_packages): """When has_running_builds=True and no token is available, nothing is started.""" spec = self._make_spec("trivial-install-test-package") pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) # num_jobs=1 writes 0 tokens to the FIFO. Only the implicit token exists. jobserver = JobServer(num_jobs=1) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite=set(), overwrite_time=0.0, capacity=2, needs_jobserver_token=True, jobserver=jobserver, explicit=set(), ) assert not result.blocked assert not result.to_start assert not result.newly_installed assert len(pending) == 1 finally: jobserver.close() def test_all_locked_returns_blocked(self, temporary_store, mock_packages, monkeypatch): """When all pending specs are locked externally, blocked_on_locks is True.""" spec = self._make_spec("trivial-install-test-package") pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) jobserver = JobServer(num_jobs=2) # Pre-register the lock in the prefix_locker cache, then patch try_acquire to fail. lock = temporary_store.prefix_locker.lock(spec) monkeypatch.setattr(lock, "try_acquire_write", lambda: False) monkeypatch.setattr(lock, "try_acquire_read", lambda: False) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite=set(), overwrite_time=0.0, capacity=2, needs_jobserver_token=False, jobserver=jobserver, explicit=set(), ) assert result.blocked assert not result.to_start assert not result.newly_installed assert len(pending) == 1 finally: jobserver.close() def test_overwrite_installed_spec_is_started(self, temporary_store, mock_packages): """A spec in the overwrite set is scheduled even when already installed.""" spec = self._make_spec("trivial-install-test-package") self._mark_installed(spec, temporary_store) pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) jobserver = JobServer(num_jobs=2) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite={spec.dag_hash()}, overwrite_time=time.time() + 100, capacity=1, needs_jobserver_token=False, jobserver=jobserver, explicit=set(), ) assert not result.blocked assert len(result.to_start) == 1 assert result.to_start[0][0] == spec.dag_hash() assert not result.newly_installed finally: for _, lock in result.to_start: lock.release_write() jobserver.close() def test_mixed_locked_unlocked(self, temporary_store, mock_packages, monkeypatch): """Only the unlocked spec enters to_start when one spec is externally locked.""" spec_a = self._make_spec("trivial-install-test-package") spec_b = self._make_spec("trivial-smoke-test") pending = [spec_a.dag_hash(), spec_b.dag_hash()] bg = _FakeBuildGraph([spec_a, spec_b]) jobserver = JobServer(num_jobs=4) # Patch spec_a's lock to always fail, simulating an external write lock. lock_a = temporary_store.prefix_locker.lock(spec_a) monkeypatch.setattr(lock_a, "try_acquire_write", lambda: False) monkeypatch.setattr(lock_a, "try_acquire_read", lambda: False) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite=set(), overwrite_time=0.0, capacity=2, needs_jobserver_token=False, jobserver=jobserver, explicit=set(), ) assert not result.blocked # spec_b was schedulable started_hashes = {h for h, _ in result.to_start} assert spec_b.dag_hash() in started_hashes assert spec_a.dag_hash() not in started_hashes assert not result.newly_installed finally: for _, lock in result.to_start: lock.release_write() jobserver.close() def test_write_locked_read_locked_installed_yields_newly_installed( self, temporary_store, mock_packages, monkeypatch ): """Write lock fails but read lock succeeds and spec is installed: treated as done. Simulates the case where another process finished building and downgraded its write lock to a read lock. The spec should appear in newly_installed. blocked remains True because no write lock was obtained, preventing the jobserver from firing unnecessarily. """ spec = self._make_spec("trivial-install-test-package") self._mark_installed(spec, temporary_store) pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) jobserver = JobServer(num_jobs=2) lock = temporary_store.prefix_locker.lock(spec) monkeypatch.setattr(lock, "try_acquire_write", lambda: False) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite=set(), overwrite_time=0.0, capacity=2, needs_jobserver_token=False, jobserver=jobserver, explicit=set(), ) assert result.blocked # no write lock was obtained; jobserver should not fire assert not result.to_start assert len(result.newly_installed) == 1 dag_hash, installed_spec, lock = result.newly_installed[0] assert dag_hash == spec.dag_hash() assert installed_spec == spec assert not pending # spec was removed from pending finally: for _, _, lock in result.newly_installed: lock.release_read() jobserver.close() def test_write_locked_read_locked_not_installed_still_blocked( self, temporary_store, mock_packages, monkeypatch ): """Write lock fails, read lock succeeds, but spec is not in DB: retry later. Simulates the case where a concurrent process was killed mid-build. The read lock is released and the spec stays in pending; blocked should remain True. """ spec = self._make_spec("trivial-install-test-package") pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) jobserver = JobServer(num_jobs=2) lock = temporary_store.prefix_locker.lock(spec) monkeypatch.setattr(lock, "try_acquire_write", lambda: False) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite=set(), overwrite_time=0.0, capacity=2, needs_jobserver_token=False, jobserver=jobserver, explicit=set(), ) assert result.blocked assert not result.to_start assert not result.newly_installed assert pending == [spec.dag_hash()] # spec stays in pending for retry finally: jobserver.close() def test_overwrite_handled_by_concurrent_process(self, temporary_store, mock_packages): """When a spec in overwrite was installed AFTER overwrite_time, another process did it.""" spec = self._make_spec("trivial-install-test-package") self._mark_installed(spec, temporary_store) # installation_time = now() pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) jobserver = JobServer(num_jobs=2) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite={spec.dag_hash()}, overwrite_time=0.0, # earlier than now() capacity=1, needs_jobserver_token=False, jobserver=jobserver, explicit=set(), ) assert not result.blocked assert not result.to_start assert len(result.newly_installed) == 1 assert result.newly_installed[0][0] == spec.dag_hash() finally: for _, _, lock in result.newly_installed: lock.release_read() jobserver.close() def test_installed_implicit_explicit_set_produces_db_update( self, temporary_store, mock_packages ): """An installed-implicit spec in explicit set produces a DbUpdate.""" spec = self._make_spec("trivial-install-test-package") temporary_store.layout.create_install_directory(spec) temporary_store.db.add(spec, explicit=False) pending = [spec.dag_hash()] bg = _FakeBuildGraph([spec]) jobserver = JobServer(num_jobs=2) try: result = schedule_builds( pending, bg, temporary_store.db, temporary_store.prefix_locker, overwrite=set(), overwrite_time=0.0, capacity=1, needs_jobserver_token=False, jobserver=jobserver, explicit={spec.dag_hash()}, ) assert len(result.to_mark_explicit) == 1 assert result.to_mark_explicit[0].spec is spec assert len(result.newly_installed) == 1 finally: for _, _, lock in result.newly_installed: lock.release_read() jobserver.close() def test_nodes_to_roots(): """Independent roots don't reach each other's exclusive nodes.""" # A - B and C - D are disconnected graphs, A, B and C are "roots". specs = create_dag(nodes=["A", "B", "C", "D"], edges=[("A", "B", "all"), ("C", "D", "all")]) a, b, c, d = specs["A"], specs["B"], specs["C"], specs["D"] node_to_roots = _node_to_roots([a, b, c]) assert node_to_roots[a.dag_hash()] == frozenset([a.dag_hash()]) assert node_to_roots[b.dag_hash()] == frozenset([a.dag_hash(), b.dag_hash()]) assert node_to_roots[c.dag_hash()] == frozenset([c.dag_hash()]) assert node_to_roots[d.dag_hash()] == frozenset([c.dag_hash()]) def test_nodes_to_roots_shared_dependency(): """A dependency shared by two roots is attributed to both.""" specs = create_dag(nodes=["A", "B", "C"], edges=[("A", "C", "all"), ("B", "C", "all")]) a, b, c = specs["A"], specs["B"], specs["C"] node_to_roots = _node_to_roots([a, b]) assert node_to_roots[a.dag_hash()] == frozenset([a.dag_hash()]) assert node_to_roots[b.dag_hash()] == frozenset([b.dag_hash()]) assert node_to_roots[c.dag_hash()] == frozenset([a.dag_hash(), b.dag_hash()]) ================================================ FILE: lib/spack/spack/test/oci/image.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest from spack.oci.image import Digest, ImageReference @pytest.mark.parametrize( "image_ref, expected", [ ( f"example.com:1234/a/b/c:tag@sha256:{'a' * 64}", ("example.com:1234", "a/b/c", "tag", Digest.from_sha256("a" * 64)), ), ("example.com:1234/a/b/c:tag", ("example.com:1234", "a/b/c", "tag", None)), ("example.com:1234/a/b/c", ("example.com:1234", "a/b/c", "latest", None)), ( f"example.com:1234/a/b/c@sha256:{'a' * 64}", ("example.com:1234", "a/b/c", "latest", Digest.from_sha256("a" * 64)), ), # ipv4 ("1.2.3.4:1234/a/b/c:tag", ("1.2.3.4:1234", "a/b/c", "tag", None)), # ipv6 ("[2001:db8::1]:1234/a/b/c:tag", ("[2001:db8::1]:1234", "a/b/c", "tag", None)), # Follow docker rules for parsing ("ubuntu:22.04", ("index.docker.io", "library/ubuntu", "22.04", None)), ("myname/myimage:abc", ("index.docker.io", "myname/myimage", "abc", None)), ("myname:1234/myimage:abc", ("myname:1234", "myimage", "abc", None)), ("localhost/myimage:abc", ("localhost", "myimage", "abc", None)), ("localhost:1234/myimage:abc", ("localhost:1234", "myimage", "abc", None)), ( "example.com/UPPERCASE/lowercase:AbC", ("example.com", "uppercase/lowercase", "AbC", None), ), ], ) def test_name_parsing(image_ref, expected): x = ImageReference.from_string(image_ref) assert (x.domain, x.name, x.tag, x.digest) == expected @pytest.mark.parametrize( "image_ref", [ # wrong order of tag and sha f"example.com:1234/a/b/c@sha256:{'a' * 64}:tag", # double tag "example.com:1234/a/b/c:tag:tag", # empty tag "example.com:1234/a/b/c:", # empty digest "example.com:1234/a/b/c@sha256:", # unsupported digest algorithm f"example.com:1234/a/b/c@sha512:{'a' * 128}", # invalid digest length f"example.com:1234/a/b/c@sha256:{'a' * 63}", # whitespace "example.com:1234/a/b/c :tag", "example.com:1234/a/b/c: tag", "example.com:1234/a/b/c:tag ", " example.com:1234/a/b/c:tag", # broken ipv4 "1.2..3:1234/a/b/c:tag", ], ) def test_parsing_failure(image_ref): with pytest.raises(ValueError): ImageReference.from_string(image_ref) def test_digest(): valid_digest = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" # Test string roundtrip assert str(Digest.from_string(f"sha256:{valid_digest}")) == f"sha256:{valid_digest}" # Invalid digest length with pytest.raises(ValueError): Digest.from_string("sha256:abcdef") # Missing algorithm with pytest.raises(ValueError): Digest.from_string(valid_digest) def test_url_with_scheme(): """Test that scheme=http translates to http:// URLs""" http = ImageReference.from_string("localhost:1234/myimage:abc", scheme="http") https = ImageReference.from_string("localhost:1234/myimage:abc", scheme="https") default = ImageReference.from_string("localhost:1234/myimage:abc") assert http != https assert https == default assert http.manifest_url() == "http://localhost:1234/v2/myimage/manifests/abc" assert https.manifest_url() == "https://localhost:1234/v2/myimage/manifests/abc" assert default.manifest_url() == "https://localhost:1234/v2/myimage/manifests/abc" ================================================ FILE: lib/spack/spack/test/oci/integration_test.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) # These are slow integration tests that do concretization, install, tarballing # and compression. They still use an in-memory OCI registry. import hashlib import json import os import pathlib import re import urllib.error from contextlib import contextmanager import pytest import spack.binary_distribution import spack.database import spack.deptypes as dt import spack.environment as ev import spack.error import spack.oci.opener import spack.spec import spack.traverse from spack.main import SpackCommand from spack.oci.image import Digest, ImageReference, default_config, default_manifest from spack.oci.oci import blob_exists, get_manifest_and_config, upload_blob, upload_manifest from spack.test.oci.mock_registry import DummyServer, InMemoryOCIRegistry, create_opener from spack.util.archive import gzip_compressed_tarfile buildcache = SpackCommand("buildcache") mirror = SpackCommand("mirror") env = SpackCommand("env") install = SpackCommand("install") @contextmanager def oci_servers(*servers: DummyServer): old_opener = spack.oci.opener.urlopen spack.oci.opener.urlopen = create_opener(*servers).open yield spack.oci.opener.urlopen = old_opener def test_buildcache_push_command(mutable_database): with oci_servers(InMemoryOCIRegistry("example.com")): mirror("add", "oci-test", "oci://example.com/image") # Push the package(s) to the OCI registry buildcache("push", "--update-index", "oci-test", "mpileaks^mpich") # Remove mpileaks from the database matches = mutable_database.query_local("mpileaks^mpich") assert len(matches) == 1 spec = matches[0] spec.package.do_uninstall() # Reinstall mpileaks from the OCI registry buildcache("install", "--unsigned", "mpileaks^mpich") # Now it should be installed again assert spec.installed # And let's check that the bin/mpileaks executable is there assert os.path.exists(os.path.join(spec.prefix, "bin", "mpileaks")) def test_buildcache_tag(install_mockery, mock_fetch, mutable_mock_env_path): """Tests whether we can create an OCI image from a full environment with multiple roots.""" env("create", "test") with ev.read("test"): install("--fake", "--add", "libelf") install("--fake", "--add", "trivial-install-test-package") registry = InMemoryOCIRegistry("example.com") with oci_servers(registry): mirror("add", "oci-test", "oci://example.com/image") with ev.read("test"): buildcache("push", "--tag", "full_env", "oci-test") name = ImageReference.from_string("example.com/image:full_env") with ev.read("test") as e: specs = [ x for x in spack.traverse.traverse_nodes( e.concrete_roots(), deptype=dt.LINK | dt.RUN ) if not x.external ] manifest, config = get_manifest_and_config(name) # without a base image, we should have one layer per spec assert len(manifest["layers"]) == len(specs) # Now create yet another tag, but with just a single selected spec as root. This should # also test the case where Spack doesn't have to upload any binaries, it just has to create # a new tag. libelf = next(s for s in specs if s.name == "libelf") with ev.read("test"): # Get libelf spec buildcache("push", "--tag", "single_spec", "oci-test", libelf.format("libelf{/hash}")) name = ImageReference.from_string("example.com/image:single_spec") manifest, config = get_manifest_and_config(name) assert len(manifest["layers"]) == len( [x for x in libelf.traverse(deptype=dt.LINK | dt.RUN) if not x.external] ) def test_buildcache_push_with_base_image_command(mutable_database, tmp_path: pathlib.Path): """Test that we can push a package with a base image to an OCI registry. This test is a bit involved, cause we have to create a small base image.""" registry_src = InMemoryOCIRegistry("src.example.com") registry_dst = InMemoryOCIRegistry("dst.example.com") base_image = ImageReference.from_string("src.example.com/my-base-image:latest") with oci_servers(registry_src, registry_dst): mirror("add", "oci-test", "oci://dst.example.com/image") # TODO: simplify creation of images... # We create a rootfs.tar.gz, a config file and a manifest file, # and upload those. config, manifest = default_config(architecture="amd64", os="linux"), default_manifest() # Create a small rootfs rootfs = tmp_path / "rootfs" rootfs.mkdir() (rootfs / "bin").mkdir() (rootfs / "bin" / "sh").touch() # Create a tarball of it. tarball = tmp_path / "base.tar.gz" with gzip_compressed_tarfile(str(tarball)) as (tar, tar_gz_checksum, tar_checksum): tar.add(str(rootfs), arcname=".") tar_gz_digest = Digest.from_sha256(tar_gz_checksum.hexdigest()) tar_digest = Digest.from_sha256(tar_checksum.hexdigest()) # Save the config file config["rootfs"]["diff_ids"] = [str(tar_digest)] config_file = tmp_path / "config.json" config_file.write_text(json.dumps(config), encoding="utf-8") config_digest = Digest.from_sha256(hashlib.sha256(config_file.read_bytes()).hexdigest()) # Register the layer in the manifest manifest["layers"].append( { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": str(tar_gz_digest), "size": tarball.stat().st_size, } ) manifest["config"]["digest"] = str(config_digest) manifest["config"]["size"] = config_file.stat().st_size # Upload the layer and config file upload_blob(base_image, str(tarball), tar_gz_digest) upload_blob(base_image, str(config_file), config_digest) # Upload the manifest upload_manifest(base_image, manifest) # END TODO # Finally... use it as a base image buildcache("push", "--base-image", str(base_image), "oci-test", "mpileaks^mpich") # Figure out what tag was produced tag = next(tag for _, tag in registry_dst.manifests.keys() if tag.startswith("mpileaks-")) assert tag is not None # Fetch the manifest and config dst_image = ImageReference.from_string(f"dst.example.com/image:{tag}") retrieved_manifest, retrieved_config = get_manifest_and_config(dst_image) # Check that the media type is OCI assert retrieved_manifest["mediaType"] == "application/vnd.oci.image.manifest.v1+json" assert ( retrieved_manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json" ) # Check that the base image layer is first. assert retrieved_manifest["layers"][0]["digest"] == str(tar_gz_digest) assert retrieved_config["rootfs"]["diff_ids"][0] == str(tar_digest) # And also check that we have layers for each link-run dependency matches = mutable_database.query_local("mpileaks^mpich") assert len(matches) == 1 spec = matches[0] num_runtime_deps = len(list(spec.traverse(root=True, deptype=("link", "run")))) # One base layer + num_runtime_deps assert len(retrieved_manifest["layers"]) == 1 + num_runtime_deps # And verify that all layers including the base layer are present for layer in retrieved_manifest["layers"]: assert blob_exists(dst_image, digest=Digest.from_string(layer["digest"])) assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.tar+gzip" def test_uploading_with_base_image_in_docker_image_manifest_v2_format( tmp_path: pathlib.Path, mutable_database ): """If the base image uses an old manifest schema, Spack should also use that. That is necessary for container images to work with Apptainer, which is rather strict about mismatching manifest/layer types.""" registry_src = InMemoryOCIRegistry("src.example.com") registry_dst = InMemoryOCIRegistry("dst.example.com") base_image = ImageReference.from_string("src.example.com/my-base-image:latest") with oci_servers(registry_src, registry_dst): mirror("add", "oci-test", "oci://dst.example.com/image") # Create a dummy base image (blob, config, manifest) in registry A in the Docker Image # Manifest V2 format. rootfs = tmp_path / "rootfs" (rootfs / "bin").mkdir(parents=True) (rootfs / "bin" / "sh").write_text("hello world") tarball = tmp_path / "base.tar.gz" with gzip_compressed_tarfile(str(tarball)) as (tar, tar_gz_checksum, tar_checksum): tar.add(rootfs, arcname=".") tar_gz_digest = Digest.from_sha256(tar_gz_checksum.hexdigest()) tar_digest = Digest.from_sha256(tar_checksum.hexdigest()) upload_blob(base_image, str(tarball), tar_gz_digest) config = { "created": "2015-10-31T22:22:56.015925234Z", "author": "Foo ", "architecture": "amd64", "os": "linux", "config": { "User": "foo", "Memory": 2048, "MemorySwap": 4096, "CpuShares": 8, "ExposedPorts": {"8080/tcp": {}}, "Env": ["PATH=/usr/bin:/bin"], "Entrypoint": ["/bin/sh"], "Cmd": ["-c", "'echo hello world'"], "Volumes": {"/x": {}}, "WorkingDir": "/", }, "rootfs": {"diff_ids": [str(tar_digest)], "type": "layers"}, "history": [ { "created": "2015-10-31T22:22:54.690851953Z", "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /", # noqa: E501 } ], } config_file = tmp_path / "config.json" config_file.write_text(json.dumps(config)) config_digest = Digest.from_sha256(hashlib.sha256(config_file.read_bytes()).hexdigest()) upload_blob(base_image, str(config_file), config_digest) manifest = { "schemaVersion": 2, "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "config": { "mediaType": "application/vnd.docker.container.image.v1+json", "size": config_file.stat().st_size, "digest": str(config_digest), }, "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": tarball.stat().st_size, "digest": str(tar_gz_digest), } ], } upload_manifest(base_image, manifest) # Finally upload some package to registry B with registry A's image as base buildcache("push", "--base-image", str(base_image), "oci-test", "mpileaks^mpich") # Should have some manifests uploaded to registry B now. assert registry_dst.manifests # Verify that all manifest are in the Docker Image Manifest V2 format, not OCI. # And also check that we're not using annotations, which is an OCI-only "feature". for m in registry_dst.manifests.values(): assert m["mediaType"] == "application/vnd.docker.distribution.manifest.v2+json" assert m["config"]["mediaType"] == "application/vnd.docker.container.image.v1+json" for layer in m["layers"]: assert layer["mediaType"] == "application/vnd.docker.image.rootfs.diff.tar.gzip" assert "annotations" not in m def test_best_effort_upload(mutable_database: spack.database.Database, monkeypatch): """Failure to upload a blob or manifest should not prevent others from being uploaded -- it should be a best-effort operation. If any runtime dep fails to upload, it results in a missing layer for dependents. But we do still create manifests for dependents, so that the build cache is maximally useful. (The downside is that container images are not runnable).""" _push_blob = spack.binary_distribution._oci_push_pkg_blob _push_manifest = spack.binary_distribution._oci_put_manifest def push_blob(image_ref, spec, tmpdir): # fail to upload the blob of mpich if spec.name == "mpich": raise Exception("Blob Server Error") return _push_blob(image_ref, spec, tmpdir) def put_manifest(base_images, checksums, image_ref, tmpdir, extra_config, annotations, *specs): # fail to upload the manifest of libdwarf if "libdwarf" in (s.name for s in specs): raise Exception("Manifest Server Error") return _push_manifest( base_images, checksums, image_ref, tmpdir, extra_config, annotations, *specs ) monkeypatch.setattr(spack.binary_distribution, "_oci_push_pkg_blob", push_blob) monkeypatch.setattr(spack.binary_distribution, "_oci_put_manifest", put_manifest) mirror("add", "oci-test", "oci://example.com/image") registry = InMemoryOCIRegistry("example.com") image = ImageReference.from_string("example.com/image") with oci_servers(registry): with pytest.raises(spack.error.SpackError, match="The following 2 errors occurred") as e: buildcache("push", "--update-index", "oci-test", "mpileaks^mpich") # mpich's blob failed to upload and libdwarf's manifest failed to upload assert re.search("mpich.+: Exception: Blob Server Error", e.value) assert re.search("libdwarf.+: Exception: Manifest Server Error", e.value) mpileaks: spack.spec.Spec = mutable_database.query_local("mpileaks^mpich")[0] without_manifest = ("mpich", "libdwarf") # Verify that manifests of mpich/libdwarf are missing due to upload failure. for name in without_manifest: tagged_img = image.with_tag(spack.binary_distribution._oci_default_tag(mpileaks[name])) with pytest.raises(urllib.error.HTTPError, match="404"): get_manifest_and_config(tagged_img) # Collect the layer digests of successfully uploaded packages. Every package should refer # to its own tarballs and those of its runtime deps that were uploaded. pkg_to_all_digests = {} pkg_to_own_digest = {} for s in mpileaks.traverse(): if s.name in without_manifest: continue if s.external: continue # This should not raise a 404. manifest, _ = get_manifest_and_config( image.with_tag(spack.binary_distribution._oci_default_tag(s)) ) # Collect layer digests pkg_to_all_digests[s.name] = {layer["digest"] for layer in manifest["layers"]} pkg_to_own_digest[s.name] = manifest["layers"][-1]["digest"] # Verify that all packages reference blobs of their runtime deps that uploaded fine. for s in mpileaks.traverse(): if s.name in without_manifest: continue if s.external: continue expected_digests = { pkg_to_own_digest[t.name] for t in s.traverse(deptype=("link", "run"), root=True) if t.name not in without_manifest } # Test with issubset, cause we don't have the blob of libdwarf as it has no manifest. assert expected_digests and expected_digests.issubset(pkg_to_all_digests[s.name]) ================================================ FILE: lib/spack/spack/test/oci/mock_registry.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import base64 import hashlib import io import json import re import urllib.error import urllib.parse import urllib.request import uuid from typing import Callable, Dict, List, Optional, Pattern, Tuple from urllib.request import Request import spack.oci.oci from spack.oci.image import Digest from spack.oci.opener import OCIAuthHandler from spack.test.conftest import MockHTTPResponse class MiddlewareError(Exception): """Thrown in a handler to return a response early.""" def __init__(self, response: MockHTTPResponse): self.response = response class Router: """This class is a small router for requests to the OCI registry. It is used to dispatch requests to a handler, and middleware can be used to transform requests, as well as return responses early (e.g. for authentication).""" def __init__(self) -> None: self.routes: List[Tuple[str, Pattern, Callable]] = [] self.middleware: List[Callable[[Request], Request]] = [] def handle(self, req: Request) -> MockHTTPResponse: """Dispatch a request to a handler.""" result = urllib.parse.urlparse(req.full_url) # Apply middleware try: for handler in self.middleware: req = handler(req) except MiddlewareError as e: return e.response for method, path_regex, handler in self.routes: if method != req.get_method(): continue match = re.fullmatch(path_regex, result.path) if not match: continue return handler(req, **match.groupdict()) return MockHTTPResponse(404, "Not found") def register(self, method, path: str, handler: Callable): self.routes.append((method, re.compile(path), handler)) def add_middleware(self, handler: Callable[[Request], Request]): self.middleware.append(handler) class DummyServer: def __init__(self, domain: str) -> None: # The domain of the server, e.g. "registry.example.com" self.domain = domain # List of (method, url) tuples self.requests: List[Tuple[str, str]] = [] # Dispatches requests to handlers self.router = Router() # Always install a request logger self.router.add_middleware(self.log_request) def handle(self, req: Request) -> MockHTTPResponse: return self.router.handle(req) def log_request(self, req: Request): path = urllib.parse.urlparse(req.full_url).path self.requests.append((req.get_method(), path)) return req def clear_log(self): self.requests = [] class InMemoryOCIRegistry(DummyServer): """This implements the basic OCI registry API, but in memory. It supports two types of blob uploads: 1. POST + PUT: the client first starts a session with POST, then does a large PUT request 2. POST: the client does a single POST request with the whole blob Option 2 is not supported by all registries, so we allow to disable it, with allow_single_post=False. A third option is to use the chunked upload, but this is not implemented here, because it's typically a major performance hit in upload speed, so we're not using it in Spack.""" def __init__( self, domain: str, allow_single_post: bool = True, tags_per_page: int = 100 ) -> None: super().__init__(domain) self.router.register("GET", r"/v2/", self.index) self.router.register("HEAD", r"/v2/(?P.+)/blobs/(?P.+)", self.head_blob) self.router.register("POST", r"/v2/(?P.+)/blobs/uploads/", self.start_session) self.router.register("PUT", r"/upload", self.put_session) self.router.register("PUT", r"/v2/(?P.+)/manifests/(?P.+)", self.put_manifest) self.router.register("GET", r"/v2/(?P.+)/manifests/(?P.+)", self.get_manifest) self.router.register("GET", r"/v2/(?P.+)/blobs/(?P.+)", self.get_blob) self.router.register("GET", r"/v2/(?P.+)/tags/list", self.list_tags) # If True, allow single POST upload, not all registries support this self.allow_single_post = allow_single_post # How many tags are returned in a single request self.tags_per_page = tags_per_page # Used for POST + PUT upload. This is a map from session ID to image name self.sessions: Dict[str, str] = {} # Set of sha256:... digests that are known to the registry self.blobs: Dict[str, bytes] = {} # Map from (name, tag) to manifest self.manifests: Dict[Tuple[str, str], dict] = {} def index(self, req: Request): return MockHTTPResponse.with_json(200, "OK", body={}) def head_blob(self, req: Request, name: str, digest: str): if digest in self.blobs: return MockHTTPResponse(200, "OK", headers={"Content-Length": "1234"}) return MockHTTPResponse(404, "Not found") def get_blob(self, req: Request, name: str, digest: str): if digest in self.blobs: return MockHTTPResponse(200, "OK", body=io.BytesIO(self.blobs[digest])) return MockHTTPResponse(404, "Not found") def start_session(self, req: Request, name: str): id = str(uuid.uuid4()) self.sessions[id] = name # Check if digest is present (single monolithic upload) result = urllib.parse.urlparse(req.full_url) query = urllib.parse.parse_qs(result.query) if self.allow_single_post and "digest" in query: return self.handle_upload( req, name=name, digest=Digest.from_string(query["digest"][0]) ) return MockHTTPResponse(202, "Accepted", headers={"Location": f"/upload?uuid={id}"}) def put_session(self, req: Request): # Do the upload. result = urllib.parse.urlparse(req.full_url) query = urllib.parse.parse_qs(result.query) # uuid param should be preserved, and digest should be present assert "uuid" in query and len(query["uuid"]) == 1 assert "digest" in query and len(query["digest"]) == 1 id = query["uuid"][0] assert id in self.sessions name, digest = self.sessions[id], Digest.from_string(query["digest"][0]) response = self.handle_upload(req, name=name, digest=digest) # End the session del self.sessions[id] return response def put_manifest(self, req: Request, name: str, ref: str): # In requests, Python runs header.capitalize(). content_type = req.get_header("Content-type") assert content_type in spack.oci.oci.all_content_type index_or_manifest = json.loads(self._require_data(req)) # Verify that we have all blobs (layers for manifest, manifests for index) if content_type in spack.oci.oci.manifest_content_type: for layer in index_or_manifest["layers"]: assert layer["digest"] in self.blobs, "Missing blob while uploading manifest" else: for manifest in index_or_manifest["manifests"]: assert (name, manifest["digest"]) in self.manifests, ( "Missing manifest while uploading index" ) self.manifests[(name, ref)] = index_or_manifest return MockHTTPResponse( 201, "Created", headers={"Location": f"/v2/{name}/manifests/{ref}"} ) def get_manifest(self, req: Request, name: str, ref: str): if (name, ref) not in self.manifests: return MockHTTPResponse(404, "Not found") manifest_or_index = self.manifests[(name, ref)] return MockHTTPResponse.with_json( 200, "OK", headers={"Content-type": manifest_or_index["mediaType"]}, body=manifest_or_index, ) def _require_data(self, req: Request) -> bytes: """Extract request.data, it's type remains a mystery""" assert req.data is not None if hasattr(req.data, "read"): return req.data.read() elif isinstance(req.data, bytes): return req.data raise ValueError("req.data should be bytes or have a read() method") def handle_upload(self, req: Request, name: str, digest: Digest): """Verify the digest, save the blob, return created status""" data = self._require_data(req) assert hashlib.sha256(data).hexdigest() == digest.digest self.blobs[str(digest)] = data return MockHTTPResponse(201, "Created", headers={"Location": f"/v2/{name}/blobs/{digest}"}) def list_tags(self, req: Request, name: str): # Paginate using Link headers, this was added to the spec in the following commit: # https://github.com/opencontainers/distribution-spec/commit/2ed79d930ecec11dd755dc8190409a3b10f01ca9 # List all tags, exclude digests. all_tags = sorted( _tag for _name, _tag in self.manifests.keys() if _name == name and ":" not in _tag ) query = urllib.parse.parse_qs(urllib.parse.urlparse(req.full_url).query) n = int(query["n"][0]) if "n" in query else self.tags_per_page if "last" in query: try: offset = all_tags.index(query["last"][0]) + 1 except ValueError: return MockHTTPResponse(404, "Not found") else: offset = 0 tags = all_tags[offset : offset + n] if offset + n < len(all_tags): headers = {"Link": f'; rel="next"'} else: headers = None return MockHTTPResponse.with_json(200, "OK", headers=headers, body={"tags": tags}) class DummyServerUrllibHandler(urllib.request.BaseHandler): """Glue between urllib and DummyServer, routing requests to the correct mock server for a given domain.""" def __init__(self) -> None: self.servers: Dict[str, DummyServer] = {} def add_server(self, domain: str, api: DummyServer): self.servers[domain] = api return self def https_open(self, req: Request): domain = urllib.parse.urlparse(req.full_url).netloc if domain not in self.servers: return MockHTTPResponse(404, "Not found") return self.servers[domain].handle(req) class InMemoryOCIRegistryWithBearerAuth(InMemoryOCIRegistry): """This is another in-memory OCI registry requiring bearer token authentication.""" def __init__( self, domain, token: Optional[str], realm: str, allow_single_post: bool = True ) -> None: super().__init__(domain, allow_single_post) self.token = token # token to accept self.realm = realm # url to the authorization server self.router.add_middleware(self.authenticate) def authenticate(self, req: Request): # Any request needs an Authorization header authorization = req.get_header("Authorization") if authorization is None: raise MiddlewareError(self.unauthorized()) # Ensure that the token is correct assert authorization.startswith("Bearer ") token = authorization[7:] if token != self.token: raise MiddlewareError(self.unauthorized()) return req def unauthorized(self): return MockHTTPResponse( 401, "Unauthorized", { "www-authenticate": f'Bearer realm="{self.realm}",' f'service="{self.domain}",' 'scope="repository:spack-registry:pull,push"' }, ) class InMemoryOCIRegistryWithBasicAuth(InMemoryOCIRegistry): """This is another in-memory OCI registry requiring basic authentication.""" def __init__( self, domain, username: str, password: str, realm: str, allow_single_post: bool = True ) -> None: super().__init__(domain, allow_single_post) self.username = username self.password = password self.realm = realm self.router.add_middleware(self.authenticate) def authenticate(self, req: Request): # Any request needs an Authorization header authorization = req.get_header("Authorization") if authorization is None: raise MiddlewareError(self.unauthorized()) # Ensure that the username and password are correct assert authorization.startswith("Basic ") auth = base64.b64decode(authorization[6:]).decode("utf-8") username, password = auth.split(":", 1) if username != self.username or password != self.password: raise MiddlewareError(self.unauthorized()) return req def unauthorized(self): return MockHTTPResponse( 401, "Unauthorized", {"www-authenticate": f'Basic realm="{self.realm}"'} ) class MockBearerTokenServer(DummyServer): """Simulates a basic server that hands out bearer tokens at the /login endpoint for the following services: public.example.com, which doesn't require Basic Auth private.example.com, which requires Basic Auth, with user:pass """ def __init__(self, domain: str) -> None: super().__init__(domain) self.router.register("GET", "/login", self.login) def login(self, req: Request): url = urllib.parse.urlparse(req.full_url) query_params = urllib.parse.parse_qs(url.query) # Verify query params, from the www-authenticate header assert query_params["client_id"] == ["spack"] assert len(query_params["service"]) == 1 assert query_params["scope"] == ["repository:spack-registry:pull,push"] service = query_params["service"][0] if service == "public.example.com": return self.public_auth(req) elif service == "private.example.com": return self.private_auth(req) elif service == "oauth.example.com": return self.oauth_auth(req) return MockHTTPResponse(404, "Not found") def public_auth(self, req: Request): # No need to login with username and password for the public registry assert req.get_header("Authorization") is None return MockHTTPResponse.with_json(200, "OK", body={"token": "public_token"}) def oauth_auth(self, req: Request): assert req.get_header("Authorization") is None return MockHTTPResponse.with_json(200, "OK", body={"access_token": "oauth_token"}) def private_auth(self, req: Request): # For the private registry we need to login with username and password auth_value = req.get_header("Authorization") if ( auth_value is None or not auth_value.startswith("Basic ") or base64.b64decode(auth_value[6:]) != b"user:pass" ): return MockHTTPResponse(401, "Unauthorized") return MockHTTPResponse.with_json(200, "OK", body={"token": "private_token"}) def create_opener(*servers: DummyServer, credentials_provider=None): """Creates a mock opener, that can be used to fake requests to a list of servers.""" opener = urllib.request.OpenerDirector() handler = DummyServerUrllibHandler() for server in servers: handler.add_server(server.domain, server) opener.add_handler(handler) opener.add_handler(urllib.request.HTTPDefaultErrorHandler()) opener.add_handler(urllib.request.HTTPErrorProcessor()) if credentials_provider is not None: opener.add_handler(OCIAuthHandler(credentials_provider)) return opener ================================================ FILE: lib/spack/spack/test/oci/urlopen.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import hashlib import json import pathlib import random import urllib.error import urllib.parse import urllib.request from urllib.request import Request import pytest import spack.mirrors.mirror from spack.oci.image import Digest, ImageReference, default_config, default_manifest from spack.oci.oci import ( copy_missing_layers, get_manifest_and_config, image_from_mirror, list_tags, upload_blob, upload_manifest, ) from spack.oci.opener import ( Challenge, RealmServiceScope, UsernamePassword, _get_basic_challenge, _get_bearer_challenge, credentials_from_mirrors, default_retry, parse_www_authenticate, ) from spack.test.conftest import MockHTTPResponse from spack.test.oci.mock_registry import ( DummyServer, DummyServerUrllibHandler, InMemoryOCIRegistry, InMemoryOCIRegistryWithBasicAuth, InMemoryOCIRegistryWithBearerAuth, MiddlewareError, MockBearerTokenServer, create_opener, ) def test_parse_www_authenticate(): """Test parsing of valid WWW-Authenticate header, check whether it's decomposed into a list of challenges with correct scheme and parameters according to RFC 7235 section 4.1""" www_authenticate = 'Bearer realm="https://spack.io/authenticate",service="spack-registry",scope="repository:spack-registry:pull,push"' assert parse_www_authenticate(www_authenticate) == [ Challenge( "bearer", [ ("realm", "https://spack.io/authenticate"), ("service", "spack-registry"), ("scope", "repository:spack-registry:pull,push"), ], ) ] assert parse_www_authenticate("Bearer") == [Challenge("bearer")] assert parse_www_authenticate("MethodA, MethodB,MethodC") == [ Challenge("methoda"), Challenge("methodb"), Challenge("methodc"), ] assert parse_www_authenticate( 'Digest realm="Digest Realm", nonce="1234567890", algorithm=MD5, qop="auth"' ) == [ Challenge( "digest", [ ("realm", "Digest Realm"), ("nonce", "1234567890"), ("algorithm", "MD5"), ("qop", "auth"), ], ) ] assert parse_www_authenticate( r'Newauth realm="apps", type=1, title="Login to \"apps\"", Basic realm="simple"' ) == [ Challenge("newauth", [("realm", "apps"), ("type", "1"), ("title", 'Login to "apps"')]), Challenge("basic", [("realm", "simple")]), ] assert parse_www_authenticate(r'BASIC REALM="simple"') == [ Challenge("basic", [("realm", "simple")]) ] @pytest.mark.parametrize( "invalid_str", [ # Not comma separated "SchemeA SchemeB SchemeC", # Unexpected eof "SchemeA, SchemeB, SchemeC, ", # Invalid auth param or scheme r"Scheme x=y, ", # Unexpected eof "Scheme key=", # Invalid token r'"Bearer"', # Invalid token r'Scheme"xyz"', # No auth param r"Scheme ", ], ) def test_invalid_www_authenticate(invalid_str): with pytest.raises(ValueError): parse_www_authenticate(invalid_str) def test_get_basic_challenge(): """Test extracting Basic challenge from a list of challenges""" # No basic challenge assert ( _get_basic_challenge( [ Challenge( "bearer", [ ("realm", "https://spack.io/authenticate"), ("service", "spack-registry"), ("scope", "repository:spack-registry:pull,push"), ], ), Challenge( "digest", [ ("realm", "Digest Realm"), ("nonce", "1234567890"), ("algorithm", "MD5"), ("qop", "auth"), ], ), ] ) is None ) # Multiple challenges, should pick the basic one and return its realm. assert ( _get_basic_challenge( [ Challenge( "dummy", [ ("realm", "https://example.com/"), ("service", "service"), ("scope", "scope"), ], ), Challenge("basic", [("realm", "simple")]), Challenge( "bearer", [ ("realm", "https://spack.io/authenticate"), ("service", "spack-registry"), ("scope", "repository:spack-registry:pull,push"), ], ), ] ) == "simple" ) def test_get_bearer_challenge(): """Test extracting Bearer challenge from a list of challenges""" # Only an incomplete bearer challenge, missing service and scope, not usable. assert ( _get_bearer_challenge( [ Challenge("bearer", [("realm", "https://spack.io/authenticate")]), Challenge("basic", [("realm", "simple")]), Challenge( "Digest", [ ("realm", "Digest Realm"), ("nonce", "1234567890"), ("algorithm", "MD5"), ("qop", "auth"), ], ), ] ) is None ) # Multiple challenges, should pick the bearer one. assert _get_bearer_challenge( [ Challenge( "dummy", [("realm", "https://example.com/"), ("service", "service"), ("scope", "scope")], ), Challenge( "bearer", [ ("realm", "https://spack.io/authenticate"), ("service", "spack-registry"), ("scope", "repository:spack-registry:pull,push"), ], ), ] ) == RealmServiceScope( "https://spack.io/authenticate", "spack-registry", "repository:spack-registry:pull,push" ) @pytest.mark.parametrize( "image_ref,token", [ ("public.example.com/spack-registry:latest", "public_token"), ("private.example.com/spack-registry:latest", "private_token"), ("oauth.example.com/spack-registry:latest", "oauth_token"), ], ) def test_automatic_oci_bearer_authentication(image_ref: str, token: str): image = ImageReference.from_string(image_ref) def credentials_provider(domain: str): return UsernamePassword("user", "pass") if domain == "private.example.com" else None opener = create_opener( InMemoryOCIRegistryWithBearerAuth( image.domain, token=token, realm="https://auth.example.com/login" ), MockBearerTokenServer("auth.example.com"), credentials_provider=credentials_provider, ) # Run this twice, as it will triggers a code path that caches the bearer token assert opener.open(image.endpoint()).status == 200 assert opener.open(image.endpoint()).status == 200 def test_automatic_oci_basic_authentication(): image = ImageReference.from_string("private.example.com/image") server = InMemoryOCIRegistryWithBasicAuth( image.domain, username="user", password="pass", realm="example.com" ) # With correct credentials we should get a 200 opener_with_correct_auth = create_opener( server, credentials_provider=lambda domain: UsernamePassword("user", "pass") ) assert opener_with_correct_auth.open(image.endpoint()).status == 200 # With wrong credentials we should get a 401 opener_with_wrong_auth = create_opener( server, credentials_provider=lambda domain: UsernamePassword("wrong", "wrong") ) with pytest.raises(urllib.error.HTTPError) as e: opener_with_wrong_auth.open(image.endpoint()) assert e.value.getcode() == 401 def test_wrong_credentials(): """Test that when wrong credentials are rejected by the auth server, we get a 401 error.""" credentials_provider = lambda domain: UsernamePassword("wrong", "wrong") image = ImageReference.from_string("private.example.com/image") opener = create_opener( InMemoryOCIRegistryWithBearerAuth( image.domain, token="something", realm="https://auth.example.com/login" ), MockBearerTokenServer("auth.example.com"), credentials_provider=credentials_provider, ) with pytest.raises(urllib.error.HTTPError) as e: opener.open(image.endpoint()) assert e.value.getcode() == 401 def test_wrong_bearer_token_returned_by_auth_server(): """When the auth server returns a wrong bearer token, we should get a 401 error when the request we attempt fails. We shouldn't go in circles getting a 401 from the registry, then a non-working token from the auth server, then a 401 from the registry, etc.""" image = ImageReference.from_string("private.example.com/image") opener = create_opener( InMemoryOCIRegistryWithBearerAuth( image.domain, token="other_token_than_token_server_provides", realm="https://auth.example.com/login", ), MockBearerTokenServer("auth.example.com"), credentials_provider=lambda domain: UsernamePassword("user", "pass"), ) with pytest.raises(urllib.error.HTTPError) as e: opener.open(image.endpoint()) assert e.value.getcode() == 401 class TrivialAuthServer(DummyServer): """A trivial auth server that hands out a bearer token at GET /login.""" def __init__(self, domain: str, token: str) -> None: super().__init__(domain) self.router.register("GET", "/login", self.login) self.token = token def login(self, req: Request): return MockHTTPResponse.with_json(200, "OK", body={"token": self.token}) def test_registry_with_short_lived_bearer_tokens(): """An issued bearer token is mostly opaque to the client, but typically it embeds a short-lived expiration date. To speed up requests to a registry, it's good not to authenticate on every request, but to cache the bearer token, however: we have to deal with the case of an expired bearer token. Here we test that when the bearer token expires, we authenticate again, and when the token is still valid, we don't re-authenticate.""" image = ImageReference.from_string("private.example.com/image") credentials_provider = lambda domain: UsernamePassword("user", "pass") auth_server = TrivialAuthServer("auth.example.com", token="token") registry_server = InMemoryOCIRegistryWithBearerAuth( image.domain, token="token", realm="https://auth.example.com/login" ) urlopen = create_opener( registry_server, auth_server, credentials_provider=credentials_provider ).open # First request, should work with token "token" assert urlopen(image.endpoint()).status == 200 # Invalidate the token on the registry registry_server.token = "new_token" auth_server.token = "new_token" # Second request: reusing the cached token should fail # but in the background we will get a new token from the auth server assert urlopen(image.endpoint()).status == 200 # Subsequent requests should work with the same token, let's do two more assert urlopen(image.endpoint()).status == 200 assert urlopen(image.endpoint()).status == 200 # And finally, we should see that we've issues exactly two requests to the auth server assert auth_server.requests == [("GET", "/login"), ("GET", "/login")] # Whereas we've done more requests to the registry assert registry_server.requests == [ ("GET", "/v2/"), # 1: without bearer token ("GET", "/v2/"), # 2: retry with bearer token ("GET", "/v2/"), # 3: with incorrect bearer token ("GET", "/v2/"), # 4: retry with new bearer token ("GET", "/v2/"), # 5: with recycled correct bearer token ("GET", "/v2/"), # 6: with recycled correct bearer token ] class InMemoryRegistryWithUnsupportedAuth(InMemoryOCIRegistry): """A registry that does set a WWW-Authenticate header, but with a challenge we don't support.""" def __init__(self, domain: str, allow_single_post: bool = True, www_authenticate=None) -> None: self.www_authenticate = www_authenticate super().__init__(domain, allow_single_post) self.router.add_middleware(self.unsupported_auth_method) def unsupported_auth_method(self, req: Request): headers = {} if self.www_authenticate: headers["WWW-Authenticate"] = self.www_authenticate raise MiddlewareError(MockHTTPResponse(401, "Unauthorized", headers=headers)) @pytest.mark.parametrize( "www_authenticate,error_message", [ # missing service and scope ('Bearer realm="https://auth.example.com/login"', "unsupported authentication scheme"), # missing realm ("Basic", "unsupported authentication scheme"), # multiple unsupported challenges ( "CustomChallenge method=unsupported, OtherChallenge method=x,param=y", "unsupported authentication scheme", ), # no challenge (None, "missing WWW-Authenticate header"), # malformed challenge, missing quotes ("Bearer realm=https://auth.example.com", "malformed WWW-Authenticate header"), # http instead of https ('Bearer realm="http://auth.example.com",scope=x,service=y', "insecure http connection"), ], ) def test_auth_method_we_cannot_handle_is_error(www_authenticate, error_message): # We can only handle WWW-Authenticate with a Bearer challenge image = ImageReference.from_string("private.example.com/image") urlopen = create_opener( InMemoryRegistryWithUnsupportedAuth(image.domain, www_authenticate=www_authenticate), TrivialAuthServer("auth.example.com", token="token"), credentials_provider=lambda domain: UsernamePassword("user", "pass"), ).open with pytest.raises(urllib.error.HTTPError, match=error_message) as e: urlopen(image.endpoint()) assert e.value.getcode() == 401 # Parametrize over single POST vs POST + PUT. @pytest.mark.parametrize("client_single_request", [True, False]) @pytest.mark.parametrize("server_single_request", [True, False]) def test_oci_registry_upload(tmp_path: pathlib.Path, client_single_request, server_single_request): opener = urllib.request.OpenerDirector() opener.add_handler( DummyServerUrllibHandler().add_server( "example.com", InMemoryOCIRegistry(server_single_request) ) ) opener.add_handler(urllib.request.HTTPDefaultErrorHandler()) opener.add_handler(urllib.request.HTTPErrorProcessor()) # Create a small blob blob = tmp_path / "blob" blob.write_text("Hello world!") image = ImageReference.from_string("example.com/image:latest") digest = Digest.from_sha256(hashlib.sha256(blob.read_bytes()).hexdigest()) # Set small file size larger than the blob iff we're doing single request small_file_size = 1024 if client_single_request else 0 # Upload once, should actually upload assert upload_blob( ref=image, file=str(blob), digest=digest, small_file_size=small_file_size, _urlopen=opener.open, ) # Second time should exit as it exists assert not upload_blob( ref=image, file=str(blob), digest=digest, small_file_size=small_file_size, _urlopen=opener.open, ) # Force upload should upload again assert upload_blob( ref=image, file=str(blob), digest=digest, force=True, small_file_size=small_file_size, _urlopen=opener.open, ) def test_copy_missing_layers(tmp_path: pathlib.Path, config): """Test copying layers from one registry to another. Creates 3 blobs, 1 config and 1 manifest in registry A and copies layers to registry B. Then checks that all layers are present in registry B. Finally it runs the copy again and checks that no new layers are uploaded.""" # NOTE: config fixture is used to disable default source mirrors # which are used in Stage(...). Otherwise this test doesn't really # rely on globals. src = ImageReference.from_string("a.example.com/image:x") dst = ImageReference.from_string("b.example.com/image:y") src_registry = InMemoryOCIRegistry(src.domain) dst_registry = InMemoryOCIRegistry(dst.domain) urlopen = create_opener(src_registry, dst_registry).open # TODO: make it a bit easier to create bunch of blobs + config + manifest? # Create a few blobs and a config file blobs = [tmp_path / f"blob{i}" for i in range(3)] for i, blob in enumerate(blobs): blob.write_text(f"Blob {i}") digests = [Digest.from_sha256(hashlib.sha256(blob.read_bytes()).hexdigest()) for blob in blobs] config = default_config(architecture="amd64", os="linux") configfile = tmp_path / "config.json" configfile.write_text(json.dumps(config)) config_digest = Digest.from_sha256(hashlib.sha256(configfile.read_bytes()).hexdigest()) for blob, digest in zip(blobs, digests): upload_blob(src, str(blob), digest, _urlopen=urlopen) upload_blob(src, str(configfile), config_digest, _urlopen=urlopen) # Then create a manifest referencing them manifest = default_manifest() for blob, digest in zip(blobs, digests): manifest["layers"].append( { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": str(digest), "size": blob.stat().st_size, } ) manifest["config"] = { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": str(config_digest), "size": configfile.stat().st_size, } upload_manifest(src, manifest, _urlopen=urlopen) # Finally, copy the image from src to dst copy_missing_layers(src, dst, architecture="amd64", _urlopen=urlopen) # Check that all layers (not config) were copied and identical assert len(dst_registry.blobs) == len(blobs) for blob, digest in zip(blobs, digests): assert dst_registry.blobs.get(str(digest)) == blob.read_bytes() is_upload = lambda method, path: method == "POST" and path == "/v2/image/blobs/uploads/" is_exists = lambda method, path: method == "HEAD" and path.startswith("/v2/image/blobs/") # Check that exactly 3 uploads were initiated, and that we don't do # double existence checks when uploading. assert sum(is_upload(method, path) for method, path in dst_registry.requests) == 3 assert sum(is_exists(method, path) for method, path in dst_registry.requests) == 3 # Check that re-uploading skips existing layers. dst_registry.clear_log() copy_missing_layers(src, dst, architecture="amd64", _urlopen=urlopen) # Check that no uploads were initiated, only existence checks were done. assert sum(is_upload(method, path) for method, path in dst_registry.requests) == 0 assert sum(is_exists(method, path) for method, path in dst_registry.requests) == 3 def test_image_from_mirror(): mirror = spack.mirrors.mirror.Mirror("oci://example.com/image") assert image_from_mirror(mirror) == ImageReference.from_string("example.com/image") def test_image_from_mirror_with_http_scheme(): image = image_from_mirror(spack.mirrors.mirror.Mirror({"url": "oci+http://example.com/image"})) assert image.scheme == "http" assert image.with_tag("latest").scheme == "http" assert image.with_digest(f"sha256:{1234:064x}").scheme == "http" def test_image_reference_str(): """Test that with_digest() works with Digest and str.""" digest_str = f"sha256:{1234:064x}" digest = Digest.from_string(digest_str) img = ImageReference.from_string("example.com/image") assert str(img.with_digest(digest)) == f"example.com/image:latest@{digest}" assert str(img.with_digest(digest_str)) == f"example.com/image:latest@{digest}" assert str(img.with_tag("hello")) == "example.com/image:hello" assert str(img.with_tag("hello").with_digest(digest)) == f"example.com/image:hello@{digest}" @pytest.mark.parametrize( "image", [ # white space issue " example.com/image", # not alpha-numeric "hello#world:latest", ], ) def test_image_reference_invalid(image): with pytest.raises(ValueError, match="Invalid image reference"): ImageReference.from_string(image) def test_default_credentials_provider(): """The default credentials provider uses a collection of configured mirrors.""" mirrors = [ # OCI mirror with push credentials spack.mirrors.mirror.Mirror( {"url": "oci://a.example.com/image", "push": {"access_pair": ["user.a", "pass.a"]}} ), # Not an OCI mirror spack.mirrors.mirror.Mirror( {"url": "https://b.example.com/image", "access_pair": ["user.b", "pass.b"]} ), # No credentials spack.mirrors.mirror.Mirror("oci://c.example.com/image"), # Top-level credentials spack.mirrors.mirror.Mirror( {"url": "oci://d.example.com/image", "access_pair": ["user.d", "pass.d"]} ), # Dockerhub short reference spack.mirrors.mirror.Mirror( {"url": "oci://user/image", "access_pair": ["dockerhub_user", "dockerhub_pass"]} ), # Localhost (not a dockerhub short reference) spack.mirrors.mirror.Mirror( {"url": "oci://localhost/image", "access_pair": ["user.localhost", "pass.localhost"]} ), ] assert credentials_from_mirrors("a.example.com", mirrors=mirrors) == UsernamePassword( "user.a", "pass.a" ) assert credentials_from_mirrors("b.example.com", mirrors=mirrors) is None assert credentials_from_mirrors("c.example.com", mirrors=mirrors) is None assert credentials_from_mirrors("d.example.com", mirrors=mirrors) == UsernamePassword( "user.d", "pass.d" ) assert credentials_from_mirrors("index.docker.io", mirrors=mirrors) == UsernamePassword( "dockerhub_user", "dockerhub_pass" ) assert credentials_from_mirrors("localhost", mirrors=mirrors) == UsernamePassword( "user.localhost", "pass.localhost" ) def test_manifest_index(tmp_path: pathlib.Path): """Test obtaining manifest + config from a registry that has an index""" urlopen = create_opener(InMemoryOCIRegistry("registry.example.com")).open img = ImageReference.from_string("registry.example.com/image") # Create two config files and manifests, for different architectures manifest_descriptors = [] manifest_and_config = {} for arch in ("amd64", "arm64"): file = tmp_path / f"config_{arch}.json" config = default_config(architecture=arch, os="linux") file.write_text(json.dumps(config)) config_digest = Digest.from_sha256(hashlib.sha256(file.read_bytes()).hexdigest()) assert upload_blob(img, str(file), config_digest, _urlopen=urlopen) manifest = { "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "digest": str(config_digest), "size": file.stat().st_size, }, "layers": [], } manifest_digest, manifest_size = upload_manifest( img, manifest, tag=False, _urlopen=urlopen ) manifest_descriptors.append( { "mediaType": "application/vnd.oci.image.manifest.v1+json", "platform": {"architecture": arch, "os": "linux"}, "digest": str(manifest_digest), "size": manifest_size, } ) manifest_and_config[arch] = (manifest, config) # And a single index. index = { "schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", "manifests": manifest_descriptors, } upload_manifest(img, index, tag=True, _urlopen=urlopen) # Check that we fetcht the correct manifest and config for each architecture for arch in ("amd64", "arm64"): assert ( get_manifest_and_config(img, architecture=arch, _urlopen=urlopen) == manifest_and_config[arch] ) # Also test max recursion with pytest.raises(Exception, match="Maximum recursion depth reached"): get_manifest_and_config(img, architecture="amd64", recurse=0, _urlopen=urlopen) class BrokenServer(DummyServer): """Dummy server that returns 500 and 429 errors twice before succeeding""" def __init__(self, domain: str) -> None: super().__init__(domain) self.router.register("GET", r"/internal-server-error/", self.internal_server_error_twice) self.router.register("GET", r"/rate-limit/", self.rate_limit_twice) self.router.register("GET", r"/not-found/", self.not_found) self.count_500 = 0 self.count_429 = 0 def internal_server_error_twice(self, request: Request): self.count_500 += 1 if self.count_500 < 3: return MockHTTPResponse(500, "Internal Server Error") else: return MockHTTPResponse(200, "OK") def rate_limit_twice(self, request: Request): self.count_429 += 1 if self.count_429 < 3: return MockHTTPResponse(429, "Rate Limit Exceeded") else: return MockHTTPResponse(200, "OK") def not_found(self, request: Request): return MockHTTPResponse(404, "Not Found") @pytest.mark.parametrize( "url,max_retries,expect_failure,expect_requests", [ # 500s should be retried ("https://example.com/internal-server-error/", 2, True, 2), ("https://example.com/internal-server-error/", 5, False, 3), # 429s should be retried ("https://example.com/rate-limit/", 2, True, 2), ("https://example.com/rate-limit/", 5, False, 3), # 404s shouldn't be retried ("https://example.com/not-found/", 3, True, 1), ], ) def test_retry(url, max_retries, expect_failure, expect_requests): server = BrokenServer("example.com") urlopen = create_opener(server).open sleep_time = [] dont_sleep = lambda t: sleep_time.append(t) # keep track of sleep times try: response = default_retry(urlopen, retries=max_retries, sleep=dont_sleep)(url) except urllib.error.HTTPError as e: if not expect_failure: assert False, f"Unexpected HTTPError: {e}" else: if expect_failure: assert False, "Expected HTTPError, but none was raised" assert response.status == 200 assert len(server.requests) == expect_requests assert sleep_time == [2**i for i in range(expect_requests - 1)] def test_list_tags(): # Follows a relatively new rewording of the OCI distribution spec, which is not yet tagged. # https://github.com/opencontainers/distribution-spec/commit/2ed79d930ecec11dd755dc8190409a3b10f01ca9 N = 20 urlopen = create_opener(InMemoryOCIRegistry("example.com", tags_per_page=5)).open image = ImageReference.from_string("example.com/image") to_tag = lambda i: f"tag-{i:02}" # Create N tags in arbitrary order _tags_to_create = [to_tag(i) for i in range(N)] random.shuffle(_tags_to_create) for tag in _tags_to_create: upload_manifest(image.with_tag(tag), default_manifest(), tag=True, _urlopen=urlopen) # list_tags should return all tags from all pages in order tags = list_tags(image, urlopen) assert len(tags) == N assert [to_tag(i) for i in range(N)] == tags # Test a single request, which should give the first 5 tags assert json.loads(urlopen(image.tags_url()).read())["tags"] == [to_tag(i) for i in range(5)] # Test response at an offset, which should exclude the `last` tag. assert json.loads(urlopen(image.tags_url() + f"?last={to_tag(N - 3)}").read())["tags"] == [ to_tag(i) for i in range(N - 2, N) ] ================================================ FILE: lib/spack/spack/test/optional_deps.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.concretize from spack.spec import Spec @pytest.fixture( params=[ # Normalize simple conditionals ("optional-dep-test", {"optional-dep-test": None}), ("optional-dep-test~a", {"optional-dep-test~a": None}), ("optional-dep-test+a", {"optional-dep-test+a": {"pkg-a": None}}), ("optional-dep-test a=true", {"optional-dep-test a=true": {"pkg-a": None}}), ("optional-dep-test a=true", {"optional-dep-test+a": {"pkg-a": None}}), ("optional-dep-test@1.1", {"optional-dep-test@1.1": {"pkg-b": None}}), ("optional-dep-test%intel", {"optional-dep-test%intel": {"pkg-c": None}}), ( "optional-dep-test%intel@64.1", {"optional-dep-test%intel@64.1": {"pkg-c": None, "pkg-d": None}}, ), ( "optional-dep-test%intel@64.1.2", {"optional-dep-test%intel@64.1.2": {"pkg-c": None, "pkg-d": None}}, ), ("optional-dep-test%clang@35", {"optional-dep-test%clang@35": {"pkg-e": None}}), # Normalize multiple conditionals ("optional-dep-test+a@1.1", {"optional-dep-test+a@1.1": {"pkg-a": None, "pkg-b": None}}), ( "optional-dep-test+a%intel", {"optional-dep-test+a%intel": {"pkg-a": None, "pkg-c": None}}, ), ( "optional-dep-test@1.1%intel", {"optional-dep-test@1.1%intel": {"pkg-b": None, "pkg-c": None}}, ), ( "optional-dep-test@1.1+a%intel@64.1.2", { "optional-dep-test@1.1+a%intel@64.1.2": { "pkg-a": None, "pkg-b": None, "pkg-c": None, "pkg-d": None, } }, ), ( "optional-dep-test@1.1+a%clang@36.5", {"optional-dep-test@1.1+a%clang@36.5": {"pkg-b": None, "pkg-a": None, "pkg-e": None}}, ), # Chained MPI ( "optional-dep-test-2+mpi", {"optional-dep-test-2+mpi": {"optional-dep-test+mpi": {"mpi": None}}}, ), # Each of these dependencies comes from a conditional # dependency on another. This requires iterating to evaluate # the whole chain. ( "optional-dep-test+f", {"optional-dep-test+f": {"pkg-f": None, "pkg-g": None, "mpi": None}}, ), ] ) def spec_and_expected(request): """Parameters for the normalization test.""" spec, d = request.param return spec, Spec.from_literal(d) def test_default_variant(config, mock_packages): spec = spack.concretize.concretize_one("optional-dep-test-3") assert "pkg-a" in spec spec = spack.concretize.concretize_one("optional-dep-test-3~var") assert "pkg-a" in spec spec = spack.concretize.concretize_one("optional-dep-test-3+var") assert "pkg-b" in spec ================================================ FILE: lib/spack/spack/test/package_class.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test class methods on PackageBase objects. This doesn't include methods on package *instances* (like do_patch(), etc.). Only methods like ``possible_dependencies()`` that deal with the static DSL metadata for packages. """ import os import pathlib import shutil import pytest import spack.binary_distribution import spack.concretize import spack.deptypes as dt import spack.error import spack.install_test import spack.llnl.util.filesystem as fs import spack.package_base import spack.spec import spack.store import spack.subprocess_context from spack.error import InstallError from spack.package_base import PackageBase from spack.solver.input_analysis import NoStaticAnalysis, StaticAnalysis from spack.version import Version @pytest.fixture(scope="module") def compiler_names(mock_packages_repo): return [spec.name for spec in mock_packages_repo.providers_for("c")] @pytest.fixture() def mpileaks_possible_deps(mock_packages, mpi_names, compiler_names): possible = { "callpath": set(["dyninst"] + mpi_names + compiler_names), "low-priority-provider": set(), "dyninst": set(["libdwarf", "libelf"] + compiler_names), "fake": set(), "gcc": set(compiler_names), "intel-parallel-studio": set(), "libdwarf": set(["libelf"] + compiler_names), "libelf": set(compiler_names), "llvm": {"gcc", "llvm"}, "mpich": set(compiler_names), "mpich2": set(compiler_names), "mpileaks": set(["callpath"] + mpi_names + compiler_names), "multi-provider-mpi": set(), "zmpi": set(["fake"] + compiler_names), "compiler-with-deps": set(["binutils-for-test", "zlib"] + compiler_names), "binutils-for-test": set(["zlib"] + compiler_names), "zlib": set(), } return possible @pytest.fixture(params=[NoStaticAnalysis, StaticAnalysis]) def mock_inspector(config, mock_packages, request): inspector_cls = request.param if inspector_cls is NoStaticAnalysis: return inspector_cls(configuration=config, repo=mock_packages) return inspector_cls( configuration=config, repo=mock_packages, store=spack.store.STORE, binary_index=spack.binary_distribution.BINARY_INDEX, ) @pytest.fixture def mpi_names(mock_inspector): return [spec.name for spec in mock_inspector.providers_for("mpi")] @pytest.mark.parametrize( "pkg_name,fn_kwargs,expected", [ ( "mpileaks", {"expand_virtuals": True, "allowed_deps": dt.ALL}, { "fake", "mpileaks", "gcc", "llvm", "compiler-with-deps", "binutils-for-test", "zlib", "multi-provider-mpi", "callpath", "dyninst", "mpich2", "libdwarf", "zmpi", "low-priority-provider", "intel-parallel-studio", "mpich", "libelf", }, ), ( "mpileaks", {"expand_virtuals": False, "allowed_deps": dt.ALL}, {"callpath", "dyninst", "libdwarf", "libelf", "mpileaks"}, ), ( "mpileaks", {"expand_virtuals": False, "allowed_deps": dt.ALL, "transitive": False}, {"callpath", "mpileaks"}, ), ("dtbuild1", {"allowed_deps": dt.LINK | dt.RUN}, {"dtbuild1", "dtrun2", "dtlink2"}), ("dtbuild1", {"allowed_deps": dt.BUILD}, {"dtbuild1", "dtbuild2", "dtlink2"}), ("dtbuild1", {"allowed_deps": dt.LINK}, {"dtbuild1", "dtlink2"}), ], ) def test_possible_dependencies(pkg_name, fn_kwargs, expected, mock_inspector): """Tests possible nodes of mpileaks, under different scenarios.""" result, *_ = mock_inspector.possible_dependencies(pkg_name, **fn_kwargs) assert expected == result def test_possible_dependencies_virtual(mock_inspector, mock_packages, mpi_names): expected = set(mpi_names) for name in mpi_names: expected.update( dep for dep in mock_packages.get_pkg_class(name).dependencies_by_name() if not mock_packages.is_virtual(dep) ) expected.update(s.name for s in mock_packages.providers_for("c")) real_pkgs, *_ = mock_inspector.possible_dependencies( "mpi", transitive=False, allowed_deps=dt.ALL ) assert expected == real_pkgs def test_possible_dependencies_missing(mock_inspector): result, *_ = mock_inspector.possible_dependencies("missing-dependency", allowed_deps=dt.ALL) assert "this-is-a-missing-dependency" not in result def test_possible_dependencies_with_multiple_classes( mock_inspector, mock_packages, mpileaks_possible_deps ): pkgs = ["dt-diamond", "mpileaks"] expected = set(mpileaks_possible_deps) expected.update({"dt-diamond", "dt-diamond-left", "dt-diamond-right", "dt-diamond-bottom"}) real_pkgs, *_ = mock_inspector.possible_dependencies(*pkgs, allowed_deps=dt.ALL) assert set(expected) == real_pkgs def setup_install_test(source_paths, test_root): """ Set up the install test by creating sources and install test roots. The convention used here is to create an empty file if the path name ends with an extension otherwise, a directory is created. """ fs.mkdirp(test_root) for path in source_paths: if os.path.splitext(path)[1]: fs.touchp(path) else: fs.mkdirp(path) @pytest.mark.parametrize( "spec,sources,extras,expect", [ ( "pkg-a", ["example/a.c"], # Source(s) ["example/a.c"], # Extra test source ["example/a.c"], ), # Test install dir source(s) ( "pkg-b", ["test/b.cpp", "test/b.hpp", "example/b.txt"], # Source(s) ["test"], # Extra test source ["test/b.cpp", "test/b.hpp"], ), # Test install dir source ( "pkg-c", ["examples/a.py", "examples/b.py", "examples/c.py", "tests/d.py"], ["examples/b.py", "tests"], ["examples/b.py", "tests/d.py"], ), ], ) def test_cache_extra_sources(install_mockery, spec, sources, extras, expect): """Test the package's cache extra test sources helper function.""" s = spack.concretize.concretize_one(spec) source_path = s.package.stage.source_path srcs = [fs.join_path(source_path, src) for src in sources] test_root = spack.install_test.install_test_root(s.package) setup_install_test(srcs, test_root) emsg_dir = "Expected {0} to be a directory" emsg_file = "Expected {0} to be a file" for src in srcs: assert os.path.exists(src), "Expected {0} to exist".format(src) if os.path.splitext(src)[1]: assert os.path.isfile(src), emsg_file.format(src) else: assert os.path.isdir(src), emsg_dir.format(src) spack.install_test.cache_extra_test_sources(s.package, extras) src_dests = [fs.join_path(test_root, src) for src in sources] exp_dests = [fs.join_path(test_root, e) for e in expect] poss_dests = set(src_dests) | set(exp_dests) msg = "Expected {0} to{1} exist" for pd in poss_dests: if pd in exp_dests: assert os.path.exists(pd), msg.format(pd, "") if os.path.splitext(pd)[1]: assert os.path.isfile(pd), emsg_file.format(pd) else: assert os.path.isdir(pd), emsg_dir.format(pd) else: assert not os.path.exists(pd), msg.format(pd, " not") # Perform a little cleanup shutil.rmtree(os.path.dirname(source_path)) def test_cache_extra_sources_fails(install_mockery, tmp_path: pathlib.Path): s = spack.concretize.concretize_one("pkg-a") with pytest.raises(InstallError) as exc_info: spack.install_test.cache_extra_test_sources(s.package, [str(tmp_path), "no-such-file"]) errors = str(exc_info.value) assert f"'{tmp_path}') must be relative" in errors assert "'no-such-file') for the copy does not exist" in errors def test_package_exes_and_libs(): with pytest.raises(spack.error.SpackError, match="defines both"): class BadDetectablePackage(PackageBase): executables = ["findme"] libraries = ["libFindMe.a"] def test_package_url_and_urls(): UrlsPackage = type( "URLsPackage", (PackageBase,), { "__module__": "spack.pkg.builtin.urls_package", "url": "https://www.example.com/url-package-1.0.tgz", "urls": ["https://www.example.com/archive"], }, ) s = spack.spec.Spec("urls-package") with pytest.raises(ValueError, match="defines both"): UrlsPackage(s) def test_package_license(): LicensedPackage = type( "LicensedPackage", (PackageBase,), {"__module__": "spack.pkg.builtin.licensed_package"} ) pkg = LicensedPackage(spack.spec.Spec("licensed-package")) assert pkg.global_license_file is None pkg.license_files = ["license.txt"] assert os.path.basename(pkg.global_license_file) == pkg.license_files[0] class BaseTestPackage(PackageBase): extendees = {} # currently a required attribute for is_extension() def test_package_version_fails(): s = spack.spec.Spec("pkg-a") pkg = BaseTestPackage(s) with pytest.raises(ValueError, match="does not have a concrete version"): pkg.version() def test_package_tester_fails(): s = spack.spec.Spec("pkg-a") pkg = BaseTestPackage(s) with pytest.raises(ValueError, match="without concrete version"): pkg.tester() def test_package_fetcher_fails(): s = spack.spec.Spec("pkg-a") pkg = BaseTestPackage(s) with pytest.raises(ValueError, match="without concrete version"): pkg.fetcher def test_package_test_no_compilers(mock_packages, monkeypatch, capfd): """Ensures that a test which needs the compiler, and build dependencies, to run, is skipped if no compiler is available. """ s = spack.spec.Spec("pkg-a") pkg = BaseTestPackage(s) pkg.test_requires_compiler = True pkg.do_test() error = capfd.readouterr()[1] assert "Skipping tests for package" in error def test_package_subscript(default_mock_concretization): """Tests that we can use the subscript notation on packages, and that it returns a package""" root = default_mock_concretization("mpileaks") root_pkg = root.package # Subscript of a virtual assert isinstance(root_pkg["mpi"], spack.package_base.PackageBase) # Subscript on concrete for d in root.traverse(): assert isinstance(root_pkg[d.name], spack.package_base.PackageBase) def test_deserialize_preserves_package_attribute(default_mock_concretization): x = default_mock_concretization("mpileaks").package assert x.spec._package is x y = spack.subprocess_context.deserialize(spack.subprocess_context.serialize(x)) assert y.spec._package is y @pytest.mark.require_provenance def test_git_provenance_commit_version(default_mock_concretization): spec = default_mock_concretization("git-ref-package@stable") assert spec.satisfies(f"commit={'c' * 40}") @pytest.mark.parametrize("version", ("main", "tag", "annotated-tag")) @pytest.mark.parametrize("pre_stage", (True, False)) @pytest.mark.require_provenance @pytest.mark.disable_clean_stage_check def test_git_provenance_find_commit_ls_remote( git, mock_git_repository, mock_packages, config, monkeypatch, version, pre_stage ): repo_path = mock_git_repository.path monkeypatch.setattr( spack.package_base.PackageBase, "git", f"file://{repo_path}", raising=False ) spec_str = f"git-test-commit@{version}" if pre_stage: spack.concretize.concretize_one(spec_str).package.do_stage(False) else: # explicitly disable ability to use stage or mirror, force url path monkeypatch.setattr( spack.package_base.PackageBase, "do_fetch", lambda *args, **kwargs: None ) spec = spack.concretize.concretize_one(spec_str) if pre_stage: # confirmation that we actually had an expanded stage to query with ls-remote assert spec.package.stage.expanded vattrs = spec.package.versions[spec.version] git_ref = vattrs.get("tag") or vattrs.get("branch") # add the ^{} suffix to the ref so it redirects to the first parent git object # for branches and lightweight tags the suffix makes no difference since it is # always a commit SHA, but for annotated tags the SHA shifts from the tag SHA # back to the commit SHA, which is what we want actual_commit = git( "-C", repo_path, "rev-parse", f"{git_ref}^{{}}", output=str, error=str ).strip() assert spec.variants["commit"].value == actual_commit @pytest.mark.require_provenance @pytest.mark.disable_clean_stage_check def test_git_provenance_cant_resolve_commit(mock_packages, monkeypatch, config, capfd, tmp_path): """Fail all attempts to resolve git commits""" repo_path = str(tmp_path / "non_existent") monkeypatch.setattr(spack.package_base.PackageBase, "git", repo_path, raising=False) monkeypatch.setattr(spack.package_base.PackageBase, "do_fetch", lambda *args, **kwargs: None) spec = spack.concretize.concretize_one("git-ref-package@develop") captured = capfd.readouterr() assert "commit" not in spec.variants assert "Warning: Unable to resolve the git commit" in captured.err @pytest.mark.parametrize( "pkg_name,preferred_version", [ # This package has a deprecated v1.1.0 which should not be the preferred ("deprecated_versions", "1.0.0"), # Python has v2.7.11 marked as preferred and newer v3 versions ("python", "2.7.11"), # This package has various versions, some deprecated, plus "main" and "develop" ("git-ref-package", "3.0.1"), ], ) def test_package_preferred_version(mock_packages, config, pkg_name, preferred_version): """Tests retrieving the preferred version of a package.""" pkg_cls = mock_packages.get_pkg_class(pkg_name) assert spack.package_base.preferred_version(pkg_cls) == Version(preferred_version) ================================================ FILE: lib/spack/spack/test/packages.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import importlib import os import pathlib import sys import pytest import spack.concretize import spack.directives import spack.error import spack.fetch_strategy import spack.package import spack.package_base import spack.repo from spack.paths import mock_packages_path from spack.spec import Spec from spack.util.naming import pkg_name_to_class_name from spack.version import VersionChecksumError class MyPrependFileLoader(spack.repo._PrependFileLoader): """Skip explicit prepending of 'spack_repo.builtin.build_systems' import.""" def __init__(self, fullname, repo, package_name): super().__init__(fullname, repo, package_name) self.prepend = b"" def pkg_factory(name): """Return a package object tied to an abstract spec""" pkg_cls = spack.repo.PATH.get_pkg_class(name) return pkg_cls(Spec(name)) @pytest.mark.usefixtures("config", "mock_packages") class TestPackage: def test_load_package(self): spack.repo.PATH.get_pkg_class("mpich") def test_package_name(self): pkg_cls = spack.repo.PATH.get_pkg_class("mpich") assert pkg_cls.name == "mpich" def test_package_filename(self): repo = spack.repo.from_path(mock_packages_path) filename = repo.filename_for_package_name("mpich") assert filename == os.path.join(mock_packages_path, "packages", "mpich", "package.py") def test_nonexisting_package_filename(self): repo = spack.repo.from_path(mock_packages_path) filename = repo.filename_for_package_name("some-nonexisting-package") assert filename == os.path.join( mock_packages_path, "packages", "some_nonexisting_package", "package.py" ) def test_package_class_names(self): assert "Mpich" == pkg_name_to_class_name("mpich") assert "PmgrCollective" == pkg_name_to_class_name("pmgr_collective") assert "PmgrCollective" == pkg_name_to_class_name("pmgr-collective") assert "Pmgrcollective" == pkg_name_to_class_name("PmgrCollective") assert "_3db" == pkg_name_to_class_name("3db") assert "_True" == pkg_name_to_class_name("true") # reserved keyword assert "_False" == pkg_name_to_class_name("false") # reserved keyword assert "_None" == pkg_name_to_class_name("none") # reserved keyword assert "Finally" == pkg_name_to_class_name("finally") # `Finally` is not reserved # Below tests target direct imports of spack packages from the spack.pkg namespace def test_import_package(self, tmp_path: pathlib.Path, monkeypatch): monkeypatch.setattr(spack.repo, "_PrependFileLoader", MyPrependFileLoader) root, _ = spack.repo.create_repo(str(tmp_path), "testing_repo", package_api=(1, 0)) pkg_path = pathlib.Path(root) / "packages" / "mpich" / "package.py" pkg_path.parent.mkdir(parents=True) pkg_path.write_text("foo = 1") with spack.repo.use_repositories(root): importlib.import_module("spack.pkg.testing_repo") assert importlib.import_module("spack.pkg.testing_repo.mpich").foo == 1 del sys.modules["spack.pkg.testing_repo"] del sys.modules["spack.pkg.testing_repo.mpich"] def test_inheritance_of_directives(self): pkg_cls = spack.repo.PATH.get_pkg_class("simple-inheritance") # Check dictionaries that should have been filled by directives dependencies = pkg_cls.dependencies_by_name() assert len(dependencies) == 4 assert "cmake" in dependencies assert "openblas" in dependencies assert "mpi" in dependencies assert len(pkg_cls.provided) == 2 # Check that Spec instantiation behaves as we expect s = spack.concretize.concretize_one("simple-inheritance") assert "^cmake" in s assert "^openblas" in s assert "+openblas" in s assert "mpi" in s s = spack.concretize.concretize_one("simple-inheritance~openblas") assert "^cmake" in s assert "^openblas" not in s assert "~openblas" in s assert "mpi" in s @pytest.mark.regression("11844") def test_inheritance_of_patches(self): # Will error if inheritor package cannot find inherited patch files _ = spack.concretize.concretize_one("patch-inheritance") @pytest.mark.regression("2737") def test_urls_for_versions(mock_packages, config): """Version directive without a 'url' argument should use default url.""" for spec_str in ("url-override@0.9.0", "url-override@1.0.0"): s = spack.concretize.concretize_one(spec_str) url = s.package.url_for_version("0.9.0") assert url == "http://www.anothersite.org/uo-0.9.0.tgz" url = s.package.url_for_version("1.0.0") assert url == "http://www.doesnotexist.org/url_override-1.0.0.tar.gz" url = s.package.url_for_version("0.8.1") assert url == "http://www.doesnotexist.org/url_override-0.8.1.tar.gz" def test_url_for_version_with_no_urls(mock_packages, config): spec = Spec("git-test") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) with pytest.raises(spack.error.NoURLError): pkg_cls(spec).url_for_version("1.0") with pytest.raises(spack.error.NoURLError): pkg_cls(spec).url_for_version("1.1") @pytest.mark.skip(reason="spack.build_systems moved out of spack/spack") def test_custom_cmake_prefix_path(mock_packages, config): pass # spec = spack.concretize.concretize_one("depends-on-define-cmake-prefix-paths") # assert spack.build_systems.cmake.get_cmake_prefix_path(spec.package) == [ # spec["define-cmake-prefix-paths"].prefix.test # ] def test_url_for_version_with_only_overrides(mock_packages, config): s = spack.concretize.concretize_one("url-only-override") # these exist and should just take the URL provided in the package assert s.package.url_for_version("1.0.0") == "http://a.example.com/url_override-1.0.0.tar.gz" assert s.package.url_for_version("0.9.0") == "http://b.example.com/url_override-0.9.0.tar.gz" assert s.package.url_for_version("0.8.1") == "http://c.example.com/url_override-0.8.1.tar.gz" # these don't exist but should still work, even if there are only overrides assert s.package.url_for_version("1.0.5") == "http://a.example.com/url_override-1.0.5.tar.gz" assert s.package.url_for_version("0.9.5") == "http://b.example.com/url_override-0.9.5.tar.gz" assert s.package.url_for_version("0.8.5") == "http://c.example.com/url_override-0.8.5.tar.gz" assert s.package.url_for_version("0.7.0") == "http://c.example.com/url_override-0.7.0.tar.gz" def test_url_for_version_with_only_overrides_with_gaps(mock_packages, config): s = spack.concretize.concretize_one("url-only-override-with-gaps") # same as for url-only-override -- these are specific assert s.package.url_for_version("1.0.0") == "http://a.example.com/url_override-1.0.0.tar.gz" assert s.package.url_for_version("0.9.0") == "http://b.example.com/url_override-0.9.0.tar.gz" assert s.package.url_for_version("0.8.1") == "http://c.example.com/url_override-0.8.1.tar.gz" # these don't have specific URLs, but should still work by extrapolation assert s.package.url_for_version("1.0.5") == "http://a.example.com/url_override-1.0.5.tar.gz" assert s.package.url_for_version("0.9.5") == "http://b.example.com/url_override-0.9.5.tar.gz" assert s.package.url_for_version("0.8.5") == "http://c.example.com/url_override-0.8.5.tar.gz" assert s.package.url_for_version("0.7.0") == "http://c.example.com/url_override-0.7.0.tar.gz" @pytest.mark.usefixtures("mock_packages", "config") @pytest.mark.parametrize( "spec_str,expected_type,expected_url", [ ( "git-top-level", spack.fetch_strategy.GitFetchStrategy, "https://example.com/some/git/repo", ), ( "svn-top-level", spack.fetch_strategy.SvnFetchStrategy, "https://example.com/some/svn/repo", ), ("hg-top-level", spack.fetch_strategy.HgFetchStrategy, "https://example.com/some/hg/repo"), ], ) def test_fetcher_url(spec_str, expected_type, expected_url): """Ensure that top-level git attribute can be used as a default.""" fetcher = spack.fetch_strategy.for_package_version(pkg_factory(spec_str), "1.0") assert isinstance(fetcher, expected_type) assert fetcher.url == expected_url @pytest.mark.usefixtures("mock_packages", "config") @pytest.mark.parametrize( "spec_str,version_str,exception_type", [ # Non-url-package ("git-top-level", "1.1", spack.fetch_strategy.ExtrapolationError), # Two VCS specified together ("git-url-svn-top-level", "1.0", spack.fetch_strategy.FetcherConflict), ("git-svn-top-level", "1.0", spack.fetch_strategy.FetcherConflict), ], ) def test_fetcher_errors(spec_str, version_str, exception_type): """Verify that we can't extrapolate versions for non-URL packages.""" with pytest.raises(exception_type): spack.fetch_strategy.for_package_version(pkg_factory(spec_str), version_str) @pytest.mark.usefixtures("mock_packages", "config") @pytest.mark.parametrize( "version_str,expected_url,digest", [ ("2.0", "https://example.com/some/tarball-2.0.tar.gz", "20"), ("2.1", "https://example.com/some/tarball-2.1.tar.gz", "21"), ("2.2", "https://www.example.com/foo2.2.tar.gz", "22"), ("2.3", "https://www.example.com/foo2.3.tar.gz", "23"), ], ) def test_git_url_top_level_url_versions(version_str, expected_url, digest): """Test URL fetch strategy inference when url is specified with git.""" # leading 62 zeros of sha256 hash leading_zeros = "0" * 62 fetcher = spack.fetch_strategy.for_package_version( pkg_factory("git-url-top-level"), version_str ) assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy) assert fetcher.url == expected_url assert fetcher.digest == leading_zeros + digest @pytest.mark.usefixtures("mock_packages", "config") @pytest.mark.parametrize( "version_str,tag,commit,branch", [ ("3.0", "v3.0", None, None), ("3.1", "v3.1", "abc31", None), ("3.2", None, None, "releases/v3.2"), ("3.3", None, "abc33", "releases/v3.3"), ("3.4", None, "abc34", None), ("submodules", None, None, None), ("develop", None, None, "develop"), ], ) def test_git_url_top_level_git_versions(version_str, tag, commit, branch): """Test git fetch strategy inference when url is specified with git.""" fetcher = spack.fetch_strategy.for_package_version( pkg_factory("git-url-top-level"), version_str ) assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy) assert fetcher.url == "https://example.com/some/git/repo" assert fetcher.tag == tag assert fetcher.commit == commit assert fetcher.branch == branch assert fetcher.url == pkg_factory("git-url-top-level").git @pytest.mark.usefixtures("mock_packages", "config") @pytest.mark.parametrize("version_str", ["1.0", "1.1", "1.2", "1.3"]) def test_git_url_top_level_conflicts(version_str): """Test git fetch strategy inference when url is specified with git.""" with pytest.raises(spack.fetch_strategy.FetcherConflict): spack.fetch_strategy.for_package_version(pkg_factory("git-url-top-level"), version_str) def test_rpath_args(mutable_database): """Test a package's rpath_args property.""" rec = mutable_database.get_record("mpich") rpath_args = rec.spec.package.rpath_args assert "-rpath" in rpath_args assert "mpich" in rpath_args def test_bundle_version_checksum(mock_directive_bundle, clear_directive_functions): """Test raising exception on a version checksum with a bundle package.""" with pytest.raises(VersionChecksumError, match="Checksums not allowed"): version = spack.directives.version("1.0", checksum="1badpkg") version(mock_directive_bundle) def test_bundle_patch_directive(mock_directive_bundle, clear_directive_functions): """Test raising exception on a patch directive with a bundle package.""" with pytest.raises( spack.directives.UnsupportedPackageDirective, match="Patches are not allowed" ): patch = spack.directives.patch("mock/patch.txt") patch(mock_directive_bundle) @pytest.mark.usefixtures("mock_packages", "config") @pytest.mark.parametrize( "version_str,digest_end,extra_options", [ ("1.0", "10", {"timeout": 42, "cookie": "foobar"}), ("1.1", "11", {"timeout": 65}), ("1.2", "12", {"cookie": "baz"}), ], ) def test_fetch_options(version_str, digest_end, extra_options): """Test fetch options inference.""" leading_zeros = "000000000000000000000000000000" fetcher = spack.fetch_strategy.for_package_version(pkg_factory("fetch-options"), version_str) assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy) assert fetcher.digest == leading_zeros + digest_end assert fetcher.extra_options == extra_options def test_package_deprecated_version(mock_packages, mock_fetch, mock_stage): spec = Spec("deprecated-versions") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) assert spack.package_base.deprecated_version(pkg_cls, "1.1.0") assert not spack.package_base.deprecated_version(pkg_cls, "1.0.0") def test_package_can_have_sparse_checkout_properties(mock_packages, mock_fetch, mock_stage): spec = Spec("git-sparsepaths-pkg") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) assert hasattr(pkg_cls, "git_sparse_paths") fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), "1.0") assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy) assert hasattr(fetcher, "git_sparse_paths") assert fetcher.git_sparse_paths == pkg_cls.git_sparse_paths def test_package_can_have_sparse_checkout_properties_with_commit_version( mock_packages, mock_fetch, mock_stage ): spec = Spec("git-sparsepaths-pkg commit=abcdefg") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) assert hasattr(pkg_cls, "git_sparse_paths") fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), "1.0") assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy) assert hasattr(fetcher, "git_sparse_paths") assert fetcher.git_sparse_paths == pkg_cls.git_sparse_paths def test_package_can_have_sparse_checkout_properties_with_gitversion( mock_packages, mock_fetch, mock_stage ): spec = Spec("git-sparsepaths-pkg") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) assert hasattr(pkg_cls, "git_sparse_paths") version = "git.foo=1.0" fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), version) assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy) assert hasattr(fetcher, "git_sparse_paths") assert fetcher.git_sparse_paths == pkg_cls.git_sparse_paths def test_package_version_can_have_sparse_checkout_properties( mock_packages, mock_fetch, mock_stage ): spec = Spec("git-sparsepaths-version") pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), version="1.0") assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy) assert fetcher.git_sparse_paths == ["foo", "bar"] fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), version="0.9") assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy) assert fetcher.git_sparse_paths is None def test_package_can_depend_on_commit_of_dependency(mock_packages, config): spec = spack.concretize.concretize_one(Spec("git-ref-commit-dep@1.0.0")) assert spec.satisfies(f"^git-ref-package commit={'a' * 40}") assert "surgical" not in spec["git-ref-package"].variants def test_package_condtional_variants_may_depend_on_commit(mock_packages, config): spec = spack.concretize.concretize_one(Spec("git-ref-commit-dep@develop")) assert spec.satisfies(f"^git-ref-package commit={'b' * 40}") conditional_variant = spec["git-ref-package"].variants.get("surgical", None) assert conditional_variant assert conditional_variant.value def test_commit_variant_finds_matches_for_commit_versions(mock_packages, config): """ test conditional dependence on `when='commit='` git-ref-commit-dep variant commit-selector depends on a specific commit of git-ref-package that commit is associated with the stable version of git-ref-package """ spec = spack.concretize.concretize_one(Spec("git-ref-commit-dep+commit-selector")) assert spec.satisfies(f"^git-ref-package commit={'c' * 40}") def test_pkg_name_can_only_be_derived_when_package_module(): """When the module prefix is not spack_repo (or legacy spack.pkg) we cannot derive a package name.""" ExamplePackage = type( "ExamplePackage", (spack.package_base.PackageBase,), {"__module__": "not.a.spack.repo.packages.example_package.package"}, ) with pytest.raises(ValueError, match="Package ExamplePackage is not a known Spack package"): ExamplePackage.name def test_spack_package_api_versioning(): """Test that the symbols in spack.package.api match the public API.""" assert spack.package.__all__ == [ symbol for symbols in spack.package.api.values() for symbol in symbols ] ================================================ FILE: lib/spack/spack/test/packaging.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This test checks the binary packaging infrastructure """ import argparse import os import pathlib import platform import shutil import urllib.error from collections import OrderedDict import pytest import spack.binary_distribution import spack.cmd.buildcache as buildcache import spack.cmd.mirror import spack.concretize import spack.config import spack.error import spack.fetch_strategy import spack.package_base import spack.stage import spack.util.gpg import spack.util.url as url_util from spack.fetch_strategy import URLFetchStrategy from spack.installer import PackageInstaller from spack.llnl.util import filesystem as fs from spack.llnl.util.filesystem import readlink, symlink from spack.paths import mock_gpg_keys_path from spack.relocate import _macho_find_paths, relocate_links, relocate_text pytestmark = pytest.mark.not_on_windows("does not run on windows") @pytest.mark.usefixtures("install_mockery", "mock_gnupghome") def test_buildcache(mock_archive, tmp_path: pathlib.Path, monkeypatch, mutable_config): # Install a test package spec = spack.concretize.concretize_one("trivial-install-test-package") monkeypatch.setattr(spec.package, "fetcher", URLFetchStrategy(url=mock_archive.url)) PackageInstaller([spec.package], explicit=True).install() pkghash = "/" + str(spec.dag_hash(7)) # Put some non-relocatable file in there dummy_txt = pathlib.Path(spec.prefix) / "dummy.txt" dummy_txt.write_text(spec.prefix) # Create an absolute symlink linkname = os.path.join(spec.prefix, "link_to_dummy.txt") symlink(dummy_txt, linkname) # Create the build cache and put it directly into the mirror mirror_path = str(tmp_path / "test-mirror") spack.cmd.mirror.create(mirror_path, specs=[]) # register mirror with spack config mirrors = {"spack-mirror-test": url_util.path_to_file_url(mirror_path)} spack.config.set("mirrors", mirrors) with spack.stage.Stage(mirrors["spack-mirror-test"], name="build_cache", keep=True): parser = argparse.ArgumentParser() buildcache.setup_parser(parser) create_args = ["create", "-f", "--rebuild-index", mirror_path, pkghash] # Create a private key to sign package with if gpg2 available spack.util.gpg.create( name="test key 1", expires="0", email="spack@googlegroups.com", comment="Spack test key", ) args = parser.parse_args(create_args) buildcache.buildcache(parser, args) # trigger overwrite warning buildcache.buildcache(parser, args) # Uninstall the package spec.package.do_uninstall(force=True) install_args = ["install", "-f", pkghash] args = parser.parse_args(install_args) # Test install buildcache.buildcache(parser, args) files = os.listdir(spec.prefix) assert "link_to_dummy.txt" in files assert "dummy.txt" in files # Validate the relocation information buildinfo = spack.binary_distribution.read_buildinfo_file(spec.prefix) assert buildinfo["relocate_textfiles"] == ["dummy.txt"] assert buildinfo["relocate_links"] == ["link_to_dummy.txt"] args = parser.parse_args(["keys"]) buildcache.buildcache(parser, args) args = parser.parse_args(["list"]) buildcache.buildcache(parser, args) args = parser.parse_args(["list"]) buildcache.buildcache(parser, args) args = parser.parse_args(["list", "trivial"]) buildcache.buildcache(parser, args) # Copy a key to the mirror to have something to download shutil.copyfile(mock_gpg_keys_path + "/external.key", mirror_path + "/external.key") args = parser.parse_args(["keys"]) buildcache.buildcache(parser, args) args = parser.parse_args(["keys", "-f"]) buildcache.buildcache(parser, args) args = parser.parse_args(["keys", "-i", "-t"]) buildcache.buildcache(parser, args) def test_relocate_text(tmp_path: pathlib.Path): """Tests that a text file containing the original directory of an installation, can be relocated to a target directory. """ original_dir = "/home/spack/opt/spack" relocation_dir = "/opt/rh/devtoolset/" dummy_txt = tmp_path / "dummy.txt" dummy_txt.write_text(original_dir) relocate_text([str(dummy_txt)], {original_dir: relocation_dir}) text = dummy_txt.read_text() assert relocation_dir in text assert original_dir not in text def test_relocate_links(tmp_path: pathlib.Path): (tmp_path / "new_prefix_a").mkdir() own_prefix_path = str(tmp_path / "prefix_a" / "file") dep_prefix_path = str(tmp_path / "prefix_b" / "file") new_own_prefix_path = str(tmp_path / "new_prefix_a" / "file") new_dep_prefix_path = str(tmp_path / "new_prefix_b" / "file") system_path = os.path.join(os.path.sep, "system", "path") fs.touchp(own_prefix_path) fs.touchp(new_own_prefix_path) fs.touchp(dep_prefix_path) fs.touchp(new_dep_prefix_path) # Old prefixes to new prefixes prefix_to_prefix = OrderedDict( [ # map /prefix_a -> /new_prefix_a (str(tmp_path / "prefix_a"), str(tmp_path / "new_prefix_a")), # map /prefix_b -> /new_prefix_b (str(tmp_path / "prefix_b"), str(tmp_path / "new_prefix_b")), # map -> /fallback/path -- this is just to see we respect order. (str(tmp_path), os.path.join(os.path.sep, "fallback", "path")), ] ) with fs.working_dir(str(tmp_path / "new_prefix_a")): # To be relocated os.symlink(own_prefix_path, "to_self") os.symlink(dep_prefix_path, "to_dependency") # To be ignored os.symlink(system_path, "to_system") os.symlink("relative", "to_self_but_relative") relocate_links(["to_self", "to_dependency", "to_system"], prefix_to_prefix) # These two are relocated assert readlink("to_self") == str(tmp_path / "new_prefix_a" / "file") assert readlink("to_dependency") == str(tmp_path / "new_prefix_b" / "file") # These two are not. assert readlink("to_system") == system_path assert readlink("to_self_but_relative") == "relative" def test_replace_paths(tmp_path: pathlib.Path): with fs.working_dir(str(tmp_path)): suffix = "dylib" if platform.system().lower() == "darwin" else "so" hash_a = "53moz6jwnw3xpiztxwhc4us26klribws" hash_b = "tk62dzu62kd4oh3h3heelyw23hw2sfee" hash_c = "hdkhduizmaddpog6ewdradpobnbjwsjl" hash_d = "hukkosc7ahff7o65h6cdhvcoxm57d4bw" hash_loco = "zy4oigsc4eovn5yhr2lk4aukwzoespob" prefix2hash = {} old_spack_dir = os.path.join(f"{tmp_path}", "Users", "developer", "spack") fs.mkdirp(old_spack_dir) oldprefix_a = os.path.join(f"{old_spack_dir}", f"pkgA-{hash_a}") oldlibdir_a = os.path.join(f"{oldprefix_a}", "lib") fs.mkdirp(oldlibdir_a) prefix2hash[str(oldprefix_a)] = hash_a oldprefix_b = os.path.join(f"{old_spack_dir}", f"pkgB-{hash_b}") oldlibdir_b = os.path.join(f"{oldprefix_b}", "lib") fs.mkdirp(oldlibdir_b) prefix2hash[str(oldprefix_b)] = hash_b oldprefix_c = os.path.join(f"{old_spack_dir}", f"pkgC-{hash_c}") oldlibdir_c = os.path.join(f"{oldprefix_c}", "lib") oldlibdir_cc = os.path.join(f"{oldlibdir_c}", "C") fs.mkdirp(oldlibdir_c) prefix2hash[str(oldprefix_c)] = hash_c oldprefix_d = os.path.join(f"{old_spack_dir}", f"pkgD-{hash_d}") oldlibdir_d = os.path.join(f"{oldprefix_d}", "lib") fs.mkdirp(oldlibdir_d) prefix2hash[str(oldprefix_d)] = hash_d oldprefix_local = os.path.join(f"{tmp_path}", "usr", "local") oldlibdir_local = os.path.join(f"{oldprefix_local}", "lib") fs.mkdirp(oldlibdir_local) prefix2hash[str(oldprefix_local)] = hash_loco libfile_a = f"libA.{suffix}" libfile_b = f"libB.{suffix}" libfile_c = f"libC.{suffix}" libfile_d = f"libD.{suffix}" libfile_loco = f"libloco.{suffix}" old_libnames = [ os.path.join(oldlibdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b), os.path.join(oldlibdir_c, libfile_c), os.path.join(oldlibdir_d, libfile_d), os.path.join(oldlibdir_local, libfile_loco), ] for old_libname in old_libnames: with open(old_libname, "a", encoding="utf-8"): os.utime(old_libname, None) hash2prefix = dict() new_spack_dir = os.path.join(f"{tmp_path}", "Users", "Shared", "spack") fs.mkdirp(new_spack_dir) prefix_a = os.path.join(new_spack_dir, f"pkgA-{hash_a}") libdir_a = os.path.join(prefix_a, "lib") fs.mkdirp(libdir_a) hash2prefix[hash_a] = str(prefix_a) prefix_b = os.path.join(new_spack_dir, f"pkgB-{hash_b}") libdir_b = os.path.join(prefix_b, "lib") fs.mkdirp(libdir_b) hash2prefix[hash_b] = str(prefix_b) prefix_c = os.path.join(new_spack_dir, f"pkgC-{hash_c}") libdir_c = os.path.join(prefix_c, "lib") libdir_cc = os.path.join(libdir_c, "C") fs.mkdirp(libdir_cc) hash2prefix[hash_c] = str(prefix_c) prefix_d = os.path.join(new_spack_dir, f"pkgD-{hash_d}") libdir_d = os.path.join(prefix_d, "lib") fs.mkdirp(libdir_d) hash2prefix[hash_d] = str(prefix_d) prefix_local = os.path.join(f"{tmp_path}", "usr", "local") libdir_local = os.path.join(prefix_local, "lib") fs.mkdirp(libdir_local) hash2prefix[hash_loco] = str(prefix_local) new_libnames = [ os.path.join(libdir_a, libfile_a), os.path.join(libdir_b, libfile_b), os.path.join(libdir_cc, libfile_c), os.path.join(libdir_d, libfile_d), os.path.join(libdir_local, libfile_loco), ] for new_libname in new_libnames: with open(new_libname, "a", encoding="utf-8"): os.utime(new_libname, None) prefix2prefix = dict() for prefix, hash in prefix2hash.items(): prefix2prefix[prefix] = hash2prefix[hash] out_dict = _macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local], [ os.path.join(oldlibdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b), os.path.join(oldlibdir_local, libfile_loco), ], os.path.join(oldlibdir_cc, libfile_c), prefix2prefix, ) assert out_dict == { oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_c: libdir_c, oldlibdir_cc: libdir_cc, libdir_local: libdir_local, os.path.join(oldlibdir_a, libfile_a): os.path.join(libdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b): os.path.join(libdir_b, libfile_b), os.path.join(oldlibdir_local, libfile_loco): os.path.join(libdir_local, libfile_loco), os.path.join(oldlibdir_cc, libfile_c): os.path.join(libdir_cc, libfile_c), } out_dict = _macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local], [ os.path.join(oldlibdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b), os.path.join(oldlibdir_cc, libfile_c), os.path.join(oldlibdir_local, libfile_loco), ], None, prefix2prefix, ) assert out_dict == { oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_c: libdir_c, oldlibdir_cc: libdir_cc, libdir_local: libdir_local, os.path.join(oldlibdir_a, libfile_a): os.path.join(libdir_a, libfile_a), os.path.join(oldlibdir_b, libfile_b): os.path.join(libdir_b, libfile_b), os.path.join(oldlibdir_local, libfile_loco): os.path.join(libdir_local, libfile_loco), os.path.join(oldlibdir_cc, libfile_c): os.path.join(libdir_cc, libfile_c), } out_dict = _macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local], [ f"@rpath/{libfile_a}", f"@rpath/{libfile_b}", f"@rpath/{libfile_c}", f"@rpath/{libfile_loco}", ], None, prefix2prefix, ) assert out_dict == { f"@rpath/{libfile_a}": f"@rpath/{libfile_a}", f"@rpath/{libfile_b}": f"@rpath/{libfile_b}", f"@rpath/{libfile_c}": f"@rpath/{libfile_c}", f"@rpath/{libfile_loco}": f"@rpath/{libfile_loco}", oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_c: libdir_c, oldlibdir_cc: libdir_cc, libdir_local: libdir_local, } out_dict = _macho_find_paths( [oldlibdir_a, oldlibdir_b, oldlibdir_d, oldlibdir_local], [f"@rpath/{libfile_a}", f"@rpath/{libfile_b}", f"@rpath/{libfile_loco}"], None, prefix2prefix, ) assert out_dict == { f"@rpath/{libfile_a}": f"@rpath/{libfile_a}", f"@rpath/{libfile_b}": f"@rpath/{libfile_b}", f"@rpath/{libfile_loco}": f"@rpath/{libfile_loco}", oldlibdir_a: libdir_a, oldlibdir_b: libdir_b, oldlibdir_d: libdir_d, libdir_local: libdir_local, } @pytest.fixture() def mock_download(monkeypatch): """Mock a failing download strategy.""" class FailedDownloadStrategy(spack.fetch_strategy.FetchStrategy): def mirror_id(self): return None def fetch(self): raise spack.fetch_strategy.FailedDownloadError( urllib.error.URLError("This FetchStrategy always fails") ) @property def fake_fn(self): return FailedDownloadStrategy() monkeypatch.setattr(spack.package_base.PackageBase, "fetcher", fake_fn) @pytest.mark.parametrize( "manual,instr", [(False, False), (False, True), (True, False), (True, True)] ) @pytest.mark.disable_clean_stage_check def test_manual_download(mock_download, default_mock_concretization, monkeypatch, manual, instr): """ Ensure expected fetcher fail message based on manual download and instr. """ @property def _instr(pkg): return f"Download instructions for {pkg.spec.name}" spec = default_mock_concretization("pkg-a") spec.package.manual_download = manual if instr: monkeypatch.setattr(spack.package_base.PackageBase, "download_instr", _instr) expected = spec.package.download_instr if manual else "All fetchers failed" with pytest.raises(spack.error.FetchError, match=expected): spec.package.do_fetch() @pytest.fixture() def fetching_not_allowed(monkeypatch): class FetchingNotAllowed(spack.fetch_strategy.FetchStrategy): def mirror_id(self): return None def fetch(self): raise Exception("Sources are fetched but shouldn't have been") monkeypatch.setattr(spack.package_base.PackageBase, "fetcher", FetchingNotAllowed()) def test_fetch_without_code_is_noop(default_mock_concretization, fetching_not_allowed): """do_fetch for packages without code should be a no-op""" pkg = default_mock_concretization("pkg-a").package pkg.has_code = False pkg.do_fetch() def test_fetch_external_package_is_noop(default_mock_concretization, fetching_not_allowed): """do_fetch for packages without code should be a no-op""" spec = default_mock_concretization("pkg-a") spec.external_path = "/some/where" assert spec.external spec.package.do_fetch() @pytest.mark.parametrize( "relocation_dict", [ {"/foo/bar/baz": "/a/b/c", "/foo/bar": "/a/b"}, # Ensure correctness does not depend on the ordering of the dict {"/foo/bar": "/a/b", "/foo/bar/baz": "/a/b/c"}, ], ) def test_macho_relocation_with_changing_projection(relocation_dict): """Tests that prefix relocation is computed correctly when the prefixes to be relocated contain a directory and its subdirectories. This happens when relocating to a new place AND changing the store projection. In that case we might have a relocation dict like: /foo/bar/baz/ -> /a/b/c /foo/bar -> /a/b What we need to check is that we don't end up in situations where we relocate to a mixture of the two schemes, like /a/b/baz. """ original_rpath = "/foo/bar/baz/abcdef" result = _macho_find_paths( [original_rpath], deps=[], idpath=None, prefix_to_prefix=relocation_dict ) assert result[original_rpath] == "/a/b/c/abcdef" ================================================ FILE: lib/spack/spack/test/patch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import filecmp import os import pathlib import shutil import sys import pytest import spack.concretize import spack.deptypes as dt import spack.error import spack.fetch_strategy import spack.patch import spack.paths import spack.repo import spack.spec import spack.stage import spack.util.url as url_util from spack.llnl.util.filesystem import mkdirp, touch, working_dir from spack.spec import Spec from spack.stage import Stage from spack.util.executable import Executable # various sha256 sums (using variables for legibility) # many file based shas will differ between Windows and other platforms # due to the use of carriage returns ('\r\n') in Windows line endings # files with contents 'foo', 'bar', and 'baz' foo_sha256 = ( "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" if sys.platform != "win32" else "bf874c7dd3a83cf370fdc17e496e341de06cd596b5c66dbf3c9bb7f6c139e3ee" ) bar_sha256 = ( "7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730" if sys.platform != "win32" else "556ddc69a75d0be0ecafc82cd4657666c8063f13d762282059c39ff5dbf18116" ) baz_sha256 = ( "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c" if sys.platform != "win32" else "d30392e66c636a063769cbb1db08cd3455a424650d4494db6379d73ea799582b" ) biz_sha256 = ( "a69b288d7393261e613c276c6d38a01461028291f6e381623acc58139d01f54d" if sys.platform != "win32" else "2f2b087a8f84834fd03d4d1d5b43584011e869e4657504ef3f8b0a672a5c222e" ) # url patches # url shas are the same on Windows url1_sha256 = "abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234" url2_sha256 = "1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd" url2_archive_sha256 = "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd" platform_url_sha = ( "252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866" if sys.platform != "win32" else "ecf44a8244a486e9ef5f72c6cb622f99718dcd790707ac91af0b8c9a4ab7a2bb" ) @pytest.fixture() def mock_patch_stage(tmp_path_factory: pytest.TempPathFactory, monkeypatch): # Don't disrupt the spack install directory with tests. mock_path = str(tmp_path_factory.mktemp("mock-patch-stage")) monkeypatch.setattr(spack.stage, "_stage_root", mock_path) return mock_path data_path = os.path.join(spack.paths.test_path, "data", "patch") @pytest.mark.not_on_windows("Line ending conflict on Windows") @pytest.mark.parametrize( "filename, sha256, archive_sha256", [ # compressed patch -- needs sha256 and archive_256 ( os.path.join(data_path, "foo.tgz"), "252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866", "4e8092a161ec6c3a1b5253176fcf33ce7ba23ee2ff27c75dbced589dabacd06e", ), # uncompressed patch -- needs only sha256 (os.path.join(data_path, "foo.patch"), platform_url_sha, None), ], ) def test_url_patch(mock_packages, mock_patch_stage, filename, sha256, archive_sha256, config): # Make a patch object url = url_util.path_to_file_url(filename) s = spack.concretize.concretize_one("patch") # make a stage with Stage(url) as stage: # TODO: url isn't used; maybe refactor Stage stage.mirror_path = mock_patch_stage mkdirp(stage.source_path) with working_dir(stage.source_path): # write a file to be patched with open("foo.txt", "w", encoding="utf-8") as f: f.write( """\ first line second line """ ) # save it for later comparison shutil.copyfile("foo.txt", "foo-original.txt") # write the expected result of patching. with open("foo-expected.txt", "w", encoding="utf-8") as f: f.write( """\ zeroth line first line third line """ ) # apply the patch and compare files patch = spack.patch.UrlPatch(s.package, url, sha256=sha256, archive_sha256=archive_sha256) patch_stage = Stage(patch.fetcher()) with patch_stage: patch_stage.create() patch_stage.fetch() patch_stage.expand_archive() spack.patch.apply_patch( stage.source_path, patch_stage.single_file, patch.level, patch.working_dir, patch.reverse, ) with working_dir(stage.source_path): assert filecmp.cmp("foo.txt", "foo-expected.txt") # apply the patch in reverse and compare files patch = spack.patch.UrlPatch( s.package, url, sha256=sha256, archive_sha256=archive_sha256, reverse=True ) patch_stage = Stage(patch.fetcher()) with patch_stage: patch_stage.create() patch_stage.fetch() patch_stage.expand_archive() spack.patch.apply_patch( stage.source_path, patch_stage.single_file, patch.level, patch.working_dir, patch.reverse, ) with working_dir(stage.source_path): assert filecmp.cmp("foo.txt", "foo-original.txt") def test_patch_in_spec(mock_packages, config): """Test whether patches in a package appear in the spec.""" spec = spack.concretize.concretize_one("patch") assert "patches" in list(spec.variants.keys()) # Here the order is bar, foo, baz. Note that MV variants order # lexicographically based on the hash, not on the position of the # patch directive. assert (bar_sha256, foo_sha256, baz_sha256) == spec.variants["patches"].value assert (foo_sha256, bar_sha256, baz_sha256) == tuple( spec.variants["patches"]._patches_in_order_of_appearance ) def test_stale_patch_cache_falls_back_to_fresh(mock_packages, config): """spec.patches returns correct patches even when the stale in-memory cache is wrong.""" spec = spack.concretize.concretize_one("patch@=1.0") pkg_cls = spack.repo.PATH.get_pkg_class("patch") # Inject a stale PatchCache: foo_sha256 points to a non-existent patch file stale_cache = spack.patch.PatchCache(repository=spack.repo.PATH) stale_cache.index = { foo_sha256: { pkg_cls.fullname: { "owner": pkg_cls.fullname, "relative_path": "stale_wrong.patch", "level": 1, "working_dir": ".", "reverse": False, } } } spack.repo.PATH._patch_index = stale_cache spack.repo.PATH._index_is_fresh = False patches = spec.patches assert len(patches) == 2 assert {p.relative_path for p in patches} == {"foo.patch", "baz.patch"} def test_patch_mixed_versions_subset_constraint(mock_packages, config): """If we have a package with mixed x.y and x.y.z versions, make sure that a patch applied to a version range of x.y.z versions is not applied to an x.y version. """ spec1 = spack.concretize.concretize_one("patch@1.0.1") assert biz_sha256 in spec1.variants["patches"].value spec2 = spack.concretize.concretize_one("patch@=1.0") assert biz_sha256 not in spec2.variants["patches"].value def test_patch_order(mock_packages, config): spec = spack.concretize.concretize_one("dep-diamond-patch-top") mid2_sha256 = ( "mid21234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234" if sys.platform != "win32" else "mid21234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234" ) mid1_sha256 = ( "0b62284961dab49887e31319843431ee5b037382ac02c4fe436955abef11f094" if sys.platform != "win32" else "aeb16c4dec1087e39f2330542d59d9b456dd26d791338ae6d80b6ffd10c89dfa" ) top_sha256 = ( "f7de2947c64cb6435e15fb2bef359d1ed5f6356b2aebb7b20535e3772904e6db" if sys.platform != "win32" else "ff34cb21271d16dbf928374f610bb5dd593d293d311036ddae86c4846ff79070" ) dep = spec["patch"] patch_order = dep.variants["patches"]._patches_in_order_of_appearance # 'mid2' comes after 'mid1' alphabetically # 'top' comes after 'mid1'/'mid2' alphabetically # 'patch' comes last of all specs in the dag, alphabetically, so the # patches of 'patch' to itself are applied last. The patches applied by # 'patch' are ordered based on their appearance in the package.py file expected_order = (mid1_sha256, mid2_sha256, top_sha256, foo_sha256, bar_sha256, baz_sha256) assert expected_order == tuple(patch_order) def test_nested_directives(mock_packages): """Ensure pkg data structures are set up properly by nested directives.""" # this ensures that the patch() directive results were removed # properly from the DirectiveMeta._directives_to_be_executed list package = spack.repo.PATH.get_pkg_class("patch-several-dependencies") assert len(package.patches) == 0 # this ensures that results of dependency patches were properly added # to Dependency objects. # package.dependencies is keyed by three when clauses assert package.dependencies.keys() == {Spec(), Spec("+foo"), Spec("@1.0")} # fake and libelf are unconditional dependencies when_unconditional = package.dependencies[Spec()] assert when_unconditional.keys() == {"fake", "libelf"} # fake has two unconditional URL patches assert when_unconditional["fake"].patches.keys() == {Spec()} assert len(when_unconditional["fake"].patches[Spec()]) == 2 # libelf has one unconditional patch assert when_unconditional["libelf"].patches.keys() == {Spec()} assert len(when_unconditional["libelf"].patches[Spec()]) == 1 # there are multiple depends_on directives for libelf under the +foo when clause; these must be # reduced to a single Dependency object. when_foo = package.dependencies[Spec("+foo")] assert when_foo.keys() == {"libelf"} assert when_foo["libelf"].spec == Spec("libelf@0.8.10") assert when_foo["libelf"].depflag == dt.BUILD | dt.LINK # there is one unconditional patch for libelf under the +foo when clause assert len(when_foo["libelf"].patches) == 1 assert len(when_foo["libelf"].patches[Spec()]) == 1 # libdwarf is a dependency when @1.0 with two patches applied from a single depends_on # statement, one conditional on the libdwarf version when_1_0 = package.dependencies[Spec("@1.0")] assert when_1_0.keys() == {"libdwarf"} assert when_1_0["libdwarf"].patches.keys() == {Spec(), Spec("@20111030")} assert len(when_1_0["libdwarf"].patches[Spec()]) == 1 assert len(when_1_0["libdwarf"].patches[Spec("@20111030")]) == 1 @pytest.mark.not_on_windows("Test requires Autotools") def test_patched_dependency(mock_packages, install_mockery, mock_fetch): """Test whether patched dependencies work.""" spec = spack.concretize.concretize_one("patch-a-dependency") assert "patches" in list(spec["libelf"].variants.keys()) # make sure the patch makes it into the dependency spec t_sha = ( "c45c1564f70def3fc1a6e22139f62cb21cd190cc3a7dbe6f4120fa59ce33dcb8" if sys.platform != "win32" else "3c5b65abcd6a3b2c714dbf7c31ff65fe3748a1adc371f030c283007ca5534f11" ) assert (t_sha,) == spec["libelf"].variants["patches"].value # make sure the patch in the dependent's directory is applied to the # dependency libelf = spec["libelf"] pkg = libelf.package pkg.do_patch() with pkg.stage: with working_dir(pkg.stage.source_path): # output a Makefile with 'echo Patched!' as the default target configure = Executable("./configure") configure() # Make sure the Makefile contains the patched text with open("Makefile", encoding="utf-8") as mf: assert "Patched!" in mf.read() def trigger_bad_patch(pkg): if not os.path.isdir(pkg.stage.source_path): os.makedirs(pkg.stage.source_path) bad_file = os.path.join(pkg.stage.source_path, ".spack_patch_failed") touch(bad_file) return bad_file def test_patch_failure_develop_spec_exits_gracefully( mock_packages, install_mockery, mock_fetch, tmp_path: pathlib.Path, mock_stage ): """ensure that a failing patch does not trigger exceptions for develop specs""" spec = spack.concretize.concretize_one(f"patch-a-dependency ^libelf dev_path={tmp_path}") libelf = spec["libelf"] assert "patches" in list(libelf.variants.keys()) pkg = libelf.package with pkg.stage: bad_patch_indicator = trigger_bad_patch(pkg) assert os.path.isfile(bad_patch_indicator) pkg.do_patch() # success if no exceptions raised def test_patch_failure_restages(mock_packages, install_mockery, mock_fetch): """ ensure that a failing patch does not trigger exceptions for non-develop specs and the source gets restaged """ spec = spack.concretize.concretize_one("patch-a-dependency") pkg = spec["libelf"].package with pkg.stage: bad_patch_indicator = trigger_bad_patch(pkg) assert os.path.isfile(bad_patch_indicator) pkg.do_patch() assert not os.path.isfile(bad_patch_indicator) def test_multiple_patched_dependencies(mock_packages, config): """Test whether multiple patched dependencies work.""" spec = spack.concretize.concretize_one("patch-several-dependencies") # basic patch on libelf assert "patches" in list(spec["libelf"].variants.keys()) # foo assert (foo_sha256,) == spec["libelf"].variants["patches"].value # URL patches assert "patches" in list(spec["fake"].variants.keys()) # urlpatch.patch, urlpatch.patch.gz assert (url2_sha256, url1_sha256) == spec["fake"].variants["patches"].value def test_conditional_patched_dependencies(mock_packages, config): """Test whether conditional patched dependencies work.""" spec = spack.concretize.concretize_one("patch-several-dependencies @1.0") # basic patch on libelf assert "patches" in list(spec["libelf"].variants.keys()) # foo assert (foo_sha256,) == spec["libelf"].variants["patches"].value # conditional patch on libdwarf assert "patches" in list(spec["libdwarf"].variants.keys()) # bar assert (bar_sha256,) == spec["libdwarf"].variants["patches"].value # baz is conditional on libdwarf version assert baz_sha256 not in spec["libdwarf"].variants["patches"].value # URL patches assert "patches" in list(spec["fake"].variants.keys()) # urlpatch.patch, urlpatch.patch.gz assert (url2_sha256, url1_sha256) == spec["fake"].variants["patches"].value def check_multi_dependency_patch_specs( libelf, libdwarf, fake, owner, package_dir, # specs ): # parent spec properties """Validate patches on dependencies of patch-several-dependencies.""" # basic patch on libelf assert "patches" in list(libelf.variants.keys()) # foo assert foo_sha256 in libelf.variants["patches"].value # conditional patch on libdwarf assert "patches" in list(libdwarf.variants.keys()) # bar assert bar_sha256 in libdwarf.variants["patches"].value # baz is conditional on libdwarf version (no guarantee on order w/conds) assert baz_sha256 in libdwarf.variants["patches"].value def get_patch(spec, ending): return next(p for p in spec.patches if p.path_or_url.endswith(ending)) # make sure file patches are reconstructed properly foo_patch = get_patch(libelf, "foo.patch") bar_patch = get_patch(libdwarf, "bar.patch") baz_patch = get_patch(libdwarf, "baz.patch") assert foo_patch.owner == owner assert foo_patch.path == os.path.join(package_dir, "foo.patch") assert foo_patch.sha256 == foo_sha256 assert bar_patch.owner == "builtin_mock.patch-several-dependencies" assert bar_patch.path == os.path.join(package_dir, "bar.patch") assert bar_patch.sha256 == bar_sha256 assert baz_patch.owner == "builtin_mock.patch-several-dependencies" assert baz_patch.path == os.path.join(package_dir, "baz.patch") assert baz_patch.sha256 == baz_sha256 # URL patches assert "patches" in list(fake.variants.keys()) # urlpatch.patch, urlpatch.patch.gz assert (url2_sha256, url1_sha256) == fake.variants["patches"].value url1_patch = get_patch(fake, "urlpatch.patch") url2_patch = get_patch(fake, "urlpatch2.patch.gz") assert url1_patch.owner == "builtin_mock.patch-several-dependencies" assert url1_patch.url == "http://example.com/urlpatch.patch" assert url1_patch.sha256 == url1_sha256 assert url2_patch.owner == "builtin_mock.patch-several-dependencies" assert url2_patch.url == "http://example.com/urlpatch2.patch.gz" assert url2_patch.sha256 == url2_sha256 assert url2_patch.archive_sha256 == url2_archive_sha256 def test_conditional_patched_deps_with_conditions(mock_packages, config): """Test whether conditional patched dependencies with conditions work.""" spec = spack.concretize.concretize_one( Spec("patch-several-dependencies @1.0 ^libdwarf@20111030") ) libelf = spec["libelf"] libdwarf = spec["libdwarf"] fake = spec["fake"] check_multi_dependency_patch_specs( libelf, libdwarf, fake, "builtin_mock.patch-several-dependencies", spec.package.package_dir ) def test_write_and_read_sub_dags_with_patched_deps(mock_packages, config): """Test whether patched dependencies are still correct after writing and reading a sub-DAG of a concretized Spec. """ spec = spack.concretize.concretize_one( Spec("patch-several-dependencies @1.0 ^libdwarf@20111030") ) # write to YAML and read back in -- new specs will *only* contain # their sub-DAGs, and won't contain the dependent that patched them libelf = spack.spec.Spec.from_yaml(spec["libelf"].to_yaml()) libdwarf = spack.spec.Spec.from_yaml(spec["libdwarf"].to_yaml()) fake = spack.spec.Spec.from_yaml(spec["fake"].to_yaml()) # make sure we can still read patches correctly for these specs check_multi_dependency_patch_specs( libelf, libdwarf, fake, "builtin_mock.patch-several-dependencies", spec.package.package_dir ) def test_patch_no_file(): # Give it the attributes we need to construct the error message FakePackage = collections.namedtuple("FakePackage", ["name", "namespace", "fullname"]) fp = FakePackage("fake-package", "test", "fake-package") with pytest.raises(ValueError, match="FilePatch:"): spack.patch.FilePatch(fp, "nonexistent_file", 0, "") patch = spack.patch.Patch(fp, "nonexistent_file", 0, "") patch.path = "test" with pytest.raises(spack.error.NoSuchPatchError, match="No such patch:"): spack.patch.apply_patch(Stage("https://example.com/foo.patch").source_path, patch.path) def test_patch_no_sha256(): # Give it the attributes we need to construct the error message FakePackage = collections.namedtuple("FakePackage", ["name", "namespace", "fullname"]) fp = FakePackage("fake-package", "test", "fake-package") url = url_util.path_to_file_url("foo.tgz") match = "Compressed patches require 'archive_sha256' and patch 'sha256' attributes: file://" with pytest.raises(spack.error.PatchDirectiveError, match=match): spack.patch.UrlPatch(fp, url, sha256="", archive_sha256="") match = "URL patches require a sha256 checksum" with pytest.raises(spack.error.PatchDirectiveError, match=match): spack.patch.UrlPatch(fp, url, sha256="", archive_sha256="abc") @pytest.mark.parametrize("level", [-1, 0.0, "1"]) def test_invalid_level(level): # Give it the attributes we need to construct the error message FakePackage = collections.namedtuple("FakePackage", ["name", "namespace"]) fp = FakePackage("fake-package", "test") with pytest.raises(ValueError, match="Patch level needs to be a non-negative integer."): spack.patch.Patch(fp, "nonexistent_file", level, "") def test_equality(): FakePackage = collections.namedtuple("FakePackage", ["name", "namespace", "fullname"]) fp = FakePackage("fake-package", "test", "fake-package") patch1 = spack.patch.UrlPatch(fp, "nonexistent_url1", sha256="abc") patch2 = spack.patch.UrlPatch(fp, "nonexistent_url2", sha256="def") assert patch1 == patch1 assert patch1 != patch2 assert patch1 != "not a patch" def test_sha256_setter(mock_packages, mock_patch_stage, config): path = os.path.join(data_path, "foo.patch") s = spack.concretize.concretize_one("patch") patch = spack.patch.FilePatch(s.package, path, level=1, working_dir=".") patch.sha256 = "abc" def test_invalid_from_dict(mock_packages, config): dictionary = {} with pytest.raises(ValueError, match="Invalid patch dictionary:"): spack.patch.from_dict(dictionary, mock_packages) dictionary = {"owner": "patch"} with pytest.raises(ValueError, match="Invalid patch dictionary:"): spack.patch.from_dict(dictionary, mock_packages) dictionary = { "owner": "patch", "relative_path": "foo.patch", "level": 1, "working_dir": ".", "reverse": False, "sha256": bar_sha256, } with pytest.raises(spack.fetch_strategy.ChecksumError, match="sha256 checksum failed for"): spack.patch.from_dict(dictionary, mock_packages) ================================================ FILE: lib/spack/spack/test/permissions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import stat import pytest import spack.llnl.util.filesystem as fs from spack.util.file_permissions import InvalidPermissionsError, set_permissions pytestmark = pytest.mark.not_on_windows("chmod unsupported on Windows") def ensure_known_group(path): """Ensure that the group of a file is one that's actually in our group list. On systems with remote groups, the primary user group may be remote and may not exist on the local system (i.e., it might just be a number). Trying to use chmod to setgid can fail silently in situations like this. """ uid = os.getuid() gid = fs.group_ids(uid)[0] os.chown(path, uid, gid) def test_chmod_real_entries_ignores_suid_sgid(tmp_path: pathlib.Path): path = tmp_path / "file" path.touch() mode = stat.S_ISUID | stat.S_ISGID | stat.S_ISVTX os.chmod(str(path), mode) mode = os.stat(str(path)).st_mode # adds a high bit we aren't concerned with perms = stat.S_IRWXU set_permissions(str(path), perms) assert os.stat(str(path)).st_mode == mode | perms & ~stat.S_IXUSR def test_chmod_rejects_group_writable_suid(tmp_path: pathlib.Path): path = tmp_path / "file" path.touch() mode = stat.S_ISUID fs.chmod_x(str(path), mode) perms = stat.S_IWGRP with pytest.raises(InvalidPermissionsError): set_permissions(str(path), perms) def test_chmod_rejects_world_writable_suid(tmp_path: pathlib.Path): path = tmp_path / "file" path.touch() mode = stat.S_ISUID fs.chmod_x(str(path), mode) perms = stat.S_IWOTH with pytest.raises(InvalidPermissionsError): set_permissions(str(path), perms) def test_chmod_rejects_world_writable_sgid(tmp_path: pathlib.Path): path = tmp_path / "file" path.touch() ensure_known_group(str(path)) mode = stat.S_ISGID fs.chmod_x(str(path), mode) perms = stat.S_IWOTH with pytest.raises(InvalidPermissionsError): set_permissions(str(path), perms) ================================================ FILE: lib/spack/spack/test/projections.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from datetime import date import spack.projections import spack.spec def test_projection_expansion(mock_packages, monkeypatch): """Test that env variables and spack config variables are expanded in projections""" monkeypatch.setenv("FOO_ENV_VAR", "test-string") projections = {"all": "{name}-{version}/$FOO_ENV_VAR/$date"} spec = spack.spec.Spec("fake@1.0") projection = spack.projections.get_projection(projections, spec) assert "{name}-{version}/test-string/%s" % date.today().strftime("%Y-%m-%d") == projection ================================================ FILE: lib/spack/spack/test/provider_index.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for provider index cache files. Tests assume that mock packages provide this:: {'blas': { blas: set([netlib-blas, openblas, openblas-with-lapack])}, 'lapack': {lapack: set([netlib-lapack, openblas-with-lapack])}, 'mpi': {mpi@:1: set([mpich@:1]), mpi@:2.0: set([mpich2]), mpi@:2.1: set([mpich2@1.1:]), mpi@:2.2: set([mpich2@1.2:]), mpi@:3: set([mpich@3:]), mpi@:10.0: set([zmpi])}, 'stuff': {stuff: set([externalvirtual])}} """ import io import spack.repo from spack.provider_index import ProviderIndex from spack.spec import Spec def test_provider_index_round_trip(mock_packages): p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) ostream = io.StringIO() p.to_json(ostream) istream = io.StringIO(ostream.getvalue()) q = ProviderIndex.from_json(istream, repository=spack.repo.PATH) assert p == q def test_providers_for_simple(mock_packages): p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) blas_providers = p.providers_for("blas") assert Spec("netlib-blas") in blas_providers assert Spec("openblas") in blas_providers assert Spec("openblas-with-lapack") in blas_providers lapack_providers = p.providers_for("lapack") assert Spec("netlib-lapack") in lapack_providers assert Spec("openblas-with-lapack") in lapack_providers def test_mpi_providers(mock_packages): p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) mpi_2_providers = p.providers_for("mpi@2") assert Spec("mpich2") in mpi_2_providers assert Spec("mpich@3:") in mpi_2_providers mpi_3_providers = p.providers_for("mpi@3") assert Spec("mpich2") not in mpi_3_providers assert Spec("mpich@3:") in mpi_3_providers assert Spec("zmpi") in mpi_3_providers def test_equal(mock_packages): p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) q = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) assert p == q def test_copy(mock_packages): p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH) q = p.copy() assert p == q def test_remove_providers(mock_packages): """Test removing providers from the index.""" p = ProviderIndex(specs=["mpich"], repository=spack.repo.PATH) # Check that mpich is a provider for mpi providers = p.providers_for("mpi") assert any(spec.name == "mpich" for spec in providers) p.remove_providers({"mpich"}) # After removal, mpich should no longer be a provider for mpi providers = p.providers_for("mpi") assert not any(spec.name == "mpich" for spec in providers) ================================================ FILE: lib/spack/spack/test/relocate.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import re import shutil import subprocess import tempfile import pytest import spack.platforms import spack.relocate import spack.relocate_text as relocate_text import spack.util.executable pytestmark = pytest.mark.not_on_windows("Tests fail on Windows") def skip_unless_linux(f): return pytest.mark.skipif( str(spack.platforms.real_host()) != "linux", reason="implementation currently requires linux", )(f) def rpaths_for(new_binary): """Return the RPATHs or RUNPATHs of a binary.""" patchelf = spack.util.executable.which("patchelf", required=True) output = patchelf("--print-rpath", str(new_binary), output=str) return output.strip() def text_in_bin(text, binary): with open(str(binary), "rb") as f: data = f.read() f.seek(0) pat = re.compile(text.encode("utf-8")) if not pat.search(data): return False return True @pytest.fixture() def make_dylib(tmp_path_factory: pytest.TempPathFactory): """Create a shared library with unfriendly qualities. - Writes the same rpath twice - Writes its install path as an absolute path """ cc = spack.util.executable.which("cc", required=True) def _factory(abs_install_name="abs", extra_rpaths=[]): assert all(extra_rpaths) tmpdir = tmp_path_factory.mktemp( abs_install_name + "-".join(extra_rpaths).replace("/", "") ) src = tmpdir / "foo.c" src.write_text("int foo() { return 1; }\n") filename = "foo.dylib" lib = tmpdir / filename args = ["-shared", str(src), "-o", str(lib)] rpaths = list(extra_rpaths) if abs_install_name.startswith("abs"): args += ["-install_name", str(lib)] else: args += ["-install_name", "@rpath/" + filename] if abs_install_name.endswith("rpath"): rpaths.append(str(tmpdir)) args.extend("-Wl,-rpath," + s for s in rpaths) cc(*args) return (str(tmpdir), filename) return _factory @pytest.fixture() def make_object_file(tmp_path: pathlib.Path): cc = spack.util.executable.which("cc", required=True) def _factory(): src = tmp_path / "bar.c" src.write_text("int bar() { return 2; }\n") filename = "bar.o" lib = tmp_path / filename args = ["-c", str(src), "-o", str(lib)] cc(*args) return (str(tmp_path), filename) return _factory @pytest.fixture() def copy_binary(prefix_like): """Returns a function that copies a binary somewhere and returns the new location. """ def _copy_somewhere(orig_binary): # Create a temporary directory temp_dir = pathlib.Path(tempfile.mkdtemp()) new_root = temp_dir / prefix_like new_root.mkdir(parents=True, exist_ok=True) new_binary = new_root / "main.x" shutil.copy(str(orig_binary), str(new_binary)) return new_binary return _copy_somewhere @pytest.mark.requires_executables("patchelf", "gcc") @skip_unless_linux def test_relocate_text_bin(binary_with_rpaths, prefix_like): prefix = "/usr/" + prefix_like prefix_bytes = prefix.encode("utf-8") new_prefix = "/foo/" + prefix_like new_prefix_bytes = new_prefix.encode("utf-8") # Compile an "Hello world!" executable and set RPATHs executable = binary_with_rpaths(rpaths=[prefix + "/lib", prefix + "/lib64"]) # Relocate the RPATHs spack.relocate.relocate_text_bin([str(executable)], {prefix_bytes: new_prefix_bytes}) # Some compilers add rpaths so ensure changes included in final result assert "%s/lib:%s/lib64" % (new_prefix, new_prefix) in rpaths_for(executable) @pytest.mark.requires_executables("patchelf", "gcc") @skip_unless_linux def test_relocate_elf_binaries_absolute_paths(binary_with_rpaths, copy_binary, prefix_tmpdir): # Create an executable, set some RPATHs, copy it to another location lib_dir = prefix_tmpdir / "lib" lib_dir.mkdir() orig_binary = binary_with_rpaths(rpaths=[str(lib_dir), "/usr/lib64"]) new_binary = copy_binary(orig_binary) spack.relocate.relocate_elf_binaries( binaries=[str(new_binary)], prefix_to_prefix={str(orig_binary.parent): "/foo"} ) # Some compilers add rpaths so ensure changes included in final result assert "/foo/lib:/usr/lib64" in rpaths_for(new_binary) @pytest.mark.requires_executables("patchelf", "gcc") @skip_unless_linux def test_relocate_text_bin_with_message(binary_with_rpaths, copy_binary, prefix_tmpdir): lib_dir = prefix_tmpdir / "lib" lib_dir.mkdir() lib64_dir = prefix_tmpdir / "lib64" lib64_dir.mkdir() orig_binary = binary_with_rpaths( rpaths=[str(lib_dir), str(lib64_dir), "/opt/local/lib"], message=str(prefix_tmpdir) ) new_binary = copy_binary(orig_binary) # Check original directory is in the executable and the new one is not assert text_in_bin(str(prefix_tmpdir), new_binary) assert not text_in_bin(str(new_binary.parent), new_binary) # Check this call succeed orig_path_bytes = str(orig_binary.parent).encode("utf-8") new_path_bytes = str(new_binary.parent).encode("utf-8") spack.relocate.relocate_text_bin([str(new_binary)], {orig_path_bytes: new_path_bytes}) # Check original directory is not there anymore and it was # substituted with the new one assert not text_in_bin(str(prefix_tmpdir), new_binary) assert text_in_bin(str(new_binary.parent), new_binary) def test_relocate_text_bin_raise_if_new_prefix_is_longer(tmp_path: pathlib.Path): short_prefix = b"/short" long_prefix = b"/much/longer" fpath = str(tmp_path / "fakebin") with open(fpath, "w", encoding="utf-8") as f: f.write("/short") with pytest.raises(relocate_text.BinaryTextReplaceError): spack.relocate.relocate_text_bin([fpath], {short_prefix: long_prefix}) @pytest.mark.requires_executables("install_name_tool", "cc") def test_fixup_macos_rpaths(make_dylib, make_object_file): # Get Apple Clang major version for XCode 15+ linker behavior try: result = subprocess.check_output(["cc", "--version"], universal_newlines=True) version_match = re.search(r"Apple clang version (\d+)", result) assert version_match, "Apple Clang version not found in output" xcode_major_version = int(version_match.group(1)) except Exception: pytest.xfail("cannot determine Apple Clang major version") return # For each of these tests except for the "correct" case, the first fixup # should make changes, and the second fixup should be a null-op. fixup_rpath = spack.relocate.fixup_macos_rpath no_rpath = [] duplicate_rpaths = ["/usr", "/usr"] bad_rpath = ["/nonexistent/path"] # Non-relocatable library id and duplicate rpaths (root, filename) = make_dylib("abs", duplicate_rpaths) # XCode 15 ships a new linker that takes care of deduplication if xcode_major_version < 15: assert fixup_rpath(root, filename) assert not fixup_rpath(root, filename) # Hardcoded but relocatable library id (but we do NOT relocate) (root, filename) = make_dylib("abs_with_rpath", no_rpath) assert not fixup_rpath(root, filename) # Library id uses rpath but there are extra duplicate rpaths (root, filename) = make_dylib("rpath", duplicate_rpaths) # XCode 15 ships a new linker that takes care of deduplication if xcode_major_version < 15: assert fixup_rpath(root, filename) assert not fixup_rpath(root, filename) # Shared library was constructed with relocatable id from the get-go (root, filename) = make_dylib("rpath", no_rpath) assert not fixup_rpath(root, filename) # Non-relocatable library id (root, filename) = make_dylib("abs", no_rpath) assert not fixup_rpath(root, filename) # Relocatable with executable paths and loader paths (root, filename) = make_dylib("rpath", ["@executable_path/../lib", "@loader_path"]) assert not fixup_rpath(root, filename) # Non-relocatable library id but nonexistent rpath (root, filename) = make_dylib("abs", bad_rpath) assert fixup_rpath(root, filename) assert not fixup_rpath(root, filename) # Duplicate nonexistent rpath will need *two* passes (root, filename) = make_dylib("rpath", bad_rpath * 2) assert fixup_rpath(root, filename) # XCode 15 ships a new linker that takes care of deduplication if xcode_major_version < 15: assert fixup_rpath(root, filename) assert not fixup_rpath(root, filename) # Test on an object file, which *also* has type 'application/x-mach-binary' # but should be ignored (no ID headers, no RPATH) # (this is a corner case for GCC installation) (root, filename) = make_object_file() assert not fixup_rpath(root, filename) ================================================ FILE: lib/spack/spack/test/relocate_text.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io from collections import OrderedDict import pytest import spack.relocate_text as relocate_text def test_text_relocation_regex_is_safe(): # Test whether prefix regex is properly escaped string = b"This does not match /a/, but this does: /[a-z]/." assert relocate_text.utf8_path_to_binary_regex("/[a-z]/").search(string).group(0) == b"/[a-z]/" def test_utf8_paths_to_single_binary_regex(): regex = relocate_text.utf8_paths_to_single_binary_regex( ["/first/path", "/second/path", "/safe/[a-z]"] ) # Match nothing assert not regex.search(b"text /neither/first/path text /the/second/path text") # Match first string = b"contains both /first/path/subdir and /second/path/sub" assert regex.search(string).group(0) == b"/first/path/subdir" # Match second string = b"contains both /not/first/path/subdir but /second/path/subdir" assert regex.search(string).group(0) == b"/second/path/subdir" # Match "unsafe" dir name string = b"don't match /safe/a/path but do match /safe/[a-z]/file" assert regex.search(string).group(0) == b"/safe/[a-z]/file" def test_ordered_replacement(): # This tests whether binary text replacement respects order, so that # a long package prefix is replaced before a shorter sub-prefix like # the root of the spack store (as a fallback). def replace_and_expect(prefix_map, before, after=None, suffix_safety_size=7): f = io.BytesIO(before) relocater = relocate_text.BinaryFilePrefixReplacer( OrderedDict(prefix_map), suffix_safety_size ) relocater.apply_to_file(f) f.seek(0) assert f.read() == after # The case of having a non-null terminated common suffix. replace_and_expect( [ (b"/old-spack/opt/specific-package", b"/first/specific-package"), (b"/old-spack/opt", b"/sec/spack/opt"), ], b"Binary with /old-spack/opt/specific-package and /old-spack/opt", b"Binary with /////////first/specific-package and /sec/spack/opt", suffix_safety_size=7, ) # The case of having a direct null terminated common suffix. replace_and_expect( [ (b"/old-spack/opt/specific-package", b"/first/specific-package"), (b"/old-spack/opt", b"/sec/spack/opt"), ], b"Binary with /old-spack/opt/specific-package\0 and /old-spack/opt\0", b"Binary with /////////first/specific-package\0 and /sec/spack/opt\0", suffix_safety_size=7, ) # Testing the order of operations (not null terminated, long enough common suffix) replace_and_expect( [ (b"/old-spack/opt", b"/s/spack/opt"), (b"/old-spack/opt/specific-package", b"/first/specific-package"), ], b"Binary with /old-spack/opt/specific-package and /old-spack/opt", b"Binary with ///s/spack/opt/specific-package and ///s/spack/opt", suffix_safety_size=7, ) # Testing the order of operations (null terminated, long enough common suffix) replace_and_expect( [ (b"/old-spack/opt", b"/s/spack/opt"), (b"/old-spack/opt/specific-package", b"/first/specific-package"), ], b"Binary with /old-spack/opt/specific-package\0 and /old-spack/opt\0", b"Binary with ///s/spack/opt/specific-package\0 and ///s/spack/opt\0", suffix_safety_size=7, ) # Null terminated within the lookahead window, common suffix long enough replace_and_expect( [(b"/old-spack/opt/specific-package", b"/opt/specific-XXXXage")], b"Binary with /old-spack/opt/specific-package/sub\0 data", b"Binary with ///////////opt/specific-XXXXage/sub\0 data", suffix_safety_size=7, ) # Null terminated within the lookahead window, common suffix too short, but # shortening is enough to spare more than 7 bytes of old suffix. replace_and_expect( [(b"/old-spack/opt/specific-package", b"/opt/specific-XXXXXge")], b"Binary with /old-spack/opt/specific-package/sub\0 data", b"Binary with /opt/specific-XXXXXge/sub\0ckage/sub\0 data", # ckage/sub = 9 bytes suffix_safety_size=7, ) # Null terminated within the lookahead window, common suffix too short, # shortening leaves exactly 7 suffix bytes untouched, amazing! replace_and_expect( [(b"/old-spack/opt/specific-package", b"/spack/specific-XXXXXge")], b"Binary with /old-spack/opt/specific-package/sub\0 data", b"Binary with /spack/specific-XXXXXge/sub\0age/sub\0 data", # age/sub = 7 bytes suffix_safety_size=7, ) # Null terminated within the lookahead window, common suffix too short, # shortening doesn't leave space for 7 bytes, sad! error_msg = "Cannot replace {!r} with {!r} in the C-string {!r}.".format( b"/old-spack/opt/specific-package", b"/snacks/specific-XXXXXge", b"/old-spack/opt/specific-package/sub", ) with pytest.raises(relocate_text.CannotShrinkCString, match=error_msg): replace_and_expect( [(b"/old-spack/opt/specific-package", b"/snacks/specific-XXXXXge")], b"Binary with /old-spack/opt/specific-package/sub\0 data", # expect failure! suffix_safety_size=7, ) # Check that it works when changing suffix_safety_size. replace_and_expect( [(b"/old-spack/opt/specific-package", b"/snacks/specific-XXXXXXe")], b"Binary with /old-spack/opt/specific-package/sub\0 data", b"Binary with /snacks/specific-XXXXXXe/sub\0ge/sub\0 data", suffix_safety_size=6, ) # Finally check the case of no shortening but a long enough common suffix. replace_and_expect( [(b"pkg-gwixwaalgczp6", b"pkg-zkesfralgczp6")], b"Binary with pkg-gwixwaalgczp6/config\0 data", b"Binary with pkg-zkesfralgczp6/config\0 data", suffix_safety_size=7, ) # Too short matching suffix, identical string length error_msg = "Cannot replace {!r} with {!r} in the C-string {!r}.".format( b"pkg-gwixwaxlgczp6", b"pkg-zkesfrzlgczp6", b"pkg-gwixwaxlgczp6" ) with pytest.raises(relocate_text.CannotShrinkCString, match=error_msg): replace_and_expect( [(b"pkg-gwixwaxlgczp6", b"pkg-zkesfrzlgczp6")], b"Binary with pkg-gwixwaxlgczp6\0 data", # expect failure suffix_safety_size=7, ) # Finally, make sure that the regex is not greedily finding the LAST null byte # it should find the first null byte in the window. In this test we put one null # at a distance where we can't keep a long enough suffix, and one where we can, # so we should expect failure when the first null is used. error_msg = "Cannot replace {!r} with {!r} in the C-string {!r}.".format( b"pkg-abcdef", b"pkg-xyzabc", b"pkg-abcdef" ) with pytest.raises(relocate_text.CannotShrinkCString, match=error_msg): replace_and_expect( [(b"pkg-abcdef", b"pkg-xyzabc")], b"Binary with pkg-abcdef\0/xx\0", # def\0/xx is 7 bytes. # expect failure suffix_safety_size=7, ) def test_inplace_text_replacement(): def replace_and_expect(prefix_to_prefix, before: bytes, after: bytes): f = io.BytesIO(before) replacer = relocate_text.TextFilePrefixReplacer(OrderedDict(prefix_to_prefix)) replacer.apply_to_file(f) f.seek(0) assert f.read() == after replace_and_expect( [ (b"/first/prefix", b"/first-replacement/prefix"), (b"/second/prefix", b"/second-replacement/prefix"), ], b"Example: /first/prefix/subdir and /second/prefix/subdir", b"Example: /first-replacement/prefix/subdir and /second-replacement/prefix/subdir", ) replace_and_expect( [ (b"/replace/in/order", b"/first"), (b"/replace/in", b"/second"), (b"/replace", b"/third"), ], b"/replace/in/order/x /replace/in/y /replace/z", b"/first/x /second/y /third/z", ) replace_and_expect( [ (b"/replace", b"/third"), (b"/replace/in", b"/second"), (b"/replace/in/order", b"/first"), ], b"/replace/in/order/x /replace/in/y /replace/z", b"/third/in/order/x /third/in/y /third/z", ) replace_and_expect( [(b"/my/prefix", b"/replacement")], b"/dont/replace/my/prefix #!/dont/replace/my/prefix", b"/dont/replace/my/prefix #!/dont/replace/my/prefix", ) replace_and_expect( [(b"/my/prefix", b"/replacement")], b"Install path: /my/prefix.", b"Install path: /replacement.", ) replace_and_expect([(b"/my/prefix", b"/replacement")], b"#!/my/prefix", b"#!/replacement") def test_relocate_text_filters_redundant_entries(): # Test that we're filtering identical old / new paths, since that's a waste. mapping = OrderedDict([("/hello", "/hello"), ("/world", "/world")]) replacer_1 = relocate_text.BinaryFilePrefixReplacer.from_strings_or_bytes(mapping) replacer_2 = relocate_text.TextFilePrefixReplacer.from_strings_or_bytes(mapping) assert not replacer_1.prefix_to_prefix assert not replacer_2.prefix_to_prefix ================================================ FILE: lib/spack/spack/test/repo.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.environment import spack.package_base import spack.paths import spack.repo import spack.schema.repos import spack.spec import spack.util.executable import spack.util.file_cache import spack.util.lock import spack.util.naming from spack.test.conftest import RepoBuilder from spack.util.naming import valid_module_name @pytest.fixture(params=["packages", "", "foo"]) def extra_repo(tmp_path_factory: pytest.TempPathFactory, request): repo_namespace = "extra_test_repo" repo_dir = tmp_path_factory.mktemp(repo_namespace) cache_dir = tmp_path_factory.mktemp("cache") (repo_dir / request.param).mkdir(parents=True, exist_ok=True) if request.param == "packages": (repo_dir / "repo.yaml").write_text( """ repo: namespace: extra_test_repo """ ) else: (repo_dir / "repo.yaml").write_text( f""" repo: namespace: extra_test_repo subdirectory: '{request.param}' """ ) repo_cache = spack.util.file_cache.FileCache(cache_dir) return spack.repo.Repo(str(repo_dir), cache=repo_cache), request.param def test_repo_getpkg(mutable_mock_repo): mutable_mock_repo.get_pkg_class("pkg-a") mutable_mock_repo.get_pkg_class("builtin_mock.pkg-a") def test_repo_multi_getpkg(mutable_mock_repo, extra_repo): mutable_mock_repo.put_first(extra_repo[0]) mutable_mock_repo.get_pkg_class("pkg-a") mutable_mock_repo.get_pkg_class("builtin_mock.pkg-a") def test_repo_multi_getpkgclass(mutable_mock_repo, extra_repo): mutable_mock_repo.put_first(extra_repo[0]) mutable_mock_repo.get_pkg_class("pkg-a") mutable_mock_repo.get_pkg_class("builtin_mock.pkg-a") def test_repo_pkg_with_unknown_namespace(mutable_mock_repo): with pytest.raises(spack.repo.UnknownNamespaceError): mutable_mock_repo.get_pkg_class("unknown.pkg-a") def test_repo_unknown_pkg(mutable_mock_repo): with pytest.raises(spack.repo.UnknownPackageError): mutable_mock_repo.get_pkg_class("builtin_mock.nonexistentpackage") def test_repo_last_mtime(mock_packages): mtime_with_package_py = [ (os.path.getmtime(p.module.__file__), p.module.__file__) for p in spack.repo.PATH.all_package_classes() ] repo_mtime = spack.repo.PATH.last_mtime() max_mtime, max_file = max(mtime_with_package_py) if max_mtime > repo_mtime: modified_after = "\n ".join( f"{path} ({mtime})" for mtime, path in mtime_with_package_py if mtime > repo_mtime ) assert max_mtime <= repo_mtime, ( f"the following files were modified while running tests:\n {modified_after}" ) assert max_mtime == repo_mtime, f"last_mtime incorrect for {max_file}" def test_repo_invisibles(mutable_mock_repo, extra_repo): with open( os.path.join(extra_repo[0].root, extra_repo[1], ".invisible"), "w", encoding="utf-8" ): pass extra_repo[0].all_package_names() @pytest.mark.regression("24552") def test_all_package_names_is_cached_correctly(mock_packages): assert "mpi" in spack.repo.all_package_names(include_virtuals=True) assert "mpi" not in spack.repo.all_package_names(include_virtuals=False) @pytest.mark.regression("29203") def test_use_repositories_doesnt_change_class(mock_packages): """Test that we don't create the same package module and class multiple times when swapping repositories. """ zlib_cls_outer = spack.repo.PATH.get_pkg_class("zlib") current_paths = [r.root for r in spack.repo.PATH.repos] with spack.repo.use_repositories(*current_paths): zlib_cls_inner = spack.repo.PATH.get_pkg_class("zlib") assert id(zlib_cls_inner) == id(zlib_cls_outer) def test_absolute_import_spack_packages_as_python_modules(mock_packages): import spack_repo.builtin_mock.packages.mpileaks.package # type: ignore[import] assert hasattr(spack_repo.builtin_mock.packages.mpileaks.package, "Mpileaks") assert isinstance( spack_repo.builtin_mock.packages.mpileaks.package.Mpileaks, spack.package_base.PackageMeta ) assert issubclass( spack_repo.builtin_mock.packages.mpileaks.package.Mpileaks, spack.package_base.PackageBase ) def test_relative_import_spack_packages_as_python_modules(mock_packages): from spack_repo.builtin_mock.packages.mpileaks.package import Mpileaks assert isinstance(Mpileaks, spack.package_base.PackageMeta) assert issubclass(Mpileaks, spack.package_base.PackageBase) def test_get_all_mock_packages(mock_packages): """Get the mock packages once each too.""" for name in mock_packages.all_package_names(): mock_packages.get_pkg_class(name) def test_repo_path_handles_package_removal(mock_packages, repo_builder: RepoBuilder): repo_builder.add_package("pkg-c") with spack.repo.use_repositories(repo_builder.root, override=False) as repos: r = repos.repo_for_pkg("pkg-c") assert r.namespace == repo_builder.namespace repo_builder.remove("pkg-c") with spack.repo.use_repositories(repo_builder.root, override=False) as repos: r = repos.repo_for_pkg("pkg-c") assert r.namespace == "builtin_mock" def test_repo_dump_virtuals( tmp_path: pathlib.Path, mutable_mock_repo, mock_packages, ensure_debug, capfd ): # Start with a package-less virtual vspec = spack.spec.Spec("something") mutable_mock_repo.dump_provenance(vspec, str(tmp_path)) captured = capfd.readouterr()[1] assert "does not have a package" in captured # Now with a virtual with a package vspec = spack.spec.Spec("externalvirtual") mutable_mock_repo.dump_provenance(vspec, str(tmp_path)) captured = capfd.readouterr()[1] assert "Installing" in captured assert "package.py" in os.listdir(str(tmp_path)), "Expected the virtual's package to be copied" @pytest.mark.parametrize("repos", [["mock"], ["extra"], ["mock", "extra"], ["extra", "mock"]]) def test_repository_construction_doesnt_use_globals( nullify_globals, tmp_path: pathlib.Path, repos, repo_builder: RepoBuilder ): def _repo_descriptors(repos): descriptors = {} for entry in repos: if entry == "mock": descriptors["builtin_mock"] = spack.repo.LocalRepoDescriptor( "builtin_mock", spack.paths.mock_packages_path ) if entry == "extra": repo_dir = tmp_path / "extra_mock" repo_dir.mkdir() descriptors[repo_builder.namespace] = spack.repo.LocalRepoDescriptor( repo_builder.namespace, repo_builder.root ) return spack.repo.RepoDescriptors(descriptors) descriptors = _repo_descriptors(repos) repo_cache = spack.util.file_cache.FileCache(tmp_path / "cache") repo_path = spack.repo.RepoPath.from_descriptors(descriptors, cache=repo_cache) assert len(repo_path.repos) == len(descriptors) assert [x.namespace for x in repo_path.repos] == list(descriptors.keys()) @pytest.mark.parametrize("method_name", ["dirname_for_package_name", "filename_for_package_name"]) def test_path_computation_with_names(method_name, mock_packages_repo): """Tests that repositories can compute the correct paths when using both fully qualified names and unqualified names. """ repo_path = spack.repo.RepoPath(mock_packages_repo) method = getattr(repo_path, method_name) unqualified = method("mpileaks") qualified = method("builtin_mock.mpileaks") assert qualified == unqualified def test_use_repositories_and_import(): """Tests that use_repositories changes the import search too""" import spack.paths repo_dir = pathlib.Path(spack.paths.test_repos_path) with spack.repo.use_repositories(str(repo_dir / "spack_repo" / "compiler_runtime_test")): import spack_repo.compiler_runtime_test.packages.gcc_runtime.package # type: ignore[import] # noqa: E501 with spack.repo.use_repositories(str(repo_dir / "spack_repo" / "builtin_mock")): import spack_repo.builtin_mock.packages.cmake.package # type: ignore[import] # noqa: F401 @pytest.mark.usefixtures("nullify_globals") class TestRepo: """Test that the Repo class work correctly, and does not depend on globals, except the REPOS_FINDER. """ def test_creation(self, mock_test_cache): repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) assert repo.config_file.endswith("repo.yaml") assert repo.namespace == "builtin_mock" @pytest.mark.parametrize( "name,expected", [("mpi", True), ("mpich", False), ("mpileaks", False)] ) def test_is_virtual(self, name, expected, mock_test_cache): repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) assert repo.is_virtual(name) is expected assert repo.is_virtual_safe(name) is expected repo_path = spack.repo.RepoPath(repo) assert repo_path.is_virtual(name) is expected assert repo_path.is_virtual_safe(name) is expected @pytest.mark.parametrize( "module_name,pkg_name", [ ("dla_future", "dla-future"), ("num7zip", "7zip"), # If no package is there, None is returned ("unknown", None), ], ) def test_real_name(self, module_name, pkg_name, mock_test_cache, tmp_path: pathlib.Path): """Test that we can correctly compute the 'real' name of a package, from the one used to import the Python module. """ path, _ = spack.repo.create_repo(str(tmp_path), package_api=(1, 0)) if pkg_name is not None: pkg_path = pathlib.Path(path) / "packages" / pkg_name / "package.py" pkg_path.parent.mkdir(parents=True) pkg_path.write_text("") repo = spack.repo.Repo( path, cache=spack.util.file_cache.FileCache(str(tmp_path / "cache")) ) assert repo.real_name(module_name) == pkg_name @pytest.mark.parametrize("name", ["mpileaks", "7zip", "dla-future"]) def test_get(self, name, mock_test_cache): repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) mock_spec = spack.spec.Spec(name) mock_spec._mark_concrete() pkg = repo.get(mock_spec) assert pkg.__class__ == repo.get_pkg_class(name) @pytest.mark.parametrize("virtual_name,expected", [("mpi", ["mpich", "zmpi"])]) def test_providers(self, virtual_name, expected, mock_test_cache): repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) provider_names = {x.name for x in repo.providers_for(virtual_name)} assert provider_names.issuperset(expected) @pytest.mark.parametrize( "extended,expected", [("python", ["py-extension1", "python-venv"]), ("perl", ["perl-extension"])], ) def test_extensions(self, extended, expected, mock_test_cache): repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): provider_names = {x.name for x in instance.extensions_for(extended)} assert provider_names.issuperset(expected) def test_all_package_names(self, mock_test_cache): repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): all_names = instance.all_package_names(include_virtuals=True) real_names = instance.all_package_names(include_virtuals=False) assert set(all_names).issuperset(real_names) for name in set(all_names) - set(real_names): assert instance.is_virtual(name) assert instance.is_virtual_safe(name) def test_packages_with_tags(self, mock_test_cache): repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache) repo_path = spack.repo.RepoPath(repo) for instance in (repo, repo_path): r1 = instance.packages_with_tags("tag1") r2 = instance.packages_with_tags("tag1", "tag2") assert "mpich" in r1 and "mpich" in r2 assert "mpich2" in r1 and "mpich2" not in r2 assert r2.issubset(r1) @pytest.mark.usefixtures("nullify_globals") class TestRepoPath: def test_creation_from_string(self, mock_test_cache): repo = spack.repo.RepoPath.from_descriptors( spack.repo.RepoDescriptors( { "builtin_mock": spack.repo.LocalRepoDescriptor( "builtin_mock", spack.paths.mock_packages_path ) } ), cache=mock_test_cache, ) assert len(repo.repos) == 1 assert repo.by_namespace["builtin_mock"] is repo.repos[0] def test_get_repo(self, mock_test_cache): repo = spack.repo.RepoPath.from_descriptors( spack.repo.RepoDescriptors( { "builtin_mock": spack.repo.LocalRepoDescriptor( "builtin_mock", spack.paths.mock_packages_path ) } ), cache=mock_test_cache, ) # builtin_mock is there assert repo.get_repo("builtin_mock") is repo.repos[0] # foo is not there, raise with pytest.raises(spack.repo.UnknownNamespaceError): repo.get_repo("foo") def test_parse_package_api_version(): """Test that we raise an error if a repository has a version that is not supported.""" # valid version assert spack.repo._parse_package_api_version( {"api": "v1.2"}, min_api=(1, 0), max_api=(2, 3) ) == (1, 2) # too new and too old with pytest.raises( spack.repo.BadRepoError, match=r"Package API v2.4 is not supported .* \(must be between v1.0 and v2.3\)", ): spack.repo._parse_package_api_version({"api": "v2.4"}, min_api=(1, 0), max_api=(2, 3)) with pytest.raises( spack.repo.BadRepoError, match=r"Package API v0.9 is not supported .* \(must be between v1.0 and v2.3\)", ): spack.repo._parse_package_api_version({"api": "v0.9"}, min_api=(1, 0), max_api=(2, 3)) # default to v1.0 if not specified assert spack.repo._parse_package_api_version({}, min_api=(1, 0), max_api=(2, 3)) == (1, 0) # if v1.0 support is dropped we should also raise with pytest.raises( spack.repo.BadRepoError, match=r"Package API v1.0 is not supported .* \(must be between v2.0 and v2.3\)", ): spack.repo._parse_package_api_version({}, min_api=(2, 0), max_api=(2, 3)) # finally test invalid input with pytest.raises(spack.repo.BadRepoError, match="Invalid Package API version"): spack.repo._parse_package_api_version({"api": "v2"}, min_api=(1, 0), max_api=(3, 3)) with pytest.raises(spack.repo.BadRepoError, match="Invalid Package API version"): spack.repo._parse_package_api_version({"api": 2.0}, min_api=(1, 0), max_api=(3, 3)) def test_repo_package_api_version(tmp_path: pathlib.Path): """Test that we can specify the API version of a repository.""" (tmp_path / "example" / "packages").mkdir(parents=True) (tmp_path / "example" / "repo.yaml").write_text( """\ repo: namespace: example """ ) cache = spack.util.file_cache.FileCache(tmp_path / "cache") assert spack.repo.Repo(str(tmp_path / "example"), cache=cache).package_api == (1, 0) def test_mod_to_pkg_name_and_reverse(): # In repo v1 the dirname/module name is the package name assert spack.util.naming.pkg_dir_to_pkg_name("zlib_ng", package_api=(1, 0)) == "zlib_ng" assert ( spack.util.naming.pkg_dir_to_pkg_name("_3example_4", package_api=(1, 0)) == "_3example_4" ) assert spack.util.naming.pkg_name_to_pkg_dir("zlib_ng", package_api=(1, 0)) == "zlib_ng" assert ( spack.util.naming.pkg_name_to_pkg_dir("_3example_4", package_api=(1, 0)) == "_3example_4" ) # In repo v2 there is a 1-1 mapping between module and package names assert spack.util.naming.pkg_dir_to_pkg_name("_3example_4", package_api=(2, 0)) == "3example-4" assert spack.util.naming.pkg_dir_to_pkg_name("zlib_ng", package_api=(2, 0)) == "zlib-ng" assert spack.util.naming.pkg_name_to_pkg_dir("zlib-ng", package_api=(2, 0)) == "zlib_ng" assert spack.util.naming.pkg_name_to_pkg_dir("3example-4", package_api=(2, 0)) == "_3example_4" # reserved names need an underscore assert spack.util.naming.pkg_dir_to_pkg_name("_finally", package_api=(2, 0)) == "finally" assert spack.util.naming.pkg_dir_to_pkg_name("_assert", package_api=(2, 0)) == "assert" assert spack.util.naming.pkg_name_to_pkg_dir("finally", package_api=(2, 0)) == "_finally" assert spack.util.naming.pkg_name_to_pkg_dir("assert", package_api=(2, 0)) == "_assert" # reserved names are case sensitive, so true/false/none are ok assert spack.util.naming.pkg_dir_to_pkg_name("true", package_api=(2, 0)) == "true" assert spack.util.naming.pkg_dir_to_pkg_name("none", package_api=(2, 0)) == "none" assert spack.util.naming.pkg_name_to_pkg_dir("true", package_api=(2, 0)) == "true" assert spack.util.naming.pkg_name_to_pkg_dir("none", package_api=(2, 0)) == "none" def test_repo_v2_invalid_module_name(tmp_path: pathlib.Path, capfd): # Create a repo with a v2 structure root, _ = spack.repo.create_repo(str(tmp_path), namespace="repo_1", package_api=(2, 0)) repo_dir = pathlib.Path(root) # Create two invalid module names (repo_dir / "packages" / "zlib-ng").mkdir() (repo_dir / "packages" / "zlib-ng" / "package.py").write_text( """ from spack.package import PackageBase class ZlibNg(PackageBase): pass """ ) (repo_dir / "packages" / "UPPERCASE").mkdir() (repo_dir / "packages" / "UPPERCASE" / "package.py").write_text( """ from spack.package import PackageBase class Uppercase(PackageBase): pass """ ) with spack.repo.use_repositories(str(repo_dir)) as repo: assert len(repo.all_package_names()) == 0 stderr = capfd.readouterr().err assert "cannot be used because `zlib-ng` is not a valid Spack package module name" in stderr assert "cannot be used because `UPPERCASE` is not a valid Spack package module name" in stderr def test_repo_v2_module_and_class_to_package_name(tmp_path: pathlib.Path): # Create a repo with a v2 structure root, _ = spack.repo.create_repo(str(tmp_path), namespace="repo_2", package_api=(2, 0)) repo_dir = pathlib.Path(root) # Create an invalid module name (repo_dir / "packages" / "_1example_2_test").mkdir() (repo_dir / "packages" / "_1example_2_test" / "package.py").write_text( """ from spack.package import PackageBase class _1example2Test(PackageBase): pass """ ) with spack.repo.use_repositories(str(repo_dir)) as repo: assert repo.exists("1example-2-test") pkg_cls = repo.get_pkg_class("1example-2-test") assert pkg_cls.name == "1example-2-test" assert pkg_cls.module.__name__ == "spack_repo.repo_2.packages._1example_2_test.package" def test_valid_module_name_v2(): api = (2, 0) # no hyphens assert not valid_module_name("zlib-ng", api) # cannot start with a number assert not valid_module_name("7zip", api) # no consecutive underscores assert not valid_module_name("zlib__ng", api) # reserved names assert not valid_module_name("finally", api) assert not valid_module_name("assert", api) # cannot contain uppercase assert not valid_module_name("False", api) assert not valid_module_name("zlib_NG", api) # reserved names are allowed when preceded by underscore assert valid_module_name("_finally", api) assert valid_module_name("_assert", api) # digits are allowed when preceded by underscore assert valid_module_name("_1example_2_test", api) # underscore is not allowed unless followed by reserved name or digit assert not valid_module_name("_zlib", api) assert not valid_module_name("_false", api) def test_namespace_is_optional_in_v2(tmp_path: pathlib.Path): """Test that a repo without a namespace is valid in v2.""" repo_yaml_dir = tmp_path / "spack_repo" / "foo" / "bar" / "baz" (repo_yaml_dir / "packages").mkdir(parents=True) (repo_yaml_dir / "repo.yaml").write_text( """\ repo: api: v2.0 """ ) cache = spack.util.file_cache.FileCache(tmp_path / "cache") repo = spack.repo.Repo(str(repo_yaml_dir), cache=cache) assert repo.namespace == "foo.bar.baz" assert repo.full_namespace == "spack_repo.foo.bar.baz.packages" assert repo.root == str(repo_yaml_dir) assert repo.packages_path == str(repo_yaml_dir / "packages") assert repo.python_path == str(tmp_path) assert repo.package_api == (2, 0) def test_subdir_in_v2(): """subdir cannot be . or empty in v2, because otherwise we cannot statically distinguish between namespace and subdir.""" with pytest.raises(spack.repo.BadRepoError, match="Use a symlink packages -> . instead"): spack.repo._validate_and_normalize_subdir(subdir="", root="root", package_api=(2, 0)) with pytest.raises(spack.repo.BadRepoError, match="Use a symlink packages -> . instead"): spack.repo._validate_and_normalize_subdir(subdir=".", root="root", package_api=(2, 0)) with pytest.raises(spack.repo.BadRepoError, match="Expected a directory name, not a path"): subdir = os.path.join("a", "b") spack.repo._validate_and_normalize_subdir(subdir=subdir, root="root", package_api=(2, 0)) with pytest.raises(spack.repo.BadRepoError, match="Must be a valid Python module name"): spack.repo._validate_and_normalize_subdir(subdir="123", root="root", package_api=(2, 0)) def test_is_package_module(): assert spack.repo.is_package_module("spack.pkg.something.something") assert spack.repo.is_package_module("spack_repo.foo.bar.baz.package") assert not spack.repo.is_package_module("spack_repo.builtin.build_systems.cmake") assert not spack.repo.is_package_module("spack.something.else") def test_environment_activation_updates_repo_path(tmp_path: pathlib.Path): """Test that the environment activation updates the repo path correctly.""" repo_root, _ = spack.repo.create_repo(str(tmp_path / "foo"), namespace="bar") (tmp_path / "spack.yaml").write_text( """\ spack: repos: bar: $env/foo/spack_repo/bar """ ) env = spack.environment.Environment(tmp_path) with env: assert any(os.path.samefile(repo_root, r.root) for r in spack.repo.PATH.repos) assert not any(os.path.samefile(repo_root, r.root) for r in spack.repo.PATH.repos) with env: assert any(os.path.samefile(repo_root, r.root) for r in spack.repo.PATH.repos) assert not any(os.path.samefile(repo_root, r.root) for r in spack.repo.PATH.repos) def test_repo_update(tmp_path: pathlib.Path): existing_root, _ = spack.repo.create_repo(str(tmp_path), namespace="foo") nonexisting_root = str(tmp_path / "nonexisting") config = {"repos": [existing_root, nonexisting_root]} assert spack.schema.repos.update(config) assert config["repos"] == { "foo": existing_root # non-existing root is removed for simplicity; would be a warning otherwise. } def test_mock_builtin_repo(mock_packages): assert spack.repo.builtin_repo() is spack.repo.PATH.get_repo("builtin_mock") def test_parse_config_descriptor_git_1(tmp_path: pathlib.Path): descriptor = spack.repo.parse_config_descriptor( name="name", descriptor={ "git": str(tmp_path / "repo.git"), "destination": str(tmp_path / "some/destination"), }, lock=spack.util.lock.Lock(str(tmp_path / "x"), enable=False), ) assert isinstance(descriptor, spack.repo.RemoteRepoDescriptor) assert descriptor.name == "name" assert descriptor.repository == str(tmp_path / "repo.git") assert descriptor.destination == str(tmp_path / "some/destination") assert descriptor.relative_paths is None def test_parse_config_descriptor_git_2(tmp_path: pathlib.Path): descriptor = spack.repo.parse_config_descriptor( name="name", descriptor={"git": str(tmp_path / "repo.git"), "paths": ["some/path"]}, lock=spack.util.lock.Lock(str(tmp_path / "x"), enable=False), ) assert isinstance(descriptor, spack.repo.RemoteRepoDescriptor) assert descriptor.relative_paths == ["some/path"] def test_remote_descriptor_no_git(tmp_path: pathlib.Path): """Test that descriptor fails without git.""" descriptor = spack.repo.parse_config_descriptor( name="name", descriptor={ "git": str(tmp_path / "repo.git"), "destination": str(tmp_path / "some/destination"), }, lock=spack.util.lock.Lock(str(tmp_path / "x"), enable=False), ) descriptor.initialize(fetch=True, git=None) assert isinstance(descriptor, spack.repo.RemoteRepoDescriptor) assert descriptor.error == "Git executable not found" def test_remote_descriptor_update_no_git(tmp_path: pathlib.Path): """Test that descriptor fails without git.""" descriptor = spack.repo.parse_config_descriptor( name="name", descriptor={ "git": str(tmp_path / "repo.git"), "destination": str(tmp_path / "some/destination"), }, lock=spack.util.lock.Lock(str(tmp_path / "x"), enable=False), ) assert isinstance(descriptor, spack.repo.RemoteRepoDescriptor) with pytest.raises(spack.repo.RepoError, match="Git executable not found"): descriptor.update(git=None) def test_parse_config_descriptor_local(tmp_path: pathlib.Path): descriptor = spack.repo.parse_config_descriptor( name="name", descriptor=str(tmp_path / "local_repo"), lock=spack.util.lock.Lock(str(tmp_path / "x"), enable=False), ) assert isinstance(descriptor, spack.repo.LocalRepoDescriptor) assert descriptor.name == "name" assert descriptor.path == str(tmp_path / "local_repo") def test_parse_config_descriptor_no_git(tmp_path: pathlib.Path): """Test that we can parse a descriptor without a git key.""" with pytest.raises(RuntimeError, match="Invalid configuration for repository"): spack.repo.parse_config_descriptor( name="name", descriptor={"destination": str(tmp_path / "some/destination"), "paths": ["some/path"]}, lock=spack.util.lock.Lock(str(tmp_path / "x"), enable=False), ) def test_repo_descriptors_construct(tmp_path: pathlib.Path): """Test the RepoDescriptors construct function. Ensure it does not raise when we cannot construct a Repo instance, e.g. due to missing repo.yaml file. Check that it parses the spack-repo-index.yaml file both when newly initialized and when already cloned.""" lock = spack.util.lock.Lock(str(tmp_path / "x"), enable=False) cache = spack.util.file_cache.FileCache(str(tmp_path / "cache")) # Construct 3 identical descriptors descriptors_1, descriptors_2, descriptors_3 = [ { "foo": spack.repo.RemoteRepoDescriptor( name="foo", repository=str(tmp_path / "foo.git"), destination=str(tmp_path / "foo_destination"), branch=None, tag=None, commit=None, relative_paths=None, lock=lock, ) } for _ in range(3) ] repos_1 = spack.repo.RepoDescriptors(descriptors_1) # type: ignore repos_2 = spack.repo.RepoDescriptors(descriptors_2) # type: ignore repos_3 = spack.repo.RepoDescriptors(descriptors_3) # type: ignore class MockGit(spack.util.executable.Executable): def __init__(self): pass def __call__(self, *args, **kwargs) -> str: # type: ignore action = args[0] if action == "ls-remote": return """\ a8eff4da7aab59bbf5996ac1720954bf82443247 HEAD 165c479984b94051c982a6be1bd850f8bae02858 refs/heads/feature-branch a8eff4da7aab59bbf5996ac1720954bf82443247 refs/heads/develop 3bd0276ab0491552247fa055921a23d2ffd9443c refs/heads/releases/v0.20""" elif action == "rev-parse": return "develop" elif action == "config": return "origin" elif action == "init": # The git repo needs a .git subdir os.makedirs(os.path.join(".git")) elif action == "checkout": # The spack-repo-index.yaml is optional; we test Spack reads from it. with open(os.path.join("spack-repo-index.yaml"), "w", encoding="utf-8") as f: f.write( """\ repo_index: paths: - spack_repo/foo """ ) return "" repo_path_1, errors_1 = repos_1.construct(cache=cache, find_git=MockGit) # Verify it cannot construct a Repo instance, and that this does *not* throw, since that would # break Spack very early on. Instead, an error is returned. Also verify that # relative_paths is read from spack-repo-index.yaml. assert len(repo_path_1.repos) == 0 assert len(errors_1) == 1 assert all("No repo.yaml" in str(err) for err in errors_1.values()), errors_1 assert descriptors_1["foo"].relative_paths == ["spack_repo/foo"] # Verify that the default branch was detected from ls-remote assert descriptors_1["foo"].branch == "develop" # Do the same test with another instance: it should *not* clone a second time. repo_path_2, errors_2 = repos_2.construct(cache=cache, find_git=MockGit) assert len(repo_path_2.repos) == 0 assert len(errors_2) == 1 assert all("No repo.yaml" in str(err) for err in errors_2.values()), errors_2 assert descriptors_1["foo"].relative_paths == ["spack_repo/foo"] # Finally fill the repo with an actual repo and check that the repo can be constructed. spack.repo.create_repo(str(tmp_path / "foo_destination"), "foo") repo_path_3, errors_3 = repos_3.construct(cache=cache, find_git=MockGit) assert not errors_3 assert len(repo_path_3.repos) == 1 assert repo_path_3.repos[0].namespace == "foo" def test_repo_descriptors_update(tmp_path: pathlib.Path): """Test the RepoDescriptors construct function. Ensure it does not raise when we cannot construct a Repo instance, e.g. due to missing repo.yaml file. Check that it parses the spack-repo-index.yaml file both when newly initialized and when already cloned.""" lock = spack.util.lock.Lock(str(tmp_path / "x"), enable=False) cache = spack.util.file_cache.FileCache(str(tmp_path / "cache")) # Construct 3 identical descriptors descriptors_1, descriptors_2, descriptors_3, descriptors_4 = [ { "foo": spack.repo.RemoteRepoDescriptor( name="foo", repository=str(tmp_path / "foo.git"), destination=str(tmp_path / "foo_destination"), branch="develop" if i == 0 else None, tag="v1.0" if i == 1 else None, commit="abc123" if i == 2 else None, relative_paths=None, lock=lock, ) } for i in range(4) ] repos_1 = spack.repo.RepoDescriptors(descriptors_1) # type: ignore repos_2 = spack.repo.RepoDescriptors(descriptors_2) # type: ignore repos_3 = spack.repo.RepoDescriptors(descriptors_3) # type: ignore repos_4 = spack.repo.RepoDescriptors(descriptors_4) # type: ignore class MockGit(spack.util.executable.Executable): def __init__(self): pass def __call__(self, *args, **kwargs) -> str: # type: ignore action = args[0] if action == "ls-remote": return """\ a8eff4da7aab59bbf5996ac1720954bf82443247 HEAD 165c479984b94051c982a6be1bd850f8bae02858 refs/heads/feature-branch a8eff4da7aab59bbf5996ac1720954bf82443247 refs/heads/develop 3bd0276ab0491552247fa055921a23d2ffd9443c refs/heads/releases/v0.20""" elif action == "rev-parse": return "develop" elif action == "config": return "origin" elif action == "init": # The git repo needs a .git subdir os.makedirs(os.path.join(".git")) elif action == "checkout": # The spack-repo-index.yaml is optional; we test Spack reads from it. with open(os.path.join("spack-repo-index.yaml"), "w", encoding="utf-8") as f: f.write( """\ repo_index: paths: - spack_repo/foo """ ) return "" spack.repo.create_repo(str(tmp_path / "foo_destination"), "foo") # branch develop _, errors_1 = repos_1.construct(cache=cache, find_git=MockGit) assert not errors_1 for descriptor in repos_1.values(): descriptor.update(git=MockGit()) # tag v1.0 _, errors_2 = repos_2.construct(cache=cache, find_git=MockGit) assert not errors_2 for descriptor in repos_2.values(): descriptor.update(git=MockGit()) # commit abc123 _, errors_3 = repos_3.construct(cache=cache, find_git=MockGit) assert not errors_3 for descriptor in repos_3.values(): descriptor.update(git=MockGit()) # default branch _, errors_4 = repos_4.construct(cache=cache, find_git=MockGit) assert not errors_4 for descriptor in repos_4.values(): descriptor.update(git=MockGit()) # Rerun construction after initialization to test early exit logic _, errors_4 = repos_4.construct(cache=cache, find_git=MockGit) assert not errors_4 def test_repo_descriptors_update_invalid(tmp_path: pathlib.Path): """Test the RepoDescriptors construct function. Ensure it does not raise when we cannot construct a Repo instance, e.g. due to missing repo.yaml file. Check that it parses the spack-repo-index.yaml file both when newly initialized and when already cloned.""" lock = spack.util.lock.Lock(str(tmp_path / "x"), enable=False) cache = spack.util.file_cache.FileCache(str(tmp_path / "cache")) # Construct 3 identical descriptors descriptors_1 = { "foo": spack.repo.RemoteRepoDescriptor( name="foo", repository=str(tmp_path / "foo.git"), destination=str(tmp_path / "foo_destination"), branch=None, tag=None, commit=None, relative_paths=None, lock=lock, ) } repos_1 = spack.repo.RepoDescriptors(descriptors_1) # type: ignore class MockGitInvalidRemote(spack.util.executable.Executable): def __init__(self): pass def __call__(self, *args, **kwargs) -> str: # type: ignore action = args[0] if action == "ls-remote": # HEAD ref exists, but no default branch (i.e. no refs/heads/*) return "a8eff4da7aab59bbf5996ac1720954bf82443247 HEAD" return "" class MockGitFailed(spack.util.executable.Executable): def __init__(self): pass def __call__(self, *args, **kwargs) -> str: # type: ignore raise spack.util.executable.ProcessError("failed") spack.repo.create_repo(str(tmp_path / "foo_destination"), "foo") _, errors_1 = repos_1.construct(cache=cache, find_git=MockGitFailed) assert len(errors_1) == 1 assert all("Failed to clone repository" in str(err) for err in errors_1.values()), errors_1 with pytest.raises(spack.repo.RepoError, match="Unable to locate a default branch"): for descriptor in repos_1.values(): descriptor.update(git=MockGitInvalidRemote()) def test_repo_use_bad_import(config, repo_builder: RepoBuilder): """Demonstrate failure when attempt to get the class for package containing a failing import (e.g., missing repository).""" package_py = pathlib.Path(repo_builder._recipe_filename("importer")) package_py.parent.mkdir(parents=True) package_py.write_text( """\ from spack_repo.missing.packages import base from spack.package import * class Importer(PackageBase): homepage = "https://www.bad-importer.com" url = "https://www.bad-importer.com/v1.0.tar.gz" version("1.0", md5="0123456789abcdef0123456789abcdef") """, encoding="utf-8", ) with spack.repo.use_repositories(repo_builder.root): with pytest.raises(spack.repo.RepoError, match="cannot load"): spack.repo.PATH.get_pkg_class("importer") def test_repo_use_bad_syntax(config, repo_builder: RepoBuilder): """Demonstrate failure when attempt to get class for package with invalid syntax.""" package_py = pathlib.Path(repo_builder._recipe_filename("erroneous")) package_py.parent.mkdir(parents=True) package_py.write_text("class 123: pass", encoding="utf-8") with spack.repo.use_repositories(repo_builder.root): with pytest.raises(spack.repo.RepoError): spack.repo.PATH.get_pkg_class("erroneous") def test_unknownpkgerror_match_fails(): """Ensure fails with basic message when get_close_matches fails.""" def _get_close_matches(*args, **kwargs): raise MemoryError("Too many packages to compare") # Confirm that the error indicates there were no matches (default). exception = spack.repo.UnknownPackageError("pkg_a", get_close_matches=_get_close_matches) assert "mean one of the following" not in str(exception) def test_unknownpkgerror_str_repo(): """Ensure reasonable error message when repo is a string.""" assert "not found in repository" in str(spack.repo.UnknownPackageError("pkg_a", "my_repo")) ================================================ FILE: lib/spack/spack/test/reporters.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.llnl.util.filesystem as fs import spack.llnl.util.tty as tty import spack.reporters.extract from spack.install_test import TestStatus from spack.reporters import CDash, CDashConfiguration # Use a path variable to appease Spack style line length checks fake_install_prefix = fs.join_path( os.sep, "usr", "spack", "spack", "opt", "spack", "linux-rhel7-broadwell", "intel-19.0.4.227", "fake-1.0", ) fake_install_test_root = fs.join_path(fake_install_prefix, ".spack", "test") fake_test_cache = fs.join_path( "usr", "spack", ".spack", "test", "abcdefg", "fake-1.0-abcdefg", "cache", "fake" ) def test_reporters_extract_basics(): # This test has a description, command, and status fake_bin = fs.join_path(fake_install_prefix, "bin", "fake") name = "test_no_status" desc = "basic description" status = TestStatus.PASSED outputs = """ ==> Testing package fake-1.0-abcdefg ==> [2022-02-15-18:44:21.250165] test: {0}: {1} ==> [2022-02-15-18:44:21.250200] '{2}' {3}: {0} """.format(name, desc, fake_bin, status).splitlines() parts = spack.reporters.extract.extract_test_parts("fake", outputs) assert len(parts) == 1 assert parts[0]["command"] == "{0}".format(fake_bin) assert parts[0]["desc"] == desc assert parts[0]["loglines"] == ["{0}: {1}".format(status, name)] assert parts[0]["status"] == status.lower() def test_reporters_extract_no_parts(capfd): # This test ticks three boxes: # 1) has Installing, which is skipped; # 2) does not define any test parts; # 3) has a status value without a part so generates a warning status = TestStatus.NO_TESTS outputs = """ ==> Testing package fake-1.0-abcdefg ==> [2022-02-11-17:14:38.875259] Installing {0} to {1} {2} """.format(fake_install_test_root, fake_test_cache, status).splitlines() parts = spack.reporters.extract.extract_test_parts("fake", outputs) err = capfd.readouterr()[1] assert len(parts) == 1 assert parts[0]["status"] == "notrun" assert "No part to add status" in err def test_reporters_extract_missing_desc(): # This test parts with and without descriptions *and* a test part that has # multiple commands fake_bin = fs.join_path(fake_install_prefix, "bin", "importer") names = ["test_fake_bin", "test_fake_util", "test_multiple_commands"] descs = ["", "import fake util module", ""] failed = TestStatus.FAILED passed = TestStatus.PASSED results = [passed, failed, passed] outputs = """ ==> Testing package fake-1.0-abcdefg ==> [2022-02-15-18:44:21.250165] test: {0}: {1} ==> [2022-02-15-18:44:21.250170] '{5}' '-c' 'import fake.bin' {2}: {0} ==> [2022-02-15-18:44:21.250185] test: {3}: {4} ==> [2022-02-15-18:44:21.250200] '{5}' '-c' 'import fake.util' {6}: {3} ==> [2022-02-15-18:44:21.250205] test: {7}: {8} ==> [2022-02-15-18:44:21.250210] 'exe1 1' ==> [2022-02-15-18:44:21.250250] 'exe2 2' {9}: {7} """.format( names[0], descs[0], results[0], names[1], descs[1], fake_bin, results[1], names[2], descs[2], results[2], ).splitlines() parts = spack.reporters.extract.extract_test_parts("fake", outputs) assert len(parts) == 3 for i, (name, desc, status) in enumerate(zip(names, descs, results)): assert parts[i]["name"] == name assert parts[i]["desc"] == desc assert parts[i]["status"] == status.lower() assert parts[2]["command"] == "exe1 1; exe2 2" @pytest.mark.parametrize("state", [("not installed"), ("external")]) def test_reporters_extract_skipped(state): expected = "Skipped {0} package".format(state) outputs = """ ==> Testing package fake-1.0-abcdefg {0} """.format(expected).splitlines() parts = spack.reporters.extract.extract_test_parts("fake", outputs) assert len(parts) == 1 assert parts[0]["completed"] == spack.reporters.extract.completed["skipped"] def test_reporters_skip_new(): outputs = """ ==> [2023-04-06-15:55:13.094025] test: test_skip: SKIPPED: test_skip: Package must be built with +python ==> [2023-04-06-15:55:13.540029] Completed testing ==> [2023-04-06-15:55:13.540275] ======================= SUMMARY: fake-1.0-abcdefg ======================== fake::test_skip .. SKIPPED =========================== 1 skipped of 1 part ========================== """.splitlines() parts = spack.reporters.extract.extract_test_parts("fake", outputs) assert len(parts) == 1 part = parts[0] assert part["name"] == "test_skip" assert part["status"] == "skipped" assert part["completed"] == "Completed" assert part["loglines"][0].startswith("SKIPPED:") def test_reporters_report_for_package_no_stdout(tmp_path: pathlib.Path, monkeypatch, capfd): class MockCDash(CDash): def upload(*args, **kwargs): # Just return (Do NOT try to upload the report to the fake site) return configuration = CDashConfiguration( upload_url="https://fake-upload", packages="fake-package", build="fake-cdash-build", site="fake-site", buildstamp=None, track="fake-track", ) monkeypatch.setattr(tty, "_debug", 1) reporter = MockCDash(configuration=configuration) pkg_data = {"name": "fake-package"} reporter.test_report_for_package(str(tmp_path), pkg_data, 0) err = capfd.readouterr()[1] assert "Skipping report for" in err assert "No generated output" in err def test_cdash_reporter_truncates_build_name_if_too_long(): build_name = "a" * 190 extra_long_build_name = build_name + "a" configuration = CDashConfiguration( upload_url="https://fake-upload", packages="fake-package", build=extra_long_build_name, site="fake-site", buildstamp=None, track="fake-track", ) reporter = CDash(configuration=configuration) new_build_name = reporter.report_build_name("fake-package") assert new_build_name != extra_long_build_name assert new_build_name == build_name ================================================ FILE: lib/spack/spack/test/rewiring.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp import os import sys import pytest import spack.concretize import spack.deptypes as dt import spack.rewiring import spack.store from spack.installer import PackageInstaller from spack.test.relocate import text_in_bin if sys.platform == "darwin": required_executables = ["/usr/bin/clang++", "install_name_tool"] else: required_executables = ["g++", "patchelf"] def check_spliced_spec_prefixes(spliced_spec): """check the file in the prefix has the correct paths""" for node in spliced_spec.traverse(root=True): text_file_path = os.path.join(node.prefix, node.name) with open(text_file_path, "r", encoding="utf-8") as f: text = f.read() for modded_spec in node.traverse(root=True, deptype=dt.ALL & ~dt.BUILD): assert modded_spec.prefix in text @pytest.mark.requires_executables(*required_executables) @pytest.mark.parametrize("transitive", [True, False]) def test_rewire_db(mock_fetch, install_mockery, transitive): """Tests basic rewiring without binary executables.""" spec = spack.concretize.concretize_one("splice-t^splice-h~foo") dep = spack.concretize.concretize_one("splice-h+foo") PackageInstaller([spec.package, dep.package], explicit=True).install() spliced_spec = spec.splice(dep, transitive=transitive) assert spec.dag_hash() != spliced_spec.dag_hash() spack.rewiring.rewire(spliced_spec) # check that the prefix exists assert os.path.exists(spliced_spec.prefix) # test that it made it into the database rec = spack.store.STORE.db.get_record(spliced_spec) installed_in_db = rec.installed if rec else False assert installed_in_db # check for correct prefix paths check_spliced_spec_prefixes(spliced_spec) @pytest.mark.requires_executables(*required_executables) @pytest.mark.parametrize("transitive", [True, False]) def test_rewire_bin(mock_fetch, install_mockery, transitive): """Tests basic rewiring with binary executables.""" spec = spack.concretize.concretize_one("quux") dep = spack.concretize.concretize_one("garply cflags=-g") PackageInstaller([spec.package, dep.package], explicit=True).install() spliced_spec = spec.splice(dep, transitive=transitive) assert spec.dag_hash() != spliced_spec.dag_hash() spack.rewiring.rewire(spliced_spec) # check that the prefix exists assert os.path.exists(spliced_spec.prefix) # test that it made it into the database rec = spack.store.STORE.db.get_record(spliced_spec) installed_in_db = rec.installed if rec else False assert installed_in_db # check the file in the prefix has the correct paths bin_names = {"garply": "garplinator", "corge": "corgegator", "quux": "quuxifier"} for node in spliced_spec.traverse(root=True): for dep in node.traverse(root=True): bin_file_path = os.path.join(dep.prefix.bin, bin_names[dep.name]) assert text_in_bin(dep.prefix, bin_file_path) @pytest.mark.requires_executables(*required_executables) def test_rewire_writes_new_metadata(mock_fetch, install_mockery): """Tests that new metadata was written during a rewire. Accuracy of metadata is left to other tests.""" spec = spack.concretize.concretize_one("quux") dep = spack.concretize.concretize_one("garply cflags=-g") PackageInstaller([spec.package, dep.package], explicit=True).install() spliced_spec = spec.splice(dep, transitive=True) spack.rewiring.rewire(spliced_spec) # test install manifests for node in spliced_spec.traverse(root=True): spack.store.STORE.layout.ensure_installed(node) manifest_file_path = os.path.join( node.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name, ) assert os.path.exists(manifest_file_path) orig_node = spec[node.name] if node == orig_node: continue orig_manifest_file_path = os.path.join( orig_node.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name, ) assert os.path.exists(orig_manifest_file_path) assert not filecmp.cmp(orig_manifest_file_path, manifest_file_path, shallow=False) specfile_path = os.path.join( node.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.spec_file_name, ) assert os.path.exists(specfile_path) orig_specfile_path = os.path.join( orig_node.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.spec_file_name, ) assert os.path.exists(orig_specfile_path) assert not filecmp.cmp(orig_specfile_path, specfile_path, shallow=False) @pytest.mark.requires_executables(*required_executables) @pytest.mark.parametrize("transitive", [True, False]) def test_uninstall_rewired_spec(mock_fetch, install_mockery, transitive): """Test that rewired packages can be uninstalled as normal.""" spec = spack.concretize.concretize_one("quux") dep = spack.concretize.concretize_one("garply cflags=-g") PackageInstaller([spec.package, dep.package], explicit=True).install() spliced_spec = spec.splice(dep, transitive=transitive) spack.rewiring.rewire(spliced_spec) spliced_spec.package.do_uninstall() assert len(spack.store.STORE.db.query(spliced_spec)) == 0 assert not os.path.exists(spliced_spec.prefix) @pytest.mark.requires_executables(*required_executables) def test_rewire_not_installed_fails(mock_fetch, install_mockery): """Tests error when an attempt is made to rewire a package that was not previously installed.""" spec = spack.concretize.concretize_one("quux") dep = spack.concretize.concretize_one("garply cflags=-g") spliced_spec = spec.splice(dep, False) with pytest.raises( spack.rewiring.PackageNotInstalledError, match="failed due to missing install of build spec", ): spack.rewiring.rewire(spliced_spec) def test_rewire_virtual(mock_fetch, install_mockery): """Check installed package can successfully splice an alternate virtual implementation""" dep = "splice-a" alt_dep = "splice-h" spec = spack.concretize.concretize_one(f"splice-vt^{dep}") alt_spec = spack.concretize.concretize_one(alt_dep) PackageInstaller([spec.package, alt_spec.package]).install() spliced_spec = spec.splice(alt_spec, True) spack.rewiring.rewire(spliced_spec) # Confirm the original spec still has the original virtual implementation. assert spec.satisfies(f"^{dep}") # Confirm the spliced spec uses the new virtual implementation. assert spliced_spec.satisfies(f"^{alt_dep}") # check for correct prefix paths check_spliced_spec_prefixes(spliced_spec) ================================================ FILE: lib/spack/spack/test/s3_fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import spack.fetch_strategy as spack_fs import spack.stage as spack_stage def test_s3fetchstrategy_downloaded(tmp_path: pathlib.Path): """Ensure fetch with archive file already downloaded is a noop.""" archive = tmp_path / "s3.tar.gz" class Archived_S3FS(spack_fs.S3FetchStrategy): @property def archive_file(self): return archive fetcher = Archived_S3FS(url="s3://example/s3.tar.gz") with spack_stage.Stage(fetcher, path=str(tmp_path)): fetcher.fetch() ================================================ FILE: lib/spack/spack/test/sbang.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """\ Test that Spack's shebang filtering works correctly. """ import filecmp import os import pathlib import shutil import stat import sys import tempfile import pytest import spack.config import spack.hooks.sbang as sbang import spack.llnl.util.filesystem as fs import spack.store import spack.util.spack_yaml as syaml from spack.util.executable import which if sys.platform != "win32": import grp pytestmark = pytest.mark.not_on_windows("does not run on windows") too_long = sbang.system_shebang_limit + 1 short_line = "#!/this/is/short/bin/bash\n" long_line = "#!/this/" + ("x" * too_long) + "/is/long\n" lua_line = "#!/this/" + ("x" * too_long) + "/is/lua\n" lua_in_text = ("line\n") * 100 + "lua\n" + ("line\n" * 100) lua_line_patched = "--!/this/" + ("x" * too_long) + "/is/lua\n" luajit_line = "#!/this/" + ("x" * too_long) + "/is/luajit\n" luajit_in_text = ("line\n") * 100 + "lua\n" + ("line\n" * 100) luajit_line_patched = "--!/this/" + ("x" * too_long) + "/is/luajit\n" node_line = "#!/this/" + ("x" * too_long) + "/is/node\n" node_in_text = ("line\n") * 100 + "lua\n" + ("line\n" * 100) node_line_patched = "//!/this/" + ("x" * too_long) + "/is/node\n" php_line = "#!/this/" + ("x" * too_long) + "/is/php\n" php_in_text = ("line\n") * 100 + "php\n" + ("line\n" * 100) php_line_patched = "\n" last_line = "last!\n" @pytest.fixture # type: ignore[no-redef] def sbang_line(): yield "#!/bin/sh %s/bin/sbang\n" % spack.store.STORE.layout.root class ScriptDirectory: """Directory full of test scripts to run sbang instrumentation on.""" def __init__(self, sbang_line): self.tempdir = tempfile.mkdtemp() self.directory = os.path.join(self.tempdir, "dir") fs.mkdirp(self.directory) # Script with short shebang self.short_shebang = os.path.join(self.tempdir, "short") with open(self.short_shebang, "w", encoding="utf-8") as f: f.write(short_line) f.write(last_line) self.make_executable(self.short_shebang) # Script with long shebang self.long_shebang = os.path.join(self.tempdir, "long") with open(self.long_shebang, "w", encoding="utf-8") as f: f.write(long_line) f.write(last_line) self.make_executable(self.long_shebang) # Non-executable script with long shebang self.nonexec_long_shebang = os.path.join(self.tempdir, "nonexec_long") with open(self.nonexec_long_shebang, "w", encoding="utf-8") as f: f.write(long_line) f.write(last_line) # Lua script with long shebang self.lua_shebang = os.path.join(self.tempdir, "lua") with open(self.lua_shebang, "w", encoding="utf-8") as f: f.write(lua_line) f.write(last_line) self.make_executable(self.lua_shebang) # Lua occurring in text, not in shebang self.lua_textbang = os.path.join(self.tempdir, "lua_in_text") with open(self.lua_textbang, "w", encoding="utf-8") as f: f.write(short_line) f.write(lua_in_text) f.write(last_line) self.make_executable(self.lua_textbang) # Luajit script with long shebang self.luajit_shebang = os.path.join(self.tempdir, "luajit") with open(self.luajit_shebang, "w", encoding="utf-8") as f: f.write(luajit_line) f.write(last_line) self.make_executable(self.luajit_shebang) # Luajit occurring in text, not in shebang self.luajit_textbang = os.path.join(self.tempdir, "luajit_in_text") with open(self.luajit_textbang, "w", encoding="utf-8") as f: f.write(short_line) f.write(luajit_in_text) f.write(last_line) self.make_executable(self.luajit_textbang) # Node script with long shebang self.node_shebang = os.path.join(self.tempdir, "node") with open(self.node_shebang, "w", encoding="utf-8") as f: f.write(node_line) f.write(last_line) self.make_executable(self.node_shebang) # Node occurring in text, not in shebang self.node_textbang = os.path.join(self.tempdir, "node_in_text") with open(self.node_textbang, "w", encoding="utf-8") as f: f.write(short_line) f.write(node_in_text) f.write(last_line) self.make_executable(self.node_textbang) # php script with long shebang self.php_shebang = os.path.join(self.tempdir, "php") with open(self.php_shebang, "w", encoding="utf-8") as f: f.write(php_line) f.write(last_line) self.make_executable(self.php_shebang) # php occurring in text, not in shebang self.php_textbang = os.path.join(self.tempdir, "php_in_text") with open(self.php_textbang, "w", encoding="utf-8") as f: f.write(short_line) f.write(php_in_text) f.write(last_line) self.make_executable(self.php_textbang) # Script already using sbang. self.has_sbang = os.path.join(self.tempdir, "shebang") with open(self.has_sbang, "w", encoding="utf-8") as f: f.write(sbang_line) f.write(long_line) f.write(last_line) self.make_executable(self.has_sbang) # Fake binary file. self.binary = os.path.join(self.tempdir, "binary") tar = which("tar", required=True) tar("czf", self.binary, self.has_sbang) self.make_executable(self.binary) def destroy(self): shutil.rmtree(self.tempdir, ignore_errors=True) def make_executable(self, path): # make a file executable st = os.stat(path) executable_mode = st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH os.chmod(path, executable_mode) st = os.stat(path) assert oct(executable_mode) == oct(st.st_mode & executable_mode) @pytest.fixture def script_dir(sbang_line): sdir = ScriptDirectory(sbang_line) yield sdir sdir.destroy() @pytest.mark.parametrize( "shebang,interpreter", [ (b"#!/path/to/interpreter argument\n", b"/path/to/interpreter"), (b"#! /path/to/interpreter truncated-argum", b"/path/to/interpreter"), (b"#! \t \t/path/to/interpreter\t \targument", b"/path/to/interpreter"), (b"#! \t \t /path/to/interpreter", b"/path/to/interpreter"), (b"#!/path/to/interpreter\0", b"/path/to/interpreter"), (b"#!/path/to/interpreter multiple args\n", b"/path/to/interpreter"), (b"#!\0/path/to/interpreter arg\n", None), (b"#!\n/path/to/interpreter arg\n", None), (b"#!", None), ], ) def test_shebang_interpreter_regex(shebang, interpreter): assert sbang.get_interpreter(shebang) == interpreter def test_shebang_handling(script_dir, sbang_line): sbang.filter_shebangs_in_directory(script_dir.tempdir) # Make sure this is untouched with open(script_dir.short_shebang, "r", encoding="utf-8") as f: assert f.readline() == short_line assert f.readline() == last_line # Make sure this got patched. with open(script_dir.long_shebang, "r", encoding="utf-8") as f: assert f.readline() == sbang_line assert f.readline() == long_line assert f.readline() == last_line # Make sure this is untouched with open(script_dir.nonexec_long_shebang, "r", encoding="utf-8") as f: assert f.readline() == long_line assert f.readline() == last_line # Make sure this got patched. with open(script_dir.lua_shebang, "r", encoding="utf-8") as f: assert f.readline() == sbang_line assert f.readline() == lua_line_patched assert f.readline() == last_line # Make sure this got patched. with open(script_dir.luajit_shebang, "r", encoding="utf-8") as f: assert f.readline() == sbang_line assert f.readline() == luajit_line_patched assert f.readline() == last_line # Make sure this got patched. with open(script_dir.node_shebang, "r", encoding="utf-8") as f: assert f.readline() == sbang_line assert f.readline() == node_line_patched assert f.readline() == last_line assert filecmp.cmp(script_dir.lua_textbang, os.path.join(script_dir.tempdir, "lua_in_text")) assert filecmp.cmp( script_dir.luajit_textbang, os.path.join(script_dir.tempdir, "luajit_in_text") ) assert filecmp.cmp(script_dir.node_textbang, os.path.join(script_dir.tempdir, "node_in_text")) assert filecmp.cmp(script_dir.php_textbang, os.path.join(script_dir.tempdir, "php_in_text")) # Make sure this is untouched with open(script_dir.has_sbang, "r", encoding="utf-8") as f: assert f.readline() == sbang_line assert f.readline() == long_line assert f.readline() == last_line def test_shebang_handles_non_writable_files(script_dir, sbang_line): # make a file non-writable st = os.stat(script_dir.long_shebang) not_writable_mode = st.st_mode & ~stat.S_IWRITE os.chmod(script_dir.long_shebang, not_writable_mode) test_shebang_handling(script_dir, sbang_line) st = os.stat(script_dir.long_shebang) assert oct(not_writable_mode) == oct(st.st_mode) @pytest.fixture(scope="function") def configure_group_perms(): # On systems with remote groups, the primary user group may be remote # and grp does not act on remote groups. # To ensure we find a group we can operate on, we get take the first group # listed which has the current user as a member. gid = fs.group_ids(os.getuid())[0] group_name = grp.getgrgid(gid).gr_name conf = syaml.load_config( """\ all: permissions: read: world write: group group: {0} """.format(group_name) ) spack.config.set("packages", conf, scope="user") yield @pytest.fixture(scope="function") def configure_user_perms(): conf = syaml.load_config( """\ all: permissions: read: world write: user """ ) spack.config.set("packages", conf, scope="user") yield def check_sbang_installation(group=False): sbang_path = sbang.sbang_install_path() sbang_bin_dir = os.path.dirname(sbang_path) assert sbang_path.startswith(spack.store.STORE.unpadded_root) assert os.path.exists(sbang_path) assert fs.is_exe(sbang_path) status = os.stat(sbang_bin_dir) mode = status.st_mode & 0o777 if group: assert mode == 0o775, "Unexpected {0}".format(oct(mode)) else: assert mode == 0o755, "Unexpected {0}".format(oct(mode)) status = os.stat(sbang_path) mode = status.st_mode & 0o777 if group: assert mode == 0o775, "Unexpected {0}".format(oct(mode)) else: assert mode == 0o755, "Unexpected {0}".format(oct(mode)) def run_test_install_sbang(group): sbang_path = sbang.sbang_install_path() sbang_bin_dir = os.path.dirname(sbang_path) assert sbang_path.startswith(spack.store.STORE.unpadded_root) assert not os.path.exists(sbang_bin_dir) sbang.install_sbang() check_sbang_installation(group) # put an invalid file in for sbang fs.mkdirp(sbang_bin_dir) with open(sbang_path, "w", encoding="utf-8") as f: f.write("foo") sbang.install_sbang() check_sbang_installation(group) # install again and make sure sbang is still fine sbang.install_sbang() check_sbang_installation(group) def test_install_group_sbang(install_mockery, configure_group_perms): run_test_install_sbang(True) def test_install_user_sbang(install_mockery, configure_user_perms): run_test_install_sbang(False) def test_install_sbang_too_long(tmp_path: pathlib.Path): root = str(tmp_path) num_extend = sbang.system_shebang_limit - len(root) - len("/bin/sbang") long_path = root while num_extend > 1: add = min(num_extend, 255) long_path = os.path.join(long_path, "e" * add) num_extend -= add with spack.store.use_store(long_path): with pytest.raises(sbang.SbangPathError) as exc_info: sbang.sbang_install_path() err = str(exc_info.value) assert "root is too long" in err assert "exceeds limit" in err assert "cannot patch" in err def test_sbang_hook_skips_nonexecutable_blobs(tmp_path: pathlib.Path): # Write a binary blob to non-executable.sh, with a long interpreter "path" # consisting of invalid UTF-8. The latter is technically not really necessary for # the test, but binary blobs accidentally starting with b'#!' usually do not contain # valid UTF-8, so we also ensure that Spack does not attempt to decode as UTF-8. contents = b"#!" + b"\x80" * sbang.system_shebang_limit file = str(tmp_path / "non-executable.sh") with open(file, "wb") as f: f.write(contents) sbang.filter_shebangs_in_directory(str(tmp_path)) # Make sure there is no sbang shebang. with open(file, "rb") as f: assert b"sbang" not in f.readline() def test_sbang_handles_non_utf8_files(tmp_path: pathlib.Path): # We have an executable with a copyright sign as filename contents = b"#!" + b"\xa9" * sbang.system_shebang_limit + b"\nand another symbol: \xa9" # Make sure it's indeed valid latin1 but invalid utf-8. assert contents.decode("latin1") with pytest.raises(UnicodeDecodeError): contents.decode("utf-8") # Put it in an executable file file = str(tmp_path / "latin1.sh") with open(file, "wb") as f: f.write(contents) # Run sbang assert sbang.filter_shebang(file) with open(file, "rb") as f: new_contents = f.read() assert contents in new_contents assert b"sbang" in new_contents @pytest.fixture def shebang_limits_system_8_spack_16(): system_limit, sbang.system_shebang_limit = sbang.system_shebang_limit, 8 spack_limit, sbang.spack_shebang_limit = sbang.spack_shebang_limit, 16 yield sbang.system_shebang_limit = system_limit sbang.spack_shebang_limit = spack_limit def test_shebang_exceeds_spack_shebang_limit( shebang_limits_system_8_spack_16, tmp_path: pathlib.Path ): """Tests whether shebangs longer than Spack's limit are skipped""" file = str(tmp_path / "longer_than_spack_limit.sh") with open(file, "wb") as f: f.write(b"#!" + b"x" * sbang.spack_shebang_limit) # Then Spack shouldn't try to add a shebang assert not sbang.filter_shebang(file) with open(file, "rb") as f: assert b"sbang" not in f.read() def test_sbang_hook_handles_non_writable_files_preserving_permissions(tmp_path: pathlib.Path): path = str(tmp_path / "file.sh") with open(path, "w", encoding="utf-8") as f: f.write(long_line) os.chmod(path, 0o555) sbang.filter_shebang(path) with open(path, "r", encoding="utf-8") as f: assert "sbang" in f.readline() assert os.stat(path).st_mode & 0o777 == 0o555 ================================================ FILE: lib/spack/spack/test/schema.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import importlib import os import pytest from spack.vendor import jsonschema import spack.schema import spack.schema.env import spack.util.spack_yaml as syaml from spack.llnl.util.lang import list_modules _draft_07_with_spack_extensions = { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "http://json-schema.org/draft-07/schema#", "title": "Core schema meta-schema", "definitions": { "schemaArray": {"type": "array", "minItems": 1, "items": {"$ref": "#"}}, "nonNegativeInteger": {"type": "integer", "minimum": 0}, "nonNegativeIntegerDefault0": { "allOf": [{"$ref": "#/definitions/nonNegativeInteger"}, {"default": 0}] }, "simpleTypes": { "enum": ["array", "boolean", "integer", "null", "number", "object", "string"] }, "stringArray": { "type": "array", "items": {"type": "string"}, "uniqueItems": True, "default": [], }, }, "type": ["object", "boolean"], "properties": { "$id": {"type": "string", "format": "uri-reference"}, "$schema": {"type": "string", "format": "uri"}, "$ref": {"type": "string", "format": "uri-reference"}, "$comment": {"type": "string"}, "title": {"type": "string"}, "description": {"type": "string"}, "default": True, "readOnly": {"type": "boolean", "default": False}, "writeOnly": {"type": "boolean", "default": False}, "examples": {"type": "array", "items": True}, "multipleOf": {"type": "number", "exclusiveMinimum": 0}, "maximum": {"type": "number"}, "exclusiveMaximum": {"type": "number"}, "minimum": {"type": "number"}, "exclusiveMinimum": {"type": "number"}, "maxLength": {"$ref": "#/definitions/nonNegativeInteger"}, "minLength": {"$ref": "#/definitions/nonNegativeIntegerDefault0"}, "pattern": {"type": "string", "format": "regex"}, "additionalItems": {"$ref": "#"}, "items": { "anyOf": [{"$ref": "#"}, {"$ref": "#/definitions/schemaArray"}], "default": True, }, "maxItems": {"$ref": "#/definitions/nonNegativeInteger"}, "minItems": {"$ref": "#/definitions/nonNegativeIntegerDefault0"}, "uniqueItems": {"type": "boolean", "default": False}, "contains": {"$ref": "#"}, "maxProperties": {"$ref": "#/definitions/nonNegativeInteger"}, "minProperties": {"$ref": "#/definitions/nonNegativeIntegerDefault0"}, "required": {"$ref": "#/definitions/stringArray"}, "additionalProperties": {"$ref": "#"}, "definitions": {"type": "object", "additionalProperties": {"$ref": "#"}, "default": {}}, "properties": {"type": "object", "additionalProperties": {"$ref": "#"}, "default": {}}, "patternProperties": { "type": "object", "additionalProperties": {"$ref": "#"}, "propertyNames": {"format": "regex"}, "default": {}, }, "dependencies": { "type": "object", "additionalProperties": { "anyOf": [{"$ref": "#"}, {"$ref": "#/definitions/stringArray"}] }, }, "propertyNames": {"$ref": "#"}, "const": True, "enum": {"type": "array", "items": True, "minItems": 1, "uniqueItems": True}, "type": { "anyOf": [ {"$ref": "#/definitions/simpleTypes"}, { "type": "array", "items": {"$ref": "#/definitions/simpleTypes"}, "minItems": 1, "uniqueItems": True, }, ] }, "format": {"type": "string"}, "contentMediaType": {"type": "string"}, "contentEncoding": {"type": "string"}, "if": {"$ref": "#"}, "then": {"$ref": "#"}, "else": {"$ref": "#"}, "allOf": {"$ref": "#/definitions/schemaArray"}, "anyOf": {"$ref": "#/definitions/schemaArray"}, "oneOf": {"$ref": "#/definitions/schemaArray"}, "not": {"$ref": "#"}, # What follows is two Spack extensions to JSON Schema Draft 7: # deprecatedProperties and additionalKeysAreSpecs "deprecatedProperties": { "type": "array", "items": { "type": "object", "properties": { "names": { "type": "array", "items": {"type": "string"}, "minItems": 1, "uniqueItems": True, }, "message": {"type": "string"}, "error": {"type": "boolean"}, }, "required": ["names", "message"], "additionalProperties": False, }, }, "additionalKeysAreSpecs": {"type": "boolean"}, }, "default": True, # note: not in draft-07, this is for catching typos "additionalProperties": False, } @pytest.fixture() def validate_spec_schema(): return { "type": "object", "additionalKeysAreSpecs": True, "patternProperties": {r"\w[\w-]*": {"type": "string"}}, } @pytest.fixture() def module_suffixes_schema(): return { "type": "object", "properties": { "tcl": { "type": "object", "patternProperties": { r"\w[\w-]*": { "type": "object", "properties": { "suffixes": { "additionalKeysAreSpecs": True, "patternProperties": {r"\w[\w-]*": {"type": "string"}}, } }, } }, } }, } @pytest.mark.regression("9857") def test_validate_spec(validate_spec_schema): v = spack.schema.Validator(validate_spec_schema) data = {"foo@3.7": "bar"} # Validate good data (the key is a spec) v.validate(data) # Check that invalid data throws data["^python@3.7@"] = "baz" with pytest.raises(jsonschema.ValidationError, match="is not a valid spec"): v.validate(data) @pytest.mark.regression("9857") def test_module_suffixes(module_suffixes_schema): v = spack.schema.Validator(module_suffixes_schema) data = {"tcl": {"all": {"suffixes": {"^python@2.7@": "py2.7"}}}} with pytest.raises(jsonschema.ValidationError, match="is not a valid spec"): v.validate(data) def test_deprecated_properties(module_suffixes_schema): # Test that an error is reported when 'error: True' msg_fmt = r"{name} is deprecated" module_suffixes_schema["deprecatedProperties"] = [ {"names": ["tcl"], "message": msg_fmt, "error": True} ] v = spack.schema.Validator(module_suffixes_schema) data = {"tcl": {"all": {"suffixes": {"^python": "py"}}}} expected_match = "tcl is deprecated" with pytest.raises(jsonschema.ValidationError, match=expected_match): v.validate(data) # Test that just a warning is reported when 'error: False' module_suffixes_schema["deprecatedProperties"] = [ {"names": ["tcl"], "message": msg_fmt, "error": False} ] v = spack.schema.Validator(module_suffixes_schema) data = {"tcl": {"all": {"suffixes": {"^python": "py"}}}} # The next validation doesn't raise anymore v.validate(data) def test_ordereddict_merge_order(): """ "Test that source keys come before dest keys in merge_yaml results.""" source = syaml.syaml_dict([("k1", "v1"), ("k2", "v2"), ("k3", "v3")]) dest = syaml.syaml_dict([("k4", "v4"), ("k3", "WRONG"), ("k5", "v5")]) result = spack.schema.merge_yaml(dest, source) assert "WRONG" not in result.values() expected_keys = ["k1", "k2", "k3", "k4", "k5"] expected_items = [("k1", "v1"), ("k2", "v2"), ("k3", "v3"), ("k4", "v4"), ("k5", "v5")] assert expected_keys == list(result.keys()) assert expected_items == list(result.items()) def test_list_merge_order(): """ "Test that source lists are prepended to dest.""" source = ["a", "b", "c"] dest = ["d", "e", "f"] result = spack.schema.merge_yaml(dest, source) assert ["a", "b", "c", "d", "e", "f"] == result def test_spack_schemas_are_valid(): """Test that the Spack schemas in spack.schema.*.schema are valid under JSON Schema Draft 7 with Spack extensions *only*.""" # Collect schema submodules, and verify we have at least a few known ones schema_submodules = ( importlib.import_module(f"spack.schema.{name}") for name in list_modules(os.path.dirname(spack.schema.__file__)) ) schemas = {m.__name__: m.schema for m in schema_submodules if hasattr(m, "schema")} assert set(schemas) >= {"spack.schema.config", "spack.schema.packages", "spack.schema.modules"} # Validate them using the meta-schema for module_name, module_schema in schemas.items(): try: jsonschema.validate(module_schema, _draft_07_with_spack_extensions) except jsonschema.ValidationError as e: raise RuntimeError(f"Invalid JSON schema in {module_name}: {e.message}") from e def test_env_schema_update_wrong_type(): """Confirm passing the wrong type to env.update() results in no changes.""" assert not spack.schema.env.update(["a/b"]) ================================================ FILE: lib/spack/spack/test/spack_yaml.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test Spack's custom YAML format.""" import io import pathlib import pytest import spack.util.spack_yaml as syaml from spack.util.spack_yaml import DictWithLineInfo @pytest.fixture() def data(): """Returns the data loaded from a test file""" test_file = """\ config_file: x86_64: foo: /path/to/foo bar: /path/to/bar baz: /path/to/baz some_list: - item 1 - item 2 - item 3 another_list: [ 1, 2, 3 ] some_key: some_string """ return syaml.load_config(test_file) def test_parse(data): expected = { "config_file": syaml.syaml_dict( [ ( "x86_64", syaml.syaml_dict( [("foo", "/path/to/foo"), ("bar", "/path/to/bar"), ("baz", "/path/to/baz")] ), ), ("some_list", ["item 1", "item 2", "item 3"]), ("another_list", [1, 2, 3]), ("some_key", "some_string"), ] ) } assert data == expected def test_dict_order(data): expected_order = ["x86_64", "some_list", "another_list", "some_key"] assert list(data["config_file"].keys()) == expected_order expected_order = ["foo", "bar", "baz"] assert list(data["config_file"]["x86_64"].keys()) == expected_order def test_line_numbers(data): def check(obj, start_line, end_line): assert obj._start_mark.line == start_line assert obj._end_mark.line == end_line check(data, 0, 12) check(data["config_file"], 1, 12) check(data["config_file"]["x86_64"], 2, 5) check(data["config_file"]["x86_64"]["foo"], 2, 2) check(data["config_file"]["x86_64"]["bar"], 3, 3) check(data["config_file"]["x86_64"]["baz"], 4, 4) check(data["config_file"]["some_list"], 6, 9) check(data["config_file"]["some_list"][0], 6, 6) check(data["config_file"]["some_list"][1], 7, 7) check(data["config_file"]["some_list"][2], 8, 8) check(data["config_file"]["another_list"], 10, 10) check(data["config_file"]["some_key"], 11, 11) def test_yaml_aliases(): aliased_list_1 = ["foo"] aliased_list_2 = [] dict_with_aliases = { "a": aliased_list_1, "b": aliased_list_1, "c": aliased_list_1, "d": aliased_list_2, "e": aliased_list_2, "f": aliased_list_2, } stringio = io.StringIO() syaml.dump(dict_with_aliases, stream=stringio) # ensure no YAML aliases appear in syaml dumps. assert "*id" not in stringio.getvalue() @pytest.mark.parametrize( "initial_content,expected_final_content", [ # List are dumped indented as the outer attribute ( """spack: #foo specs: # bar - zlib """, None, ), ( """spack: #foo specs: # bar - zlib """, """spack: #foo specs: # bar - zlib """, ), ], ) @pytest.mark.not_on_windows(reason="fails on Windows") def test_round_trip_configuration(initial_content, expected_final_content, tmp_path: pathlib.Path): """Test that configuration can be loaded and dumped without too many changes""" file = tmp_path / "test.yaml" file.write_text(initial_content) final_content = io.StringIO() data = syaml.load_config(file) syaml.dump_config(data, stream=final_content) if expected_final_content is None: expected_final_content = initial_content assert final_content.getvalue() == expected_final_content def test_sorted_dict(): assert syaml.sorted_dict( { "z": 0, "y": [{"x": 0, "w": [2, 1, 0]}, 0], "v": ({"u": 0, "t": 0, "s": 0}, 0, {"r": 0, "q": 0}), "p": 0, } ) == { "p": 0, "v": ({"s": 0, "t": 0, "u": 0}, 0, {"q": 0, "r": 0}), "y": [{"w": [2, 1, 0], "x": 0}, 0], "z": 0, } def test_deepcopy_to_native(): yaml = """\ a: b: 1 c: 1.0 d: - e: false - f: null - "string" 2.0: "float key" 1: "int key" """ allowed_types = {str, int, float, bool, type(None), DictWithLineInfo, list} original = syaml.load(yaml) copied = syaml.deepcopy_as_builtin(original) assert original == copied assert type(copied["a"]["b"]) is int assert type(copied["a"]["c"]) is float assert type(copied["a"]["d"][0]["e"]) is bool # edge case: bool is subclass of int assert type(copied["a"]["d"][1]["f"]) is type(None) assert type(copied["a"]["d"][2]) is str stack = [copied] while stack: obj = stack.pop() assert type(obj) in allowed_types if type(obj) is DictWithLineInfo: stack.extend(obj.keys()) stack.extend(obj.values()) elif type(obj) is list: stack.extend(obj) ================================================ FILE: lib/spack/spack/test/spec_dag.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ These tests check Spec DAG operations using dummy packages. """ import pytest import spack.concretize import spack.deptypes as dt import spack.error import spack.installer import spack.repo import spack.solver.asp import spack.util.hash as hashutil import spack.version from spack.dependency import Dependency from spack.spec import Spec from spack.test.conftest import RepoBuilder def check_links(spec_to_check): for spec in spec_to_check.traverse(): for dependent in spec.dependents(): assert dependent.edges_to_dependencies(name=spec.name) for dependency in spec.dependencies(): assert dependency.edges_from_dependents(name=spec.name) @pytest.fixture() def saved_deps(): """Returns a dictionary to save the dependencies.""" return {} @pytest.fixture() def set_dependency(saved_deps, monkeypatch): """Returns a function that alters the dependency information for a package in the ``saved_deps`` fixture. """ def _mock(pkg_name, spec): """Alters dependence information for a package. Adds a dependency on to pkg. Use this to mock up constraints. """ spec = Spec(spec) # Save original dependencies before making any changes. pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) if pkg_name not in saved_deps: saved_deps[pkg_name] = (pkg_cls, pkg_cls.dependencies.copy()) cond = Spec(pkg_cls.name) dependency = Dependency(pkg_cls, spec) monkeypatch.setitem(pkg_cls.dependencies, cond, {spec.name: dependency}) return _mock @pytest.mark.usefixtures("config") def test_test_deptype(repo_builder: RepoBuilder): """Ensure that test-only dependencies are only included for specified packages in the following spec DAG:: w /| x y | z w->y deptypes are (link, build), w->x and y->z deptypes are (test) """ repo_builder.add_package("x") repo_builder.add_package("z") repo_builder.add_package("y", dependencies=[("z", "test", None)]) repo_builder.add_package("w", dependencies=[("x", "test", None), ("y", None, None)]) with spack.repo.use_repositories(repo_builder.root): spec = spack.concretize.concretize_one("w", tests=("w",)) assert "x" in spec assert "z" not in spec def test_installed_deps(monkeypatch, install_mockery): """Ensure that concrete specs and their build deps don't constrain solves. Preinstall a package ``c`` that has a constrained build dependency on ``d``, then install ``a`` and ensure that neither: * ``c``'s package constraints, nor * the concrete ``c``'s build dependencies constrain ``a``'s dependency on ``d``. """ # see installed-deps-[abcde] test packages. # a # / \ # b c b --> d build/link # |\ /| b --> e build/link # |/ \| c --> d build # d e c --> e build/link # a, b, c, d, e = [f"installed-deps-{s}" for s in "abcde"] # install C, which will force d's version to be 2 # BUT d is only a build dependency of C, so it won't constrain # link/run dependents of C when C is depended on as an existing # (concrete) installation. c_spec = spack.concretize.concretize_one(c) assert c_spec[d].version == spack.version.Version("2") spack.installer.PackageInstaller([c_spec.package], fake=True, explicit=True).install() # install A, which depends on B, C, D, and E, and force A to # use the installed C. It should *not* force A to use the installed D # *if* we're doing a fresh installation. a_spec = spack.concretize.concretize_one(f"{a} ^/{c_spec.dag_hash()}") assert spack.version.Version("2") == a_spec[c][d].version assert spack.version.Version("2") == a_spec[e].version assert spack.version.Version("3") == a_spec[b][d].version assert spack.version.Version("3") == a_spec[d].version @pytest.mark.usefixtures("config") def test_specify_preinstalled_dep(monkeypatch, repo_builder: RepoBuilder): """Specify the use of a preinstalled package during concretization with a transitive dependency that is only supplied by the preinstalled package. """ repo_builder.add_package("pkg-c") repo_builder.add_package("pkg-b", dependencies=[("pkg-c", None, None)]) repo_builder.add_package("pkg-a", dependencies=[("pkg-b", None, None)]) with spack.repo.use_repositories(repo_builder.root): b_spec = spack.concretize.concretize_one("pkg-b") monkeypatch.setattr(Spec, "installed", property(lambda x: x.name != "pkg-a")) a_spec = Spec("pkg-a") a_spec._add_dependency(b_spec, depflag=dt.BUILD | dt.LINK, virtuals=()) a_spec = spack.concretize.concretize_one(a_spec) assert {x.name for x in a_spec.traverse()} == {"pkg-a", "pkg-b", "pkg-c"} @pytest.mark.usefixtures("config") @pytest.mark.parametrize( "spec_str,expr_str,expected", [("x ^y@2", "y@2", True), ("x@1", "y", False), ("x", "y@3", True)], ) def test_conditional_dep_with_user_constraints( spec_str, expr_str, expected, repo_builder: RepoBuilder ): """This sets up packages X->Y such that X depends on Y conditionally. It then constructs a Spec with X but with no constraints on X, so that the initial normalization pass cannot determine whether the constraints are met to add the dependency; this checks whether a user-specified constraint on Y is applied properly. """ repo_builder.add_package("y") repo_builder.add_package("x", dependencies=[("y", None, "x@2:")]) with spack.repo.use_repositories(repo_builder.root): spec = spack.concretize.concretize_one(spec_str) result = expr_str in spec assert result is expected, "{0} in {1}".format(expr_str, spec) @pytest.mark.usefixtures("mutable_mock_repo", "config") class TestSpecDag: def test_conflicting_package_constraints(self, set_dependency): set_dependency("mpileaks", "mpich@1.0") set_dependency("callpath", "mpich@2.0") spec = Spec("mpileaks ^mpich ^callpath ^dyninst ^libelf ^libdwarf") with pytest.raises(spack.error.UnsatisfiableSpecError): spack.concretize.concretize_one(spec) @pytest.mark.parametrize( "pairs,traverse_kwargs", [ # Preorder node traversal ( [ (0, "mpileaks"), (1, "callpath"), (2, "compiler-wrapper"), (2, "dyninst"), (3, "gcc"), (3, "gcc-runtime"), (3, "libdwarf"), (4, "libelf"), (2, "zmpi"), (3, "fake"), ], {}, ), # Preorder edge traversal ( [ (0, "mpileaks"), (1, "callpath"), (2, "compiler-wrapper"), (2, "dyninst"), (3, "compiler-wrapper"), (3, "gcc"), (3, "gcc-runtime"), (4, "gcc"), (3, "libdwarf"), (4, "compiler-wrapper"), (4, "gcc"), (4, "gcc-runtime"), (4, "libelf"), (5, "compiler-wrapper"), (5, "gcc"), (5, "gcc-runtime"), (3, "libelf"), (2, "gcc"), (2, "gcc-runtime"), (2, "zmpi"), (3, "compiler-wrapper"), (3, "fake"), (3, "gcc"), (3, "gcc-runtime"), (1, "compiler-wrapper"), (1, "gcc"), (1, "gcc-runtime"), (1, "zmpi"), ], {"cover": "edges"}, ), # Preorder path traversal ( [ (0, "mpileaks"), (1, "callpath"), (2, "compiler-wrapper"), (2, "dyninst"), (3, "compiler-wrapper"), (3, "gcc"), (3, "gcc-runtime"), (4, "gcc"), (3, "libdwarf"), (4, "compiler-wrapper"), (4, "gcc"), (4, "gcc-runtime"), (5, "gcc"), (4, "libelf"), (5, "compiler-wrapper"), (5, "gcc"), (5, "gcc-runtime"), (6, "gcc"), (3, "libelf"), (4, "compiler-wrapper"), (4, "gcc"), (4, "gcc-runtime"), (5, "gcc"), (2, "gcc"), (2, "gcc-runtime"), (3, "gcc"), (2, "zmpi"), (3, "compiler-wrapper"), (3, "fake"), (3, "gcc"), (3, "gcc-runtime"), (4, "gcc"), (1, "compiler-wrapper"), (1, "gcc"), (1, "gcc-runtime"), (2, "gcc"), (1, "zmpi"), (2, "compiler-wrapper"), (2, "fake"), (2, "gcc"), (2, "gcc-runtime"), (3, "gcc"), ], {"cover": "paths"}, ), # Postorder node traversal ( [ (2, "compiler-wrapper"), (3, "gcc"), (3, "gcc-runtime"), (4, "libelf"), (3, "libdwarf"), (2, "dyninst"), (3, "fake"), (2, "zmpi"), (1, "callpath"), (0, "mpileaks"), ], {"order": "post"}, ), # Postorder edge traversal ( [ (2, "compiler-wrapper"), (3, "compiler-wrapper"), (3, "gcc"), (4, "gcc"), (3, "gcc-runtime"), (4, "compiler-wrapper"), (4, "gcc"), (4, "gcc-runtime"), (5, "compiler-wrapper"), (5, "gcc"), (5, "gcc-runtime"), (4, "libelf"), (3, "libdwarf"), (3, "libelf"), (2, "dyninst"), (2, "gcc"), (2, "gcc-runtime"), (3, "compiler-wrapper"), (3, "fake"), (3, "gcc"), (3, "gcc-runtime"), (2, "zmpi"), (1, "callpath"), (1, "compiler-wrapper"), (1, "gcc"), (1, "gcc-runtime"), (1, "zmpi"), (0, "mpileaks"), ], {"cover": "edges", "order": "post"}, ), # Postorder path traversal ( [ (2, "compiler-wrapper"), (3, "compiler-wrapper"), (3, "gcc"), (4, "gcc"), (3, "gcc-runtime"), (4, "compiler-wrapper"), (4, "gcc"), (5, "gcc"), (4, "gcc-runtime"), (5, "compiler-wrapper"), (5, "gcc"), (6, "gcc"), (5, "gcc-runtime"), (4, "libelf"), (3, "libdwarf"), (4, "compiler-wrapper"), (4, "gcc"), (5, "gcc"), (4, "gcc-runtime"), (3, "libelf"), (2, "dyninst"), (2, "gcc"), (3, "gcc"), (2, "gcc-runtime"), (3, "compiler-wrapper"), (3, "fake"), (3, "gcc"), (4, "gcc"), (3, "gcc-runtime"), (2, "zmpi"), (1, "callpath"), (1, "compiler-wrapper"), (1, "gcc"), (2, "gcc"), (1, "gcc-runtime"), (2, "compiler-wrapper"), (2, "fake"), (2, "gcc"), (3, "gcc"), (2, "gcc-runtime"), (1, "zmpi"), (0, "mpileaks"), ], {"cover": "paths", "order": "post"}, ), ], ) def test_traversal(self, pairs, traverse_kwargs, default_mock_concretization): r"""Tests different traversals of the following graph o mpileaks@2.3/3qeg7jx |\ | |\ | | |\ | | | |\ | | | | |\ | | | | | o callpath@1.0/4gilijr | |_|_|_|/| |/| |_|_|/| | |/| |_|/| | | |/| |/| | | | |/|/| | | | | | o dyninst@8.2/u4oymb3 | | |_|_|/| | |/| |_|/| | | |/| |/| | | | |/|/| | | | | | |\ o | | | | | | mpich@3.0.4/g734fu6 |\| | | | | | |\ \ \ \ \ \ \ | |_|/ / / / / |/| | | | | | | |\ \ \ \ \ \ | | |_|/ / / / | |/| | | | | | | |/ / / / | | | | | o libdwarf@20130729/q5r7l2r | |_|_|_|/| |/| |_|_|/| | |/| |_|/| | | |/| |/| | | | |/|/ | | | | o libelf@0.8.13/i2x6pya | |_|_|/| |/| |_|/| | |/| |/| | | |/|/ | | o | compiler-wrapper@1.0/njdili2 | | / o | | gcc-runtime@10.5.0/iyytqeo |\| | | |/ |/| | o gcc@10.5.0/ljeisd4 | o glibc@2.31/tbyn33w """ dag = default_mock_concretization("mpileaks ^zmpi") names = [x for _, x in pairs] traversal = dag.traverse(**traverse_kwargs, depth=True) assert [(x, y.name) for x, y in traversal] == pairs traversal = dag.traverse(**traverse_kwargs) assert [x.name for x in traversal] == names def test_dependents_and_dependencies_are_correct(self): spec = Spec.from_literal( { "mpileaks": { "callpath": { "dyninst": {"libdwarf": {"libelf": None}, "libelf": None}, "mpi": None, }, "mpi": None, } } ) check_links(spec) concrete = spack.concretize.concretize_one(spec) check_links(concrete) @pytest.mark.parametrize( "constraint_str,spec_str", [ ("mpich@1.0", "mpileaks ^mpich@3.0"), ("mpich%gcc", "mpileaks ^mpich%intel"), ("mpich%gcc@2.0", "mpileaks ^mpich%gcc@3.0"), ], ) def test_unsatisfiable_cases(self, set_dependency, constraint_str, spec_str): """Tests that synthetic cases of conflicting requirements raise an UnsatisfiableSpecError when concretizing. """ set_dependency("mpileaks", constraint_str) with pytest.raises(spack.error.UnsatisfiableSpecError): spack.concretize.concretize_one(spec_str) @pytest.mark.parametrize( "spec_str", ["libelf ^mpich", "libelf ^libdwarf", "mpich ^dyninst ^libelf"] ) def test_invalid_dep(self, spec_str): spec = Spec(spec_str) with pytest.raises(spack.solver.asp.InvalidDependencyError): spack.concretize.concretize_one(spec) def test_equal(self): # Different spec structures to test for equality flat = Spec.from_literal({"mpileaks ^callpath ^libelf ^libdwarf": None}) flat_init = Spec.from_literal( {"mpileaks": {"callpath": None, "libdwarf": None, "libelf": None}} ) flip_flat = Spec.from_literal( {"mpileaks": {"libelf": None, "libdwarf": None, "callpath": None}} ) dag = Spec.from_literal({"mpileaks": {"callpath": {"libdwarf": {"libelf": None}}}}) flip_dag = Spec.from_literal({"mpileaks": {"callpath": {"libelf": {"libdwarf": None}}}}) # All these are equal to each other with regular == specs = (flat, flat_init, flip_flat, dag, flip_dag) for lhs, rhs in zip(specs, specs): assert lhs == rhs assert str(lhs) == str(rhs) # Same DAGs constructed different ways are equal assert flat.eq_dag(flat_init) # order at same level does not matter -- (dep on same parent) assert flat.eq_dag(flip_flat) # DAGs should be unequal if nesting is different assert not flat.eq_dag(dag) assert not flat.eq_dag(flip_dag) assert not flip_flat.eq_dag(dag) assert not flip_flat.eq_dag(flip_dag) assert not dag.eq_dag(flip_dag) def test_contains(self): spec = Spec("mpileaks ^mpi ^libelf@1.8.11 ^libdwarf") assert Spec("mpi") in spec assert Spec("libelf") in spec assert Spec("libelf@1.8.11") in spec assert Spec("libelf@1.8.12") not in spec assert Spec("libdwarf") in spec assert Spec("libgoblin") not in spec assert Spec("mpileaks") in spec def test_copy_simple(self): orig = Spec("mpileaks") copy = orig.copy() check_links(copy) assert orig == copy assert orig.eq_dag(copy) assert orig._concrete == copy._concrete # ensure no shared nodes bt/w orig and copy. orig_ids = set(id(s) for s in orig.traverse()) copy_ids = set(id(s) for s in copy.traverse()) assert not orig_ids.intersection(copy_ids) def test_copy_concretized(self): orig = spack.concretize.concretize_one("mpileaks") copy = orig.copy() check_links(copy) assert orig == copy assert orig.eq_dag(copy) assert orig._concrete == copy._concrete # ensure no shared nodes bt/w orig and copy. orig_ids = set(id(s) for s in orig.traverse()) copy_ids = set(id(s) for s in copy.traverse()) assert not orig_ids.intersection(copy_ids) def test_copy_through_spec_build_interface(self): """Check that copying dependencies using id(node) as a fast identifier of the node works when the spec is wrapped in a SpecBuildInterface object. """ s = spack.concretize.concretize_one("mpileaks") c0 = s.copy() assert c0 == s # Single indirection c1 = s["mpileaks"].copy() assert c0 == c1 == s # Double indirection c2 = s["mpileaks"]["mpileaks"].copy() assert c0 == c1 == c2 == s # Here is the graph with deptypes labeled (assume all packages have a 'dt' # prefix). Arrows are marked with the deptypes ('b' for 'build', 'l' for # 'link', 'r' for 'run'). # use -bl-> top # top -b-> build1 # top -bl-> link1 # top -r-> run1 # build1 -b-> build2 # build1 -bl-> link2 # build1 -r-> run2 # link1 -bl-> link3 # run1 -bl-> link5 # run1 -r-> run3 # link3 -b-> build2 # link3 -bl-> link4 # run3 -b-> build3 @pytest.mark.parametrize( "spec_str,deptypes,expected", [ ( "dtuse", ("build", "link"), [ "dtuse", "dttop", "dtbuild1", "dtbuild2", "dtlink2", "dtlink1", "dtlink3", "dtlink4", ], ), ( "dttop", ("build", "link"), ["dttop", "dtbuild1", "dtbuild2", "dtlink2", "dtlink1", "dtlink3", "dtlink4"], ), ( "dttop", all, [ "dttop", "dtbuild1", "dtbuild2", "dtlink2", "dtrun2", "dtlink1", "dtlink3", "dtlink4", "dtrun1", "dtlink5", "dtrun3", "dtbuild3", ], ), ("dttop", "run", ["dttop", "dtrun1", "dtrun3"]), ], ) def test_deptype_traversal(self, spec_str, deptypes, expected): dag = spack.concretize.concretize_one(spec_str) traversal = dag.traverse(deptype=deptypes) assert [x.name for x in traversal] == expected def test_hash_bits(self): """Ensure getting first n bits of a base32-encoded DAG hash works.""" # RFC 4648 base32 decode table b32 = dict((j, i) for i, j in enumerate("abcdefghijklmnopqrstuvwxyz")) b32.update(dict((j, i) for i, j in enumerate("234567", 26))) # some package hashes tests = [ "35orsd4cenv743hg4i5vxha2lzayycby", "6kfqtj7dap3773rxog6kkmoweix5gpwo", "e6h6ff3uvmjbq3azik2ckr6ckwm3depv", "snz2juf4ij7sv77cq3vs467q6acftmur", "4eg47oedi5bbkhpoxw26v3oe6vamkfd7", "vrwabwj6umeb5vjw6flx2rnft3j457rw", ] for test_hash in tests: # string containing raw bits of hash ('1' and '0') expected = "".join([format(b32[c], "#07b").replace("0b", "") for c in test_hash]) for bits in (1, 2, 3, 4, 7, 8, 9, 16, 64, 117, 128, 160): actual_int = hashutil.base32_prefix_bits(test_hash, bits) fmt = "#0%sb" % (bits + 2) actual = format(actual_int, fmt).replace("0b", "") assert expected[:bits] == actual with pytest.raises(ValueError): hashutil.base32_prefix_bits(test_hash, 161) with pytest.raises(ValueError): hashutil.base32_prefix_bits(test_hash, 256) def test_traversal_directions(self): """Make sure child and parent traversals of specs work.""" # Mock spec - d is used for a diamond dependency spec = Spec.from_literal( {"a": {"b": {"c": {"d": None}, "e": None}, "f": {"g": {"d": None}}}} ) assert ["a", "b", "c", "d", "e", "f", "g"] == [ s.name for s in spec.traverse(direction="children") ] assert ["g", "f", "a"] == [s.name for s in spec["g"].traverse(direction="parents")] assert ["d", "c", "b", "a", "g", "f"] == [ s.name for s in spec["d"].traverse(direction="parents") ] def test_edge_traversals(self): """Make sure child and parent traversals of specs work.""" # Mock spec - d is used for a diamond dependency spec = Spec.from_literal( {"a": {"b": {"c": {"d": None}, "e": None}, "f": {"g": {"d": None}}}} ) assert ["a", "b", "c", "d", "e", "f", "g"] == [ s.name for s in spec.traverse(direction="children") ] assert ["g", "f", "a"] == [s.name for s in spec["g"].traverse(direction="parents")] assert ["d", "c", "b", "a", "g", "f"] == [ s.name for s in spec["d"].traverse(direction="parents") ] def test_copy_dependencies(self): s1 = Spec("mpileaks ^mpich2@1.1") s2 = s1.copy() assert "^mpich2@1.1" in s2 assert "^mpich2" in s2 def test_construct_spec_with_deptypes(self): """Ensure that it is possible to construct a spec with explicit dependency types.""" s = Spec.from_literal( {"a": {"b": {"c:build": None}, "d": {"e:build,link": {"f:run": None}}}} ) assert s["b"].edges_to_dependencies(name="c")[0].depflag == dt.BUILD assert s["d"].edges_to_dependencies(name="e")[0].depflag == dt.BUILD | dt.LINK assert s["e"].edges_to_dependencies(name="f")[0].depflag == dt.RUN # The subscript follows link/run transitive deps or direct build/test deps, therefore # we need an extra step to get to "c" assert s["b"]["c"].edges_from_dependents(name="b")[0].depflag == dt.BUILD assert s["e"].edges_from_dependents(name="d")[0].depflag == dt.BUILD | dt.LINK assert s["f"].edges_from_dependents(name="e")[0].depflag == dt.RUN def check_diamond_deptypes(self, spec): """Validate deptypes in dt-diamond spec. This ensures that concretization works properly when two packages depend on the same dependency in different ways. """ assert ( spec["dt-diamond"].edges_to_dependencies(name="dt-diamond-left")[0].depflag == dt.BUILD | dt.LINK ) assert ( spec["dt-diamond"].edges_to_dependencies(name="dt-diamond-right")[0].depflag == dt.BUILD | dt.LINK ) assert ( spec["dt-diamond-left"].edges_to_dependencies(name="dt-diamond-bottom")[0].depflag == dt.BUILD ) assert ( spec["dt-diamond-right"].edges_to_dependencies(name="dt-diamond-bottom")[0].depflag == dt.BUILD | dt.LINK | dt.RUN ) def test_concretize_deptypes(self): """Ensure that dependency types are preserved after concretization.""" s = spack.concretize.concretize_one("dt-diamond") self.check_diamond_deptypes(s) def test_copy_deptypes(self): """Ensure that dependency types are preserved by spec copy.""" s1 = spack.concretize.concretize_one("dt-diamond") self.check_diamond_deptypes(s1) s2 = s1.copy() self.check_diamond_deptypes(s2) def test_getitem_query(self): s = spack.concretize.concretize_one("mpileaks") # Check a query to a non-virtual package a = s["callpath"] query = a.last_query assert query.name == "callpath" assert len(query.extra_parameters) == 0 assert not query.isvirtual # Check a query to a virtual package a = s["mpi"] query = a.last_query assert query.name == "mpi" assert len(query.extra_parameters) == 0 assert query.isvirtual # Check a query to a virtual package with # extra parameters after query a = s["mpi:cxx,fortran"] query = a.last_query assert query.name == "mpi" assert len(query.extra_parameters) == 2 assert "cxx" in query.extra_parameters assert "fortran" in query.extra_parameters assert query.isvirtual def test_getitem_exceptional_paths(self): s = spack.concretize.concretize_one("mpileaks") # Needed to get a proxy object q = s["mpileaks"] # Test that the attribute is read-only with pytest.raises(AttributeError): q.libs = "foo" with pytest.raises(AttributeError): q.libs def test_canonical_deptype(self): # special values assert dt.canonicalize(all) == dt.ALL assert dt.canonicalize("all") == dt.ALL with pytest.raises(ValueError): dt.canonicalize(None) with pytest.raises(ValueError): dt.canonicalize([None]) # everything in all_types is canonical for v in dt.ALL_TYPES: assert dt.canonicalize(v) == dt.flag_from_string(v) # tuples assert dt.canonicalize(("build",)) == dt.BUILD assert dt.canonicalize(("build", "link", "run")) == dt.BUILD | dt.LINK | dt.RUN assert dt.canonicalize(("build", "link")) == dt.BUILD | dt.LINK assert dt.canonicalize(("build", "run")) == dt.BUILD | dt.RUN # lists assert dt.canonicalize(["build", "link", "run"]) == dt.BUILD | dt.LINK | dt.RUN assert dt.canonicalize(["build", "link"]) == dt.BUILD | dt.LINK assert dt.canonicalize(["build", "run"]) == dt.BUILD | dt.RUN # sorting assert dt.canonicalize(("run", "build", "link")) == dt.BUILD | dt.LINK | dt.RUN assert dt.canonicalize(("run", "link", "build")) == dt.BUILD | dt.LINK | dt.RUN assert dt.canonicalize(("run", "link")) == dt.LINK | dt.RUN assert dt.canonicalize(("link", "build")) == dt.BUILD | dt.LINK # deduplication assert dt.canonicalize(("run", "run", "link")) == dt.RUN | dt.LINK assert dt.canonicalize(("run", "link", "link")) == dt.RUN | dt.LINK # can't put 'all' in tuple or list with pytest.raises(ValueError): dt.canonicalize(["all"]) with pytest.raises(ValueError): dt.canonicalize(("all",)) # invalid values with pytest.raises(ValueError): dt.canonicalize("foo") with pytest.raises(ValueError): dt.canonicalize(("foo", "bar")) with pytest.raises(ValueError): dt.canonicalize(("foo",)) def test_invalid_literal_spec(self): # Can't give type 'build' to a top-level spec with pytest.raises(spack.error.SpecSyntaxError): Spec.from_literal({"foo:build": None}) # Can't use more than one ':' separator with pytest.raises(KeyError): Spec.from_literal({"foo": {"bar:build:link": None}}) def test_spec_tree_respect_deptypes(self): # Version-test-root uses version-test-pkg as a build dependency s = spack.concretize.concretize_one("version-test-root") out = s.tree(deptypes="all") assert "version-test-pkg" in out out = s.tree(deptypes=("link", "run")) assert "version-test-pkg" not in out @pytest.mark.parametrize( "query,expected_length,expected_satisfies", [ ({"virtuals": ["mpi"]}, 1, ["mpich", "mpi"]), ({"depflag": dt.BUILD}, 4, ["mpich", "mpi", "callpath"]), ({"depflag": dt.BUILD, "virtuals": ["mpi"]}, 1, ["mpich", "mpi"]), ({"depflag": dt.LINK}, 3, ["mpich", "mpi", "callpath"]), ({"depflag": dt.BUILD | dt.LINK}, 5, ["mpich", "mpi", "callpath"]), ({"virtuals": ["lapack"]}, 0, []), ], ) def test_query_dependency_edges( self, default_mock_concretization, query, expected_length, expected_satisfies ): """Tests querying edges to dependencies on the following DAG: - [ ] mpileaks@2.3 - [bl ] ^callpath@1.0 - [bl ] ^dyninst@8.2 - [bl ] ^libdwarf@20130729 - [bl ] ^libelf@0.8.13 [e] [b ] ^gcc@10.1.0 - [ l ] ^gcc-runtime@10.1.0 - [bl ] ^mpich@3.0.4~debug """ mpileaks = default_mock_concretization("mpileaks") edges = mpileaks.edges_to_dependencies(**query) assert len(edges) == expected_length for constraint in expected_satisfies: assert any(x.spec.satisfies(constraint) for x in edges) def test_query_dependents_edges(self, default_mock_concretization): """Tests querying edges from dependents""" mpileaks = default_mock_concretization("mpileaks") mpich = mpileaks["mpich"] # Recover the root with 2 different queries edges_of_link_type = mpich.edges_from_dependents(depflag=dt.LINK) edges_with_mpi = mpich.edges_from_dependents(virtuals=["mpi"]) assert edges_with_mpi == edges_of_link_type # Check a node depended upon by 2 parents assert len(mpileaks["libelf"].edges_from_dependents(depflag=dt.LINK)) == 2 def test_tree_cover_nodes_reduce_deptype(): """Test that tree output with deptypes sticks to the sub-dag of interest, instead of looking at in-edges from nodes not reachable from the root.""" a, b, c, d = Spec("a"), Spec("b"), Spec("c"), Spec("d") a.add_dependency_edge(d, depflag=dt.BUILD, virtuals=()) a.add_dependency_edge(b, depflag=dt.LINK, virtuals=()) b.add_dependency_edge(d, depflag=dt.LINK, virtuals=()) c.add_dependency_edge(d, depflag=dt.RUN | dt.TEST, virtuals=()) assert ( a.tree(cover="nodes", show_types=True) == """\ [ ] a [ l ] ^b [bl ] ^d """ ) assert ( c.tree(cover="nodes", show_types=True) == """\ [ ] c [ rt] ^d """ ) def test_synthetic_construction_of_split_dependencies_from_same_package(mock_packages, config): # Construct in a synthetic way (i.e. without using the solver) # the following spec: # # pkg-b # build / \ link,run # pkg-c@2.0 pkg-c@1.0 # # To demonstrate that a spec can now hold two direct # dependencies from the same package root = spack.concretize.concretize_one("pkg-b") link_run_spec = spack.concretize.concretize_one("pkg-c@=1.0") build_spec = spack.concretize.concretize_one("pkg-c@=2.0") root.add_dependency_edge(link_run_spec, depflag=dt.LINK, virtuals=()) root.add_dependency_edge(link_run_spec, depflag=dt.RUN, virtuals=()) root.add_dependency_edge(build_spec, depflag=dt.BUILD, virtuals=()) # Check dependencies from the perspective of root assert len(root.dependencies()) == 5 assert len([x for x in root.dependencies() if x.name == "pkg-c"]) == 2 assert "@2.0" in root.dependencies(name="pkg-c", deptype=dt.BUILD)[0] assert "@1.0" in root.dependencies(name="pkg-c", deptype=dt.LINK | dt.RUN)[0] # Check parent from the perspective of the dependencies assert len(build_spec.dependents()) == 1 assert len(link_run_spec.dependents()) == 1 assert build_spec.dependents() == link_run_spec.dependents() assert build_spec != link_run_spec def test_synthetic_construction_bootstrapping(mock_packages, config): # Construct the following spec: # # pkg-b@2.0 # | build # pkg-b@1.0 # root = spack.concretize.concretize_one("pkg-b@=2.0") bootstrap = spack.concretize.concretize_one("pkg-b@=1.0") root.add_dependency_edge(bootstrap, depflag=dt.BUILD, virtuals=()) assert len([x for x in root.dependencies() if x.name == "pkg-b"]) == 1 assert root.name == "pkg-b" def test_addition_of_different_deptypes_in_multiple_calls(mock_packages, config): # Construct the following spec: # # pkg-b@2.0 # | build,link,run # pkg-b@1.0 # # with three calls and check we always have a single edge root = spack.concretize.concretize_one("pkg-b@=2.0") bootstrap = spack.concretize.concretize_one("pkg-b@=1.0") for current_depflag in (dt.BUILD, dt.LINK, dt.RUN): root.add_dependency_edge(bootstrap, depflag=current_depflag, virtuals=()) # Check edges in dependencies assert len(root.edges_to_dependencies(name="pkg-b")) == 1 forward_edge = root.edges_to_dependencies(depflag=current_depflag, name="pkg-b")[0] assert current_depflag & forward_edge.depflag assert id(forward_edge.parent) == id(root) assert id(forward_edge.spec) == id(bootstrap) # Check edges from dependents assert len(bootstrap.edges_from_dependents()) == 1 backward_edge = bootstrap.edges_from_dependents(depflag=current_depflag)[0] assert current_depflag & backward_edge.depflag assert id(backward_edge.parent) == id(root) assert id(backward_edge.spec) == id(bootstrap) @pytest.mark.parametrize( "c1_depflag,c2_depflag", [(dt.LINK, dt.BUILD | dt.LINK), (dt.LINK | dt.RUN, dt.BUILD | dt.LINK)], ) def test_adding_same_deptype_with_the_same_name_raises( mock_packages, config, c1_depflag, c2_depflag ): p = spack.concretize.concretize_one("pkg-b@=2.0") c1 = spack.concretize.concretize_one("pkg-b@=1.0") c2 = spack.concretize.concretize_one("pkg-b@=2.0") p.add_dependency_edge(c1, depflag=c1_depflag, virtuals=()) with pytest.raises(spack.error.SpackError): p.add_dependency_edge(c2, depflag=c2_depflag, virtuals=()) @pytest.mark.regression("33499") def test_indexing_prefers_direct_or_transitive_link_deps(): """Tests whether spec indexing prefers direct/transitive link/run type deps over deps of build/test deps. """ root = Spec("root") # Use a and z to since we typically traverse by edges sorted alphabetically. a1 = Spec("a1") a2 = Spec("a2") z1 = Spec("z1") z2 = Spec("z2") # Same package, different spec. z3_flavor_1 = Spec("z3 +through_a1") z3_flavor_2 = Spec("z3 +through_z1") root.add_dependency_edge(a1, depflag=dt.BUILD | dt.TEST, virtuals=()) # unique package as a dep of a build/run/test type dep. a1.add_dependency_edge(a2, depflag=dt.ALL, virtuals=()) a1.add_dependency_edge(z3_flavor_1, depflag=dt.ALL, virtuals=()) # chain of link type deps root -> z1 -> z2 -> z3 root.add_dependency_edge(z1, depflag=dt.LINK, virtuals=()) z1.add_dependency_edge(z2, depflag=dt.LINK, virtuals=()) z2.add_dependency_edge(z3_flavor_2, depflag=dt.LINK, virtuals=()) # Indexing should prefer the link-type dep. assert "through_z1" in root["z3"].variants assert "through_a1" in a1["z3"].variants # Ensure that only the runtime sub-DAG can be searched with pytest.raises(KeyError): root["a2"] # Check consistency of __contains__ with __getitem__ assert "z3 +through_z1" in root assert "z3 +through_a1" in a1 assert "a2" not in root def test_getitem_sticks_to_subdag(): """Test that indexing on Spec by virtual does not traverse outside the dag, which happens in the unlikely case someone would rewrite __getitem__ in terms of edges_from_dependents instead of edges_to_dependencies.""" x, y, z = Spec("x"), Spec("y"), Spec("z") x.add_dependency_edge(z, depflag=dt.LINK, virtuals=("virtual",)) y.add_dependency_edge(z, depflag=dt.LINK, virtuals=()) assert x["virtual"].name == "z" with pytest.raises(KeyError): y["virtual"] def test_getitem_finds_transitive_virtual(): x, y, z = Spec("x"), Spec("y"), Spec("z") x.add_dependency_edge(z, depflag=dt.LINK, virtuals=()) x.add_dependency_edge(y, depflag=dt.LINK, virtuals=()) y.add_dependency_edge(z, depflag=dt.LINK, virtuals=("virtual",)) assert x["virtual"].name == "z" ================================================ FILE: lib/spack/spack/test/spec_list.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import itertools import pytest import spack.concretize from spack.environment.list import SpecListParser from spack.installer import PackageInstaller from spack.spec import Spec DEFAULT_EXPANSION = [ "mpileaks", "zmpi@1.0", "mpich@3.0", {"matrix": [["hypre"], ["%gcc@4.5.0", "%clang@3.3"]]}, "libelf", ] DEFAULT_CONSTRAINTS = [ [Spec("mpileaks")], [Spec("zmpi@1.0")], [Spec("mpich@3.0")], [Spec("hypre"), Spec("%gcc@4.5.0")], [Spec("hypre"), Spec("%clang@3.3")], [Spec("libelf")], ] DEFAULT_SPECS = [ Spec("mpileaks"), Spec("zmpi@1.0"), Spec("mpich@3.0"), Spec("hypre%gcc@4.5.0"), Spec("hypre%clang@3.3"), Spec("libelf"), ] @pytest.fixture() def parser_and_speclist(): """Default configuration of parser and user spec list for tests""" parser = SpecListParser() parser.parse_definitions( data=[ {"gccs": ["%gcc@4.5.0"]}, {"clangs": ["%clang@3.3"]}, {"mpis": ["zmpi@1.0", "mpich@3.0"]}, ] ) result = parser.parse_user_specs( name="specs", yaml_list=["mpileaks", "$mpis", {"matrix": [["hypre"], ["$gccs", "$clangs"]]}, "libelf"], ) return parser, result @pytest.mark.usefixtures("mock_packages") class TestSpecList: @pytest.mark.regression("28749") @pytest.mark.parametrize( "specs,expected", [ # Constraints are ordered randomly ( [ { "matrix": [ ["^zmpi"], ["%gcc@4.5.0"], ["hypre", "libelf"], ["~shared"], ["cflags=-O3", 'cflags="-g -O0"'], ["^foo"], ] } ], [ "hypre cflags=-O3 ~shared %gcc@4.5.0 ^foo ^zmpi", 'hypre cflags="-g -O0" ~shared %gcc@4.5.0 ^foo ^zmpi', "libelf cflags=-O3 ~shared %gcc@4.5.0 ^foo ^zmpi", 'libelf cflags="-g -O0" ~shared %gcc@4.5.0 ^foo ^zmpi', ], ), # A constraint affects both the root and a dependency ( [{"matrix": [["version-test-root"], ["%gcc"], ["^version-test-pkg%gcc"]]}], ["version-test-root%gcc ^version-test-pkg%gcc"], ), ], ) def test_spec_list_constraint_ordering(self, specs, expected): result = SpecListParser().parse_user_specs(name="specs", yaml_list=specs) assert result.specs == [Spec(x) for x in expected] def test_mock_spec_list(self, parser_and_speclist): """Tests expected properties on the default mock spec list""" parser, mock_list = parser_and_speclist assert mock_list.specs_as_yaml_list == DEFAULT_EXPANSION assert mock_list.specs_as_constraints == DEFAULT_CONSTRAINTS assert mock_list.specs == DEFAULT_SPECS def test_spec_list_add(self, parser_and_speclist): parser, mock_list = parser_and_speclist mock_list.add("libdwarf") assert mock_list.specs_as_yaml_list == DEFAULT_EXPANSION + ["libdwarf"] assert mock_list.specs_as_constraints == DEFAULT_CONSTRAINTS + [[Spec("libdwarf")]] assert mock_list.specs == DEFAULT_SPECS + [Spec("libdwarf")] def test_spec_list_remove(self, parser_and_speclist): parser, mock_list = parser_and_speclist mock_list.remove("libelf") assert mock_list.specs_as_yaml_list + ["libelf"] == DEFAULT_EXPANSION assert mock_list.specs_as_constraints + [[Spec("libelf")]] == DEFAULT_CONSTRAINTS assert mock_list.specs + [Spec("libelf")] == DEFAULT_SPECS def test_spec_list_extension(self, parser_and_speclist): parser, mock_list = parser_and_speclist other_list = parser.parse_user_specs( name="specs", yaml_list=[{"matrix": [["callpath"], ["%intel@18"]]}] ) mock_list.extend(other_list) assert mock_list.specs_as_yaml_list == (DEFAULT_EXPANSION + other_list.specs_as_yaml_list) assert mock_list.specs == DEFAULT_SPECS + other_list.specs def test_spec_list_nested_matrices(self, parser_and_speclist): parser, _ = parser_and_speclist inner_matrix = [{"matrix": [["zlib", "libelf"], ["%gcc", "%intel"]]}] outer_addition = ["+shared", "~shared"] outer_matrix = [{"matrix": [inner_matrix, outer_addition]}] result = parser.parse_user_specs(name="specs", yaml_list=outer_matrix) expected_components = itertools.product( ["zlib", "libelf"], ["%gcc", "%intel"], ["+shared", "~shared"] ) def _reduce(*, combo): root = Spec(combo[0]) for x in combo[1:]: root.constrain(x) return root expected = [_reduce(combo=combo) for combo in expected_components] assert set(result.specs) == set(expected) @pytest.mark.regression("16897") def test_spec_list_recursion_specs_as_constraints(self): input = ["mpileaks", "$mpis", {"matrix": [["hypre"], ["$%gccs", "$%clangs"]]}, "libelf"] definitions = [ {"gccs": ["gcc@4.5.0"]}, {"clangs": ["clang@3.3"]}, {"mpis": ["zmpi@1.0", "mpich@3.0"]}, ] parser = SpecListParser() parser.parse_definitions(data=definitions) result = parser.parse_user_specs(name="specs", yaml_list=input) assert result.specs_as_yaml_list == DEFAULT_EXPANSION assert result.specs_as_constraints == DEFAULT_CONSTRAINTS assert result.specs == DEFAULT_SPECS @pytest.mark.regression("16841") def test_spec_list_matrix_exclude(self): parser = SpecListParser() result = parser.parse_user_specs( name="specs", yaml_list=[ { "matrix": [["multivalue-variant"], ["foo=bar", "foo=baz"]], "exclude": ["foo=bar"], } ], ) assert len(result.specs) == 1 def test_spec_list_exclude_with_abstract_hashes(self, install_mockery): # Put mpich in the database so it can be referred to by hash. mpich_1 = spack.concretize.concretize_one("mpich+debug") mpich_2 = spack.concretize.concretize_one("mpich~debug") PackageInstaller([mpich_1.package, mpich_2.package], explicit=True, fake=True).install() # Create matrix and exclude +debug, which excludes the first mpich after its abstract hash # is resolved. parser = SpecListParser() result = parser.parse_user_specs( name="specs", yaml_list=[ { "matrix": [ ["mpileaks"], ["^callpath"], [f"^mpich/{mpich_1.dag_hash(5)}", f"^mpich/{mpich_2.dag_hash(5)}"], ], "exclude": ["^mpich+debug"], } ], ) # Ensure that only mpich~debug is selected, and that the assembled spec remains abstract. assert len(result.specs) == 1 assert result.specs[0] == Spec(f"mpileaks ^callpath ^mpich/{mpich_2.dag_hash(5)}") @pytest.mark.regression("51703") def test_exclusion_with_conditional_dependencies(self): """Tests that we can exclude some spec using conditional dependencies in the exclusion.""" parser = SpecListParser() result = parser.parse_user_specs( name="specs", yaml_list=[ { "matrix": [["libunwind"], ["%[when=%c]c=gcc", "%[when=%c]c=llvm"]], "exclude": ["libunwind %[when=%c]c=gcc"], } ], ) assert len(result.specs) == 1 ================================================ FILE: lib/spack/spack/test/spec_semantics.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib import pytest import spack.concretize import spack.deptypes as dt import spack.directives import spack.llnl.util.lang import spack.package_base import spack.paths import spack.repo import spack.solver.asp import spack.spec import spack.spec_parser import spack.store import spack.variant import spack.version as vn from spack.enums import PropagationPolicy from spack.error import SpecError, UnsatisfiableSpecError from spack.llnl.util.tty.color import colorize from spack.spec import ArchSpec, DependencySpec, Spec, SpecFormatSigilError, SpecFormatStringError from spack.variant import ( InvalidVariantValueError, MultipleValuesInExclusiveVariantError, UnknownVariantError, ) @pytest.fixture() def setup_complex_splice(monkeypatch): r"""Fixture to set up splicing for two complex specs. a_red is a spec in which every node has the variant color=red c_blue is a spec in which every node has the variant color=blue a_red structure: a - / \ \ b c \ /|\ / \ | e | d g@2 \|/ g@1 c_blue structure: c /|\ d f \ / |\ \ g@2 e \ \ \| / g@3 This is not intended for use in tests that use virtuals, so ``_splice_match`` is monkeypatched to avoid needing package files for each spec. """ def splice_match(self, other, self_root, other_root): return self.name == other.name def virtuals_provided(self, root): return [] monkeypatch.setattr(Spec, "_splice_match", splice_match) monkeypatch.setattr(Spec, "_virtuals_provided", virtuals_provided) g1_red = Spec("pkg-g color=red") g1_red.versions = vn.VersionList([vn.Version("1")]) g2_red = Spec("pkg-g color=red") g2_red.versions = vn.VersionList([vn.Version("2")]) g2_blue = Spec("pkg-g color=blue") g2_blue.versions = vn.VersionList([vn.Version("2")]) g3_blue = Spec("pkg-g color=blue") g3_blue.versions = vn.VersionList([vn.Version("3")]) depflag = dt.LINK | dt.BUILD e_red = Spec("pkg-e color=red") e_red._add_dependency(g1_red, depflag=depflag, virtuals=()) e_blue = Spec("pkg-e color=blue") e_blue._add_dependency(g3_blue, depflag=depflag, virtuals=()) d_red = Spec("pkg-d color=red") d_red._add_dependency(g1_red, depflag=depflag, virtuals=()) d_blue = Spec("pkg-d color=blue") d_blue._add_dependency(g2_blue, depflag=depflag, virtuals=()) b_red = Spec("pkg-b color=red") b_red._add_dependency(e_red, depflag=depflag, virtuals=()) b_red._add_dependency(d_red, depflag=depflag, virtuals=()) b_red._add_dependency(g1_red, depflag=depflag, virtuals=()) f_blue = Spec("pkg-f color=blue") f_blue._add_dependency(e_blue, depflag=depflag, virtuals=()) f_blue._add_dependency(g3_blue, depflag=depflag, virtuals=()) c_red = Spec("pkg-c color=red") c_red._add_dependency(d_red, depflag=depflag, virtuals=()) c_red._add_dependency(g2_red, depflag=depflag, virtuals=()) c_blue = Spec("pkg-c color=blue") c_blue._add_dependency(d_blue, depflag=depflag, virtuals=()) c_blue._add_dependency(f_blue, depflag=depflag, virtuals=()) c_blue._add_dependency(g3_blue, depflag=depflag, virtuals=()) a_red = Spec("pkg-a color=red") a_red._add_dependency(b_red, depflag=depflag, virtuals=()) a_red._add_dependency(c_red, depflag=depflag, virtuals=()) a_red._add_dependency(g2_red, depflag=depflag, virtuals=()) for spec in [e_red, e_blue, d_red, d_blue, b_red, f_blue, c_red, c_blue, a_red]: spec.versions = vn.VersionList([vn.Version("1")]) a_red._mark_concrete() c_blue._mark_concrete() return a_red, c_blue @pytest.mark.usefixtures("config", "mock_packages") class TestSpecSemantics: """Test satisfies(), intersects(), constrain() and other semantic operations on specs.""" @pytest.mark.parametrize( "lhs,rhs,expected", [ ("libelf@0.8.13", "@0:1", "libelf@0.8.13"), ("libdwarf^libelf@0.8.13", "^libelf@0:1", "libdwarf^libelf@0.8.13"), ("libelf", Spec(), "libelf"), ("libdwarf", Spec(), "libdwarf"), ("%intel", Spec(), "%intel"), ("^mpi", Spec(), "^mpi"), ("+debug", Spec(), "+debug"), ("@3:", Spec(), "@3:"), # Versions ("libelf@0:2.5", "libelf@2.1:3", "libelf@2.1:2.5"), ("libelf@0:2.5%gcc@2:4.6", "libelf@2.1:3%gcc@4.5:4.7", "libelf@2.1:2.5%gcc@4.5:4.6"), # Namespaces ("builtin.mpich", "mpich", "builtin.mpich"), ("builtin.mock.mpich", "mpich", "builtin.mock.mpich"), ("builtin.mpich", "builtin.mpich", "builtin.mpich"), ("mpileaks ^builtin.mock.mpich", "^mpich", "mpileaks ^builtin.mock.mpich"), # Virtual dependencies are fully resolved during concretization, so we can constrain # abstract specs but that would result in a new node ("mpileaks ^builtin.mock.mpich", "^mpi", "mpileaks ^mpi ^builtin.mock.mpich"), ( "mpileaks ^builtin.mock.mpich", "^builtin.mock.mpich", "mpileaks ^builtin.mock.mpich", ), # Compilers ("foo%gcc", "%gcc", "foo%gcc"), ("foo%intel", "%intel", "foo%intel"), ("foo%gcc", "%gcc@4.7.2", "foo%gcc@4.7.2"), ("foo%intel", "%intel@4.7.2", "foo%intel@4.7.2"), ("foo%gcc@4.5", "%gcc@4.4:4.6", "foo%gcc@4.5"), ("foo@2.0%gcc@4.5", "@1:3%gcc@4.4:4.6", "foo@2.0%gcc@4.5"), ("foo %gcc@4.7.3", "%gcc@4.7", "foo %gcc@4.7.3"), ("libelf %gcc@4.4.7", "libelf %gcc@4.4.7", "libelf %gcc@4.4.7"), ("libelf", "libelf %gcc@4.4.7", "libelf %gcc@4.4.7"), # Architecture ("foo platform=test", "platform=test", "foo platform=test"), ("foo platform=linux", "platform=linux", "foo platform=linux"), ( "foo platform=test", "platform=test target=frontend", "foo platform=test target=frontend", ), ( "foo platform=test", "platform=test os=frontend target=frontend", "foo platform=test os=frontend target=frontend", ), ( "foo platform=test os=frontend target=frontend", "platform=test", "foo platform=test os=frontend target=frontend", ), ("foo arch=test-None-None", "platform=test", "foo platform=test"), ( "foo arch=test-None-frontend", "platform=test target=frontend", "foo platform=test target=frontend", ), ( "foo arch=test-frontend-frontend", "platform=test os=frontend target=frontend", "foo platform=test os=frontend target=frontend", ), ( "foo arch=test-frontend-frontend", "platform=test", "foo platform=test os=frontend target=frontend", ), ( "foo platform=test target=backend os=backend", "platform=test target=backend os=backend", "foo platform=test target=backend os=backend", ), ( "libelf target=default_target os=default_os", "libelf target=default_target os=default_os", "libelf target=default_target os=default_os", ), # Dependencies ("mpileaks ^mpich", "^mpich", "mpileaks ^mpich"), ("mpileaks ^mpich@2.0", "^mpich@1:3", "mpileaks ^mpich@2.0"), ( "mpileaks ^mpich@2.0 ^callpath@1.5", "^mpich@1:3 ^callpath@1.4:1.6", "mpileaks^mpich@2.0^callpath@1.5", ), ("mpileaks ^mpi", "^mpi", "mpileaks ^mpi"), ("mpileaks ^mpi", "^mpich", "mpileaks ^mpi ^mpich"), ("mpileaks^mpi@1.5", "^mpi@1.2:1.6", "mpileaks^mpi@1.5"), ("mpileaks^mpi@2:", "^mpich", "mpileaks^mpi@2: ^mpich"), ("mpileaks^mpi@2:", "^mpich@3.0.4", "mpileaks^mpi@2: ^mpich@3.0.4"), # Variants ("mpich+foo", "mpich+foo", "mpich+foo"), ("mpich++foo", "mpich++foo", "mpich++foo"), ("mpich~foo", "mpich~foo", "mpich~foo"), ("mpich~~foo", "mpich~~foo", "mpich~~foo"), ("mpich foo=1", "mpich foo=1", "mpich foo=1"), ("mpich foo==1", "mpich foo==1", "mpich foo==1"), ("mpich+foo", "mpich foo=True", "mpich+foo"), ("mpich++foo", "mpich foo=True", "mpich+foo"), ("mpich foo=true", "mpich+foo", "mpich+foo"), ("mpich foo==true", "mpich++foo", "mpich++foo"), ("mpich~foo", "mpich foo=FALSE", "mpich~foo"), ("mpich~~foo", "mpich foo=FALSE", "mpich~foo"), ("mpich foo=False", "mpich~foo", "mpich~foo"), ("mpich foo==False", "mpich~foo", "mpich~foo"), ("mpich foo=*", "mpich~foo", "mpich~foo"), ("mpich+foo", "mpich foo=*", "mpich+foo"), ( 'multivalue-variant foo="bar,baz"', "multivalue-variant foo=bar,baz", "multivalue-variant foo=bar,baz", ), ( 'multivalue-variant foo="bar,baz"', "multivalue-variant foo=*", "multivalue-variant foo=bar,baz", ), ( 'multivalue-variant foo="bar,baz"', "multivalue-variant foo=bar", "multivalue-variant foo=bar,baz", ), ( 'multivalue-variant foo="bar,baz"', "multivalue-variant foo=baz", "multivalue-variant foo=bar,baz", ), ( 'multivalue-variant foo="bar,baz,barbaz"', "multivalue-variant foo=bar,baz", "multivalue-variant foo=bar,baz,barbaz", ), ( 'multivalue-variant foo="bar,baz"', 'foo="baz,bar"', # Order of values doesn't matter "multivalue-variant foo=bar,baz", ), ("mpich+foo", "mpich", "mpich+foo"), ("mpich~foo", "mpich", "mpich~foo"), ("mpich foo=1", "mpich", "mpich foo=1"), ("mpich", "mpich++foo", "mpich++foo"), ("libelf+debug", "libelf+foo", "libelf+debug+foo"), ("libelf+debug", "libelf+debug+foo", "libelf+debug+foo"), ("libelf debug=2", "libelf foo=1", "libelf debug=2 foo=1"), ("libelf debug=2", "libelf debug=2 foo=1", "libelf debug=2 foo=1"), ("libelf+debug", "libelf~foo", "libelf+debug~foo"), ("libelf+debug", "libelf+debug~foo", "libelf+debug~foo"), ("libelf++debug", "libelf+debug+foo", "libelf+debug+foo"), ("libelf debug==2", "libelf foo=1", "libelf debug==2 foo=1"), ("libelf debug==2", "libelf debug=2 foo=1", "libelf debug=2 foo=1"), ("libelf++debug", "libelf++debug~foo", "libelf++debug~foo"), ("libelf foo=bar,baz", "libelf foo=*", "libelf foo=bar,baz"), ("libelf foo=*", "libelf foo=bar,baz", "libelf foo=bar,baz"), ( 'multivalue-variant foo="bar"', 'multivalue-variant foo="baz"', 'multivalue-variant foo="bar,baz"', ), ( 'multivalue-variant foo="bar,barbaz"', 'multivalue-variant foo="baz"', 'multivalue-variant foo="bar,baz,barbaz"', ), # Namespace (special case, but like variants ("builtin.libelf", "namespace=builtin", "builtin.libelf"), ("libelf", "namespace=builtin", "builtin.libelf"), # Flags ("mpich ", 'mpich cppflags="-O3"', 'mpich cppflags="-O3"'), ( 'mpich cppflags="-O3 -Wall"', 'mpich cppflags="-O3 -Wall"', 'mpich cppflags="-O3 -Wall"', ), ('mpich cppflags=="-O3"', 'mpich cppflags=="-O3"', 'mpich cppflags=="-O3"'), ( 'libelf cflags="-O3"', 'libelf cppflags="-Wall"', 'libelf cflags="-O3" cppflags="-Wall"', ), ( 'libelf cflags="-O3"', 'libelf cppflags=="-Wall"', 'libelf cflags="-O3" cppflags=="-Wall"', ), ( 'libelf cflags=="-O3"', 'libelf cppflags=="-Wall"', 'libelf cflags=="-O3" cppflags=="-Wall"', ), ( 'libelf cflags="-O3"', 'libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3" cppflags="-Wall"', ), ( "libelf patches=ba5e334fe247335f3a116decfb5284100791dc302b5571ff5e664d8f9a6806c2", "libelf patches=ba5e3", # constrain by a patch sha256 prefix # TODO: the result below is not ideal. Prefix satisfies() works for patches, but # constrain() isn't similarly special-cased to do the same thing ( "libelf patches=ba5e3," "ba5e334fe247335f3a116decfb5284100791dc302b5571ff5e664d8f9a6806c2" ), ), # deptypes on direct deps ( "mpileaks %[deptypes=build] mpich", "mpileaks %[deptypes=link] mpich", "mpileaks %[deptypes=build,link] mpich", ), # conditional edges ( "libelf", "%[when='%c' virtuals=c]gcc ^[when='+mpi' virtuals=mpi]mpich", "libelf %[when='%c' virtuals=c]gcc ^[when='+mpi' virtuals=mpi]mpich", ), ( "libelf %[when='%c' virtuals=c]gcc", "%[when='%c' virtuals=c]gcc@10.3.1", "libelf%[when='%c' virtuals=c]gcc@10.3.1", ), ( "libelf %[when='%c' virtuals=c]gcc", "%[when='%c' virtuals=c]gcc@10.3.1 ^[when='+mpi'] mpich", "libelf%[when='%c' virtuals=c]gcc@10.3.1 ^[when='+mpi']mpich", ), ( "libelf %[when='%c' virtuals=c]gcc", "%[when='%cxx' virtuals=cxx]gcc@10.3.1", "libelf%[when='%c' virtuals=c]gcc %[when='%cxx' virtuals=cxx]gcc@10.3.1", ), ( "libelf %[when='+c' virtuals=c]gcc", "%[when='%c' virtuals=c]gcc@10.3.1", "libelf %[when='+c' virtuals=c]gcc %[when='%c' virtuals=c]gcc@10.3.1", ), ], ) def test_abstract_specs_can_constrain_each_other(self, lhs, rhs, expected): """Test that lhs and rhs intersect with each other, and that they can be constrained with each other. Also check that the constrained result match the expected spec. """ lhs, rhs, expected = Spec(lhs), Spec(rhs), Spec(expected) assert lhs.intersects(rhs) assert rhs.intersects(lhs) c1, c2 = lhs.copy(), rhs.copy() c1.constrain(rhs) c2.constrain(lhs) assert c1 == c2 assert c1 == expected @pytest.mark.parametrize( "lhs,rhs,expected_lhs,expected_rhs,propagated_lhs,propagated_rhs", [ ( 'mpich cppflags="-O3"', 'mpich cppflags="-O2"', 'mpich cppflags="-O3 -O2"', 'mpich cppflags="-O2 -O3"', [], [], ), ( 'mpich cflags="-O3 -g"', 'mpich cflags=="-O3"', 'mpich cflags="-O3 -g"', 'mpich cflags="-O3 -g"', [], [], ), ( 'mpich cflags=="-O3 -g"', 'mpich cflags=="-O3"', 'mpich cflags=="-O3 -g"', 'mpich cflags=="-O3 -g"', [("cflags", "-O3"), ("cflags", "-g")], [("cflags", "-O3"), ("cflags", "-g")], ), ], ) def test_constrain_compiler_flags( self, lhs, rhs, expected_lhs, expected_rhs, propagated_lhs, propagated_rhs ): """Constraining is asymmetric for compiler flags.""" lhs, rhs, expected_lhs, expected_rhs = ( Spec(lhs), Spec(rhs), Spec(expected_lhs), Spec(expected_rhs), ) assert lhs.intersects(rhs) assert rhs.intersects(lhs) c1, c2 = lhs.copy(), rhs.copy() c1.constrain(rhs) c2.constrain(lhs) assert c1 == expected_lhs assert c2 == expected_rhs for x in [c1, c2]: assert x.satisfies(lhs) assert x.satisfies(rhs) def _propagated_flags(_spec): result = set() for flagtype in _spec.compiler_flags: for flag in _spec.compiler_flags[flagtype]: if flag.propagate: result.add((flagtype, flag)) return result assert set(propagated_lhs) <= _propagated_flags(c1) assert set(propagated_rhs) <= _propagated_flags(c2) def test_constrain_specs_by_hash(self, default_mock_concretization, database): """Test that Specs specified only by their hashes can constrain each other.""" mpich_dag_hash = "/" + database.query_one("mpich").dag_hash() spec = Spec(mpich_dag_hash[:7]) assert spec.constrain(Spec(mpich_dag_hash)) is False assert spec.abstract_hash == mpich_dag_hash[1:] def test_mismatched_constrain_spec_by_hash(self, default_mock_concretization, database): """Test that Specs specified only by their incompatible hashes fail appropriately.""" lhs = "/" + database.query_one("callpath ^mpich").dag_hash() rhs = "/" + database.query_one("callpath ^mpich2").dag_hash() with pytest.raises(spack.spec.InvalidHashError): Spec(lhs).constrain(Spec(rhs)) with pytest.raises(spack.spec.InvalidHashError): Spec(lhs[:7]).constrain(Spec(rhs)) @pytest.mark.parametrize( "lhs,rhs", [("libelf", Spec()), ("libelf", "@0:1"), ("libelf", "@0:1 %gcc")] ) def test_concrete_specs_which_satisfies_abstract(self, lhs, rhs, default_mock_concretization): """Test that constraining an abstract spec by a compatible concrete one makes the abstract spec concrete, and equal to the one it was constrained with. """ lhs, rhs = default_mock_concretization(lhs), Spec(rhs) assert lhs.intersects(rhs) assert rhs.intersects(lhs) assert lhs.satisfies(rhs) assert not rhs.satisfies(lhs) assert lhs.constrain(rhs) is False assert rhs.constrain(lhs) is True assert rhs.concrete assert lhs.satisfies(rhs) assert rhs.satisfies(lhs) assert lhs == rhs @pytest.mark.parametrize( "lhs,rhs", [ ("foo platform=linux", "platform=test os=redhat6 target=x86"), ("foo os=redhat6", "platform=test os=debian6 target=x86_64"), ("foo target=x86_64", "platform=test os=redhat6 target=x86"), ("foo%gcc@4.3", "%gcc@4.4:4.6"), ("foo@4.0%gcc", "@1:3%gcc"), ("foo@4.0%gcc@4.5", "@1:3%gcc@4.4:4.6"), ("builtin.mock.mpich", "builtin.mpich"), ("mpileaks ^builtin.mock.mpich", "^builtin.mpich"), ("mpileaks^mpich@1.2", "^mpich@2.0"), ("mpileaks^mpich@4.0^callpath@1.5", "^mpich@1:3^callpath@1.4:1.6"), ("mpileaks^mpich@2.0^callpath@1.7", "^mpich@1:3^callpath@1.4:1.6"), ("mpileaks^mpich@4.0^callpath@1.7", "^mpich@1:3^callpath@1.4:1.6"), ("mpileaks^mpi@3", "^mpi@1.2:1.6"), ("mpileaks^mpi@3:", "^mpich2@1.4"), ("mpileaks^mpi@3:", "^mpich2"), ("mpileaks^mpi@3:", "^mpich@1.0"), ("mpich~foo", "mpich+foo"), ("mpich+foo", "mpich~foo"), ("mpich foo=True", "mpich foo=False"), ("mpich~~foo", "mpich++foo"), ("mpich++foo", "mpich~~foo"), ("mpich foo==True", "mpich foo==False"), ("libelf@0:2.0", "libelf@2.1:3"), ("libelf@0:2.5%gcc@4.8:4.9", "libelf@2.1:3%gcc@4.5:4.7"), ("libelf+debug", "libelf~debug"), ("libelf+debug~foo", "libelf+debug+foo"), ("libelf debug=True", "libelf debug=False"), ("namespace=builtin.mock", "namespace=builtin"), ], ) def test_constraining_abstract_specs_with_empty_intersection(self, lhs, rhs): """Check that two abstract specs with an empty intersection cannot be constrained with each other. """ lhs, rhs = Spec(lhs), Spec(rhs) assert not lhs.intersects(rhs) assert not rhs.intersects(lhs) with pytest.raises(UnsatisfiableSpecError): lhs.constrain(rhs) with pytest.raises(UnsatisfiableSpecError): rhs.constrain(lhs) @pytest.mark.parametrize( "lhs,rhs", [ ("mpich", "mpich +foo"), ("mpich", "mpich~foo"), ("mpich", "mpich foo=1"), ("multivalue-variant foo=bar", "multivalue-variant +foo"), ("multivalue-variant foo=bar", "multivalue-variant ~foo"), ("multivalue-variant fee=bar", "multivalue-variant fee=baz"), ], ) def test_concrete_specs_which_do_not_satisfy_abstract( self, lhs, rhs, default_mock_concretization ): lhs, rhs = default_mock_concretization(lhs), Spec(rhs) assert lhs.intersects(rhs) is False assert rhs.intersects(lhs) is False assert not lhs.satisfies(rhs) assert not rhs.satisfies(lhs) with pytest.raises(UnsatisfiableSpecError): assert lhs.constrain(rhs) with pytest.raises(UnsatisfiableSpecError): assert rhs.constrain(lhs) @pytest.mark.parametrize( "lhs,rhs", [("mpich", "mpich++foo"), ("mpich", "mpich~~foo"), ("mpich", "mpich foo==1")] ) def test_concrete_specs_which_satisfy_abstract(self, lhs, rhs, default_mock_concretization): lhs, rhs = default_mock_concretization(lhs), Spec(rhs) assert lhs.intersects(rhs) assert rhs.intersects(lhs) assert lhs.satisfies(rhs) s1 = lhs.copy() s1.constrain(rhs) assert s1 == lhs and s1.satisfies(lhs) s2 = rhs.copy() s2.constrain(lhs) assert s2 == lhs and s2.satisfies(lhs) @pytest.mark.parametrize( "lhs,rhs,expected,constrained", [ # hdf5++mpi satisfies hdf5, and vice versa, because of the non-contradiction semantic ("hdf5++mpi", "hdf5", True, "hdf5++mpi"), ("hdf5", "hdf5++mpi", True, "hdf5++mpi"), # Same holds true for arbitrary propagated variants ("hdf5++mpi", "hdf5++shared", True, "hdf5++mpi++shared"), # Here hdf5+mpi satisfies hdf5++mpi but not vice versa ("hdf5++mpi", "hdf5+mpi", False, "hdf5+mpi"), ("hdf5+mpi", "hdf5++mpi", True, "hdf5+mpi"), # Non contradiction is violated ("hdf5 ^foo~mpi", "hdf5++mpi", False, "hdf5++mpi ^foo~mpi"), ("hdf5++mpi", "hdf5 ^foo~mpi", False, "hdf5++mpi ^foo~mpi"), ], ) def test_abstract_specs_with_propagation(self, lhs, rhs, expected, constrained): """Tests (and documents) behavior of variant propagation on abstract specs. Propagated variants do not comply with subset semantic, making it difficult to give precise definitions. Here we document the behavior that has been decided for the practical cases we face. """ lhs, rhs, constrained = Spec(lhs), Spec(rhs), Spec(constrained) assert lhs.satisfies(rhs) is expected c = lhs.copy() c.constrain(rhs) assert c == constrained c = rhs.copy() c.constrain(lhs) assert c == constrained def test_basic_satisfies_conditional_dep(self, default_mock_concretization): """Tests basic semantic of satisfies with conditional dependencies, on a concrete spec""" concrete = default_mock_concretization("mpileaks ^mpich") # This branch exists, so the condition is met, and is satisfied assert concrete.satisfies("^[virtuals=mpi] mpich") assert concrete.satisfies("^[when='^notapackage' virtuals=mpi] mpich") assert concrete.satisfies("^[when='^mpi' virtuals=mpi] mpich") # This branch does not exist, but the condition is not met assert not concrete.satisfies("^zmpi") assert concrete.satisfies("^[when='^notapackage'] zmpi") assert not concrete.satisfies("^[when='^mpi'] zmpi") def test_concrete_satisfies_does_not_consult_repo( self, default_mock_concretization, monkeypatch ): """Tests that `satisfies()` on a concrete lhs doesn't need the provider index, when the rhs contains a virtual name. """ concrete = default_mock_concretization("mpileaks ^mpich") # Reset the index, will raise if the `_provider_index` is ever removed as an attribute monkeypatch.setattr(spack.repo.PATH, "_provider_index", None) # Basic match and mismatch cases. assert concrete.satisfies("mpileaks") assert not concrete.satisfies("zlib") # Virtuals on a direct edge assert concrete.satisfies("%mpi") assert concrete.satisfies("%mpi@3") assert not concrete.satisfies("%mpi@5") assert concrete.satisfies("%mpi=mpich") assert not concrete.satisfies("%lapack") # Virtuals on a transitive edge assert concrete.satisfies("^mpi") assert concrete.satisfies("^mpi=mpich") assert not concrete.satisfies("^lapack") # Concrete spec asking about one of its concrete deps. mpich = concrete["mpich"] assert mpich.satisfies("mpich") assert mpich.satisfies("mpi") # We should not create again the index assert spack.repo.PATH._provider_index is None def test_concrete_contains_does_not_consult_repo( self, default_mock_concretization, monkeypatch ): """Tests that `foo in spec` on a concrete spec doesn't need the provider index, when the item contains a virtual name. """ concrete = default_mock_concretization("mpileaks ^mpich") # Reset the index, will raise if the `_provider_index` is ever removed as an attribute monkeypatch.setattr(spack.repo.PATH, "_provider_index", None) assert "mpi" in concrete assert "c" in concrete # We should not create again the index assert spack.repo.PATH._provider_index is None def test_abstract_satisfies_with_lhs_provider_rhs_virtual(self): """If the left-hand side mentions a provider among dependencies and the right-hand side mentions a virtual among its deps, we only have satisfaction if the edge attribute specifies this virtual is provided.""" assert not Spec("mpileaks ^mpich").satisfies("mpileaks ^mpi") assert not Spec("mpileaks %mpich").satisfies("mpileaks %mpi") assert Spec("mpileaks ^[virtuals=mpi] mpich").satisfies("mpileaks ^mpi") assert Spec("mpileaks %[virtuals=mpi] mpich").satisfies("mpileaks ^mpi") assert Spec("mpileaks %[virtuals=mpi] mpich").satisfies("mpileaks %mpi") def test_concrete_checks_on_virtual_names_dont_need_repo( self, default_mock_concretization, monkeypatch ): """Tests that ``%mpi`` or similar on a concrete spec doesn't need the repo""" concrete = default_mock_concretization("mpileaks ^mpich") # We don't need the repo monkeypatch.setattr(spack.repo, "PATH", None) assert concrete.satisfies("%mpi") assert concrete.satisfies("%c") assert concrete.satisfies("%c=gcc") assert concrete.satisfies("%mpi=mpich") assert not concrete.satisfies("%c,mpi=mpich") def test_satisfies_single_valued_variant(self): """Tests that the case reported in https://github.com/spack/spack/pull/2386#issuecomment-282147639 is handled correctly. """ a = spack.concretize.concretize_one("pkg-a foobar=bar") assert a.satisfies("foobar=bar") assert a.satisfies("foobar=*") # Assert that an autospec generated from a literal # gives the right result for a single valued variant assert "foobar=bar" in a assert "foobar==bar" in a assert "foobar=baz" not in a assert "foobar=fee" not in a # ... and for a multi valued variant assert "foo=bar" in a # Check that conditional dependencies are treated correctly assert "^pkg-b" in a def test_unsatisfied_single_valued_variant(self): a = spack.concretize.concretize_one("pkg-a foobar=baz") assert "^pkg-b" not in a mv = spack.concretize.concretize_one("multivalue-variant") assert "pkg-a@1.0" not in mv def test_indirect_unsatisfied_single_valued_variant(self): spec = spack.concretize.concretize_one("singlevalue-variant-dependent") assert "pkg-a@1.0" not in spec def test_satisfied_namespace(self): spec = spack.concretize.concretize_one("zlib") assert spec.satisfies("namespace=builtin_mock") assert not spec.satisfies("namespace=builtin") @pytest.mark.parametrize( "spec_string", [ "tcl namespace==foobar", "tcl arch==foobar", "tcl os==foobar", "tcl patches==foobar", "tcl dev_path==foobar", ], ) def test_propagate_reserved_variant_names(self, spec_string): with pytest.raises(spack.spec_parser.SpecParsingError, match="Propagation"): Spec(spec_string) def test_multivalued_variant_1(self, default_mock_concretization): # Semantics for a multi-valued variant is different # Depending on whether the spec is concrete or not a = default_mock_concretization("multivalue-variant foo=bar") b = Spec("multivalue-variant foo=bar,baz") assert not a.satisfies(b) def test_multivalued_variant_2(self): a = Spec("multivalue-variant foo=bar") b = Spec("multivalue-variant foo=bar,baz") # The specs are abstract and they **could** be constrained assert b.satisfies(a) and not a.satisfies(b) # An abstract spec can instead be constrained assert a.constrain(b) def test_multivalued_variant_3(self, default_mock_concretization): a = default_mock_concretization("multivalue-variant foo=bar,baz") b = Spec("multivalue-variant foo=bar,baz,quux") assert not a.satisfies(b) def test_multivalued_variant_4(self): a = Spec("multivalue-variant foo=bar,baz") b = Spec("multivalue-variant foo=bar,baz,quux") # The specs are abstract and they **could** be constrained assert a.intersects(b) # An abstract spec can instead be constrained assert a.constrain(b) # ...but will fail during concretization if there are # values in the variant that are not allowed with pytest.raises(InvalidVariantValueError): spack.concretize.concretize_one(a) def test_multivalued_variant_5(self): # This time we'll try to set a single-valued variant a = Spec("multivalue-variant fee=bar") b = Spec("multivalue-variant fee=baz") # The specs are abstract and they **could** be constrained, # as before concretization I don't know which type of variant # I have (if it is not a BV) assert a.intersects(b) # A variant cannot be parsed as single-valued until we try to # concretize. This means that we can constrain the variant above assert a.constrain(b) # ...but will fail during concretization if there are # multiple values set with pytest.raises(MultipleValuesInExclusiveVariantError): spack.concretize.concretize_one(a) def test_copy_satisfies_transitive(self): spec = spack.concretize.concretize_one("dttop") copy = spec.copy() for s, t in zip(spec.traverse(), copy.traverse()): assert s.satisfies(t) assert t.satisfies(s) def test_intersects_virtual(self): assert Spec("mpich").intersects(Spec("mpi")) assert Spec("mpich2").intersects(Spec("mpi")) assert Spec("zmpi").intersects(Spec("mpi")) def test_intersects_virtual_providers(self): """Tests that we can always intersect virtual providers from abstract specs. Concretization will give meaning to virtuals, and eventually forbid certain configurations. """ assert Spec("netlib-lapack ^openblas").intersects("netlib-lapack ^openblas") assert Spec("netlib-lapack ^netlib-blas").intersects("netlib-lapack ^openblas") assert Spec("netlib-lapack ^openblas").intersects("netlib-lapack ^netlib-blas") assert Spec("netlib-lapack ^netlib-blas").intersects("netlib-lapack ^netlib-blas") def test_intersectable_concrete_specs_must_have_the_same_hash(self): """Ensure that concrete specs are matched *exactly* by hash.""" s1 = spack.concretize.concretize_one("mpileaks") s2 = s1.copy() assert s1.satisfies(s2) assert s2.satisfies(s1) assert s1.intersects(s2) # Simulate specs that were installed before and after a change to # Spack's hashing algorithm. This just reverses s2's hash. s2._hash = s1.dag_hash()[-1::-1] assert not s1.satisfies(s2) assert not s2.satisfies(s1) assert not s1.intersects(s2) # ======================================================================== # Indexing specs # ======================================================================== def test_self_index(self): s = Spec("callpath") assert s["callpath"] == s def test_dep_index(self, default_mock_concretization): """Tests __getitem__ and __contains__ for specs.""" s = default_mock_concretization("callpath") assert s["callpath"] == s # Real dependencies for key in ("dyninst", "libdwarf", "libelf"): assert isinstance(s[key], Spec) assert s[key].name == key assert key in s # Virtual dependencies assert s["mpi"].name == "mpich" assert "mpi" in s @pytest.mark.usefixtures("config") def test_virtual_index(self): s = spack.concretize.concretize_one("callpath") s_mpich = spack.concretize.concretize_one("callpath ^mpich") s_mpich2 = spack.concretize.concretize_one("callpath ^mpich2") s_zmpi = spack.concretize.concretize_one("callpath ^zmpi") assert s["mpi"].name != "mpi" assert s_mpich["mpi"].name == "mpich" assert s_mpich2["mpi"].name == "mpich2" assert s_zmpi["zmpi"].name == "zmpi" for spec in [s, s_mpich, s_mpich2, s_zmpi]: assert "mpi" in spec @pytest.mark.parametrize( "lhs,rhs", [ ("libelf", "@1.0"), ("libelf", "@1.0:5.0"), ("libelf", "%gcc"), ("libelf%gcc", "%gcc@4.5"), ("libelf", "+debug"), ("libelf", "debug=*"), ("libelf", "~debug"), ("libelf", "debug=2"), ("libelf", 'cppflags="-O3"'), ("libelf", 'cppflags=="-O3"'), ("libelf^foo", "libelf^foo@1.0"), ("libelf^foo", "libelf^foo@1.0:5.0"), ("libelf^foo", "libelf^foo%gcc"), ("libelf^foo%gcc", "libelf^foo%gcc@4.5"), ("libelf^foo", "libelf^foo+debug"), ("libelf^foo", "libelf^foo~debug"), ("libelf", "^foo"), ("mpileaks ^callpath %gcc@14", "mpileaks ^callpath %gcc@14.1"), ("mpileaks %[deptypes=build] mpich", "mpileaks %[deptypes=link] mpich"), ("mpileaks %mpich", "mpileaks %[deptypes=link] mpich"), ], ) def test_lhs_is_changed_when_constraining(self, lhs, rhs): lhs, rhs = Spec(lhs), Spec(rhs) assert lhs.intersects(rhs) assert rhs.intersects(lhs) assert not lhs.satisfies(rhs) assert lhs.constrain(rhs) is True assert lhs.satisfies(rhs) @pytest.mark.parametrize( "lhs,rhs", [ ("libelf", "libelf"), ("libelf@1.0", "@1.0"), ("libelf@1.0:5.0", "@1.0:5.0"), ("libelf%gcc", "%gcc"), ("libelf%gcc@4.5", "%gcc@4.5"), ("libelf+debug", "+debug"), ("libelf~debug", "~debug"), ("libelf debug=2", "debug=2"), ("libelf debug=2", "debug=*"), ('libelf cppflags="-O3"', 'cppflags="-O3"'), ('libelf cppflags=="-O3"', 'cppflags=="-O3"'), ("libelf^foo@1.0", "libelf^foo@1.0"), ("libelf^foo@1.0:5.0", "libelf^foo@1.0:5.0"), ("libelf^foo%gcc", "libelf^foo%gcc"), ("libelf^foo%gcc@4.5", "libelf^foo%gcc@4.5"), ("libelf^foo+debug", "libelf^foo+debug"), ("libelf^foo~debug", "libelf^foo~debug"), ('libelf^foo cppflags="-O3"', 'libelf^foo cppflags="-O3"'), ("mpileaks ^callpath %gcc@14.1", "mpileaks ^callpath %gcc@14"), ("mpileaks %[deptypes=build] gcc@14.1", "mpileaks %gcc@14"), ], ) def test_lhs_is_not_changed_when_constraining(self, lhs, rhs): lhs, rhs = Spec(lhs), Spec(rhs) assert lhs.intersects(rhs) assert rhs.intersects(lhs) assert lhs.satisfies(rhs) assert lhs.constrain(rhs) is False def test_exceptional_paths_for_constructor(self): with pytest.raises(TypeError): Spec((1, 2)) with pytest.raises(ValueError): Spec("libelf foo") def test_spec_formatting(self, default_mock_concretization): spec = default_mock_concretization("multivalue-variant cflags=-O2") # Testing named strings ie {string} and whether we get # the correct component # Mixed case intentional to test both # Fields are as follow # fmt_str: the format string to test # sigil: the portion that is a sigil (may be empty string) # prop: the property to get # component: subcomponent of spec from which to get property package_segments = [ ("{NAME}", "", "name", lambda spec: spec), ("{VERSION}", "", "version", lambda spec: spec), ("{compiler}", "", "compiler", lambda spec: spec), ("{compiler_flags}", "", "compiler_flags", lambda spec: spec), ("{variants}", "", "variants", lambda spec: spec), ("{architecture}", "", "architecture", lambda spec: spec), ("{@VERSIONS}", "@", "versions", lambda spec: spec), ("{%compiler}", "%", "compiler", lambda spec: spec), ("{arch=architecture}", "arch=", "architecture", lambda spec: spec), ("{namespace=namespace}", "namespace=", "namespace", lambda spec: spec), ("{compiler.name}", "", "name", lambda spec: spec.compiler), ("{compiler.version}", "", "version", lambda spec: spec.compiler), ( "{compiler.version.up_to_1}", "", "up_to_1", lambda spec: spec.compiler.version.up_to(1), ), ("{%compiler.name}", "%", "name", lambda spec: spec.compiler), ("{@compiler.version}", "@", "version", lambda spec: spec.compiler), ("{architecture.platform}", "", "platform", lambda spec: spec.architecture), ("{architecture.os}", "", "os", lambda spec: spec.architecture), ("{architecture.target}", "", "target", lambda spec: spec.architecture), ("{prefix}", "", "prefix", lambda spec: spec), ("{external}", "", "external", lambda spec: spec), # test we print "False" ] hash_segments = [ ("{hash:7}", "", lambda s: s.dag_hash(7)), ("{/hash}", "/", lambda s: "/" + s.dag_hash()), ] variants_segments = [ ("{variants.debug}", spec, "debug"), ("{variants.foo}", spec, "foo"), ("{^pkg-a.variants.bvv}", spec["pkg-a"], "bvv"), ("{^pkg-a.variants.foo}", spec["pkg-a"], "foo"), ] other_segments = [ ("{spack_root}", spack.paths.spack_root), ("{spack_install}", spack.store.STORE.layout.root), ] def depify(depname, fmt_str, sigil): sig = len(sigil) opening = fmt_str[: 1 + sig] closing = fmt_str[1 + sig :] return spec[depname], opening + f"^{depname}." + closing def check_prop(check_spec, fmt_str, prop, getter): actual = spec.format(fmt_str) expected = getter(check_spec) assert actual == str(expected).strip() for named_str, sigil, prop, get_component in package_segments: getter = lambda s: sigil + str(getattr(get_component(s), prop, "")) check_prop(spec, named_str, prop, getter) mpi, fmt_str = depify("mpi", named_str, sigil) check_prop(mpi, fmt_str, prop, getter) for named_str, sigil, getter in hash_segments: assert spec.format(named_str) == getter(spec) callpath, fmt_str = depify("callpath", named_str, sigil) assert spec.format(fmt_str) == getter(callpath) for named_str, test_spec, variant_name in variants_segments: assert test_spec.format(named_str) == str(test_spec.variants[variant_name]) assert test_spec.format(named_str[:-1] + ".value}") == str( test_spec.variants[variant_name].value ) for named_str, expected in other_segments: actual = spec.format(named_str) assert expected == actual @pytest.mark.parametrize( "fmt_str", [ "{name}", "{version}", "{@version}", "{namespace}", "{ namespace=namespace}", "{ namespace =namespace}", "{ name space =namespace}", "{arch}", "{architecture}", "{arch=architecture}", "{ arch=architecture}", "{ arch =architecture}", ], ) def test_spec_format_null_attributes(self, fmt_str): """Ensure that attributes format to empty strings when their values are null.""" spec = spack.spec.Spec() assert spec.format(fmt_str) == "" def test_spec_formatting_spaces_in_key(self, default_mock_concretization): spec = default_mock_concretization("multivalue-variant cflags=-O2") # test that spaces are preserved, if they come after some other text, otherwise # they are trimmed. # TODO: should we be trimming whitespace from formats? Probably not. assert spec.format("x{ arch=architecture}") == f"x arch={spec.architecture}" assert spec.format("x{ namespace=namespace}") == f"x namespace={spec.namespace}" assert spec.format("x{ name space =namespace}") == f"x name space ={spec.namespace}" assert spec.format("x{ os =os}") == f"x os ={spec.os}" @pytest.mark.parametrize( "fmt_str", ["{@name}", "{@version.concrete}", "{%compiler.version}", "{/hashd}"] ) def test_spec_formatting_sigil_mismatches(self, default_mock_concretization, fmt_str): spec = default_mock_concretization("multivalue-variant cflags=-O2") with pytest.raises(SpecFormatSigilError): spec.format(fmt_str) @pytest.mark.parametrize( "fmt_str", [ r"{}", r"name}", r"\{name}", r"{name", r"{name\}", r"{_concrete}", r"{dag_hash}", r"{foo}", r"{+variants.debug}", r"{variants.this_variant_does_not_exist}", ], ) def test_spec_formatting_bad_formats(self, default_mock_concretization, fmt_str): spec = default_mock_concretization("multivalue-variant cflags=-O2") with pytest.raises(SpecFormatStringError): spec.format(fmt_str) def test_wildcard_is_invalid_variant_value(self): """The spec string x=* is parsed as a multi-valued variant with values the empty set. That excludes * as a literal variant value.""" with pytest.raises(spack.spec_parser.SpecParsingError, match="cannot use reserved value"): Spec("multivalue-variant foo=*,bar") def test_errors_in_variant_directive(self): variant = spack.directives.variant.__wrapped__ class Pkg: name = "PKG" # We can't use names that are reserved by Spack fn = variant("patches") with pytest.raises(spack.directives.DirectiveError) as exc_info: fn(Pkg()) assert "The name 'patches' is reserved" in str(exc_info.value) # We can't have conflicting definitions for arguments fn = variant("foo", values=spack.variant.any_combination_of("fee", "foom"), default="bar") with pytest.raises(spack.directives.DirectiveError) as exc_info: fn(Pkg()) assert " it is handled by an attribute of the 'values' argument" in str(exc_info.value) # We can't leave None as a default value fn = variant("foo", default=None) with pytest.raises(spack.directives.DirectiveError) as exc_info: fn(Pkg()) assert "either a default was not explicitly set, or 'None' was used" in str(exc_info.value) # We can't use an empty string as a default value fn = variant("foo", default="") with pytest.raises(spack.directives.DirectiveError) as exc_info: fn(Pkg()) assert "the default cannot be an empty string" in str(exc_info.value) def test_abstract_spec_prefix_error(self): spec = Spec("libelf") with pytest.raises(SpecError): spec.prefix def test_forwarding_of_architecture_attributes(self): spec = spack.concretize.concretize_one("libelf target=x86_64") # Check that we can still access each member through # the architecture attribute assert "test" in spec.architecture assert "debian" in spec.architecture assert "x86_64" in spec.architecture # Check that we forward the platform and os attribute correctly assert spec.platform == "test" assert spec.os == "debian6" # Check that the target is also forwarded correctly and supports # all the operators we expect assert spec.target == "x86_64" assert spec.target.family == "x86_64" assert "avx512" not in spec.target assert spec.target < "broadwell" @pytest.mark.parametrize("transitive", [True, False]) def test_splice(self, transitive, default_mock_concretization): # Tests the new splice function in Spec using a somewhat simple case # with a variant with a conditional dependency. spec = default_mock_concretization("splice-t") dep = default_mock_concretization("splice-h+foo") # Sanity checking that these are not the same thing. assert dep.dag_hash() != spec["splice-h"].dag_hash() # Do the splice. out = spec.splice(dep, transitive) # Returned spec should still be concrete. assert out.concrete # Traverse the spec and assert that all dependencies are accounted for. for node in spec.traverse(): assert node.name in out # If the splice worked, then the dag hash of the spliced dep should # now match the dag hash of the build spec of the dependency from the # returned spec. out_h_build = out["splice-h"].build_spec assert out_h_build.dag_hash() == dep.dag_hash() # Transitivity should determine whether the transitive dependency was # changed. expected_z = dep["splice-z"] if transitive else spec["splice-z"] assert out["splice-z"].dag_hash() == expected_z.dag_hash() # Sanity check build spec of out should be the original spec. assert out["splice-t"].build_spec.dag_hash() == spec["splice-t"].dag_hash() # Finally, the spec should know it's been spliced: assert out.spliced def test_splice_intransitive_complex(self, setup_complex_splice): a_red, c_blue = setup_complex_splice spliced = a_red.splice(c_blue, transitive=False) assert spliced.satisfies( "pkg-a color=red ^pkg-b color=red ^pkg-c color=blue " "^pkg-d color=red ^pkg-e color=red ^pkg-f color=blue ^pkg-g@2 color=red" ) assert set(spliced.dependencies(deptype=dt.BUILD)) == set() assert spliced.build_spec == a_red # We cannot check spliced["b"].build_spec is spliced["b"] because Spec.__getitem__ creates # a new wrapper object on each invocation. So we select once and check on that object # For the rest of the unchanged specs we will just check the s._build_spec is None. b = spliced["pkg-b"] assert b == a_red["pkg-b"] assert b.build_spec is b assert set(b.dependents()) == {spliced} assert spliced["pkg-c"].satisfies( "pkg-c color=blue ^pkg-d color=red ^pkg-e color=red " "^pkg-f color=blue ^pkg-g@2 color=red" ) assert set(spliced["pkg-c"].dependencies(deptype=dt.BUILD)) == set() assert spliced["pkg-c"].build_spec == c_blue assert set(spliced["pkg-c"].dependents()) == {spliced} assert spliced["pkg-d"] == a_red["pkg-d"] assert spliced["pkg-d"]._build_spec is None # Since D had a parent changed, it has a split edge for link vs build dependent # note: spliced["b"] == b_red, referenced differently to preserve logic assert set(spliced["pkg-d"].dependents()) == { spliced["pkg-b"], spliced["pkg-c"], a_red["pkg-c"], } assert set(spliced["pkg-d"].dependents(deptype=dt.BUILD)) == { a_red["pkg-b"], a_red["pkg-c"], } assert spliced["pkg-e"] == a_red["pkg-e"] assert spliced["pkg-e"]._build_spec is None # Because a copy of e is used, it does not have dependnets in the original specs assert set(spliced["pkg-e"].dependents()) == {spliced["pkg-b"], spliced["pkg-f"]} # Build dependent edge to f because f originally depended on the e this was copied from assert set(spliced["pkg-e"].dependents(deptype=dt.BUILD)) == {spliced["pkg-b"]} assert spliced["pkg-f"].satisfies("pkg-f color=blue ^pkg-e color=red ^pkg-g@2 color=red") assert set(spliced["pkg-f"].dependencies(deptype=dt.BUILD)) == set() assert spliced["pkg-f"].build_spec == c_blue["pkg-f"] assert set(spliced["pkg-f"].dependents()) == {spliced["pkg-c"]} # spliced["pkg-g"] is g2, but spliced["pkg-b"]["pkg-g"] is g1 assert spliced["pkg-g"] == a_red["pkg-g"] assert spliced["pkg-g"]._build_spec is None assert set(spliced["pkg-g"].dependents(deptype=dt.LINK)) == { spliced, spliced["pkg-c"], spliced["pkg-f"], a_red["pkg-c"], } assert spliced["pkg-b"]["pkg-g"] == a_red["pkg-b"]["pkg-g"] assert spliced["pkg-b"]["pkg-g"]._build_spec is None assert set(spliced["pkg-b"]["pkg-g"].dependents()) == { spliced["pkg-b"], spliced["pkg-d"], spliced["pkg-e"], } for edge in spliced.traverse_edges(cover="edges", deptype=dt.LINK | dt.RUN): # traverse_edges creates a synthetic edge with no deptypes to the root if edge.depflag: depflag = dt.LINK if not edge.parent.spliced: depflag |= dt.BUILD assert edge.depflag == depflag def test_splice_transitive_complex(self, setup_complex_splice): a_red, c_blue = setup_complex_splice spliced = a_red.splice(c_blue, transitive=True) assert spliced.satisfies( "pkg-a color=red ^pkg-b color=red ^pkg-c color=blue ^pkg-d color=blue " "^pkg-e color=blue ^pkg-f color=blue ^pkg-g@3 color=blue" ) assert set(spliced.dependencies(deptype=dt.BUILD)) == set() assert spliced.build_spec == a_red assert spliced["pkg-b"].satisfies( "pkg-b color=red ^pkg-d color=blue ^pkg-e color=blue ^pkg-g@2 color=blue" ) assert set(spliced["pkg-b"].dependencies(deptype=dt.BUILD)) == set() assert spliced["pkg-b"].build_spec == a_red["pkg-b"] assert set(spliced["pkg-b"].dependents()) == {spliced} # We cannot check spliced["c"].build_spec is spliced["c"] because Spec.__getitem__ creates # a new wrapper object on each invocation. So we select once and check on that object # For the rest of the unchanged specs we will just check the s._build_spec is None. c = spliced["pkg-c"] assert c == c_blue assert c.build_spec is c assert set(c.dependents()) == {spliced} assert spliced["pkg-d"] == c_blue["pkg-d"] assert spliced["pkg-d"]._build_spec is None assert set(spliced["pkg-d"].dependents()) == {spliced["pkg-b"], spliced["pkg-c"]} assert spliced["pkg-e"] == c_blue["pkg-e"] assert spliced["pkg-e"]._build_spec is None assert set(spliced["pkg-e"].dependents()) == {spliced["pkg-b"], spliced["pkg-f"]} assert spliced["pkg-f"] == c_blue["pkg-f"] assert spliced["pkg-f"]._build_spec is None assert set(spliced["pkg-f"].dependents()) == {spliced["pkg-c"]} # spliced["g"] is g3, but spliced["d"]["g"] is g1 assert spliced["pkg-g"] == c_blue["pkg-g"] assert spliced["pkg-g"]._build_spec is None assert set(spliced["pkg-g"].dependents(deptype=dt.LINK)) == { spliced, spliced["pkg-b"], spliced["pkg-c"], spliced["pkg-e"], spliced["pkg-f"], } # Because a copy of g3 is used, it does not have dependents in the original specs # It has build dependents on these spliced specs because it is an unchanged dependency # for them assert set(spliced["pkg-g"].dependents(deptype=dt.BUILD)) == { spliced["pkg-c"], spliced["pkg-e"], spliced["pkg-f"], } assert spliced["pkg-d"]["pkg-g"] == c_blue["pkg-d"]["pkg-g"] assert spliced["pkg-d"]["pkg-g"]._build_spec is None assert set(spliced["pkg-d"]["pkg-g"].dependents()) == {spliced["pkg-d"]} for edge in spliced.traverse_edges(cover="edges", deptype=dt.LINK | dt.RUN): # traverse_edges creates a synthetic edge with no deptypes to the root if edge.depflag: depflag = dt.LINK if not edge.parent.spliced: depflag |= dt.BUILD assert edge.depflag == depflag @pytest.mark.parametrize("transitive", [True, False]) def test_splice_with_cached_hashes(self, default_mock_concretization, transitive): spec = default_mock_concretization("splice-t") dep = default_mock_concretization("splice-h+foo") # monkeypatch hashes so we can test that they are cached spec._hash = "aaaaaa" dep._hash = "bbbbbb" spec["splice-h"]._hash = "cccccc" spec["splice-z"]._hash = "dddddd" dep["splice-z"]._hash = "eeeeee" out = spec.splice(dep, transitive=transitive) out_z_expected = (dep if transitive else spec)["splice-z"] assert out.dag_hash() != spec.dag_hash() assert (out["splice-h"].dag_hash() == dep.dag_hash()) == transitive assert out["splice-z"].dag_hash() == out_z_expected.dag_hash() @pytest.mark.parametrize("transitive", [True, False]) def test_splice_input_unchanged(self, default_mock_concretization, transitive): spec = default_mock_concretization("splice-t") dep = default_mock_concretization("splice-h+foo") orig_spec_hash = spec.dag_hash() orig_dep_hash = dep.dag_hash() spec.splice(dep, transitive) # Post-splice, dag hash should still be different; no changes should be # made to these specs. assert spec.dag_hash() == orig_spec_hash assert dep.dag_hash() == orig_dep_hash @pytest.mark.parametrize("transitive", [True, False]) def test_splice_subsequent(self, default_mock_concretization, transitive): spec = default_mock_concretization("splice-t") dep = default_mock_concretization("splice-h+foo") out = spec.splice(dep, transitive) # Now we attempt a second splice. dep = default_mock_concretization("splice-z+bar") # Transitivity shouldn't matter since Splice Z has no dependencies. out2 = out.splice(dep, transitive) assert out2.concrete assert out2["splice-z"].dag_hash() != spec["splice-z"].dag_hash() assert out2["splice-z"].dag_hash() != out["splice-z"].dag_hash() assert out2["splice-t"].build_spec.dag_hash() == spec["splice-t"].dag_hash() assert out2.spliced @pytest.mark.parametrize("transitive", [True, False]) def test_splice_dict(self, default_mock_concretization, transitive): spec = default_mock_concretization("splice-t") dep = default_mock_concretization("splice-h+foo") out = spec.splice(dep, transitive) # Sanity check all hashes are unique... assert spec.dag_hash() != dep.dag_hash() assert out.dag_hash() != dep.dag_hash() assert out.dag_hash() != spec.dag_hash() node_list = out.to_dict()["spec"]["nodes"] root_nodes = [n for n in node_list if n["hash"] == out.dag_hash()] build_spec_nodes = [n for n in node_list if n["hash"] == spec.dag_hash()] assert spec.dag_hash() == out.build_spec.dag_hash() assert len(root_nodes) == 1 assert len(build_spec_nodes) == 1 @pytest.mark.parametrize("transitive", [True, False]) def test_splice_dict_roundtrip(self, default_mock_concretization, transitive): spec = default_mock_concretization("splice-t") dep = default_mock_concretization("splice-h+foo") out = spec.splice(dep, transitive) # Sanity check all hashes are unique... assert spec.dag_hash() != dep.dag_hash() assert out.dag_hash() != dep.dag_hash() assert out.dag_hash() != spec.dag_hash() out_rt_spec = Spec.from_dict(out.to_dict()) # rt is "round trip" assert out_rt_spec.dag_hash() == out.dag_hash() out_rt_spec_bld_hash = out_rt_spec.build_spec.dag_hash() out_rt_spec_h_bld_hash = out_rt_spec["splice-h"].build_spec.dag_hash() out_rt_spec_z_bld_hash = out_rt_spec["splice-z"].build_spec.dag_hash() # In any case, the build spec for splice-t (root) should point to the # original spec, preserving build provenance. assert spec.dag_hash() == out_rt_spec_bld_hash assert out_rt_spec.dag_hash() != out_rt_spec_bld_hash # The build spec for splice-h should always point to the introduced # spec, since that is the spec spliced in. assert dep["splice-h"].dag_hash() == out_rt_spec_h_bld_hash # The build spec for splice-z will depend on whether or not the splice # was transitive. expected_z_bld_hash = ( dep["splice-z"].dag_hash() if transitive else spec["splice-z"].dag_hash() ) assert expected_z_bld_hash == out_rt_spec_z_bld_hash @pytest.mark.parametrize( "spec,constraint,expected_result", [ ("libelf target=haswell", "target=broadwell", False), ("libelf target=haswell", "target=haswell", True), ("libelf target=haswell", "target=x86_64:", True), ("libelf target=haswell", "target=:haswell", True), ("libelf target=haswell", "target=icelake,:nocona", False), ("libelf target=haswell", "target=haswell,:nocona", True), # Check that a single target is not treated as the start # or the end of an open range ("libelf target=haswell", "target=x86_64", False), ("libelf target=x86_64", "target=haswell", False), ], ) @pytest.mark.regression("13111") def test_target_constraints(self, spec, constraint, expected_result): s = Spec(spec) assert s.intersects(constraint) is expected_result @pytest.mark.regression("13124") def test_error_message_unknown_variant(self): s = Spec("mpileaks +unknown") with pytest.raises(UnknownVariantError): spack.concretize.concretize_one(s) @pytest.mark.regression("18527") def test_satisfies_dependencies_ordered(self): d = Spec("zmpi ^fake") s = Spec("mpileaks") s._add_dependency(d, depflag=0, virtuals=()) assert s.satisfies("mpileaks ^zmpi ^fake") @pytest.mark.parametrize("transitive", [True, False]) def test_splice_swap_names(self, default_mock_concretization, transitive): spec = default_mock_concretization("splice-vt") dep = default_mock_concretization("splice-a+foo") out = spec.splice(dep, transitive) assert dep.name in out assert transitive == ("+foo" in out["splice-z"]) @pytest.mark.parametrize("transitive", [True, False]) def test_splice_swap_names_mismatch_virtuals(self, default_mock_concretization, transitive): vt = default_mock_concretization("splice-vt") vh = default_mock_concretization("splice-vh+foo") with pytest.raises(spack.spec.SpliceError, match="virtual"): vt.splice(vh, transitive) def test_adaptor_optflags(self): """Tests that we can obtain the list of optflags, and debugflags, from the compiler adaptor, and that this list is taken from the appropriate compiler package. """ # pkg-a depends on c, so only the gcc compiler should be chosen spec = spack.concretize.concretize_one(Spec("pkg-a %gcc")) assert "-Otestopt" in spec.package.compiler.opt_flags # This is not set, make sure we get an empty list for x in spec.package.compiler.debug_flags: pass def test_spec_override(self): init_spec = Spec("pkg-a foo=baz foobar=baz cflags=-O3 cxxflags=-O1") change_spec = Spec("pkg-a foo=fee cflags=-O2") new_spec = spack.concretize.concretize_one(Spec.override(init_spec, change_spec)) assert "foo=fee" in new_spec # This check fails without concretizing: apparently if both specs are # abstract, then the spec will always be considered to satisfy # 'variant=value' (regardless of whether it in fact does). assert "foo=baz" not in new_spec assert "foobar=baz" in new_spec assert new_spec.compiler_flags["cflags"] == ["-O2"] assert new_spec.compiler_flags["cxxflags"] == ["-O1"] def test_spec_override_with_nonexisting_variant(self): init_spec = Spec("pkg-a foo=baz foobar=baz cflags=-O3 cxxflags=-O1") change_spec = Spec("pkg-a baz=fee") with pytest.raises(ValueError): Spec.override(init_spec, change_spec) def test_spec_override_with_variant_not_in_init_spec(self): init_spec = Spec("pkg-a foo=baz foobar=baz cflags=-O3 cxxflags=-O1") change_spec = Spec("pkg-a +bvv ~lorem_ipsum") new_spec = spack.concretize.concretize_one(Spec.override(init_spec, change_spec)) assert "+bvv" in new_spec assert "~lorem_ipsum" in new_spec @pytest.mark.parametrize( "spec_str,specs_in_dag", [ ("hdf5 ^[virtuals=mpi] mpich", [("mpich", "mpich"), ("mpi", "mpich")]), # Try different combinations with packages that provides a # disjoint set of virtual dependencies ( "netlib-scalapack ^mpich ^openblas-with-lapack", [ ("mpi", "mpich"), ("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack"), ], ), ( "netlib-scalapack ^[virtuals=mpi] mpich ^openblas-with-lapack", [ ("mpi", "mpich"), ("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack"), ], ), ( "netlib-scalapack ^mpich ^[virtuals=lapack] openblas-with-lapack", [ ("mpi", "mpich"), ("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack"), ], ), ( "netlib-scalapack ^[virtuals=mpi] mpich ^[virtuals=lapack] openblas-with-lapack", [ ("mpi", "mpich"), ("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack"), ], ), # Test that we can mix dependencies that provide an overlapping # sets of virtual dependencies ( "netlib-scalapack ^[virtuals=mpi] intel-parallel-studio " "^[virtuals=lapack] openblas-with-lapack", [ ("mpi", "intel-parallel-studio"), ("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack"), ], ), ( "netlib-scalapack ^[virtuals=mpi] intel-parallel-studio ^openblas-with-lapack", [ ("mpi", "intel-parallel-studio"), ("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack"), ], ), ( "netlib-scalapack ^intel-parallel-studio ^[virtuals=lapack] openblas-with-lapack", [ ("mpi", "intel-parallel-studio"), ("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack"), ], ), # Test that we can bind more than one virtual to the same provider ( "netlib-scalapack ^[virtuals=lapack,blas] openblas-with-lapack", [("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack")], ), ], ) def test_virtual_deps_bindings(self, default_mock_concretization, spec_str, specs_in_dag): s = default_mock_concretization(spec_str) for label, expected in specs_in_dag: assert label in s assert s[label].satisfies(expected), label @pytest.mark.parametrize( "spec_str", [ # openblas-with-lapack needs to provide blas and lapack together "netlib-scalapack ^[virtuals=blas] intel-parallel-studio ^openblas-with-lapack", # intel-* provides blas and lapack together, openblas can provide blas only "netlib-scalapack ^[virtuals=lapack] intel-parallel-studio ^openblas", ], ) def test_unsatisfiable_virtual_deps_bindings(self, spec_str): with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): spack.concretize.concretize_one(spec_str) @pytest.mark.parametrize( "spec_str,abstract_tests,concrete_tests", [ # Ensure the 'when=+debug' is referred to 'callpath', and not to 'mpileaks', # and that we can concretize the spec despite 'callpath' has no debug variant ( "mpileaks+debug ^callpath %[when=+debug virtuals=mpi] zmpi", [ ("^zmpi", False), ("^mpich", False), ("mpileaks+debug %[when=+debug virtuals=mpi] zmpi", False), ], [("^zmpi", False), ("^[virtuals=mpi] mpich", True)], ), # Ensure we don't skip conditional edges when testing because we associate them # with the wrong node (e.g. mpileaks instead of mpich) ( "mpileaks~debug ^mpich+debug %[when=+debug virtuals=c] llvm", [("^mpich+debug %[when=+debug virtuals=c] gcc", False)], [("^mpich %[virtuals=c] gcc", False), ("^mpich %[virtuals=c] llvm", True)], ), ], ) def test_conditional_dependencies_satisfies( self, spec_str, abstract_tests, concrete_tests, default_mock_concretization ): """Tests satisfaction semantics for conditional specs, in different scenarios.""" s = Spec(spec_str) for c, result in abstract_tests: assert s.satisfies(c) is result concrete = default_mock_concretization(spec_str) for c, result in concrete_tests: assert concrete.satisfies(c) is result @pytest.mark.parametrize( "spec_str,format_str,expected", [ ("git-test@git.foo/bar", "{name}-{version}", str(pathlib.Path("git-test-git.foo_bar"))), ("git-test@git.foo/bar", "{name}-{version}-{/hash}", None), ("git-test@git.foo/bar", "{name}/{version}", str(pathlib.Path("git-test", "git.foo_bar"))), # {compiler} is 'none' if a package does not depend on C, C++, or Fortran ( f"git-test@{'a' * 40}=1.0%gcc", "{name}/{version}/{compiler}", str(pathlib.Path("git-test", f"{'a' * 40}_1.0", "none")), ), ( "git-test@git.foo/bar=1.0%gcc", "{name}/{version}/{compiler}", str(pathlib.Path("git-test", "git.foo_bar_1.0", "none")), ), ], ) def test_spec_format_path(spec_str, format_str, expected, mock_git_test_package): _check_spec_format_path(spec_str, format_str, expected) def _check_spec_format_path(spec_str, format_str, expected, path_ctor=None): spec = Spec(spec_str) if not expected: with pytest.raises((spack.spec.SpecFormatPathError, spack.spec.SpecFormatStringError)): spec.format_path(format_str, _path_ctor=path_ctor) else: formatted = spec.format_path(format_str, _path_ctor=path_ctor) assert formatted == expected @pytest.mark.parametrize( "spec_str,format_str,expected", [ ( "git-test@git.foo/bar", r"C:\\installroot\{name}\{version}", r"C:\installroot\git-test\git.foo_bar", ), ( "git-test@git.foo/bar", r"\\hostname\sharename\{name}\{version}", r"\\hostname\sharename\git-test\git.foo_bar", ), # leading '/' is preserved on windows but converted to '\' # note that it's still not "absolute" -- absolute windows paths start with a drive. ( "git-test@git.foo/bar", r"/installroot/{name}/{version}", r"\installroot\git-test\git.foo_bar", ), ], ) def test_spec_format_path_windows(spec_str, format_str, expected, mock_git_test_package): _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PureWindowsPath) @pytest.mark.parametrize( "spec_str,format_str,expected", [ ( "git-test@git.foo/bar", r"/installroot/{name}/{version}", "/installroot/git-test/git.foo_bar", ), ( "git-test@git.foo/bar", r"//installroot/{name}/{version}", "//installroot/git-test/git.foo_bar", ), # This is likely unintentional on Linux: Firstly, "\" is not a # path separator for POSIX, so this is treated as a single path # component (containing literal "\" characters); secondly, # Spec.format treats "\" as an escape character, so is # discarded (unless directly following another "\") ( "git-test@git.foo/bar", r"C:\\installroot\package-{name}-{version}", r"C__installrootpackage-git-test-git.foo_bar", ), # "\" is not a POSIX separator, and Spec.format treats "\{" as a literal # "{", which means that the resulting format string is invalid ("git-test@git.foo/bar", r"package\{name}\{version}", None), ], ) def test_spec_format_path_posix(spec_str, format_str, expected, mock_git_test_package): _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PurePosixPath) @pytest.mark.regression("3887") @pytest.mark.parametrize("spec_str", ["py-extension2", "extension1", "perl-extension"]) def test_is_extension_after_round_trip_to_dict(config, mock_packages, spec_str): # x is constructed directly from string, y from a # round-trip to dict representation x = spack.concretize.concretize_one(spec_str) y = Spec.from_dict(x.to_dict()) # Using 'y' since the round-trip make us lose build dependencies for d in y.traverse(): assert x[d.name].package.is_extension == y[d.name].package.is_extension def test_malformed_spec_dict(): # FIXME: This test was really testing the specific implementation with an ad-hoc test with pytest.raises(SpecError, match="malformed"): Spec.from_dict( {"spec": {"_meta": {"version": 2}, "nodes": [{"dependencies": {"name": "foo"}}]}} ) def test_spec_dict_hashless_dep(): # FIXME: This test was really testing the specific implementation with an ad-hoc test with pytest.raises(SpecError, match="Couldn't parse"): Spec.from_dict( { "spec": { "_meta": {"version": 2}, "nodes": [ {"name": "foo", "hash": "thehash", "dependencies": [{"name": "bar"}]} ], } } ) @pytest.mark.parametrize( "anonymous,named,expected", [ ("+plumed", "gromacs", "gromacs+plumed"), ("+plumed ^plumed%gcc", "gromacs", "gromacs+plumed ^plumed%gcc"), ("+plumed", "builtin.gromacs", "builtin.gromacs+plumed"), ], ) def test_merge_anonymous_spec_with_named_spec(anonymous, named, expected): s = Spec(anonymous) changed = s.constrain(named) assert changed assert s == Spec(expected) def test_spec_installed(default_mock_concretization, database): """Test whether Spec.installed works.""" # a known installed spec should say that it's installed specs = database.query() spec = specs[0] assert spec.installed assert spec.copy().installed # an abstract spec should say it's not installed spec = Spec("not-a-real-package") assert not spec.installed # pkg-a is not in the mock DB and is not installed spec = default_mock_concretization("pkg-a") assert not spec.installed @pytest.mark.regression("30678") def test_call_dag_hash_on_old_dag_hash_spec(mock_packages, default_mock_concretization): # create a concrete spec a = default_mock_concretization("pkg-a") dag_hashes = {spec.name: spec.dag_hash() for spec in a.traverse()} # make it look like an old DAG hash spec with no package hash on the spec. for spec in a.traverse(): assert spec.concrete spec._package_hash = None for spec in a.traverse(): assert dag_hashes[spec.name] == spec.dag_hash() with pytest.raises(ValueError, match="Cannot call package_hash()"): spec.package_hash() def test_spec_trim(mock_packages, config): top = spack.concretize.concretize_one("dt-diamond") top.trim("dt-diamond-left") remaining = {x.name for x in top.traverse()} assert { "compiler-wrapper", "dt-diamond", "dt-diamond-right", "dt-diamond-bottom", "gcc-runtime", "gcc", } == remaining top.trim("dt-diamond-right") remaining = {x.name for x in top.traverse()} assert {"compiler-wrapper", "dt-diamond", "gcc-runtime", "gcc"} == remaining @pytest.mark.regression("30861") def test_concretize_partial_old_dag_hash_spec(mock_packages, config): # create an "old" spec with no package hash bottom = spack.concretize.concretize_one("dt-diamond-bottom") delattr(bottom, "_package_hash") dummy_hash = "zd4m26eis2wwbvtyfiliar27wkcv3ehk" bottom._hash = dummy_hash # add it to an abstract spec as a dependency top = Spec("dt-diamond") top.add_dependency_edge(bottom, depflag=0, virtuals=()) # concretize with the already-concrete dependency top = spack.concretize.concretize_one(top) for spec in top.traverse(): assert spec.concrete # make sure dag_hash is untouched assert spec["dt-diamond-bottom"].dag_hash() == dummy_hash assert spec["dt-diamond-bottom"]._hash == dummy_hash # make sure package hash is NOT recomputed assert not getattr(spec["dt-diamond-bottom"], "_package_hash", None) def test_package_hash_affects_dunder_and_dag_hash(mock_packages, default_mock_concretization): a1 = default_mock_concretization("pkg-a") a2 = default_mock_concretization("pkg-a") assert hash(a1) == hash(a2) assert a1.dag_hash() == a2.dag_hash() a1.clear_caches() a2.clear_caches() # tweak the dag hash of one of these specs new_hash = "00000000000000000000000000000000" if new_hash == a1._package_hash: new_hash = "11111111111111111111111111111111" a1._package_hash = new_hash assert hash(a1) != hash(a2) assert a1.dag_hash() != a2.dag_hash() def test_intersects_and_satisfies_on_concretized_spec(default_mock_concretization): """Test that a spec obtained by concretizing an abstract spec, satisfies the abstract spec but not vice-versa. """ a1 = default_mock_concretization("pkg-a@1.0") a2 = Spec("pkg-a@1.0") assert a1.intersects(a2) assert a2.intersects(a1) assert a1.satisfies(a2) assert not a2.satisfies(a1) @pytest.mark.parametrize( "abstract_spec,spec_str", [ ("v1-provider", "v1-consumer ^conditional-provider+disable-v1"), ("conditional-provider", "v1-consumer ^conditional-provider+disable-v1"), ("^v1-provider", "v1-consumer ^conditional-provider+disable-v1"), ("^conditional-provider", "v1-consumer ^conditional-provider+disable-v1"), ], ) @pytest.mark.regression("35597") def test_abstract_provider_in_spec(abstract_spec, spec_str, default_mock_concretization): s = default_mock_concretization(spec_str) assert abstract_spec in s @pytest.mark.parametrize( "lhs,rhs,expected", [("a", "a", True), ("a", "a@1.0", True), ("a@1.0", "a", False)] ) def test_abstract_contains_semantic(lhs, rhs, expected, mock_packages): s, t = Spec(lhs), Spec(rhs) result = s in t assert result is expected @pytest.mark.parametrize( "factory,lhs_str,rhs_str,results", [ # Architecture (ArchSpec, "None-ubuntu20.04-None", "None-None-x86_64", (True, False, False)), (ArchSpec, "None-ubuntu20.04-None", "linux-None-x86_64", (True, False, False)), (ArchSpec, "None-None-x86_64:", "linux-None-haswell", (True, False, True)), (ArchSpec, "None-None-x86_64:haswell", "linux-None-icelake", (False, False, False)), (ArchSpec, "linux-None-None", "linux-None-None", (True, True, True)), (ArchSpec, "darwin-None-None", "linux-None-None", (False, False, False)), (ArchSpec, "None-ubuntu20.04-None", "None-ubuntu20.04-None", (True, True, True)), (ArchSpec, "None-ubuntu20.04-None", "None-ubuntu22.04-None", (False, False, False)), # Compiler (Spec, "gcc", "clang", (False, False, False)), (Spec, "gcc", "gcc@5", (True, False, True)), (Spec, "gcc@5", "gcc@5.3", (True, False, True)), (Spec, "gcc@5", "gcc@5-tag", (True, False, True)), # Flags (flags are a map, so for convenience we initialize a full Spec) # Note: the semantic is that of sv variants, not mv variants (Spec, "cppflags=-foo", "cppflags=-bar", (True, False, False)), (Spec, "cppflags='-bar -foo'", "cppflags=-bar", (True, True, False)), (Spec, "cppflags=-foo", "cppflags=-foo", (True, True, True)), (Spec, "cppflags=-foo", "cflags=-foo", (True, False, False)), # Versions (Spec, "@0.94h", "@:0.94i", (True, True, False)), # Different virtuals intersect if there is at least package providing both (Spec, "mpi", "lapack", (True, False, False)), (Spec, "mpi", "pkgconfig", (False, False, False)), # Intersection among target ranges for different architectures (Spec, "target=x86_64:", "target=ppc64le:", (False, False, False)), (Spec, "target=x86_64:", "target=:power9", (False, False, False)), (Spec, "target=:haswell", "target=:power9", (False, False, False)), (Spec, "target=:haswell", "target=ppc64le:", (False, False, False)), # Intersection among target ranges for the same architecture (Spec, "target=:haswell", "target=x86_64:", (True, True, True)), (Spec, "target=:haswell", "target=x86_64_v4:", (False, False, False)), # Edge case of uarch that split in a diamond structure, from a common ancestor (Spec, "target=:cascadelake", "target=:cannonlake", (False, False, False)), # Spec with compilers (Spec, "mpileaks %gcc@5", "mpileaks %gcc@6", (False, False, False)), (Spec, "mpileaks ^callpath %gcc@5", "mpileaks ^callpath %gcc@6", (False, False, False)), (Spec, "mpileaks ^callpath %gcc@5", "mpileaks ^callpath %gcc@5.4", (True, False, True)), ], ) def test_intersects_and_satisfies(mock_packages, factory, lhs_str, rhs_str, results): lhs = factory(lhs_str) rhs = factory(rhs_str) intersects, lhs_satisfies_rhs, rhs_satisfies_lhs = results assert lhs.intersects(rhs) is intersects assert rhs.intersects(lhs) is lhs.intersects(rhs) assert lhs.satisfies(rhs) is lhs_satisfies_rhs assert rhs.satisfies(lhs) is rhs_satisfies_lhs @pytest.mark.parametrize( "factory,lhs_str,rhs_str,result,constrained_str", [ # Architecture (ArchSpec, "None-ubuntu20.04-None", "None-None-x86_64", True, "None-ubuntu20.04-x86_64"), (ArchSpec, "None-None-x86_64", "None-None-x86_64", False, "None-None-x86_64"), ( ArchSpec, "None-None-x86_64:icelake", "None-None-x86_64:icelake", False, "None-None-x86_64:icelake", ), (ArchSpec, "None-ubuntu20.04-None", "linux-None-x86_64", True, "linux-ubuntu20.04-x86_64"), ( ArchSpec, "None-ubuntu20.04-nocona:haswell", "None-None-x86_64:icelake", False, "None-ubuntu20.04-nocona:haswell", ), ( ArchSpec, "None-ubuntu20.04-nocona,haswell", "None-None-x86_64:icelake", False, "None-ubuntu20.04-nocona,haswell", ), # Compiler (Spec, "foo %gcc@5", "foo %gcc@5-tag", True, "foo %gcc@5-tag"), (Spec, "foo %gcc@5", "foo %gcc@5", False, "foo %gcc@5"), # Flags (Spec, "cppflags=-foo", "cppflags=-foo", False, "cppflags=-foo"), (Spec, "cppflags=-foo", "cflags=-foo", True, "cppflags=-foo cflags=-foo"), # Target ranges (Spec, "target=x86_64:", "target=x86_64:", False, "target=x86_64:"), (Spec, "target=x86_64:", "target=:haswell", True, "target=x86_64:haswell"), ( Spec, "target=x86_64:haswell", "target=x86_64_v2:icelake", True, "target=x86_64_v2:haswell", ), ], ) def test_constrain(factory, lhs_str, rhs_str, result, constrained_str): lhs = factory(lhs_str) rhs = factory(rhs_str) assert lhs.constrain(rhs) is result assert lhs == factory(constrained_str) # The intersection must be the same, so check that invariant too lhs = factory(lhs_str) rhs = factory(rhs_str) rhs.constrain(lhs) assert rhs == factory(constrained_str) def test_constrain_dependencies_copies(mock_packages): """Tests that constraining a spec with new deps makes proper copies, and does not accidentally share dependency instances, leading to corruption of unrelated Spec instances.""" x = Spec("root") y = Spec("^foo") z = Spec("%foo +bar") assert x.constrain(y) assert x == Spec("root ^foo") assert x.constrain(z) assert x == Spec("root %foo +bar") assert not x.constrain(Spec("root %foo +bar")) # no new constraints # now, double check that we did not mutate `y` after constraining `x` with `z`. assert y == Spec("^foo") def test_abstract_hash_intersects_and_satisfies(default_mock_concretization): concrete: Spec = default_mock_concretization("pkg-a") hash = concrete.dag_hash() hash_5 = hash[:5] hash_6 = hash[:6] # abstract hash that doesn't have a common prefix with the others. hash_other = f"{'a' if hash_5[0] == 'b' else 'b'}{hash_5[1:]}" abstract_5 = Spec(f"pkg-a/{hash_5}") abstract_6 = Spec(f"pkg-a/{hash_6}") abstract_none = Spec(f"pkg-a/{hash_other}") abstract = Spec("pkg-a") def assert_subset(a: Spec, b: Spec): assert a.intersects(b) and b.intersects(a) and a.satisfies(b) and not b.satisfies(a) def assert_disjoint(a: Spec, b: Spec): assert ( not a.intersects(b) and not b.intersects(a) and not a.satisfies(b) and not b.satisfies(a) ) # left-hand side is more constrained, so its # concretization space is a subset of the right-hand side's assert_subset(concrete, abstract_5) assert_subset(abstract_6, abstract_5) assert_subset(abstract_5, abstract) # disjoint concretization space assert_disjoint(abstract_none, concrete) assert_disjoint(abstract_none, abstract_5) def test_edge_equality_does_not_depend_on_virtual_order(): """Tests that two edges that are constructed with just a different order of the virtuals in the input parameters are equal to each other. """ parent, child = Spec("parent"), Spec("child") edge1 = DependencySpec(parent, child, depflag=0, virtuals=("mpi", "lapack")) edge2 = DependencySpec(parent, child, depflag=0, virtuals=("lapack", "mpi")) assert edge1 == edge2 assert tuple(sorted(edge1.virtuals)) == edge1.virtuals assert tuple(sorted(edge2.virtuals)) == edge1.virtuals def test_update_virtuals(): parent, child = Spec("parent"), Spec("child") edge = DependencySpec(parent, child, depflag=0, virtuals=("mpi", "lapack")) assert edge.update_virtuals("blas") assert edge.virtuals == ("blas", "lapack", "mpi") assert edge.update_virtuals(("c", "fortran", "mpi", "lapack")) assert edge.virtuals == ("blas", "c", "fortran", "lapack", "mpi") assert not edge.update_virtuals("mpi") assert not edge.update_virtuals(("c", "fortran", "mpi", "lapack")) assert edge.virtuals == ("blas", "c", "fortran", "lapack", "mpi") def test_virtual_queries_work_for_strings_and_lists(): """Ensure that ``dependencies()`` works with both virtuals=str and virtuals=[str, ...].""" parent, child = Spec("parent"), Spec("child") parent._add_dependency( child, depflag=dt.BUILD, virtuals=("cxx", "fortran"), # multi-char dep names ) assert not parent.dependencies(virtuals="c") # not in virtuals but shares a char with cxx for lang in ["cxx", "fortran"]: assert parent.dependencies(virtuals=lang) # string arg assert parent.edges_to_dependencies(virtuals=lang) # string arg assert parent.dependencies(virtuals=[lang]) # list arg assert parent.edges_to_dependencies(virtuals=[lang]) # string arg def test_old_format_strings_trigger_error(default_mock_concretization): s = spack.concretize.concretize_one("pkg-a") with pytest.raises(SpecFormatStringError): s.format("${PACKAGE}-${VERSION}-${HASH}") @pytest.mark.regression("47362") @pytest.mark.parametrize( "lhs,rhs", [ ("hdf5 +mpi", "hdf5++mpi"), ("hdf5 cflags==-g", "hdf5 cflags=-g"), ("hdf5 +mpi ++shared", "hdf5+mpi +shared"), ("hdf5 +mpi cflags==-g", "hdf5++mpi cflag=-g"), ], ) def test_equality_discriminate_on_propagation(lhs, rhs): """Tests that == can discriminate abstract specs based on their 'propagation' status""" s, t = Spec(lhs), Spec(rhs) assert s != t assert len({s, t}) == 2 def test_comparison_multivalued_variants(): assert Spec("x=a") < Spec("x=a,b") < Spec("x==a,b") < Spec("x==a,b,c") @pytest.mark.parametrize( "specs_in_expected_order", [ ("a", "b", "c", "d", "e"), ("a@1.0", "a@2.0", "b", "c@3.0", "c@4.0"), ("a^d", "b^c", "c^b", "d^a"), ("e^a", "e^b", "e^c", "e^d"), ("e^a@1.0", "e^a@2.0", "e^a@3.0", "e^a@4.0"), ("e^a@1.0 +a", "e^a@1.0 +b", "e^a@1.0 +c", "e^a@1.0 +c"), ("a^b%c", "a^b%d", "a^b%e", "a^b%f"), ("a^b%c@1.0", "a^b%c@2.0", "a^b%c@3.0", "a^b%c@4.0"), ("a^b%c@1.0 +a", "a^b%c@1.0 +b", "a^b%c@1.0 +c", "a^b%c@1.0 +d"), ("a cflags=-O1", "a cflags=-O2", "a cflags=-O3"), ("a %cmake@1.0 ^b %cmake@2.0", "a %cmake@2.0 ^b %cmake@1.0"), ("a^b^c^d", "a^b^c^e", "a^b^c^f"), ("a^b^c^d", "a^b^c^e", "a^b^c^e", "a^b^c^f"), ("a%b%c%d", "a%b%c%e", "a%b%c%e", "a%b%c%f"), ("d.a", "c.b", "b.c", "a.d"), # names before namespaces ], ) def test_spec_ordering(specs_in_expected_order): specs_in_expected_order = [Spec(s) for s in specs_in_expected_order] assert sorted(specs_in_expected_order) == specs_in_expected_order assert sorted(reversed(specs_in_expected_order)) == specs_in_expected_order for i in range(len(specs_in_expected_order) - 1): lhs, rhs = specs_in_expected_order[i : i + 2] assert lhs <= rhs assert (lhs < rhs and lhs != rhs) or lhs == rhs assert rhs >= lhs assert (rhs > lhs and rhs != lhs) or rhs == lhs EMPTY_VER = vn.VersionList(":") EMPTY_VAR = Spec().variants EMPTY_FLG = Spec().compiler_flags @pytest.mark.parametrize( "spec,expected_tuplified", [ # simple, no dependencies [("a"), ((("a", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),), ())], # with some node attributes [ ("a@1.0 +foo cflags='-O3 -g'"), ( ( ( "a", None, vn.VersionList(["1.0"]), Spec("+foo").variants, Spec("cflags='-O3 -g'").compiler_flags, None, None, None, ), ), (), ), ], # single edge case [ ("a^b"), ( ( ("a", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("b", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ), ((0, 1, 0, (), False, Spec()),), ), ], # root with multiple deps [ ("a^b^c^d"), ( ( ("a", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("b", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("c", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("d", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ), ( (0, 1, 0, (), False, Spec()), (0, 2, 0, (), False, Spec()), (0, 3, 0, (), False, Spec()), ), ), ], # root with multiple build deps [ ("a%b%c%d"), ( ( ("a", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("b", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("c", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("d", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ), ( (0, 1, 0, (), True, Spec()), (0, 2, 0, (), True, Spec()), (0, 3, 0, (), True, Spec()), ), ), ], # dependencies with dependencies [ ("a ^b %c %d ^e %f %g"), ( ( ("a", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("b", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("e", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("c", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("d", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("f", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ("g", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None), ), ( (0, 1, 0, (), False, Spec()), (0, 2, 0, (), False, Spec()), (1, 3, 0, (), True, Spec()), (1, 4, 0, (), True, Spec()), (2, 5, 0, (), True, Spec()), (2, 6, 0, (), True, Spec()), ), ), ], ], ) def test_spec_canonical_comparison_form(spec, expected_tuplified): """Tests a few expected canonical comparison form of specs""" assert spack.llnl.util.lang.tuplify(Spec(spec)._cmp_iter) == expected_tuplified def test_comparison_after_breaking_hash_change(): # We simulate a breaking change in DAG hash computation in Spack. We have two specs that are # entirely equal modulo DAG hash. When deserializing these specs, we don't want them to compare # as equal, because DAG hash is used throughout in Spack to distinguish between specs # (e.g. database, build caches, install dir). s = Spec("example@=1.0") s._mark_concrete(True) # compute the dag hash and a change to it dag_hash = s.dag_hash() new_dag_hash = f"{'b' if dag_hash[0] == 'a' else 'a'}{dag_hash[1:]}" before_breakage = s.to_dict() after_breakage = s.to_dict() after_breakage["spec"]["nodes"][0]["hash"] = new_dag_hash assert before_breakage != after_breakage x = Spec.from_dict(before_breakage) y = Spec.from_dict(after_breakage) assert x != y assert len({x, y}) == 2 def test_satisfies_and_subscript_with_compilers(default_mock_concretization): """Tests the semantic of "satisfies" and __getitem__ for the following spec: [ ] multivalue-variant@2.3 [bl ] ^callpath@1.0 [bl ] ^dyninst@8.2 [bl ] ^libdwarf@20130729 [bl ] ^libelf@0.8.13 [b ] ^gcc@10.2.1 [ l ] ^gcc-runtime@10.2.1 [bl ] ^mpich@3.0.4 [bl ] ^pkg-a@2.0 [b ] ^gmake@4.4 [bl ] ^pkg-b@1.0 """ s = default_mock_concretization("multivalue-variant") # Check a direct build/link dependency assert s.satisfies("^pkg-a") assert s.dependencies(name="pkg-a")[0] == s["pkg-a"] # Transitive build/link dependency assert s.satisfies("^libelf") assert s["libdwarf"].dependencies(name="libelf")[0] == s["libelf"] # Direct build dependencies assert s.satisfies("^[virtuals=c] gcc") assert s.satisfies("%[virtuals=c] gcc") assert s.dependencies(name="gcc")[0] == s["gcc"] assert s.dependencies(name="gcc")[0] == s["c"] # Transitive build dependencies assert not s.satisfies("^gmake") # "gmake" is not in the link/run subdag + direct build deps with pytest.raises(KeyError): _ = s["gmake"] # We need to pass through "pkg-a" to get "gmake" with [] notation assert s["pkg-a"].dependencies(name="gmake")[0] == s["pkg-a"]["gmake"] @pytest.mark.parametrize( "spec_str,spec_fmt,expected", [ # Depends on C ("mpileaks", "{name}-{compiler.name}", "mpileaks-gcc"), ("mpileaks", "{name}-{compiler.name}-{compiler.version}", "mpileaks-gcc-10.2.1"), # No compiler ("pkg-c", "{name}-{compiler.name}", "pkg-c-none"), ("pkg-c", "{name}-{compiler.name}-{compiler.version}", "pkg-c-none-none"), ], ) def test_spec_format_with_compiler_adaptors( spec_str, spec_fmt, expected, default_mock_concretization ): """Tests the output of spec format, when involving `Spec.compiler` adaptors""" s = default_mock_concretization(spec_str) assert s.format(spec_fmt) == expected @pytest.mark.parametrize( "lhs,rhs,expected", [ ("mpich %gcc", "mpich %gcc", True), ("mpich %gcc", "mpich ^gcc", False), ("mpich ^callpath %gcc", "mpich %gcc ^callpath", False), ], ) def test_specs_equality(lhs, rhs, expected): """Tests the semantic of == for abstract specs""" lhs, rhs = Spec(lhs), Spec(rhs) assert (lhs == rhs) is expected def test_edge_equality_accounts_for_when_condition(): """Tests that edges can be distinguished by their 'when' condition.""" parent, child = Spec("parent"), Spec("child") edge1 = DependencySpec(parent, child, depflag=0, virtuals=(), when=Spec("%c")) edge2 = DependencySpec(parent, child, depflag=0, virtuals=()) assert edge1 != edge2 def test_long_spec(): """Test that long_spec preserves dependency types and has correct ordering.""" assert Spec("foo %m %l ^k %n %j").long_spec == "foo %l %m ^k %j %n" @pytest.mark.parametrize( "constraints,expected", [ # Anonymous specs without dependencies (["+baz", "+bar"], "+baz+bar"), (["@2.0:", "@:5.1", "+bar"], "@2.0:5.1 +bar"), # Anonymous specs with dependencies (["^mpich@3.2", "^mpich@:4.0+foo"], "^mpich@3.2 +foo"), # Mix a real package with a virtual one. This test # should fail if we start using the repository (["^mpich@3.2", "^mpi+foo"], "^mpich@3.2 ^mpi+foo"), # Non direct dependencies + direct dependencies (["^mpich", "%mpich"], "%mpich"), (["^foo", "^bar %foo"], "^foo ^bar%foo"), (["^foo", "%bar %foo"], "%bar%foo"), ], ) def test_constrain_symbolically(constraints, expected): """Tests the semantics of constraining a spec when we don't resolve virtuals.""" merged = Spec() for c in constraints: merged._constrain_symbolically(c) assert merged == Spec(expected) reverse_order = Spec() for c in reversed(constraints): reverse_order._constrain_symbolically(c) assert reverse_order == Spec(expected) @pytest.mark.parametrize( "parent_str,child_str,kwargs,expected_str,expected_repr", [ ( "mpileaks", "callpath", {"virtuals": ()}, "mpileaks ^callpath", "DependencySpec('mpileaks', 'callpath', depflag=0, virtuals=())", ), ( "mpileaks", "callpath", {"virtuals": ("mpi", "lapack")}, "mpileaks ^[virtuals=lapack,mpi] callpath", "DependencySpec('mpileaks', 'callpath', depflag=0, virtuals=('lapack', 'mpi'))", ), ( "", "callpath", {"virtuals": ("mpi", "lapack"), "direct": True}, " %[virtuals=lapack,mpi] callpath", "DependencySpec('', 'callpath', depflag=0, virtuals=('lapack', 'mpi'), direct=True)", ), ( "", "callpath", { "virtuals": ("mpi", "lapack"), "direct": True, "propagation": PropagationPolicy.PREFERENCE, }, " %%[virtuals=lapack,mpi] callpath", "DependencySpec('', 'callpath', depflag=0, virtuals=('lapack', 'mpi'), direct=True," " propagation=PropagationPolicy.PREFERENCE)", ), ( "", "callpath", {"virtuals": (), "direct": True, "propagation": PropagationPolicy.PREFERENCE}, " %%callpath", "DependencySpec('', 'callpath', depflag=0, virtuals=(), direct=True," " propagation=PropagationPolicy.PREFERENCE)", ), ( "mpileaks+foo", "callpath+bar", {"virtuals": (), "direct": True, "propagation": PropagationPolicy.PREFERENCE}, "mpileaks+foo %%callpath+bar", "DependencySpec('mpileaks+foo', 'callpath+bar', depflag=0, virtuals=(), direct=True," " propagation=PropagationPolicy.PREFERENCE)", ), ], ) def test_edge_representation(parent_str, child_str, kwargs, expected_str, expected_repr): """Tests the string representations of edges.""" parent = Spec(parent_str) or Spec() child = Spec(child_str) or Spec() edge = DependencySpec(parent, child, depflag=0, **kwargs) assert str(edge) == expected_str assert repr(edge) == expected_repr @pytest.mark.parametrize( "spec_str,assertions", [ # Check =* semantics for a "regular" variant ("mpileaks foo=abc", [("foo=*", True), ("bar=*", False)]), # Check the semantics for architecture related key value pairs ( "mpileaks", [ ("target=*", False), ("os=*", False), ("platform=*", False), ("target=* platform=*", False), ], ), ( "mpileaks target=x86_64", [ ("target=*", True), ("os=*", False), ("platform=*", False), ("target=* platform=*", False), ], ), ("mpileaks os=debian6", [("target=*", False), ("os=*", True), ("platform=*", False)]), ("mpileaks platform=linux", [("target=*", False), ("os=*", False), ("platform=*", True)]), ("mpileaks platform=linux", [("target=*", False), ("os=*", False), ("platform=*", True)]), ( "mpileaks platform=linux target=x86_64", [ ("target=*", True), ("os=*", False), ("platform=*", True), ("target=* platform=*", True), ], ), ], ) def test_attribute_existence_in_satisfies(spec_str, assertions, mock_packages, config): """Tests the semantics of =* when used in Spec.satisfies""" s = Spec(spec_str) for test, expected in assertions: assert s.satisfies(test) is expected @pytest.mark.regression("51768") @pytest.mark.parametrize("spec_str", ["mpi", "%mpi", "^mpi", "%foo", "%c=gcc", "%[when=%c]c=gcc"]) def test_specs_semantics_on_self(spec_str, mock_packages, config): """Tests that an abstract spec satisfies and intersects with itself.""" s = Spec(spec_str) assert s.satisfies(s) assert s.intersects(s) @pytest.mark.parametrize( "spec_str,expected_fmt", [ ("mpileaks@2.2", "mpileaks@_R{@=2.2}"), ("mpileaks@2.3", "mpileaks@c{@=2.3}"), ("mpileaks+debug", "@_R{+debug}"), ], ) def test_highlighting_spec_parts(spec_str, expected_fmt, default_mock_concretization): """Tests correct highlighting of non-default versions and variants""" s = default_mock_concretization(spec_str) expected = colorize(expected_fmt, color=True) colorized_str = s.format( color=True, highlight_version_fn=spack.package_base.non_preferred_version, highlight_variant_fn=spack.package_base.non_default_variant, ) assert expected in colorized_str ================================================ FILE: lib/spack/spack/test/spec_syntax.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import itertools import os import pathlib import re import sys import pytest import spack.binary_distribution import spack.cmd import spack.concretize import spack.error import spack.llnl.util.filesystem as fs import spack.platforms.test import spack.repo import spack.solver.asp import spack.spec from spack.spec_parser import ( UNIX_FILENAME, WINDOWS_FILENAME, SpecParser, SpecParsingError, SpecTokenizationError, SpecTokens, expand_toolchains, parse_one_or_raise, ) from spack.tokenize import Token SKIP_ON_WINDOWS = pytest.mark.skipif(sys.platform == "win32", reason="Unix style path on Windows") SKIP_ON_UNIX = pytest.mark.skipif(sys.platform != "win32", reason="Windows style path on Unix") def simple_package_name(name): """A simple package name in canonical form""" return name, [Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=name)], name def dependency_with_version(text): root, rest = text.split("^") dependency, version = rest.split("@") return ( text, [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=root.strip()), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=dependency.strip()), Token(SpecTokens.VERSION, value=f"@{version}"), ], text, ) @pytest.fixture() def specfile_for(default_mock_concretization): def _specfile_for(spec_str, filename): s = default_mock_concretization(spec_str) is_json = str(filename).endswith(".json") is_yaml = str(filename).endswith(".yaml") if not is_json and not is_yaml: raise ValueError("wrong extension used for specfile") with filename.open("w") as f: if is_json: f.write(s.to_json()) else: f.write(s.to_yaml()) return s return _specfile_for @pytest.mark.parametrize( "spec_str,tokens,expected_roundtrip", [ # Package names simple_package_name("mvapich"), simple_package_name("mvapich_foo"), simple_package_name("_mvapich_foo"), simple_package_name("3dtk"), simple_package_name("ns-3-dev"), # Single token anonymous specs ("@2.7", [Token(SpecTokens.VERSION, value="@2.7")], "@2.7"), ("@2.7:", [Token(SpecTokens.VERSION, value="@2.7:")], "@2.7:"), ("@:2.7", [Token(SpecTokens.VERSION, value="@:2.7")], "@:2.7"), ("+foo", [Token(SpecTokens.BOOL_VARIANT, value="+foo")], "+foo"), ("~foo", [Token(SpecTokens.BOOL_VARIANT, value="~foo")], "~foo"), ("-foo", [Token(SpecTokens.BOOL_VARIANT, value="-foo")], "~foo"), ( "platform=test", [Token(SpecTokens.KEY_VALUE_PAIR, value="platform=test")], "platform=test", ), # Multiple tokens anonymous specs ( "%intel", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "intel"), ], "%intel", ), ( "languages=go @4.2:", [ Token(SpecTokens.KEY_VALUE_PAIR, value="languages=go"), Token(SpecTokens.VERSION, value="@4.2:"), ], "@4.2: languages=go", ), ( "@4.2: languages=go", [ Token(SpecTokens.VERSION, value="@4.2:"), Token(SpecTokens.KEY_VALUE_PAIR, value="languages=go"), ], "@4.2: languages=go", ), ( "^zlib", [ Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="zlib"), ], "^zlib", ), # Specs with simple dependencies ( "openmpi ^hwloc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="openmpi"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="hwloc"), ], "openmpi ^hwloc", ), ( "openmpi ^hwloc ^libunwind", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="openmpi"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="hwloc"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="libunwind"), ], "openmpi ^hwloc ^libunwind", ), ( "openmpi ^hwloc^libunwind", [ # White spaces are tested Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="openmpi"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="hwloc"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="libunwind"), ], "openmpi ^hwloc ^libunwind", ), # Version after compiler ( "foo @2.0 %bar@1.0", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="foo"), Token(SpecTokens.VERSION, value="@2.0"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="bar"), Token(SpecTokens.VERSION, value="@1.0"), ], "foo@2.0 %bar@1.0", ), # Single dependency with version dependency_with_version("openmpi ^hwloc@1.2e6"), dependency_with_version("openmpi ^hwloc@1.2e6:"), dependency_with_version("openmpi ^hwloc@:1.4b7-rc3"), dependency_with_version("openmpi ^hwloc@1.2e6:1.4b7-rc3"), # Complex specs with multiple constraints ( "mvapich_foo ^_openmpi@1.2:1.4,1.6+debug~qt_4 %intel@12.1 ^stackwalker@8.1_1e", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"), Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"), Token(SpecTokens.BOOL_VARIANT, value="+debug"), Token(SpecTokens.BOOL_VARIANT, value="~qt_4"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), Token(SpecTokens.VERSION, value="@12.1"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"), Token(SpecTokens.VERSION, value="@8.1_1e"), ], "mvapich_foo ^_openmpi@1.2:1.4,1.6+debug~qt_4 %intel@12.1 ^stackwalker@8.1_1e", ), ( "mvapich_foo ^_openmpi@1.2:1.4,1.6~qt_4 debug=2 %intel@12.1 ^stackwalker@8.1_1e", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"), Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"), Token(SpecTokens.BOOL_VARIANT, value="~qt_4"), Token(SpecTokens.KEY_VALUE_PAIR, value="debug=2"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), Token(SpecTokens.VERSION, value="@12.1"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"), Token(SpecTokens.VERSION, value="@8.1_1e"), ], "mvapich_foo ^_openmpi@1.2:1.4,1.6~qt_4 debug=2 %intel@12.1 ^stackwalker@8.1_1e", ), ( "mvapich_foo ^_openmpi@1.2:1.4,1.6 cppflags=-O3 +debug~qt_4 %intel@12.1 " "^stackwalker@8.1_1e", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"), Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"), Token(SpecTokens.KEY_VALUE_PAIR, value="cppflags=-O3"), Token(SpecTokens.BOOL_VARIANT, value="+debug"), Token(SpecTokens.BOOL_VARIANT, value="~qt_4"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), Token(SpecTokens.VERSION, value="@12.1"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"), Token(SpecTokens.VERSION, value="@8.1_1e"), ], "mvapich_foo ^_openmpi@1.2:1.4,1.6 cppflags=-O3 +debug~qt_4 %intel@12.1" " ^stackwalker@8.1_1e", ), # Specs containing YAML or JSON in the package name ( "yaml-cpp@0.1.8%intel@12.1 ^boost@3.1.4", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="yaml-cpp"), Token(SpecTokens.VERSION, value="@0.1.8"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), Token(SpecTokens.VERSION, value="@12.1"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="boost"), Token(SpecTokens.VERSION, value="@3.1.4"), ], "yaml-cpp@0.1.8 %intel@12.1 ^boost@3.1.4", ), ( r"builtin.yaml-cpp%gcc", [ Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="builtin.yaml-cpp"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), ], "yaml-cpp %gcc", ), ( r"testrepo.yaml-cpp%gcc", [ Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="testrepo.yaml-cpp"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), ], "yaml-cpp %gcc", ), ( r"builtin.yaml-cpp@0.1.8%gcc@7.2.0 ^boost@3.1.4", [ Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="builtin.yaml-cpp"), Token(SpecTokens.VERSION, value="@0.1.8"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.VERSION, value="@7.2.0"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="boost"), Token(SpecTokens.VERSION, value="@3.1.4"), ], "yaml-cpp@0.1.8 %gcc@7.2.0 ^boost@3.1.4", ), ( r"builtin.yaml-cpp ^testrepo.boost ^zlib", [ Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="builtin.yaml-cpp"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="testrepo.boost"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="zlib"), ], "yaml-cpp ^boost ^zlib", ), # Canonicalization of the string representation ( r"mvapich ^stackwalker ^_openmpi", # Dependencies are reordered [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"), ], "mvapich ^_openmpi ^stackwalker", ), ( r"y~f+e~d+c~b+a", # Variants are reordered [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="y"), Token(SpecTokens.BOOL_VARIANT, value="~f"), Token(SpecTokens.BOOL_VARIANT, value="+e"), Token(SpecTokens.BOOL_VARIANT, value="~d"), Token(SpecTokens.BOOL_VARIANT, value="+c"), Token(SpecTokens.BOOL_VARIANT, value="~b"), Token(SpecTokens.BOOL_VARIANT, value="+a"), ], "y+a~b+c~d+e~f", ), # Things that evaluate to Spec() # TODO: consider making these format to "*" instead of "" ("@:", [Token(SpecTokens.VERSION, value="@:")], r""), ("*", [Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="*")], r""), # virtual assignment on a dep of an anonymous spec (more of these later) ( "%foo=bar", [Token(SpecTokens.DEPENDENCY, value="%foo=bar", virtuals="foo", substitute="bar")], "%foo=bar", ), ( "^foo=bar", [Token(SpecTokens.DEPENDENCY, value="^foo=bar", virtuals="foo", substitute="bar")], "^foo=bar", ), # anonymous dependencies with variants ( "^*foo=bar", [ Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="*"), Token(SpecTokens.KEY_VALUE_PAIR, value="foo=bar"), ], "^*foo=bar", ), ( "%*foo=bar", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="*"), Token(SpecTokens.KEY_VALUE_PAIR, value="foo=bar"), ], "%*foo=bar", ), ( "^*+foo", [ Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="*"), Token(SpecTokens.BOOL_VARIANT, value="+foo"), ], "^+foo", ), ( "^*~foo", [ Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="*"), Token(SpecTokens.BOOL_VARIANT, value="~foo"), ], "^~foo", ), ( "%*+foo", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="*"), Token(SpecTokens.BOOL_VARIANT, value="+foo"), ], "%+foo", ), ( "%*~foo", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="*"), Token(SpecTokens.BOOL_VARIANT, value="~foo"), ], "%~foo", ), # version range and list ("@1.6,1.2:1.4", [Token(SpecTokens.VERSION, value="@1.6,1.2:1.4")], r"@1.2:1.4,1.6"), ( r"os=fe", # Various translations associated with the architecture [Token(SpecTokens.KEY_VALUE_PAIR, value="os=fe")], "platform=test os=debian6", ), ( r"os=default_os", [Token(SpecTokens.KEY_VALUE_PAIR, value="os=default_os")], "platform=test os=debian6", ), ( r"target=be", [Token(SpecTokens.KEY_VALUE_PAIR, value="target=be")], f"platform=test target={spack.platforms.test.Test.default}", ), ( r"target=default_target", [Token(SpecTokens.KEY_VALUE_PAIR, value="target=default_target")], f"platform=test target={spack.platforms.test.Test.default}", ), ( r"platform=linux", [Token(SpecTokens.KEY_VALUE_PAIR, value="platform=linux")], r"platform=linux", ), # Version hash pair ( rf"develop-branch-version@{'abc12' * 8}=develop", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="develop-branch-version"), Token(SpecTokens.VERSION_HASH_PAIR, value=f"@{'abc12' * 8}=develop"), ], rf"develop-branch-version@{'abc12' * 8}=develop", ), # Redundant specs ( r"x ^y@foo ^y@foo", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="x"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="y"), Token(SpecTokens.VERSION, value="@foo"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="y"), Token(SpecTokens.VERSION, value="@foo"), ], r"x ^y@foo", ), ( r"x ^y@foo ^y+bar", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="x"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="y"), Token(SpecTokens.VERSION, value="@foo"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="y"), Token(SpecTokens.BOOL_VARIANT, value="+bar"), ], r"x ^y@foo+bar", ), ( r"x ^y@foo +bar ^y@foo", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="x"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="y"), Token(SpecTokens.VERSION, value="@foo"), Token(SpecTokens.BOOL_VARIANT, value="+bar"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="y"), Token(SpecTokens.VERSION, value="@foo"), ], r"x ^y@foo+bar", ), # Ambiguous variant specification ( r"_openmpi +debug-qt_4", # Parse as a single bool variant [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"), Token(SpecTokens.BOOL_VARIANT, value="+debug-qt_4"), ], r"_openmpi+debug-qt_4", ), ( r"_openmpi +debug -qt_4", # Parse as two variants [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"), Token(SpecTokens.BOOL_VARIANT, value="+debug"), Token(SpecTokens.BOOL_VARIANT, value="-qt_4"), ], r"_openmpi+debug~qt_4", ), ( r"_openmpi +debug~qt_4", # Parse as two variants [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"), Token(SpecTokens.BOOL_VARIANT, value="+debug"), Token(SpecTokens.BOOL_VARIANT, value="~qt_4"), ], r"_openmpi+debug~qt_4", ), # Key value pairs with ":" and "," in the value ( r"target=:broadwell,icelake", [Token(SpecTokens.KEY_VALUE_PAIR, value="target=:broadwell,icelake")], r"target=:broadwell,icelake", ), # Hash pair version followed by a variant ( f"develop-branch-version@git.{'a' * 40}=develop+var1+var2", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="develop-branch-version"), Token(SpecTokens.VERSION_HASH_PAIR, value=f"@git.{'a' * 40}=develop"), Token(SpecTokens.BOOL_VARIANT, value="+var1"), Token(SpecTokens.BOOL_VARIANT, value="+var2"), ], f"develop-branch-version@git.{'a' * 40}=develop+var1+var2", ), # Compiler with version ranges ( "%gcc@10.2.1:", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.VERSION, value="@10.2.1:"), ], "%gcc@10.2.1:", ), ( "%gcc@:10.2.1", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.VERSION, value="@:10.2.1"), ], "%gcc@:10.2.1", ), ( "%gcc@10.2.1:12.1.0", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.VERSION, value="@10.2.1:12.1.0"), ], "%gcc@10.2.1:12.1.0", ), ( "%gcc@10.1.0,12.2.1:", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.VERSION, value="@10.1.0,12.2.1:"), ], "%gcc@10.1.0,12.2.1:", ), ( "%gcc@:8.4.3,10.2.1:12.1.0", [ Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.VERSION, value="@:8.4.3,10.2.1:12.1.0"), ], "%gcc@:8.4.3,10.2.1:12.1.0", ), # Special key value arguments ("dev_path=*", [Token(SpecTokens.KEY_VALUE_PAIR, value="dev_path=*")], "dev_path='*'"), ( "dev_path=none", [Token(SpecTokens.KEY_VALUE_PAIR, value="dev_path=none")], "dev_path=none", ), ( "dev_path=../relpath/work", [Token(SpecTokens.KEY_VALUE_PAIR, value="dev_path=../relpath/work")], "dev_path=../relpath/work", ), ( "dev_path=/abspath/work", [Token(SpecTokens.KEY_VALUE_PAIR, value="dev_path=/abspath/work")], "dev_path=/abspath/work", ), # One liner for flags like 'a=b=c' that are injected ( "cflags=a=b=c", [Token(SpecTokens.KEY_VALUE_PAIR, value="cflags=a=b=c")], "cflags='a=b=c'", ), ( "cflags=a=b=c", [Token(SpecTokens.KEY_VALUE_PAIR, value="cflags=a=b=c")], "cflags='a=b=c'", ), ( "cflags=a=b=c+~", [Token(SpecTokens.KEY_VALUE_PAIR, value="cflags=a=b=c+~")], "cflags='a=b=c+~'", ), ( "cflags=-Wl,a,b,c", [Token(SpecTokens.KEY_VALUE_PAIR, value="cflags=-Wl,a,b,c")], "cflags=-Wl,a,b,c", ), # Multi quoted ( 'cflags=="-O3 -g"', [Token(SpecTokens.PROPAGATED_KEY_VALUE_PAIR, value='cflags=="-O3 -g"')], "cflags=='-O3 -g'", ), # Whitespace is allowed in version lists ("@1.2:1.4 , 1.6 ", [Token(SpecTokens.VERSION, value="@1.2:1.4 , 1.6")], "@1.2:1.4,1.6"), # But not in ranges. `a@1:` and `b` are separate specs, not a single `a@1:b`. ( "a@1: b", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="a"), Token(SpecTokens.VERSION, value="@1:"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="b"), ], "a@1:", ), ( "+ debug % intel @ 12.1:12.6", [ Token(SpecTokens.BOOL_VARIANT, value="+ debug"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), Token(SpecTokens.VERSION, value="@ 12.1:12.6"), ], "+debug %intel@12.1:12.6", ), ( "@ 12.1:12.6 + debug - qt_4", [ Token(SpecTokens.VERSION, value="@ 12.1:12.6"), Token(SpecTokens.BOOL_VARIANT, value="+ debug"), Token(SpecTokens.BOOL_VARIANT, value="- qt_4"), ], "@12.1:12.6+debug~qt_4", ), ( "@10.4.0:10,11.3.0:target=aarch64:", [ Token(SpecTokens.VERSION, value="@10.4.0:10,11.3.0:"), Token(SpecTokens.KEY_VALUE_PAIR, value="target=aarch64:"), ], "@10.4.0:10,11.3.0: target=aarch64:", ), ( "@:0.4 % nvhpc", [ Token(SpecTokens.VERSION, value="@:0.4"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="nvhpc"), ], "@:0.4 %nvhpc", ), ( "^[virtuals=mpi] openmpi", [ Token(SpecTokens.START_EDGE_PROPERTIES, value="^["), Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=mpi"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="openmpi"), ], "^mpi=openmpi", ), ( "^mpi=openmpi", [ Token( SpecTokens.DEPENDENCY, value="^mpi=openmpi", virtuals="mpi", substitute="openmpi", ) ], "^mpi=openmpi", ), # Allow merging attributes, if deptypes match ( "^[virtuals=mpi] openmpi+foo ^[virtuals=lapack] openmpi+bar", [ Token(SpecTokens.START_EDGE_PROPERTIES, value="^["), Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=mpi"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="openmpi"), Token(SpecTokens.BOOL_VARIANT, value="+foo"), Token(SpecTokens.START_EDGE_PROPERTIES, value="^["), Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=lapack"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="openmpi"), Token(SpecTokens.BOOL_VARIANT, value="+bar"), ], "^lapack,mpi=openmpi+bar+foo", ), ( "^lapack,mpi=openmpi+foo+bar", [ Token( SpecTokens.DEPENDENCY, value="^lapack,mpi=openmpi", virtuals="lapack,mpi", substitute="openmpi", ), Token(SpecTokens.BOOL_VARIANT, value="+foo"), Token(SpecTokens.BOOL_VARIANT, value="+bar"), ], "^lapack,mpi=openmpi+bar+foo", ), ( "^[deptypes=link,build] zlib", [ Token(SpecTokens.START_EDGE_PROPERTIES, value="^["), Token(SpecTokens.KEY_VALUE_PAIR, value="deptypes=link,build"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="zlib"), ], "^[deptypes=build,link] zlib", ), ( "^[deptypes=link] zlib ^[deptypes=build] zlib", [ Token(SpecTokens.START_EDGE_PROPERTIES, value="^["), Token(SpecTokens.KEY_VALUE_PAIR, value="deptypes=link"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="zlib"), Token(SpecTokens.START_EDGE_PROPERTIES, value="^["), Token(SpecTokens.KEY_VALUE_PAIR, value="deptypes=build"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="zlib"), ], "^[deptypes=link] zlib ^[deptypes=build] zlib", ), ( "git-test@git.foo/bar", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "git-test"), Token(SpecTokens.GIT_VERSION, "@git.foo/bar"), ], "git-test@git.foo/bar", ), # Variant propagation ( "zlib ++foo", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token(SpecTokens.PROPAGATED_BOOL_VARIANT, "++foo"), ], "zlib++foo", ), ( "zlib ~~foo", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token(SpecTokens.PROPAGATED_BOOL_VARIANT, "~~foo"), ], "zlib~~foo", ), ( "zlib foo==bar", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token(SpecTokens.PROPAGATED_KEY_VALUE_PAIR, "foo==bar"), ], "zlib foo==bar", ), # Compilers specifying virtuals ( "zlib %[virtuals=c] gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=c"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), ], "zlib %c=gcc", ), ( "zlib %c=gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token(SpecTokens.DEPENDENCY, value="%c=gcc", virtuals="c", substitute="gcc"), ], "zlib %c=gcc", ), ( "zlib %[virtuals=c,cxx] gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=c,cxx"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), ], "zlib %c,cxx=gcc", ), ( "zlib %c,cxx=gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token( SpecTokens.DEPENDENCY, value="%c,cxx=gcc", virtuals="c,cxx", substitute="gcc" ), ], "zlib %c,cxx=gcc", ), ( "zlib %[virtuals=c,cxx] gcc@14.1", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=c,cxx"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.VERSION, value="@14.1"), ], "zlib %c,cxx=gcc@14.1", ), ( "zlib %c,cxx=gcc@14.1", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token( SpecTokens.DEPENDENCY, value="%c,cxx=gcc", virtuals="c,cxx", substitute="gcc" ), Token(SpecTokens.VERSION, value="@14.1"), ], "zlib %c,cxx=gcc@14.1", ), ( "zlib %[virtuals=fortran] gcc@14.1 %[virtuals=c,cxx] clang", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=fortran"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.VERSION, value="@14.1"), Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=c,cxx"), Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="clang"), ], "zlib %fortran=gcc@14.1 %c,cxx=clang", ), ( "zlib %fortran=gcc@14.1 %c,cxx=clang", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), Token( SpecTokens.DEPENDENCY, value="%fortran=gcc", virtuals="fortran", substitute="gcc", ), Token(SpecTokens.VERSION, value="@14.1"), Token( SpecTokens.DEPENDENCY, value="%c,cxx=clang", virtuals="c,cxx", substitute="clang", ), ], "zlib %fortran=gcc@14.1 %c,cxx=clang", ), # test := and :== syntax for key value pairs ( "gcc languages:=c,c++", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "gcc"), Token(SpecTokens.KEY_VALUE_PAIR, "languages:=c,c++"), ], "gcc languages:='c,c++'", ), ( "gcc languages:==c,c++", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "gcc"), Token(SpecTokens.PROPAGATED_KEY_VALUE_PAIR, "languages:==c,c++"), ], "gcc languages:=='c,c++'", ), # test etc. after % ( "mvapich %gcc languages:=c,c++ target=x86_64", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "mvapich"), Token(SpecTokens.DEPENDENCY, "%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "gcc"), Token(SpecTokens.KEY_VALUE_PAIR, "languages:=c,c++"), Token(SpecTokens.KEY_VALUE_PAIR, "target=x86_64"), ], "mvapich %gcc languages:='c,c++' target=x86_64", ), # Test conditional dependencies ( "foo ^[when='%c' virtuals=c] gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "foo"), Token(SpecTokens.START_EDGE_PROPERTIES, "^["), Token(SpecTokens.KEY_VALUE_PAIR, "when='%c'"), Token(SpecTokens.KEY_VALUE_PAIR, "virtuals=c"), Token(SpecTokens.END_EDGE_PROPERTIES, "]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "gcc"), ], "foo ^[when='%c'] c=gcc", ), ( "foo ^[when='%c' virtuals=c]gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "foo"), Token(SpecTokens.START_EDGE_PROPERTIES, "^["), Token(SpecTokens.KEY_VALUE_PAIR, "when='%c'"), Token(SpecTokens.KEY_VALUE_PAIR, "virtuals=c"), Token(SpecTokens.END_EDGE_PROPERTIES, "]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "gcc"), ], "foo ^[when='%c'] c=gcc", ), ( "foo ^[when='%c'] c=gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "foo"), Token(SpecTokens.START_EDGE_PROPERTIES, "^["), Token(SpecTokens.KEY_VALUE_PAIR, "when='%c'"), Token(SpecTokens.END_EDGE_PROPERTIES, "] c=gcc", virtuals="c", substitute="gcc"), ], "foo ^[when='%c'] c=gcc", ), # Test dependency propagation ( "foo %%gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "foo"), Token(SpecTokens.DEPENDENCY, "%%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "gcc"), ], "foo %%gcc", ), ( "foo %%c,cxx=gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "foo"), Token(SpecTokens.DEPENDENCY, "%%c,cxx=gcc", virtuals="c,cxx", substitute="gcc"), ], "foo %%c,cxx=gcc", ), ( "foo %%[when='%c'] c=gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "foo"), Token(SpecTokens.START_EDGE_PROPERTIES, "%%["), Token(SpecTokens.KEY_VALUE_PAIR, "when='%c'"), Token(SpecTokens.END_EDGE_PROPERTIES, "] c=gcc", virtuals="c", substitute="gcc"), ], "foo %%[when='%c'] c=gcc", ), ( "foo %%[when='%c' virtuals=c] gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "foo"), Token(SpecTokens.START_EDGE_PROPERTIES, "%%["), Token(SpecTokens.KEY_VALUE_PAIR, "when='%c'"), Token(SpecTokens.KEY_VALUE_PAIR, "virtuals=c"), Token(SpecTokens.END_EDGE_PROPERTIES, "]"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "gcc"), ], "foo %%[when='%c'] c=gcc", ), ], ) def test_parse_single_spec(spec_str, tokens, expected_roundtrip, mock_git_test_package): parser = SpecParser(spec_str) assert tokens == parser.tokens() assert expected_roundtrip == str(parser.next_spec()) @pytest.mark.parametrize( "text,tokens,expected_specs", [ ( "mvapich emacs", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), ], ["mvapich", "emacs"], ), ( "mvapich cppflags='-O3 -fPIC' emacs", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"), Token(SpecTokens.KEY_VALUE_PAIR, value="cppflags='-O3 -fPIC'"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), ], ["mvapich cppflags='-O3 -fPIC'", "emacs"], ), ( "mvapich cppflags=-O3 emacs", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"), Token(SpecTokens.KEY_VALUE_PAIR, value="cppflags=-O3"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), ], ["mvapich cppflags=-O3", "emacs"], ), ( "mvapich emacs @1.1.1 cflags=-O3 %intel", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), Token(SpecTokens.VERSION, value="@1.1.1"), Token(SpecTokens.KEY_VALUE_PAIR, value="cflags=-O3"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), ], ["mvapich", "emacs @1.1.1 cflags=-O3 %intel"], ), ( 'mvapich cflags="-O3 -fPIC" emacs^ncurses%intel', [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"), Token(SpecTokens.KEY_VALUE_PAIR, value='cflags="-O3 -fPIC"'), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="ncurses"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), ], ['mvapich cflags="-O3 -fPIC"', "emacs ^ncurses%intel"], ), ( "mvapich %gcc languages=c,c++ emacs ^ncurses%gcc languages:=c", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.KEY_VALUE_PAIR, value="languages=c,c++"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="ncurses"), Token(SpecTokens.DEPENDENCY, value="%"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), Token(SpecTokens.KEY_VALUE_PAIR, value="languages:=c"), ], ["mvapich %gcc languages=c,c++", "emacs ^ncurses%gcc languages:=c"], ), ], ) def test_parse_multiple_specs(text, tokens, expected_specs): total_parser = SpecParser(text) assert total_parser.tokens() == tokens for single_spec_text in expected_specs: single_spec_parser = SpecParser(single_spec_text) assert str(total_parser.next_spec()) == str(single_spec_parser.next_spec()) @pytest.mark.parametrize( "args,expected", [ # Test that CLI-quoted flags/variant values are preserved (["zlib", "cflags=-O3 -g", "+bar", "baz"], "zlib cflags='-O3 -g' +bar baz"), # Test that CLI-quoted propagated flags/variant values are preserved (["zlib", "cflags==-O3 -g", "+bar", "baz"], "zlib cflags=='-O3 -g' +bar baz"), # An entire string passed on the CLI with embedded quotes also works (["zlib cflags='-O3 -g' +bar baz"], "zlib cflags='-O3 -g' +bar baz"), # Entire string *without* quoted flags splits -O3/-g (-g interpreted as a variant) (["zlib cflags=-O3 -g +bar baz"], "zlib cflags=-O3 +bar~g baz"), # If the entirety of "-O3 -g +bar baz" is quoted on the CLI, it's all taken as flags (["zlib", "cflags=-O3 -g +bar baz"], "zlib cflags='-O3 -g +bar baz'"), # If the string doesn't start with key=, it needs internal quotes for flags (["zlib", " cflags=-O3 -g +bar baz"], "zlib cflags=-O3 +bar~g baz"), # Internal quotes for quoted CLI args are considered part of *one* arg (["zlib", 'cflags="-O3 -g" +bar baz'], """zlib cflags='"-O3 -g" +bar baz'"""), # Use double quotes if internal single quotes are present (["zlib", "cflags='-O3 -g' +bar baz"], '''zlib cflags="'-O3 -g' +bar baz"'''), # Use single quotes and escape single quotes with internal single and double quotes (["zlib", "cflags='-O3 -g' \"+bar baz\""], 'zlib cflags="\'-O3 -g\' \\"+bar baz\\""'), # Ensure that empty strings are handled correctly on CLI (["zlib", "ldflags=", "+pic"], "zlib+pic"), # These flags are assumed to be quoted by the shell, but the space doesn't matter because # flags are space-separated. (["zlib", "ldflags= +pic"], "zlib ldflags='+pic'"), (["ldflags= +pic"], "ldflags='+pic'"), # If the name is not a flag name, the space is preserved verbatim, because variant values # are comma-separated. (["zlib", "foo= +pic"], "zlib foo=' +pic'"), (["foo= +pic"], "foo=' +pic'"), # You can ensure no quotes are added parse_specs() by starting your string with space, # but you still need to quote empty strings properly. ([" ldflags= +pic"], SpecTokenizationError), ([" ldflags=", "+pic"], SpecTokenizationError), ([" ldflags='' +pic"], "+pic"), ([" ldflags=''", "+pic"], "+pic"), # Ensure that empty strings are handled properly in quoted strings (["zlib ldflags='' +pic"], "zlib+pic"), # Ensure that $ORIGIN is handled correctly (["zlib", "ldflags=-Wl,-rpath=$ORIGIN/_libs"], "zlib ldflags='-Wl,-rpath=$ORIGIN/_libs'"), # Ensure that passing escaped quotes on the CLI raises a tokenization error (["zlib", '"-g', '-O2"'], SpecTokenizationError), ], ) def test_cli_spec_roundtrip(args, expected): if isinstance(expected, type) and issubclass(expected, BaseException): with pytest.raises(expected): spack.cmd.parse_specs(args) return specs = spack.cmd.parse_specs(args) output_string = " ".join(str(spec) for spec in specs) assert expected == output_string @pytest.mark.parametrize( ["spec_str", "toolchain", "expected_roundtrip"], [ ( "foo%my_toolchain", {"my_toolchain": "%[when='%c' virtuals=c]gcc"}, ["foo %[when='%c'] c=gcc"], ), ("foo%my_toolchain", {"my_toolchain": "%[when='%c'] c=gcc"}, ["foo %[when='%c'] c=gcc"]), ( "foo%my_toolchain", {"my_toolchain": "+bar cflags=baz %[when='%c' virtuals=c]gcc"}, ["foo cflags=baz +bar %[when='%c'] c=gcc"], ), ( "foo%my_toolchain", {"my_toolchain": "+bar cflags=baz %[when='%c']c=gcc"}, ["foo cflags=baz +bar %[when='%c'] c=gcc"], ), ( "foo%my_toolchain2", {"my_toolchain2": "%[when='%c' virtuals=c]gcc %[when='+mpi' virtuals=mpi]mpich"}, ["foo %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich"], ), ( "foo%my_toolchain2", {"my_toolchain2": "%[when='%c'] c=gcc %[when='+mpi'] mpi=mpich"}, ["foo %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich"], ), ( "foo%my_toolchain bar%my_toolchain2", { "my_toolchain": "%[when='%c' virtuals=c]gcc", "my_toolchain2": "%[when='%c' virtuals=c]gcc %[when='+mpi' virtuals=mpi]mpich", }, ["foo %[when='%c'] c=gcc", "bar %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich"], ), ( "foo%my_toolchain bar%my_toolchain2", { "my_toolchain": "%[when='%c'] c=gcc", "my_toolchain2": "%[when='%c'] c=gcc %[when='+mpi']mpi=mpich", }, ["foo %[when='%c'] c=gcc", "bar %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich"], ), ( "foo%my_toolchain2", { "my_toolchain2": [ {"spec": "%[virtuals=c]gcc", "when": "%c"}, {"spec": "%[virtuals=mpi]mpich", "when": "+mpi"}, ] }, ["foo %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich"], ), ( "foo%my_toolchain2", { "my_toolchain2": [ {"spec": "%c=gcc", "when": "%c"}, {"spec": "%mpi=mpich", "when": "+mpi"}, ] }, ["foo %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich"], ), ( "foo%my_toolchain2", {"my_toolchain2": [{"spec": "%[virtuals=c]gcc %[virtuals=mpi]mpich", "when": "%c"}]}, ["foo %[when='%c'] c=gcc %[when='%c'] mpi=mpich"], ), ( "foo%my_toolchain2", {"my_toolchain2": [{"spec": "%c=gcc %mpi=mpich", "when": "%c"}]}, ["foo %[when='%c'] c=gcc %[when='%c'] mpi=mpich"], ), # Test that we don't get caching wrong in the parser ( "foo %gcc-mpich ^bar%gcc-mpich", { "gcc-mpich": [ {"spec": "%[virtuals=c] gcc", "when": "%c"}, {"spec": "%[virtuals=mpi] mpich", "when": "%mpi"}, ] }, [ "foo %[when='%c'] c=gcc %[when='%mpi'] mpi=mpich " "^bar %[when='%c'] c=gcc %[when='%mpi'] mpi=mpich" ], ), ( "foo %gcc-mpich ^bar%gcc-mpich", { "gcc-mpich": [ {"spec": "%c=gcc", "when": "%c"}, {"spec": "%mpi=mpich", "when": "%mpi"}, ] }, [ "foo %[when='%c'] c=gcc %[when='%mpi'] mpi=mpich " "^bar %[when='%c'] c=gcc %[when='%mpi'] mpi=mpich" ], ), ], ) def test_parse_toolchain(spec_str, toolchain, expected_roundtrip, mutable_config): """Tests that toolchains are expanded correctly""" parser = SpecParser(spec_str) for expected in expected_roundtrip: result = parser.next_spec() expand_toolchains(result, toolchain) assert expected == str(result) @pytest.mark.parametrize( "text,expected_in_error", [ ("x@@1.2", r"x@@1.2\n ^"), ("y ^x@@1.2", r"y ^x@@1.2\n ^"), ("x@1.2::", r"x@1.2::\n ^"), ("x::", r"x::\n ^^"), ("cflags=''-Wl,a,b,c''", r"cflags=''-Wl,a,b,c''\n ^ ^ ^ ^^"), ("@1.2: develop = foo", r"@1.2: develop = foo\n ^^"), ("@1.2:develop = foo", r"@1.2:develop = foo\n ^^"), ], ) def test_error_reporting(text, expected_in_error): parser = SpecParser(text) with pytest.raises(SpecTokenizationError) as exc: parser.tokens() assert expected_in_error in str(exc), parser.tokens() @pytest.mark.parametrize( "text,tokens", [ ("/abcde", [Token(SpecTokens.DAG_HASH, value="/abcde")]), ( "foo/abcde", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="foo"), Token(SpecTokens.DAG_HASH, value="/abcde"), ], ), ( "foo@1.2.3 /abcde", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="foo"), Token(SpecTokens.VERSION, value="@1.2.3"), Token(SpecTokens.DAG_HASH, value="/abcde"), ], ), ], ) def test_spec_by_hash_tokens(text, tokens): parser = SpecParser(text) assert parser.tokens() == tokens @pytest.mark.db def test_spec_by_hash(database, monkeypatch, config): mpileaks = database.query_one("mpileaks ^zmpi") b = spack.concretize.concretize_one("pkg-b") monkeypatch.setattr(spack.binary_distribution, "update_cache_and_get_specs", lambda: [b]) hash_str = f"/{mpileaks.dag_hash()}" parsed_spec = SpecParser(hash_str).next_spec() parsed_spec.replace_hash() assert parsed_spec == mpileaks short_hash_str = f"/{mpileaks.dag_hash()[:5]}" parsed_spec = SpecParser(short_hash_str).next_spec() parsed_spec.replace_hash() assert parsed_spec == mpileaks name_version_and_hash = f"{mpileaks.name}@{mpileaks.version} /{mpileaks.dag_hash()[:5]}" parsed_spec = SpecParser(name_version_and_hash).next_spec() parsed_spec.replace_hash() assert parsed_spec == mpileaks b_hash = f"/{b.dag_hash()}" parsed_spec = SpecParser(b_hash).next_spec() parsed_spec.replace_hash() assert parsed_spec == b @pytest.mark.db def test_dep_spec_by_hash(database, config): mpileaks_zmpi = database.query_one("mpileaks ^zmpi") zmpi = database.query_one("zmpi") fake = database.query_one("fake") assert "fake" in mpileaks_zmpi assert "zmpi" in mpileaks_zmpi mpileaks_hash_fake = SpecParser(f"mpileaks ^/{fake.dag_hash()} ^zmpi").next_spec() mpileaks_hash_fake.replace_hash() assert "fake" in mpileaks_hash_fake assert mpileaks_hash_fake["fake"] == fake assert "zmpi" in mpileaks_hash_fake assert mpileaks_hash_fake["zmpi"] == spack.spec.Spec("zmpi") mpileaks_hash_zmpi = SpecParser(f"mpileaks ^ /{zmpi.dag_hash()}").next_spec() mpileaks_hash_zmpi.replace_hash() assert "zmpi" in mpileaks_hash_zmpi assert mpileaks_hash_zmpi["zmpi"] == zmpi mpileaks_hash_fake_and_zmpi = SpecParser( f"mpileaks ^/{fake.dag_hash()[:4]} ^ /{zmpi.dag_hash()[:5]}" ).next_spec() mpileaks_hash_fake_and_zmpi.replace_hash() assert "zmpi" in mpileaks_hash_fake_and_zmpi assert mpileaks_hash_fake_and_zmpi["zmpi"] == zmpi assert "fake" in mpileaks_hash_fake_and_zmpi assert mpileaks_hash_fake_and_zmpi["fake"] == fake @pytest.mark.db def test_multiple_specs_with_hash(database, config): mpileaks_zmpi = database.query_one("mpileaks ^zmpi") callpath_mpich2 = database.query_one("callpath ^mpich2") # name + hash + separate hash specs = SpecParser( f"mpileaks /{mpileaks_zmpi.dag_hash()} /{callpath_mpich2.dag_hash()}" ).all_specs() assert len(specs) == 2 # 2 separate hashes specs = SpecParser(f"/{mpileaks_zmpi.dag_hash()} /{callpath_mpich2.dag_hash()}").all_specs() assert len(specs) == 2 # 2 separate hashes + name specs = SpecParser( f"/{mpileaks_zmpi.dag_hash()} /{callpath_mpich2.dag_hash()} callpath" ).all_specs() assert len(specs) == 3 # hash + 2 names specs = SpecParser(f"/{mpileaks_zmpi.dag_hash()} callpath callpath").all_specs() assert len(specs) == 3 # hash + name + hash specs = SpecParser( f"/{mpileaks_zmpi.dag_hash()} callpath /{callpath_mpich2.dag_hash()}" ).all_specs() assert len(specs) == 2 @pytest.mark.db def test_ambiguous_hash(mutable_database): """Test that abstract hash ambiguity is delayed until concretization. In the past this ambiguity error would happen during parse time.""" # This is a very sketchy as manually setting hashes easily breaks invariants x1 = spack.concretize.concretize_one("pkg-a") x2 = x1.copy() x1._hash = "xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" x2._hash = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" assert x1 != x2 # doesn't hold when only the dag hash is modified. mutable_database.add(x1) mutable_database.add(x2) # ambiguity in first hash character s1 = SpecParser("/x").next_spec() with pytest.raises(spack.spec.AmbiguousHashError): s1.lookup_hash() # ambiguity in first hash character AND spec name s2 = SpecParser("pkg-a/x").next_spec() with pytest.raises(spack.spec.AmbiguousHashError): s2.lookup_hash() @pytest.mark.db def test_invalid_hash(database, config): zmpi = database.query_one("zmpi") mpich = database.query_one("mpich") # name + incompatible hash with pytest.raises(spack.spec.InvalidHashError): parsed_spec = SpecParser(f"zmpi /{mpich.dag_hash()}").next_spec() parsed_spec.replace_hash() with pytest.raises(spack.spec.InvalidHashError): parsed_spec = SpecParser(f"mpich /{zmpi.dag_hash()}").next_spec() parsed_spec.replace_hash() # name + dep + incompatible hash with pytest.raises(spack.spec.InvalidHashError): parsed_spec = SpecParser(f"mpileaks ^zmpi /{mpich.dag_hash()}").next_spec() parsed_spec.replace_hash() def test_invalid_hash_dep(database, config): mpich = database.query_one("mpich") hash = mpich.dag_hash() with pytest.raises(spack.spec.InvalidHashError): spack.spec.Spec(f"callpath ^zlib/{hash}").replace_hash() @pytest.mark.db def test_nonexistent_hash(database, config): """Ensure we get errors for non existent hashes.""" specs = database.query() # This hash shouldn't be in the test DB. What are the odds :) no_such_hash = "aaaaaaaaaaaaaaa" hashes = [s._hash for s in specs] assert no_such_hash not in [h[: len(no_such_hash)] for h in hashes] with pytest.raises(spack.spec.InvalidHashError): parsed_spec = SpecParser(f"/{no_such_hash}").next_spec() parsed_spec.replace_hash() @pytest.mark.parametrize( "spec1,spec2,constraint", [ ("zlib", "hdf5", None), ("zlib+shared", "zlib~shared", "+shared"), ("hdf5+mpi^zmpi", "hdf5~mpi", "^zmpi"), ("hdf5+mpi^mpich+debug", "hdf5+mpi^mpich~debug", "^mpich+debug"), ], ) def test_disambiguate_hash_by_spec(spec1, spec2, constraint, mock_packages, monkeypatch, config): spec1_concrete = spack.concretize.concretize_one(spec1) spec2_concrete = spack.concretize.concretize_one(spec2) spec1_concrete._hash = "spec1" spec2_concrete._hash = "spec2" monkeypatch.setattr( spack.binary_distribution, "update_cache_and_get_specs", lambda: [spec1_concrete, spec2_concrete], ) # Ordering is tricky -- for constraints we want after, for names we want before if not constraint: spec = spack.spec.Spec(spec1 + "/spec") else: spec = spack.spec.Spec("/spec" + constraint) assert spec.lookup_hash() == spec1_concrete @pytest.mark.parametrize( "text,match_string", [ # Duplicate variants ("x@1.2+debug+debug", "variant"), ("x ^y@1.2+debug debug=true", "variant"), ("x ^y@1.2 debug=false debug=true", "variant"), ("x ^y@1.2 debug=false ~debug", "variant"), # Multiple versions ("x@1.2@2.3", "version"), ("x@1.2:2.3@1.4", "version"), ("x@1.2@2.3:2.4", "version"), ("x@1.2@2.3,2.4", "version"), ("x@1.2 +foo~bar @2.3", "version"), ("x@1.2%y@1.2@2.3:2.4", "version"), # Duplicate dependency ("x ^y@1 ^y@2", "Cannot depend on incompatible specs"), # Duplicate Architectures ("x arch=linux-rhel7-x86_64 arch=linux-rhel7-x86_64", "two architectures"), ("x arch=linux-rhel7-x86_64 arch=linux-rhel7-ppc64le", "two architectures"), ("x arch=linux-rhel7-ppc64le arch=linux-rhel7-x86_64", "two architectures"), ("y ^x arch=linux-rhel7-x86_64 arch=linux-rhel7-x86_64", "two architectures"), ("y ^x arch=linux-rhel7-x86_64 arch=linux-rhel7-ppc64le", "two architectures"), ("x os=redhat6 os=debian6", "'os'"), ("x os=debian6 os=redhat6", "'os'"), ("x target=core2 target=x86_64", "'target'"), ("x target=x86_64 target=core2", "'target'"), ("x platform=test platform=test", "'platform'"), # TODO: these two seem wrong: need to change how arch is initialized (should fail on os) ("x os=debian6 platform=test target=default_target os=redhat6", "two architectures"), ("x target=default_target platform=test os=redhat6 os=debian6", "'platform'"), # Dependencies ("^[@foo] zlib", "edge attributes"), ("x ^[deptypes=link]foo ^[deptypes=run]foo", "conflicting dependency types"), ("x ^[deptypes=build,link]foo ^[deptypes=link]foo", "conflicting dependency types"), # TODO: Remove this as soon as use variants are added and we can parse custom attributes ("^[foo=bar] zlib", "edge attributes"), # Propagating reserved names generates a parse error ("x namespace==foo.bar.baz", "Propagation"), ("x arch==linux-rhel9-x86_64", "Propagation"), ("x architecture==linux-rhel9-x86_64", "Propagation"), ("x os==rhel9", "Propagation"), ("x operating_system==rhel9", "Propagation"), ("x target==x86_64", "Propagation"), ("x dev_path==/foo/bar/baz", "Propagation"), ("x patches==abcde12345,12345abcde", "Propagation"), ], ) def test_error_conditions(text, match_string): with pytest.raises(SpecParsingError, match=match_string): SpecParser(text).next_spec() @pytest.mark.parametrize( "text,exc_cls", [ # Specfile related errors pytest.param( "/bogus/path/libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS ), pytest.param( "../../libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS ), pytest.param("./libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS), pytest.param( "libfoo ^/bogus/path/libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS, ), pytest.param( "libfoo ^../../libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS ), pytest.param( "libfoo ^./libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS ), pytest.param( "/bogus/path/libdwarf.yamlfoobar", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS, ), pytest.param( "libdwarf^/bogus/path/libelf.yamlfoobar ^/path/to/bogus.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS, ), pytest.param( "c:\\bogus\\path\\libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX ), pytest.param("..\\..\\libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX), pytest.param(".\\libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX), pytest.param( "libfoo ^c:\\bogus\\path\\libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX, ), pytest.param( "libfoo ^..\\..\\libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX ), pytest.param( "libfoo ^.\\libdwarf.yaml", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX ), pytest.param( "c:\\bogus\\path\\libdwarf.yamlfoobar", spack.error.SpecFilenameError, marks=SKIP_ON_UNIX, ), pytest.param( "libdwarf^c:\\bogus\\path\\libelf.yamlfoobar ^c:\\path\\to\\bogus.yaml", spack.error.SpecFilenameError, marks=SKIP_ON_UNIX, ), ], ) def test_specfile_error_conditions_windows(text, exc_cls): with pytest.raises(exc_cls): SpecParser(text).all_specs() @pytest.mark.parametrize( "filename,regex", [ (r"c:\abs\windows\\path.yaml", WINDOWS_FILENAME), (r".\\relative\\dot\\win\\path.yaml", WINDOWS_FILENAME), (r"relative\\windows\\path.yaml", WINDOWS_FILENAME), ("/absolute/path/to/file.yaml", UNIX_FILENAME), ("relative/path/to/file.yaml", UNIX_FILENAME), ("./dot/rel/to/file.yaml", UNIX_FILENAME), ], ) def test_specfile_parsing(filename, regex): match = re.match(regex, filename) assert match assert match.end() == len(filename) def test_parse_specfile_simple(specfile_for, tmp_path: pathlib.Path): specfile = tmp_path / "libdwarf.json" s = specfile_for("libdwarf", specfile) spec = SpecParser(str(specfile)).next_spec() assert spec == s # Check we can mix literal and spec-file in text specs = SpecParser(f"mvapich_foo {str(specfile)}").all_specs() assert len(specs) == 2 @pytest.mark.parametrize("filename", ["libelf.yaml", "libelf.json"]) def test_parse_filename_missing_slash_as_spec(specfile_for, tmp_path: pathlib.Path, filename): """Ensure that libelf(.yaml|.json) parses as a spec, NOT a file.""" specfile = tmp_path / filename specfile_for(filename.split(".")[0], specfile) # Move to where the specfile is located so that libelf.yaml is there with fs.working_dir(str(tmp_path)): specs = SpecParser("libelf.yaml").all_specs() assert len(specs) == 1 spec = specs[0] assert spec.name == "yaml" assert spec.namespace == "libelf" assert spec.fullname == "libelf.yaml" # Check that if we concretize this spec, we get a good error # message that mentions we might've meant a file. with pytest.raises(spack.repo.UnknownEntityError) as exc_info: spack.concretize.concretize_one(spec) assert exc_info.value.long_message assert ( "Did you mean to specify a filename with './libelf.yaml'?" in exc_info.value.long_message ) # make sure that only happens when the spec ends in yaml with pytest.raises(spack.solver.asp.UnsatisfiableSpecError) as exc_info: spack.concretize.concretize_one("builtin_mock.doesnotexist") assert not exc_info.value.long_message or ( "Did you mean to specify a filename with" not in exc_info.value.long_message ) def test_parse_specfile_dependency(default_mock_concretization, tmp_path: pathlib.Path): """Ensure we can use a specfile as a dependency""" s = default_mock_concretization("libdwarf") specfile = tmp_path / "libelf.json" with open(specfile, "w", encoding="utf-8") as f: f.write(s["libelf"].to_json()) # Make sure we can use yaml path as dependency, e.g.: # "spack spec libdwarf ^ /path/to/libelf.json" spec = SpecParser(f"libdwarf ^ {str(specfile)}").next_spec() assert spec and spec["libelf"] == s["libelf"] with fs.working_dir(str(tmp_path)): # Make sure this also works: "spack spec ./libelf.yaml" spec = SpecParser(f"libdwarf^.{os.path.sep}{specfile.name}").next_spec() assert spec and spec["libelf"] == s["libelf"] # Should also be accepted: "spack spec ..//libelf.yaml" spec = SpecParser( f"libdwarf^..{os.path.sep}{specfile.parent.name}{os.path.sep}{specfile.name}" ).next_spec() assert spec and spec["libelf"] == s["libelf"] def test_parse_specfile_relative_paths(specfile_for, tmp_path: pathlib.Path): specfile = tmp_path / "libdwarf.json" s = specfile_for("libdwarf", specfile) basename = specfile.name parent_dir = specfile.parent with fs.working_dir(str(parent_dir)): # Make sure this also works: "spack spec ./libelf.yaml" spec = SpecParser(f".{os.path.sep}{basename}").next_spec() assert spec == s # Should also be accepted: "spack spec ..//libelf.yaml" spec = SpecParser(f"..{os.path.sep}{parent_dir.name}{os.path.sep}{basename}").next_spec() assert spec == s # Should also handle mixed clispecs and relative paths, e.g.: # "spack spec mvapich_foo ..//libelf.yaml" specs = SpecParser( f"mvapich_foo ..{os.path.sep}{parent_dir.name}{os.path.sep}{basename}" ).all_specs() assert len(specs) == 2 assert specs[1] == s def test_parse_specfile_relative_subdir_path(specfile_for, tmp_path: pathlib.Path): subdir = tmp_path / "subdir" subdir.mkdir() specfile = subdir / "libdwarf.json" s = specfile_for("libdwarf", specfile) with fs.working_dir(str(tmp_path)): spec = SpecParser(f"subdir{os.path.sep}{specfile.name}").next_spec() assert spec == s @pytest.mark.regression("20310") def test_compare_abstract_specs(): """Spec comparisons must be valid for abstract specs. Check that the spec cmp_key appropriately handles comparing specs for which some attributes are None in exactly one of two specs """ # Add fields in order they appear in `Spec._cmp_node` constraints = [ "foo", "foo.foo", "foo.foo@foo", "foo.foo@foo+foo", "foo.foo@foo+foo arch=foo-foo-foo", "foo.foo@foo+foo arch=foo-foo-foo %foo", "foo.foo@foo+foo arch=foo-foo-foo cflags=foo %foo", ] specs = [SpecParser(s).next_spec() for s in constraints] for a, b in itertools.product(specs, repeat=2): # Check that we can compare without raising an error assert a <= b or b < a @pytest.mark.parametrize( "lhs_str,rhs_str,expected", [ # Git shasum vs generic develop ( f"develop-branch-version@git.{'a' * 40}=develop", "develop-branch-version@develop", (True, True, False), ), # Two different shasums ( f"develop-branch-version@git.{'a' * 40}=develop", f"develop-branch-version@git.{'b' * 40}=develop", (False, False, False), ), # Git shasum vs. git tag ( f"develop-branch-version@git.{'a' * 40}=develop", "develop-branch-version@git.0.2.15=develop", (False, False, False), ), # Git tag vs. generic develop ( "develop-branch-version@git.0.2.15=develop", "develop-branch-version@develop", (True, True, False), ), ], ) def test_git_ref_spec_equivalences(mock_packages, lhs_str, rhs_str, expected): lhs = SpecParser(lhs_str).next_spec() rhs = SpecParser(rhs_str).next_spec() intersect, lhs_sat_rhs, rhs_sat_lhs = expected assert lhs.intersects(rhs) is intersect assert rhs.intersects(lhs) is intersect assert lhs.satisfies(rhs) is lhs_sat_rhs assert rhs.satisfies(lhs) is rhs_sat_lhs @pytest.mark.regression("32471") @pytest.mark.parametrize("spec_str", ["target=x86_64", "os=redhat6", "target=x86_64:"]) def test_platform_is_none_if_not_present(spec_str): s = SpecParser(spec_str).next_spec() assert s.architecture.platform is None, s def test_parse_one_or_raise_error_message(): with pytest.raises(ValueError) as exc: parse_one_or_raise(" x y z") msg = """\ expected a single spec, but got more: x y z ^\ """ assert str(exc.value) == msg with pytest.raises(ValueError, match="expected a single spec, but got none"): parse_one_or_raise(" ") @pytest.mark.parametrize( "input_args,expected", [ # mpileaks %[virtuals=c deptypes=build] gcc ( ["mpileaks", "%[virtuals=c", "deptypes=build]", "gcc"], ["mpileaks %[virtuals=c deptypes=build] gcc"], ), # mpileaks %[ virtuals=c deptypes=build] gcc ( ["mpileaks", "%[", "virtuals=c", "deptypes=build]", "gcc"], ["mpileaks %[virtuals=c deptypes=build] gcc"], ), # mpileaks %[ virtuals=c deptypes=build ] gcc ( ["mpileaks", "%[", "virtuals=c", "deptypes=build", "]", "gcc"], ["mpileaks %[virtuals=c deptypes=build] gcc"], ), ], ) def test_parse_multiple_edge_attributes(input_args, expected): """Tests that we can parse correctly multiple edge attributes within square brackets, from the command line. The input are strings as they would be parsed from argparse.REMAINDER """ s, *_ = spack.cmd.parse_specs(input_args) for c in expected: assert s.satisfies(c) ================================================ FILE: lib/spack/spack/test/spec_yaml.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test YAML and JSON serialization for specs. The YAML and JSON formats preserve DAG information in the spec. """ import collections import collections.abc import gzip import io import json import os import pathlib import pickle import pytest import spack.vendor.ruamel.yaml import spack.concretize import spack.config import spack.hash_types as ht import spack.paths import spack.repo import spack.spec import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml from spack.spec import Spec, save_dependency_specfiles from spack.test.conftest import RepoBuilder from spack.util.spack_yaml import SpackYAMLError, syaml_dict def check_yaml_round_trip(spec): yaml_text = spec.to_yaml() spec_from_yaml = Spec.from_yaml(yaml_text) assert spec.eq_dag(spec_from_yaml) def check_json_round_trip(spec): json_text = spec.to_json() spec_from_json = Spec.from_json(json_text) assert spec.eq_dag(spec_from_json) def test_read_spec_from_signed_json(): spec_dir = os.path.join(spack.paths.test_path, "data", "mirrors", "signed_json") file_name = ( "linux-ubuntu18.04-haswell-gcc-8.4.0-" "zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb.spec.json.sig" ) spec_path = os.path.join(spec_dir, file_name) def check_spec(spec_to_check): assert spec_to_check.name == "zlib" assert spec_to_check._hash == "g7otk5dra3hifqxej36m5qzm7uyghqgb" with open(spec_path, encoding="utf-8") as fd: s = Spec.from_signed_json(fd) check_spec(s) with open(spec_path, encoding="utf-8") as fd: s = Spec.from_signed_json(fd.read()) check_spec(s) @pytest.mark.parametrize( "invalid_yaml", ["playing_playlist: {{ action }} playlist {{ playlist_name }}"] ) def test_invalid_yaml_spec(invalid_yaml): with pytest.raises(SpackYAMLError, match="error parsing YAML") as e: Spec.from_yaml(invalid_yaml) assert invalid_yaml in str(e) @pytest.mark.parametrize("invalid_json, error_message", [("{13:", "Expecting property name")]) def test_invalid_json_spec(invalid_json, error_message): with pytest.raises(sjson.SpackJSONError) as e: Spec.from_json(invalid_json) exc_msg = str(e.value) assert exc_msg.startswith("error parsing JSON spec:") assert error_message in exc_msg @pytest.mark.parametrize( "abstract_spec", [ # Externals "externaltool", "externaltest", # Ambiguous version spec "mpileaks@1.0:5.0,6.1,7.3+debug~opt", # Variants "mpileaks+debug~opt", 'multivalue-variant foo="bar,baz"', # Virtuals on edges "callpath", "mpileaks", ], ) def test_roundtrip_concrete_specs(abstract_spec, default_mock_concretization): check_yaml_round_trip(Spec(abstract_spec)) check_json_round_trip(Spec(abstract_spec)) concrete_spec = default_mock_concretization(abstract_spec) check_yaml_round_trip(concrete_spec) check_json_round_trip(concrete_spec) def test_yaml_subdag(config, mock_packages): spec = spack.concretize.concretize_one("mpileaks^mpich+debug") yaml_spec = Spec.from_yaml(spec.to_yaml()) json_spec = Spec.from_json(spec.to_json()) for dep in ("callpath", "mpich", "dyninst", "libdwarf", "libelf"): assert spec[dep].eq_dag(yaml_spec[dep]) assert spec[dep].eq_dag(json_spec[dep]) @pytest.mark.parametrize("spec_str", ["mpileaks ^zmpi", "dttop", "dtuse"]) def test_using_ordered_dict(default_mock_concretization, spec_str): """Checks that we use syaml_dicts for spec serialization. Necessary to make sure that dag_hash is stable across python versions and processes. """ def descend_and_check(iterable, level=0): if isinstance(iterable, collections.abc.Mapping): assert type(iterable) in (syaml_dict, dict) return descend_and_check(iterable.values(), level=level + 1) max_level = level for value in iterable: if isinstance(value, collections.abc.Iterable) and not isinstance(value, str): nlevel = descend_and_check(value, level=level + 1) if nlevel > max_level: max_level = nlevel return max_level s = default_mock_concretization(spec_str) level = descend_and_check(s.to_node_dict()) # level just makes sure we are doing something here assert level >= 5 @pytest.mark.parametrize("spec_str", ["mpileaks ^zmpi", "dttop", "dtuse"]) def test_ordered_read_not_required_for_consistent_dag_hash( spec_str, mutable_config: spack.config.Configuration, mock_packages ): """Make sure ordered serialization isn't required to preserve hashes. For consistent hashes, we require that YAML and JSON serializations have their keys in a deterministic order. However, we don't want to require them to be serialized in order. This ensures that is not required.""" # Make sure that `extra_attributes` of externals is order independent for hashing. extra_attributes = { "compilers": {"c": "/some/path/bin/cc", "cxx": "/some/path/bin/c++"}, "foo": "bar", "baz": "qux", } mutable_config.set( "packages:dtuse", { "buildable": False, "externals": [ {"spec": "dtuse@=1.0", "prefix": "/usr", "extra_attributes": extra_attributes} ], }, ) spec = spack.concretize.concretize_one(spec_str) if spec_str == "dtuse": assert spec.external and spec.extra_attributes == extra_attributes spec_dict = spec.to_dict(hash=ht.dag_hash) spec_yaml = spec.to_yaml() spec_json = spec.to_json() # Make a spec with dict keys reversed recursively spec_dict_rev = reverse_all_dicts(spec_dict) # Dump to YAML and JSON yaml_string = syaml.dump(spec_dict, default_flow_style=False) yaml_string_rev = syaml.dump(spec_dict_rev, default_flow_style=False) json_string = sjson.dump(spec_dict) json_string_rev = sjson.dump(spec_dict_rev) # spec yaml is ordered like the spec dict assert yaml_string == spec_yaml assert json_string == spec_json # reversed string is different from the original, so it *would* generate a different hash assert yaml_string != yaml_string_rev assert json_string != json_string_rev # build specs from the "wrongly" ordered data from_yaml = Spec.from_yaml(yaml_string) from_json = Spec.from_json(json_string) from_yaml_rev = Spec.from_yaml(yaml_string_rev) from_json_rev = Spec.from_json(json_string_rev) # Strip spec if we stripped the yaml spec = spec.copy(deps=ht.dag_hash.depflag) # specs and their hashes are equal to the original assert ( spec.dag_hash() == from_yaml.dag_hash() == from_json.dag_hash() == from_yaml_rev.dag_hash() == from_json_rev.dag_hash() ) assert spec == from_yaml == from_json == from_yaml_rev == from_json_rev def reverse_all_dicts(data): """Descend into data and reverse all the dictionaries""" if isinstance(data, dict): return type(data)((k, reverse_all_dicts(v)) for k, v in reversed(list(data.items()))) elif isinstance(data, (list, tuple)): return type(data)(reverse_all_dicts(elt) for elt in data) return data def check_specs_equal(original_spec, spec_yaml_path): with open(spec_yaml_path, "r", encoding="utf-8") as fd: spec_yaml = fd.read() spec_from_yaml = Spec.from_yaml(spec_yaml) return original_spec.eq_dag(spec_from_yaml) def test_save_dependency_spec_jsons_subset( tmp_path: pathlib.Path, config, repo_builder: RepoBuilder ): output_path = tmp_path / "spec_jsons" output_path.mkdir() repo_builder.add_package("pkg-g") repo_builder.add_package("pkg-f") repo_builder.add_package("pkg-e") repo_builder.add_package("pkg-d", dependencies=[("pkg-f", None, None), ("pkg-g", None, None)]) repo_builder.add_package("pkg-c") repo_builder.add_package("pkg-b", dependencies=[("pkg-d", None, None), ("pkg-e", None, None)]) repo_builder.add_package("pkg-a", dependencies=[("pkg-b", None, None), ("pkg-c", None, None)]) with spack.repo.use_repositories(repo_builder.root): spec_a = spack.concretize.concretize_one("pkg-a") b_spec = spec_a["pkg-b"] c_spec = spec_a["pkg-c"] save_dependency_specfiles(spec_a, str(output_path), [Spec("pkg-b"), Spec("pkg-c")]) assert check_specs_equal(b_spec, str(output_path / "pkg-b.json")) assert check_specs_equal(c_spec, str(output_path / "pkg-c.json")) def test_legacy_yaml(install_mockery, mock_packages): """Tests a simple legacy YAML with a dependency and ensures spec survives concretization.""" yaml = """ spec: - a: version: '2.0' arch: platform: linux platform_os: rhel7 target: x86_64 compiler: name: gcc version: 8.3.0 namespace: builtin.mock parameters: bvv: true foo: - bar foobar: bar cflags: [] cppflags: [] cxxflags: [] fflags: [] ldflags: [] ldlibs: [] dependencies: b: hash: iaapywazxgetn6gfv2cfba353qzzqvhn type: - build - link hash: obokmcsn3hljztrmctbscmqjs3xclazz full_hash: avrk2tqsnzxeabmxa6r776uq7qbpeufv build_hash: obokmcsn3hljztrmctbscmqjs3xclazy - b: version: '1.0' arch: platform: linux platform_os: rhel7 target: x86_64 compiler: name: gcc version: 8.3.0 namespace: builtin.mock parameters: cflags: [] cppflags: [] cxxflags: [] fflags: [] ldflags: [] ldlibs: [] hash: iaapywazxgetn6gfv2cfba353qzzqvhn full_hash: qvsxvlmjaothtpjluqijv7qfnni3kyyg build_hash: iaapywazxgetn6gfv2cfba353qzzqvhy """ spec = Spec.from_yaml(yaml) concrete_spec = spack.concretize.concretize_one(spec) assert concrete_spec.eq_dag(spec) #: A well ordered Spec dictionary, using ``OrderdDict``. #: Any operation that transforms Spec dictionaries should #: preserve this order. ordered_spec = collections.OrderedDict( [ ( "arch", collections.OrderedDict( [ ("platform", "darwin"), ("platform_os", "bigsur"), ( "target", collections.OrderedDict( [ ( "features", [ "adx", "aes", "avx", "avx2", "bmi1", "bmi2", "clflushopt", "f16c", "fma", "mmx", "movbe", "pclmulqdq", "popcnt", "rdrand", "rdseed", "sse", "sse2", "sse4_1", "sse4_2", "ssse3", "xsavec", "xsaveopt", ], ), ("generation", 0), ("name", "skylake"), ("parents", ["broadwell"]), ("vendor", "GenuineIntel"), ] ), ), ] ), ), ("compiler", collections.OrderedDict([("name", "apple-clang"), ("version", "13.0.0")])), ("name", "zlib"), ("namespace", "builtin"), ( "parameters", collections.OrderedDict( [ ("cflags", []), ("cppflags", []), ("cxxflags", []), ("fflags", []), ("ldflags", []), ("ldlibs", []), ("optimize", True), ("pic", True), ("shared", True), ] ), ), ("version", "1.2.11"), ] ) @pytest.mark.parametrize( "specfile,expected_hash,reader_cls", [ # First version supporting JSON format for specs ("specfiles/hdf5.v013.json.gz", "vglgw4reavn65vx5d4dlqn6rjywnq76d", spack.spec.SpecfileV1), # Introduces full hash in the format, still has 3 hashes ("specfiles/hdf5.v016.json.gz", "stp45yvzte43xdauknaj3auxlxb4xvzs", spack.spec.SpecfileV1), # Introduces "build_specs", see https://github.com/spack/spack/pull/22845 ("specfiles/hdf5.v017.json.gz", "xqh5iyjjtrp2jw632cchacn3l7vqzf3m", spack.spec.SpecfileV2), # Use "full hash" everywhere, see https://github.com/spack/spack/pull/28504 ("specfiles/hdf5.v019.json.gz", "iulacrbz7o5v5sbj7njbkyank3juh6d3", spack.spec.SpecfileV3), # Add properties on edges, see https://github.com/spack/spack/pull/34821 ("specfiles/hdf5.v020.json.gz", "vlirlcgazhvsvtundz4kug75xkkqqgou", spack.spec.SpecfileV4), ], ) def test_load_json_specfiles(specfile, expected_hash, reader_cls): fullpath = os.path.join(spack.paths.test_path, "data", specfile) with gzip.open(fullpath, "rt", encoding="utf-8") as f: data = json.load(f) s1 = Spec.from_dict(data) s2 = reader_cls.load(data) assert s2.dag_hash() == expected_hash assert s1.dag_hash() == s2.dag_hash() assert s1 == s2 assert Spec.from_json(s2.to_json()).dag_hash() == s2.dag_hash() openmpi_edges = s2.edges_to_dependencies(name="openmpi") assert len(openmpi_edges) == 1 # Check that virtuals have been reconstructed for specfiles conforming to # version 4 on. if reader_cls.SPEC_VERSION >= spack.spec.SpecfileV4.SPEC_VERSION: assert "mpi" in openmpi_edges[0].virtuals # The virtuals attribute must be a tuple, when read from a # JSON or YAML file, not a list for edge in s2.traverse_edges(): assert isinstance(edge.virtuals, tuple), edge # Ensure we can format {compiler} tokens assert s2.format("{compiler}") != "none" assert s2.format("{compiler.name}") == "gcc" assert s2.format("{compiler.version}") != "none" # Ensure satisfies works with compilers and direct dependencies assert s2.satisfies("%gcc") assert s2.satisfies("%gcc@9.4.0") assert s2.satisfies("%zlib") def test_anchorify_1(): """Test that anchorify replaces duplicate values with references to a single instance, and that that results in anchors in the output YAML.""" before = {"a": [1, 2, 3], "b": [1, 2, 3]} after = {"a": [1, 2, 3], "b": [1, 2, 3]} syaml.anchorify(after) assert before == after assert after["a"] is after["b"] # Check if anchors are used out = io.StringIO() spack.vendor.ruamel.yaml.YAML().dump(after, out) assert ( out.getvalue() == """\ a: &id001 - 1 - 2 - 3 b: *id001 """ ) def test_anchorify_2(): before = {"a": {"b": {"c": True}}, "d": {"b": {"c": True}}, "e": {"c": True}} after = {"a": {"b": {"c": True}}, "d": {"b": {"c": True}}, "e": {"c": True}} syaml.anchorify(after) assert before == after assert after["a"] is after["d"] assert after["a"]["b"] is after["e"] # Check if anchors are used out = io.StringIO() spack.vendor.ruamel.yaml.YAML().dump(after, out) assert ( out.getvalue() == """\ a: &id001 b: &id002 c: true d: *id001 e: *id002 """ ) @pytest.mark.parametrize( "spec_str", [ "hdf5 ++mpi", "hdf5 cflags==-g", "hdf5 foo==bar", "hdf5~~mpi++shared", "hdf5 cflags==-g foo==bar cxxflags==-O3", "hdf5 cflags=-g foo==bar cxxflags==-O3", "hdf5%gcc", "hdf5%cmake", "hdf5^gcc", "hdf5^cmake", ], ) def test_pickle_roundtrip_for_abstract_specs(spec_str): """Tests that abstract specs correctly round trip when pickled. This test compares both spec objects and their string representation, due to some inconsistencies in how `Spec.__eq__` is implemented. """ s = spack.spec.Spec(spec_str) t = pickle.loads(pickle.dumps(s)) assert s == t assert str(s) == str(t) def test_specfile_alias_is_updated(): """Tests that the SpecfileLatest alias gets updated on a Specfile version bump""" specfile_class_name = f"SpecfileV{spack.spec.SPECFILE_FORMAT_VERSION}" specfile_cls = getattr(spack.spec, specfile_class_name) assert specfile_cls is spack.spec.SpecfileLatest @pytest.mark.parametrize("spec_str", ["mpileaks %gcc", "mpileaks ^zmpi ^callpath%gcc"]) def test_direct_edges_and_round_tripping_to_dict(spec_str, default_mock_concretization): """Tests that we preserve edge information when round-tripping to dict""" original = Spec(spec_str) reconstructed = Spec.from_dict(original.to_dict()) assert original == reconstructed assert original.to_dict() == reconstructed.to_dict() concrete = default_mock_concretization(spec_str) concrete_reconstructed = Spec.from_dict(concrete.to_dict()) assert concrete == concrete_reconstructed assert concrete.to_dict() == concrete_reconstructed.to_dict() # Ensure we don't get 'direct' in concrete JSON specs, for the time being d = concrete.to_dict() for node in d["spec"]["nodes"]: if "dependencies" not in node: continue for dependency_data in node["dependencies"]: assert "direct" not in dependency_data["parameters"] def test_pickle_preserves_identity_and_prefix(default_mock_concretization): """When pickling multiple specs that share dependencies, the identity of those dependencies should be preserved when unpickling.""" mpileaks_before: Spec = default_mock_concretization("mpileaks") callpath_before = mpileaks_before.dependencies("callpath")[0] callpath_before.set_prefix("/fake/prefix/callpath") specs_before = [mpileaks_before, callpath_before] specs_after = pickle.loads(pickle.dumps(specs_before)) mpileaks_after, callpath_after = specs_after # Test whether the mpileaks<->callpath link is preserved and corresponds to the same object assert mpileaks_after is callpath_after.dependents("mpileaks")[0] assert callpath_after is mpileaks_after.dependencies("callpath")[0] # Test that we have the exact same number of unique Spec objects before and after pickling num_unique_specs = lambda specs: len({id(s) for r in specs for s in r.traverse()}) assert num_unique_specs(specs_before) == num_unique_specs(specs_after) # Test that the specs are the same as dicts assert mpileaks_before.to_dict() == mpileaks_after.to_dict() ================================================ FILE: lib/spack/spack/test/stage.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test that the Stage class works correctly.""" import collections import errno import getpass import os import pathlib import shutil import stat import sys import pytest import spack.config import spack.error import spack.fetch_strategy import spack.stage import spack.util.executable import spack.util.path import spack.util.url as url_util from spack.llnl.util.filesystem import getuid, mkdirp, partition_path, readlink, touch, working_dir from spack.resource import Resource from spack.stage import DevelopStage, ResourceStage, Stage, StageComposite from spack.util.path import canonicalize_path # The following values are used for common fetch and stage mocking fixtures: _archive_base = "test-files" _archive_fn = "%s.tar.gz" % _archive_base _extra_fn = "extra.sh" _hidden_fn = ".hidden" _readme_fn = "README.txt" _extra_contents = "#!/bin/sh\n" _hidden_contents = "" _readme_contents = "hello world!\n" # TODO: Replace the following with an enum once guarantee supported (or # include enum34 for python versions < 3.4. _include_readme = 1 _include_hidden = 2 _include_extra = 3 # Mock fetch directories are expected to appear as follows: # # TMPDIR/ # _archive_fn archive_url = file:///path/to/_archive_fn # # Mock expanded stage directories are expected to have one of two forms, # depending on how the tarball expands. Non-exploding tarballs are expected # to have the following structure: # # TMPDIR/ temp stage dir # spack-src/ well-known stage source directory # _readme_fn Optional test_readme (contains _readme_contents) # _hidden_fn Optional hidden file (contains _hidden_contents) # _archive_fn archive_url = file:///path/to/_archive_fn # # while exploding tarball directories are expected to be structured as follows: # # TMPDIR/ temp stage dir # spack-src/ well-known stage source directory # archive_name/ archive dir # _readme_fn test_readme (contains _readme_contents) # _extra_fn test_extra file (contains _extra_contents) # _archive_fn archive_url = file:///path/to/_archive_fn # @pytest.fixture def clear_stage_root(monkeypatch): """Ensure spack.stage._stage_root is not set at test start.""" monkeypatch.setattr(spack.stage, "_stage_root", None) yield def check_expand_archive(stage, stage_name, expected_file_list): """ Ensure the expanded archive directory contains the expected structure and files as described in the module-level comments above. """ stage_path = get_stage_path(stage, stage_name) archive_dir = spack.stage._source_path_subdir stage_contents = os.listdir(stage_path) assert _archive_fn in stage_contents assert archive_dir in stage_contents source_path = os.path.join(stage_path, archive_dir) assert source_path == stage.source_path source_contents = os.listdir(source_path) for _include in expected_file_list: if _include == _include_hidden: # The hidden file represent the HFS metadata associated with Mac # OS X tar files so is expected to be in the same directory as # the archive directory. assert _hidden_fn in stage_contents fn = os.path.join(stage_path, _hidden_fn) contents = _hidden_contents elif _include == _include_readme: # The standard README.txt file will be in the source directory if # the tarball didn't explode; otherwise, it will be in the # original archive subdirectory of it. if _archive_base in source_contents: fn = os.path.join(source_path, _archive_base, _readme_fn) else: fn = os.path.join(source_path, _readme_fn) contents = _readme_contents elif _include == _include_extra: assert _extra_fn in source_contents fn = os.path.join(source_path, _extra_fn) contents = _extra_contents else: assert False assert os.path.isfile(fn) with open(fn, encoding="utf-8") as _file: assert _file.read() == contents def check_fetch(stage, stage_name): """ Ensure the fetch resulted in a properly placed archive file as described in the module-level comments. """ stage_path = get_stage_path(stage, stage_name) assert _archive_fn in os.listdir(stage_path) assert os.path.join(stage_path, _archive_fn) == stage.fetcher.archive_file def check_destroy(stage, stage_name): """Figure out whether a stage was destroyed correctly.""" stage_path = get_stage_path(stage, stage_name) # check that the stage dir/link was removed. assert not os.path.exists(stage_path) # tmp stage needs to remove tmp dir too. target = os.path.realpath(stage_path) assert not os.path.exists(target) def check_setup(stage, stage_name, archive): """Figure out whether a stage was set up correctly.""" stage_path = get_stage_path(stage, stage_name) # Ensure stage was created in the spack stage directory assert os.path.isdir(stage_path) # Make sure it points to a valid directory target = os.path.realpath(stage_path) assert os.path.isdir(target) assert not os.path.islink(target) # Make sure the directory is in the place we asked it to # be (see setUp, tearDown, and use_tmp) assert target.startswith(str(archive.stage_path)) def get_stage_path(stage, stage_name): """Figure out where a stage should be living. This depends on whether it's named. """ stage_path = spack.stage.get_stage_root() if stage_name is not None: # If it is a named stage, we know where the stage should be return os.path.join(stage_path, stage_name) else: # If it's unnamed, ensure that we ran mkdtemp in the right spot. assert stage.path is not None assert stage.path.startswith(stage_path) return stage.path # TODO: Revisit use of the following fixture (and potentially leveraging # the `mock_stage` path in `mock_stage_archive`) per discussions in # #12857. See also #13065. @pytest.fixture def tmp_build_stage_dir(tmp_path: pathlib.Path, clear_stage_root): """Use a temporary test directory for the stage root.""" test_path = str(tmp_path / "stage") with spack.config.override("config:build_stage", test_path): yield tmp_path, spack.stage.get_stage_root() shutil.rmtree(test_path) @pytest.fixture def mock_stage_archive(tmp_build_stage_dir): """Create the directories and files for the staged mock archive.""" # Mock up a stage area that looks like this: # # tmp_path/ test_files_dir # stage/ test_stage_path (where stage should be) # <_archive_base>/ archive_dir_path # <_readme_fn> Optional test_readme (contains _readme_contents) # <_extra_fn> Optional extra file (contains _extra_contents) # <_hidden_fn> Optional hidden file (contains _hidden_contents) # <_archive_fn> archive_url = file:///path/to/<_archive_fn> # def create_stage_archive(expected_file_list=[_include_readme]): tmp_build_dir, test_stage_path = tmp_build_stage_dir mkdirp(test_stage_path) # Create the archive directory and associated file archive_dir = tmp_build_dir / _archive_base archive = tmp_build_dir / _archive_fn archive_url = url_util.path_to_file_url(str(archive)) archive_dir.mkdir(exist_ok=True) # Create the optional files as requested and make sure expanded # archive peers are included. tar_args = ["czf", str(_archive_fn), _archive_base] for _include in expected_file_list: if _include == _include_hidden: # The hidden file case stands in for the way Mac OS X tar files # represent HFS metadata. Locate in the same directory as the # archive file. tar_args.append(_hidden_fn) fn, contents = (tmp_build_dir / _hidden_fn, _hidden_contents) elif _include == _include_readme: # The usual README.txt file is contained in the archive dir. fn, contents = (archive_dir / _readme_fn, _readme_contents) elif _include == _include_extra: # The extra file stands in for exploding tar files so needs # to be in the same directory as the archive file. tar_args.append(_extra_fn) fn, contents = (tmp_build_dir / _extra_fn, _extra_contents) else: break fn.write_text(contents) # Create the archive file with working_dir(str(tmp_build_dir)): tar = spack.util.executable.which("tar", required=True) tar(*tar_args) Archive = collections.namedtuple("Archive", ["url", "tmpdir", "stage_path", "archive_dir"]) return Archive( url=archive_url, tmpdir=tmp_build_dir, stage_path=test_stage_path, archive_dir=archive_dir, ) return create_stage_archive @pytest.fixture def mock_noexpand_resource(tmp_path: pathlib.Path): """Set up a non-expandable resource in the tmp_path prior to staging.""" test_resource = tmp_path / "resource-no-expand.sh" test_resource.write_text("an example resource") return str(test_resource) @pytest.fixture def mock_expand_resource(tmp_path: pathlib.Path): """Sets up an expandable resource in tmp_path prior to staging.""" # Mock up an expandable resource: # # tmp_path/ test_files_dir # resource-expand/ resource source dir # resource-file.txt resource contents (contains 'test content') # resource.tar.gz archive of resource content # subdir = "resource-expand" resource_dir = tmp_path / subdir resource_dir.mkdir() archive_name = "resource.tar.gz" archive = tmp_path / archive_name archive_url = url_util.path_to_file_url(str(archive)) filename = "resource-file.txt" test_file = resource_dir / filename test_file.write_text("test content\n") with working_dir(str(tmp_path)): tar = spack.util.executable.which("tar", required=True) tar("czf", str(archive_name), subdir) MockResource = collections.namedtuple("MockResource", ["url", "files"]) return MockResource(archive_url, [filename]) @pytest.fixture def composite_stage_with_expanding_resource(mock_stage_archive, mock_expand_resource): """Sets up a composite for expanding resources prior to staging.""" composite_stage = StageComposite() archive = mock_stage_archive() root_stage = Stage(archive.url) composite_stage.append(root_stage) test_resource_fetcher = spack.fetch_strategy.from_kwargs(url=mock_expand_resource.url) # Specify that the resource files are to be placed in the 'resource-dir' # directory test_resource = Resource("test_resource", test_resource_fetcher, "", "resource-dir") resource_stage = ResourceStage(test_resource_fetcher, root_stage, test_resource) composite_stage.append(resource_stage) return composite_stage, root_stage, resource_stage, mock_expand_resource @pytest.fixture def failing_search_fn(): """Returns a search function that fails! Always!""" def _mock(): raise Exception("This should not have been called") return _mock class FailingFetchStrategy(spack.fetch_strategy.FetchStrategy): def fetch(self): raise spack.fetch_strategy.FailedDownloadError( "", "This implementation of FetchStrategy always fails" ) @pytest.fixture def search_fn(): """Returns a search function that always succeeds.""" class _Mock: performed_search = False def __call__(self): self.performed_search = True return [] return _Mock() def check_stage_dir_perms(prefix, path): """Check the stage directory perms to ensure match expectations.""" # Ensure the path's subdirectories -- to `$user` -- have their parent's # perms while those from `$user` on are owned and restricted to the # user. assert path.startswith(prefix) user = getpass.getuser() prefix_status = os.stat(prefix) uid = getuid() # Obtain lists of ancestor and descendant paths of the $user node, if any. # # Skip processing prefix ancestors since no guarantee they will be in the # required group (e.g. $TEMPDIR on HPC machines). skip = prefix if prefix.endswith(os.sep) else prefix + os.sep group_paths, user_node, user_paths = partition_path(path.replace(skip, ""), user) for p in group_paths: p_status = os.stat(os.path.join(prefix, p)) assert p_status.st_gid == prefix_status.st_gid assert p_status.st_mode == prefix_status.st_mode # Add the path ending with the $user node to the user paths to ensure paths # from $user (on down) meet the ownership and permission requirements. if user_node: user_paths.insert(0, user_node) for p in user_paths: p_status = os.stat(os.path.join(prefix, p)) assert uid == p_status.st_uid assert p_status.st_mode & stat.S_IRWXU == stat.S_IRWXU @pytest.mark.usefixtures("mock_packages") class TestStage: stage_name = "spack-test-stage" def test_setup_and_destroy_name_with_tmp(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: check_setup(stage, self.stage_name, archive) check_destroy(stage, self.stage_name) def test_setup_and_destroy_name_without_tmp(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: check_setup(stage, self.stage_name, archive) check_destroy(stage, self.stage_name) def test_setup_and_destroy_no_name_with_tmp(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url) as stage: check_setup(stage, None, archive) check_destroy(stage, None) def test_noexpand_stage_file(self, mock_stage_archive, mock_noexpand_resource): """When creating a stage with a nonexpanding URL, the 'archive_file' property of the stage should refer to the path of that file. """ test_noexpand_fetcher = spack.fetch_strategy.from_kwargs( url=url_util.path_to_file_url(mock_noexpand_resource), expand=False ) with Stage(test_noexpand_fetcher) as stage: stage.fetch() stage.expand_archive() assert os.path.exists(stage.archive_file) @pytest.mark.disable_clean_stage_check def test_composite_stage_with_noexpand_resource( self, mock_stage_archive, mock_noexpand_resource ): archive = mock_stage_archive() composite_stage = StageComposite() root_stage = Stage(archive.url) composite_stage.append(root_stage) resource_dst_name = "resource-dst-name.sh" test_resource_fetcher = spack.fetch_strategy.from_kwargs( url=url_util.path_to_file_url(mock_noexpand_resource), expand=False ) test_resource = Resource("test_resource", test_resource_fetcher, resource_dst_name, None) resource_stage = ResourceStage(test_resource_fetcher, root_stage, test_resource) composite_stage.append(resource_stage) composite_stage.create() composite_stage.fetch() composite_stage.expand_archive() assert composite_stage.expanded # Archive is expanded assert os.path.exists(os.path.join(composite_stage.source_path, resource_dst_name)) @pytest.mark.disable_clean_stage_check def test_composite_stage_with_expand_resource(self, composite_stage_with_expanding_resource): (composite_stage, root_stage, resource_stage, mock_resource) = ( composite_stage_with_expanding_resource ) composite_stage.create() composite_stage.fetch() composite_stage.expand_archive() assert composite_stage.expanded # Archive is expanded for fname in mock_resource.files: file_path = os.path.join(root_stage.source_path, "resource-dir", fname) assert os.path.exists(file_path) # Perform a little cleanup shutil.rmtree(root_stage.path) @pytest.mark.disable_clean_stage_check def test_composite_stage_with_expand_resource_default_placement( self, composite_stage_with_expanding_resource ): """For a resource which refers to a compressed archive which expands to a directory, check that by default the resource is placed in the source_path of the root stage with the name of the decompressed directory. """ (composite_stage, root_stage, resource_stage, mock_resource) = ( composite_stage_with_expanding_resource ) resource_stage.resource.placement = None composite_stage.create() composite_stage.fetch() composite_stage.expand_archive() for fname in mock_resource.files: file_path = os.path.join(root_stage.source_path, "resource-expand", fname) assert os.path.exists(file_path) # Perform a little cleanup shutil.rmtree(root_stage.path) def test_setup_and_destroy_no_name_without_tmp(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url) as stage: check_setup(stage, None, archive) check_destroy(stage, None) @pytest.mark.parametrize("debug", [False, True]) def test_fetch(self, mock_stage_archive, debug): archive = mock_stage_archive() with spack.config.override("config:debug", debug): with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() check_setup(stage, self.stage_name, archive) check_fetch(stage, self.stage_name) check_destroy(stage, self.stage_name) def test_no_search_if_default_succeeds(self, mock_stage_archive, failing_search_fn): archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, search_fn=failing_search_fn) with stage: stage.fetch() check_destroy(stage, self.stage_name) def test_no_search_mirror_only(self, failing_search_fn): stage = Stage(FailingFetchStrategy(), name=self.stage_name, search_fn=failing_search_fn) with stage: try: stage.fetch(mirror_only=True) except spack.error.FetchError: pass check_destroy(stage, self.stage_name) @pytest.mark.parametrize( "err_msg,expected", [ ("Fetch from fetch.test.com", "Fetch from fetch.test.com"), (None, "All fetchers failed"), ], ) def test_search_if_default_fails(self, search_fn, err_msg, expected): stage = Stage(FailingFetchStrategy(), name=self.stage_name, search_fn=search_fn) with stage: with pytest.raises(spack.error.FetchError, match=expected): stage.fetch(mirror_only=False, err_msg=err_msg) check_destroy(stage, self.stage_name) assert search_fn.performed_search def test_ensure_one_stage_entry(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() stage_path = get_stage_path(stage, self.stage_name) spack.fetch_strategy._ensure_one_stage_entry(stage_path) check_destroy(stage, self.stage_name) @pytest.mark.parametrize( "expected_file_list", [ [], [_include_readme], [_include_extra, _include_readme], [_include_hidden, _include_readme], ], ) def test_expand_archive(self, expected_file_list, mock_stage_archive): archive = mock_stage_archive(expected_file_list) with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() check_setup(stage, self.stage_name, archive) check_fetch(stage, self.stage_name) stage.expand_archive() check_expand_archive(stage, self.stage_name, expected_file_list) check_destroy(stage, self.stage_name) def test_expand_archive_extra_expand(self, mock_stage_archive): """Test expand with an extra expand after expand (i.e., no-op).""" archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() check_setup(stage, self.stage_name, archive) check_fetch(stage, self.stage_name) stage.expand_archive() stage.fetcher.expand() check_expand_archive(stage, self.stage_name, [_include_readme]) check_destroy(stage, self.stage_name) def test_restage(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() stage.expand_archive() with working_dir(stage.source_path): check_expand_archive(stage, self.stage_name, [_include_readme]) # Try to make a file in the old archive dir with open("foobar", "w", encoding="utf-8") as file: file.write("this file is to be destroyed.") assert "foobar" in os.listdir(stage.source_path) # Make sure the file is not there after restage. stage.restage() check_fetch(stage, self.stage_name) assert "foobar" not in os.listdir(stage.source_path) check_destroy(stage, self.stage_name) def test_no_keep_without_exceptions(self, mock_stage_archive): archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, keep=False) with stage: pass check_destroy(stage, self.stage_name) @pytest.mark.disable_clean_stage_check def test_keep_without_exceptions(self, mock_stage_archive): archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, keep=True) with stage: pass path = get_stage_path(stage, self.stage_name) assert os.path.isdir(path) @pytest.mark.disable_clean_stage_check def test_no_keep_with_exceptions(self, mock_stage_archive): class ThisMustFailHere(Exception): pass archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, keep=False) try: with stage: raise ThisMustFailHere() except ThisMustFailHere: path = get_stage_path(stage, self.stage_name) assert os.path.isdir(path) @pytest.mark.disable_clean_stage_check def test_keep_exceptions(self, mock_stage_archive): class ThisMustFailHere(Exception): pass archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, keep=True) try: with stage: raise ThisMustFailHere() except ThisMustFailHere: path = get_stage_path(stage, self.stage_name) assert os.path.isdir(path) def test_source_path_available(self, mock_stage_archive): """Ensure source path available but does not exist on instantiation.""" archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name) source_path = stage.source_path assert source_path assert source_path.endswith(spack.stage._source_path_subdir) assert not os.path.exists(source_path) @pytest.mark.not_on_windows("Windows file permission erroring is not yet supported") @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_first_accessible_path(self, tmp_path: pathlib.Path): """Test _first_accessible_path names.""" spack_dir = tmp_path / "paths" name = str(spack_dir) files = [os.path.join(os.path.sep, "no", "such", "path"), name] # Ensure the tmp_path path is returned since the user should have access path = spack.stage._first_accessible_path(files) assert path == name assert os.path.isdir(path) check_stage_dir_perms(str(tmp_path), path) # Ensure an existing path is returned spack_subdir = spack_dir / "existing" spack_subdir.mkdir(parents=True) subdir = str(spack_subdir) path = spack.stage._first_accessible_path([subdir]) assert path == subdir # Ensure a path with a `$user` node has the right permissions # for its subdirectories. user = getpass.getuser() user_dir = spack_dir / user / "has" / "paths" user_path = str(user_dir) path = spack.stage._first_accessible_path([user_path]) assert path == user_path check_stage_dir_perms(str(tmp_path), path) # Cleanup shutil.rmtree(str(name)) def test_create_stage_root(self, tmp_path: pathlib.Path, no_path_access): """Test create_stage_root permissions.""" test_dir = tmp_path / "path" test_path = str(test_dir) try: if getpass.getuser() in str(test_path).split(os.sep): # Simply ensure directory created if tmp_path includes user spack.stage.create_stage_root(test_path) assert os.path.exists(test_path) p_stat = os.stat(test_path) assert p_stat.st_mode & stat.S_IRWXU == stat.S_IRWXU else: # Ensure an OS Error is raised on created, non-user directory with pytest.raises(OSError) as exc_info: spack.stage.create_stage_root(test_path) assert exc_info.value.errno == errno.EACCES finally: try: shutil.rmtree(test_path) except OSError: pass def test_resolve_paths(self, monkeypatch): """Test _resolve_paths.""" assert spack.stage._resolve_paths([]) == [] user = "testuser" monkeypatch.setattr(spack.util.path, "get_user", lambda: user) # Test that user is appended to path if not present (except on Windows) if sys.platform == "win32": path = r"C:\spack-test\a\b\c" expected = path else: path = "/spack-test/a/b/c" expected = os.path.join(path, user) assert spack.stage._resolve_paths([path]) == [expected] # Test that user is NOT appended if already present if sys.platform == "win32": path_with_user = rf"C:\spack-test\spack-{user}\stage" else: path_with_user = f"/spack-test/spack-{user}/stage" assert spack.stage._resolve_paths([path_with_user]) == [path_with_user] canonicalized_tempdir = canonicalize_path("$tempdir") temp_has_user = user in canonicalized_tempdir.split(os.sep) paths = [ os.path.join("$tempdir", "stage"), os.path.join("$tempdir", "$user"), os.path.join("$tempdir", "$user", "$user"), os.path.join("$tempdir", "$user", "stage", "$user"), ] res_paths = [canonicalize_path(p) for p in paths] if temp_has_user: res_paths[1] = canonicalized_tempdir res_paths[2] = os.path.join(canonicalized_tempdir, user) res_paths[3] = os.path.join(canonicalized_tempdir, "stage", user) elif sys.platform != "win32": res_paths[0] = os.path.join(res_paths[0], user) assert spack.stage._resolve_paths(paths) == res_paths @pytest.mark.not_on_windows("Windows file permission erroring is not yet supported") @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_get_stage_root_bad_path(self, clear_stage_root): """Ensure an invalid stage path root raises a StageError.""" with spack.config.override("config:build_stage", "/no/such/path"): with pytest.raises(spack.stage.StageError, match="No accessible stage paths in"): spack.stage.get_stage_root() # Make sure the cached stage path values are unchanged. assert spack.stage._stage_root is None @pytest.mark.parametrize( "path,purged", [ ("spack-stage-1234567890abcdef1234567890abcdef", True), ("spack-stage-anything-goes-here", True), ("stage-spack", False), ], ) def test_stage_purge(self, tmp_path: pathlib.Path, clear_stage_root, path, purged): """Test purging of stage directories.""" stage_config_path = str(tmp_path / "stage") with spack.config.override("config:build_stage", stage_config_path): stage_root = spack.stage.get_stage_root() test_dir = pathlib.Path(stage_root) / path test_dir.mkdir(parents=True) test_path = str(test_dir) spack.stage.purge() if purged: assert not os.path.exists(test_path) else: assert os.path.exists(test_path) shutil.rmtree(test_path) def test_stage_constructor_no_fetcher(self): """Ensure Stage constructor with no URL or fetch strategy fails.""" with pytest.raises(ValueError): with Stage(None): pass def test_stage_constructor_with_path(self, tmp_path: pathlib.Path): """Ensure Stage constructor with a path uses it.""" testpath = str(tmp_path) with Stage("file:///does-not-exist", path=testpath) as stage: assert stage.path == testpath def _create_files_from_tree(base, tree): for name, content in tree.items(): sub_base = os.path.join(base, name) if isinstance(content, dict): os.mkdir(sub_base) _create_files_from_tree(sub_base, content) else: assert (content is None) or (isinstance(content, str)) with open(sub_base, "w", encoding="utf-8") as f: if content: f.write(content) def _create_tree_from_dir_recursive(path): if os.path.islink(path): return readlink(path) elif os.path.isdir(path): tree = {} for name in os.listdir(path): sub_path = os.path.join(path, name) tree[name] = _create_tree_from_dir_recursive(sub_path) return tree else: with open(path, "r", encoding="utf-8") as f: content = f.read() or None return content @pytest.fixture def develop_path(tmp_path: pathlib.Path): dir_structure = {"a1": {"b1": None, "b2": "b1content"}, "a2": None} srcdir = str(tmp_path / "test-src") os.mkdir(srcdir) _create_files_from_tree(srcdir, dir_structure) yield dir_structure, srcdir class TestDevelopStage: def test_sanity_check_develop_path(self, develop_path): _, srcdir = develop_path with open(os.path.join(srcdir, "a1", "b2"), encoding="utf-8") as f: assert f.read() == "b1content" assert os.path.exists(os.path.join(srcdir, "a2")) def test_develop_stage(self, develop_path, tmp_build_stage_dir): """Check that (a) develop stages update the given `dev_path` with a symlink that points to the stage dir and (b) that destroying the stage does not destroy `dev_path` """ devtree, srcdir = develop_path stage = DevelopStage("test-stage", srcdir, reference_link="link-to-stage") assert not os.path.exists(stage.reference_link) stage.create() assert os.path.exists(stage.reference_link) srctree1 = _create_tree_from_dir_recursive(stage.source_path) assert os.path.samefile(srctree1["link-to-stage"], stage.path) del srctree1["link-to-stage"] assert srctree1 == devtree stage.destroy() assert not os.path.exists(stage.reference_link) # Make sure destroying the stage doesn't change anything # about the path assert not os.path.exists(stage.path) srctree2 = _create_tree_from_dir_recursive(srcdir) assert srctree2 == devtree def test_develop_stage_without_reference_link(self, develop_path, tmp_build_stage_dir): """Check that develop stages can be created without creating a reference link""" devtree, srcdir = develop_path stage = DevelopStage("test-stage", srcdir, reference_link=None) stage.create() srctree1 = _create_tree_from_dir_recursive(stage.source_path) assert srctree1 == devtree stage.destroy() # Make sure destroying the stage doesn't change anything # about the path assert not os.path.exists(stage.path) srctree2 = _create_tree_from_dir_recursive(srcdir) assert srctree2 == devtree def test_stage_create_replace_path(tmp_build_stage_dir): """Ensure stage creation replaces a non-directory path.""" _, test_stage_path = tmp_build_stage_dir mkdirp(test_stage_path) nondir = os.path.join(test_stage_path, "afile") touch(nondir) path = url_util.path_to_file_url(str(nondir)) stage = Stage(path, name="afile") stage.create() # Ensure the stage path is "converted" to a directory assert os.path.isdir(nondir) def test_cannot_access(capfd): """Ensure can_access dies with the expected error.""" with pytest.raises(SystemExit): # It's far more portable to use a non-existent filename. spack.stage.ensure_access("/no/such/file") captured = capfd.readouterr() assert "Insufficient permissions" in str(captured) def test_override_keep_in_composite_stage(): stage_1 = Stage("file:///does-not-exist", keep=True) stage_2 = Stage("file:///does-not-exist", keep=False) stage_3 = Stage("file:///does-not-exist", keep=True) stages = spack.stage.StageComposite.from_iterable((stage_1, stage_2, stage_3)) # The getter for the composite stage just returns the value of the first stage # its just there so we have a setter too. assert stages.keep assert stage_1.keep assert not stage_2.keep assert stage_3.keep # This should override all stages stages.keep = False assert not stages.keep assert not stage_1.keep assert not stage_2.keep assert not stage_3.keep ================================================ FILE: lib/spack/spack/test/svn_fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.concretize import spack.config from spack.fetch_strategy import SvnFetchStrategy from spack.llnl.util.filesystem import mkdirp, touch, working_dir from spack.stage import Stage from spack.util.executable import which from spack.version import Version pytestmark = [ pytest.mark.skipif( not which("svn") or not which("svnadmin"), reason="requires subversion to be installed" ), pytest.mark.not_on_windows("does not run on windows"), ] @pytest.mark.parametrize("type_of_test", ["default", "rev0"]) @pytest.mark.parametrize("secure", [True, False]) def test_fetch(type_of_test, secure, mock_svn_repository, config, mutable_mock_repo, monkeypatch): """Tries to: 1. Fetch the repo using a fetch strategy constructed with supplied args (they depend on type_of_test). 2. Check if the test_file is in the checked out repository. 3. Assert that the repository is at the revision supplied. 4. Add and remove some files, then reset the repo, and ensure it's all there again. """ # Retrieve the right test parameters t = mock_svn_repository.checks[type_of_test] h = mock_svn_repository.hash # Construct the package under test s = spack.concretize.concretize_one("svn-test") monkeypatch.setitem(s.package.versions, Version("svn"), t.args) # Enter the stage directory and check some properties with s.package.stage: with spack.config.override("config:verify_ssl", secure): s.package.do_stage() with working_dir(s.package.stage.source_path): assert h() == t.revision file_path = os.path.join(s.package.stage.source_path, t.file) assert os.path.isdir(s.package.stage.source_path) assert os.path.isfile(file_path) os.unlink(file_path) assert not os.path.isfile(file_path) untracked_file = "foobarbaz" touch(untracked_file) assert os.path.isfile(untracked_file) s.package.do_restage() assert not os.path.isfile(untracked_file) assert os.path.isdir(s.package.stage.source_path) assert os.path.isfile(file_path) assert h() == t.revision def test_svn_extra_fetch(tmp_path: pathlib.Path): """Ensure a fetch after downloading is effectively a no-op.""" testpath = str(tmp_path) fetcher = SvnFetchStrategy(svn="file:///not-a-real-svn-repo") assert fetcher is not None with Stage(fetcher, path=testpath) as stage: assert stage is not None source_path = stage.source_path mkdirp(source_path) fetcher.fetch() ================================================ FILE: lib/spack/spack/test/tag.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for tag index cache files.""" import io import pytest import spack.cmd.tags import spack.repo import spack.tag from spack.main import SpackCommand install = SpackCommand("install") # Alternate representation tags_json = """ { "tags": { "no-version": [ "noversion", "noversion-bundle" ], "no-source": [ "nosource" ] } } """ more_tags_json = """ { "tags": { "merge": [ "check" ] } } """ def test_tag_get_all_available(mock_packages): for skip in [False, True]: all_pkgs = spack.cmd.tags.packages_with_tags(["tag1", "tag2", "tag3"], False, skip) assert sorted(all_pkgs["tag1"]) == ["mpich", "mpich2"] assert all_pkgs["tag2"] == ["mpich"] assert all_pkgs["tag3"] == ["mpich2"] def ensure_tags_results_equal(results, expected): if expected: assert sorted(results.keys()) == sorted(expected.keys()) for tag in results: assert sorted(results[tag]) == sorted(expected[tag]) else: assert results == expected @pytest.mark.parametrize( "tags,expected", [ (["tag1"], {"tag1": ["mpich", "mpich2"]}), (["tag2"], {"tag2": ["mpich"]}), (["tag3"], {"tag3": ["mpich2"]}), (["nosuchpackage"], {"nosuchpackage": {}}), ], ) def test_tag_get_available(tags, expected, mock_packages): # Ensure results for all tags all_tag_pkgs = spack.cmd.tags.packages_with_tags(tags, False, False) ensure_tags_results_equal(all_tag_pkgs, expected) # Ensure results for tags expecting results since skipping otherwise only_pkgs = spack.cmd.tags.packages_with_tags(tags, False, True) if expected[tags[0]]: ensure_tags_results_equal(only_pkgs, expected) else: assert not only_pkgs def test_tag_get_installed_packages(mock_packages, mock_archive, mock_fetch, install_mockery): install("--fake", "mpich") for skip in [False, True]: all_pkgs = spack.cmd.tags.packages_with_tags(["tag1", "tag2", "tag3"], True, skip) assert sorted(all_pkgs["tag1"]) == ["mpich"] assert all_pkgs["tag2"] == ["mpich"] assert skip or all_pkgs["tag3"] == [] def test_tag_index_round_trip(mock_packages): # Assumes at least two packages -- mpich and mpich2 -- have tags mock_index = spack.repo.PATH.tag_index assert mock_index.tags ostream = io.StringIO() mock_index.to_json(ostream) istream = io.StringIO(ostream.getvalue()) new_index = spack.tag.TagIndex.from_json(istream) assert mock_index.tags == new_index.tags def test_tag_equal(mock_packages): first_index = spack.tag.TagIndex.from_json(io.StringIO(tags_json)) second_index = spack.tag.TagIndex.from_json(io.StringIO(tags_json)) assert first_index.tags == second_index.tags def test_tag_merge(mock_packages): first_index = spack.tag.TagIndex.from_json(io.StringIO(tags_json)) second_index = spack.tag.TagIndex.from_json(io.StringIO(more_tags_json)) assert first_index != second_index tags1 = list(first_index.tags.keys()) tags2 = list(second_index.tags.keys()) all_tags = sorted(list(set(tags1 + tags2))) first_index.merge(second_index) tag_keys = sorted(first_index.tags.keys()) assert tag_keys == all_tags # Merge again to make sure the index does not retain duplicates first_index.merge(second_index) tag_keys = sorted(first_index.tags.keys()) assert tag_keys == all_tags def test_tag_not_dict(mock_packages): list_json = "[]" with pytest.raises(spack.tag.TagIndexError) as e: spack.tag.TagIndex.from_json(io.StringIO(list_json)) assert "not a dict" in str(e) def test_tag_no_tags(mock_packages): pkg_json = '{"packages": []}' with pytest.raises(spack.tag.TagIndexError) as e: spack.tag.TagIndex.from_json(io.StringIO(pkg_json)) assert "does not start with" in str(e) def test_tag_update_package(mock_packages): mock_index = mock_packages.tag_index index = spack.tag.TagIndex() index.update_packages(set(spack.repo.all_package_names()), repo=mock_packages) ensure_tags_results_equal(mock_index.tags, index.tags) ================================================ FILE: lib/spack/spack/test/tengine.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.config import spack.tengine as tengine from spack.util.path import canonicalize_path class TestContext: class A(tengine.Context): @tengine.context_property def foo(self): return 1 class B(tengine.Context): @tengine.context_property def bar(self): return 2 class C(A, B): @tengine.context_property def foobar(self): return 3 @tengine.context_property def foo(self): return 10 def test_to_dict(self): """Tests that all the context properties in a hierarchy are considered when building the context dictionary. """ # A derives directly from Context a = TestContext.A() d = a.to_dict() assert len(d) == 1 assert "foo" in d assert d["foo"] == 1 # So does B b = TestContext.B() d = b.to_dict() assert len(d) == 1 assert "bar" in d assert d["bar"] == 2 # C derives from both and overrides 'foo' c = TestContext.C() d = c.to_dict() assert len(d) == 3 for x in ("foo", "bar", "foobar"): assert x in d assert d["foo"] == 10 assert d["bar"] == 2 assert d["foobar"] == 3 @pytest.mark.usefixtures("config") class TestTengineEnvironment: def test_template_retrieval(self): """Tests the template retrieval mechanism hooked into config files""" # Check the directories are correct template_dirs = spack.config.get("config:template_dirs") template_dirs = tuple([canonicalize_path(x) for x in template_dirs]) assert len(template_dirs) == 3 env = tengine.make_environment(template_dirs) # Retrieve a.txt, which resides in the second # template directory specified in the mock configuration template = env.get_template("a.txt") text = template.render({"word": "world"}) assert "Hello world!" == text # Retrieve b.txt, which resides in the third # template directory specified in the mock configuration template = env.get_template("b.txt") text = template.render({"word": "world"}) assert "Howdy world!" == text ================================================ FILE: lib/spack/spack/test/test_suite.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import os import pathlib import sys import pytest import spack.concretize import spack.config import spack.install_test import spack.spec import spack.util.executable from spack.install_test import TestStatus from spack.llnl.util.filesystem import touch from spack.util.executable import which def _true(*args, **kwargs): """Generic monkeypatch function that always returns True.""" return True def ensure_results(filename, expected, present=True): assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as fd: lines = fd.readlines() have = False for line in lines: if expected in line: have = True break if present: assert have, f"Expected '{expected}' in the file" else: assert not have, f"Expected '{expected}' NOT to be in the file" def test_test_log_name(mock_packages, config): """Ensure test log path is reasonable.""" spec = spack.concretize.concretize_one("libdwarf") test_name = "test_name" test_suite = spack.install_test.TestSuite([spec], test_name) logfile = test_suite.log_file_for_spec(spec) assert test_suite.stage in logfile assert test_suite.test_log_name(spec) in logfile def test_test_ensure_stage(mock_test_stage, mock_packages): """Make sure test stage directory is properly set up.""" spec = spack.concretize.concretize_one("libdwarf") test_name = "test_name" test_suite = spack.install_test.TestSuite([spec], test_name) test_suite.ensure_stage() assert os.path.isdir(test_suite.stage) assert mock_test_stage in test_suite.stage def test_write_test_result(mock_packages, mock_test_stage): """Ensure test results written to a results file.""" spec = spack.concretize.concretize_one("libdwarf") result = "TEST" test_name = "write-test" test_suite = spack.install_test.TestSuite([spec], test_name) test_suite.ensure_stage() results_file = test_suite.results_file test_suite.write_test_result(spec, result) with open(results_file, "r", encoding="utf-8") as f: lines = f.readlines() assert len(lines) == 1 msg = lines[0] assert result in msg assert spec.name in msg def test_test_not_installed(mock_packages, install_mockery, mock_test_stage): """Attempt to perform stand-alone test for not_installed package.""" spec = spack.concretize.concretize_one("trivial-smoke-test") test_suite = spack.install_test.TestSuite([spec]) test_suite() ensure_results(test_suite.results_file, "SKIPPED") ensure_results(test_suite.log_file_for_spec(spec), "Skipped not installed") @pytest.mark.parametrize( "arguments,status,msg", [({}, TestStatus.SKIPPED, "Skipped"), ({"externals": True}, TestStatus.NO_TESTS, "No tests")], ) def test_test_external( mock_packages, install_mockery, mock_test_stage, monkeypatch, arguments, status, msg ): name = "trivial-smoke-test" spec = spack.concretize.concretize_one(name) spec.external_path = "/path/to/external/{0}".format(name) monkeypatch.setattr(spack.spec.Spec, "installed", _true) test_suite = spack.install_test.TestSuite([spec]) test_suite(**arguments) ensure_results(test_suite.results_file, str(status)) if arguments: ensure_results(test_suite.log_file_for_spec(spec), msg) def test_test_stage_caches(mock_packages, install_mockery, mock_test_stage): def ensure_current_cache_fail(test_suite): with pytest.raises(spack.install_test.TestSuiteSpecError): _ = test_suite.current_test_cache_dir with pytest.raises(spack.install_test.TestSuiteSpecError): _ = test_suite.current_test_data_dir spec = spack.concretize.concretize_one("libelf") test_suite = spack.install_test.TestSuite([spec], "test-cache") # Check no current specs yield failure ensure_current_cache_fail(test_suite) # Check no current base spec yields failure test_suite.current_base_spec = None test_suite.current_test_spec = spec ensure_current_cache_fail(test_suite) # Check no current test spec yields failure test_suite.current_base_spec = spec test_suite.current_test_spec = None ensure_current_cache_fail(test_suite) def test_test_spec_run_once(mock_packages, install_mockery, mock_test_stage): spec = spack.concretize.concretize_one("libelf") test_suite = spack.install_test.TestSuite([spec], "test-dups") (test_suite.specs[0]).package.test_suite = test_suite with pytest.raises(spack.install_test.TestSuiteFailure): test_suite() @pytest.mark.not_on_windows("Cannot find echo executable") def test_test_spec_passes(mock_packages, install_mockery, mock_test_stage, monkeypatch): spec = spack.concretize.concretize_one("simple-standalone-test") monkeypatch.setattr(spack.spec.Spec, "installed", _true) test_suite = spack.install_test.TestSuite([spec]) test_suite() ensure_results(test_suite.results_file, "PASSED") ensure_results(test_suite.log_file_for_spec(spec), "simple stand-alone") ensure_results(test_suite.log_file_for_spec(spec), "standalone-ifc", present=False) def test_get_test_suite(): assert not spack.install_test.get_test_suite("nothing") def test_get_test_suite_no_name(mock_packages, mock_test_stage): with pytest.raises(spack.install_test.TestSuiteNameError) as exc_info: spack.install_test.get_test_suite("") assert "name is required" in str(exc_info) def test_get_test_suite_too_many(mock_packages, mock_test_stage): test_suites = [] name = "duplicate-alias" def add_suite(package): spec = spack.concretize.concretize_one(package) suite = spack.install_test.TestSuite([spec], name) suite.ensure_stage() spack.install_test.write_test_suite_file(suite) test_suites.append(suite) add_suite("libdwarf") suite = spack.install_test.get_test_suite(name) assert suite.alias == name add_suite("libelf") with pytest.raises(spack.install_test.TestSuiteNameError) as exc_info: spack.install_test.get_test_suite(name) assert "many suites named" in str(exc_info) @pytest.mark.parametrize( "virtuals,expected", [(False, ["Mpich.test_mpich"]), (True, ["Mpi.test_hello", "Mpich.test_mpich"])], ) def test_test_function_names(mock_packages, install_mockery, virtuals, expected): """Confirm test_function_names works as expected with/without virtuals.""" spec = spack.concretize.concretize_one("mpich") tests = spack.install_test.test_function_names(spec.package, add_virtuals=virtuals) assert sorted(tests) == sorted(expected) def test_test_functions_pkgless(mock_packages, install_mockery, ensure_debug, capfd): """Confirm works for package providing a package-less virtual.""" spec = spack.concretize.concretize_one("simple-standalone-test") fns = spack.install_test.test_functions(spec.package, add_virtuals=True) out = capfd.readouterr() assert len(fns) == 2, "Expected two test functions" for f in fns: assert f[1].__name__ in ["test_echo", "test_skip"] assert "virtual does not appear to have a package file" in out[1] # TODO: This test should go away when compilers as dependencies is supported def test_test_virtuals(): """Confirm virtuals picks up non-unique, provided compilers.""" # This is an unrealistic case but it is set up to retrieve all possible # virtual names in a single call. def satisfies(spec): return True # Ensure spec will pick up the llvm+clang virtual compiler package names. VirtualSpec = collections.namedtuple("VirtualSpec", ["name", "satisfies"]) vspec = VirtualSpec("llvm", satisfies) # Ensure the package name is in the list that provides c, cxx, and fortran # to pick up the three associated compilers and that virtuals provided will # be deduped. MyPackage = collections.namedtuple("MyPackage", ["name", "spec", "virtuals_provided"]) pkg = MyPackage("gcc", vspec, [vspec, vspec]) # This check assumes the method will not provide a unique set of compilers v_names = spack.install_test.virtuals(pkg) for name, number in [("c", 2), ("cxx", 2), ("fortran", 1), ("llvm", 1)]: assert v_names.count(name) == number, "Expected {0} of '{1}'".format(number, name) def test_package_copy_test_files_fails(mock_packages): """Confirm copy_test_files fails as expected without package or test_suite.""" vspec = spack.spec.Spec("something") # Try without a package with pytest.raises(spack.install_test.TestSuiteError) as exc_info: spack.install_test.copy_test_files(None, vspec) assert "without a package" in str(exc_info) # Try with a package without a test suite MyPackage = collections.namedtuple("MyPackage", ["name", "spec", "test_suite"]) pkg = MyPackage("SomePackage", vspec, None) with pytest.raises(spack.install_test.TestSuiteError) as exc_info: spack.install_test.copy_test_files(pkg, vspec) assert "test suite is missing" in str(exc_info) def test_package_copy_test_files_skips(mock_packages, ensure_debug, capfd): """Confirm copy_test_files errors as expected if no package class found.""" # Try with a non-concrete spec and package with a test suite MockSuite = collections.namedtuple("TestSuite", ["specs"]) MyPackage = collections.namedtuple("MyPackage", ["name", "spec", "test_suite"]) vspec = spack.spec.Spec("something") pkg = MyPackage("SomePackage", vspec, MockSuite([])) spack.install_test.copy_test_files(pkg, vspec) out = capfd.readouterr()[1] assert "skipping test data copy" in out assert "no package class found" in out def test_process_test_parts(mock_packages): """Confirm process_test_parts fails as expected without package or test_suite.""" # Try without a package with pytest.raises(spack.install_test.TestSuiteError) as exc_info: spack.install_test.process_test_parts(None, []) assert "without a package" in str(exc_info) # Try with a package without a test suite MyPackage = collections.namedtuple("MyPackage", ["name", "test_suite"]) pkg = MyPackage("SomePackage", None) with pytest.raises(spack.install_test.TestSuiteError) as exc_info: spack.install_test.process_test_parts(pkg, []) assert "test suite is missing" in str(exc_info) def test_test_part_fail(tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage): """Confirm test_part with a ProcessError results in FAILED status.""" s = spack.concretize.concretize_one("trivial-smoke-test") pkg = s.package pkg.tester.test_log_file = str(tmp_path / "test-log.txt") touch(pkg.tester.test_log_file) name = "test_fail" with spack.install_test.test_part(pkg, name, "fake ProcessError"): raise spack.util.executable.ProcessError("Mock failure") for part_name, status in pkg.tester.test_parts.items(): assert part_name.endswith(name) assert status == TestStatus.FAILED def test_test_part_pass(install_mockery, mock_fetch, mock_test_stage): """Confirm test_part that succeeds results in PASSED status.""" s = spack.concretize.concretize_one("trivial-smoke-test") pkg = s.package name = "test_echo" msg = "nothing" with spack.install_test.test_part(pkg, name, "echo"): if sys.platform == "win32": print(msg) else: echo = which("echo", required=True) echo(msg) for part_name, status in pkg.tester.test_parts.items(): assert part_name.endswith(name) assert status == TestStatus.PASSED def test_test_part_skip(install_mockery, mock_fetch, mock_test_stage): """Confirm test_part that raises SkipTest results in test status SKIPPED.""" s = spack.concretize.concretize_one("trivial-smoke-test") pkg = s.package name = "test_skip" with spack.install_test.test_part(pkg, name, "raise SkipTest"): raise spack.install_test.SkipTest("Skipping the test") for part_name, status in pkg.tester.test_parts.items(): assert part_name.endswith(name) assert status == TestStatus.SKIPPED def test_test_part_missing_exe_fail_fast( tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage ): """Confirm test_part with fail fast enabled raises exception.""" s = spack.concretize.concretize_one("trivial-smoke-test") pkg = s.package pkg.tester.test_log_file = str(tmp_path / "test-log.txt") touch(pkg.tester.test_log_file) name = "test_fail_fast" with spack.config.override("config:fail_fast", True): with pytest.raises(spack.install_test.TestFailure, match="object is not callable"): with spack.install_test.test_part(pkg, name, "fail fast"): missing = which("no-possible-program") missing() # type: ignore test_parts = pkg.tester.test_parts assert len(test_parts) == 1 for part_name, status in test_parts.items(): assert part_name.endswith(name) assert status == TestStatus.FAILED def test_test_part_missing_exe( tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage ): """Confirm test_part with missing executable fails.""" s = spack.concretize.concretize_one("trivial-smoke-test") pkg = s.package pkg.tester.test_log_file = str(tmp_path / "test-log.txt") touch(pkg.tester.test_log_file) name = "test_missing_exe" with spack.install_test.test_part(pkg, name, "missing exe"): missing = which("no-possible-program") missing() # type: ignore test_parts = pkg.tester.test_parts assert len(test_parts) == 1 for part_name, status in test_parts.items(): assert part_name.endswith(name) assert status == TestStatus.FAILED # TODO (embedded test parts): Update this once embedded test part tracking # TODO (embedded test parts): properly handles the nested context managers. @pytest.mark.parametrize( "current,substatuses,expected", [ (TestStatus.PASSED, [TestStatus.PASSED, TestStatus.PASSED], TestStatus.PASSED), (TestStatus.FAILED, [TestStatus.PASSED, TestStatus.PASSED], TestStatus.FAILED), (TestStatus.SKIPPED, [TestStatus.PASSED, TestStatus.PASSED], TestStatus.SKIPPED), (TestStatus.NO_TESTS, [TestStatus.PASSED, TestStatus.PASSED], TestStatus.NO_TESTS), (TestStatus.PASSED, [TestStatus.PASSED, TestStatus.SKIPPED], TestStatus.PASSED), (TestStatus.PASSED, [TestStatus.PASSED, TestStatus.FAILED], TestStatus.FAILED), (TestStatus.PASSED, [TestStatus.SKIPPED, TestStatus.SKIPPED], TestStatus.SKIPPED), ], ) def test_embedded_test_part_status( install_mockery, mock_fetch, mock_test_stage, current, substatuses, expected ): """Check to ensure the status of the enclosing test part reflects summary of embedded parts.""" s = spack.concretize.concretize_one("trivial-smoke-test") pkg = s.package base_name = "test_example" part_name = f"{pkg.__class__.__name__}::{base_name}" pkg.tester.test_parts[part_name] = current for i, status in enumerate(substatuses): pkg.tester.test_parts[f"{part_name}_{i}"] = status pkg.tester.status(base_name, current) assert pkg.tester.test_parts[part_name] == expected @pytest.mark.parametrize( "statuses,expected", [ ([TestStatus.PASSED, TestStatus.PASSED], TestStatus.PASSED), ([TestStatus.PASSED, TestStatus.SKIPPED], TestStatus.PASSED), ([TestStatus.PASSED, TestStatus.FAILED], TestStatus.FAILED), ([TestStatus.SKIPPED, TestStatus.SKIPPED], TestStatus.SKIPPED), ([], TestStatus.NO_TESTS), ], ) def test_write_tested_status( tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage, statuses, expected ): """Check to ensure the status of the enclosing test part reflects summary of embedded parts.""" s = spack.concretize.concretize_one("trivial-smoke-test") pkg = s.package for i, status in enumerate(statuses): pkg.tester.test_parts[f"test_{i}"] = status pkg.tester.counts[status] += 1 pkg.tester.tested_file = str(tmp_path / "test-log.txt") pkg.tester.write_tested_status() with open(pkg.tester.tested_file, "r", encoding="utf-8") as f: status = int(f.read().strip("\n")) assert TestStatus(status) == expected @pytest.mark.regression("37840") def test_write_tested_status_no_repeats( tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage ): """Emulate re-running the same stand-alone tests a second time.""" s = spack.concretize.concretize_one("trivial-smoke-test") pkg = s.package statuses = [TestStatus.PASSED, TestStatus.PASSED] for i, status in enumerate(statuses): pkg.tester.test_parts[f"test_{i}"] = status pkg.tester.counts[status] += 1 pkg.tester.tested_file = str(tmp_path / "test-log.txt") pkg.tester.write_tested_status() pkg.tester.write_tested_status() # The test should NOT result in a ValueError: invalid literal for int() # with base 10: '2\n2' (i.e., the results being appended instead of # written to the file). with open(pkg.tester.tested_file, "r", encoding="utf-8") as f: status_no = int(f.read().strip("\n")) assert TestStatus(status_no) == TestStatus.PASSED def test_check_special_outputs(tmp_path: pathlib.Path): """This test covers two related helper methods""" contents = """CREATE TABLE packages ( name varchar(80) primary key, has_code integer, url varchar(160)); INSERT INTO packages VALUES('sqlite',1,'https://www.sqlite.org'); INSERT INTO packages VALUES('readline',1,'https://tiswww.case.edu/php/chet/readline/rltop.html'); INSERT INTO packages VALUES('xsdk',0,'http://xsdk.info'); COMMIT; """ filename = tmp_path / "special.txt" with open(filename, "w", encoding="utf-8") as f: f.write(contents) expected = spack.install_test.get_escaped_text_output(str(filename)) spack.install_test.check_outputs(expected, contents) # Let's also cover case where something expected is NOT in the output expected.append("should not find me") with pytest.raises(RuntimeError, match="Expected"): spack.install_test.check_outputs(expected, contents) def test_find_required_file(tmp_path: pathlib.Path): filename = "myexe" for d in ["a", "b"]: path = tmp_path / d / filename os.makedirs(path.parent, exist_ok=True) path.touch() path = tmp_path / "c" / "d" / filename os.makedirs(path.parent, exist_ok=True) path.touch() # First just find a single path results = spack.install_test.find_required_file( str(tmp_path / "c"), filename, expected=1, recursive=True ) assert isinstance(results, str) # Ensure none file if do not recursively search that directory with pytest.raises(spack.install_test.SkipTest, match="Expected 1"): spack.install_test.find_required_file( str(tmp_path / "c"), filename, expected=1, recursive=False ) # Now make sure we get all of the files results = spack.install_test.find_required_file( str(tmp_path), filename, expected=3, recursive=True ) assert isinstance(results, list) and len(results) == 3 def test_packagetest_fails(mock_packages): MyPackage = collections.namedtuple("MyPackage", ["spec"]) s = spack.spec.Spec("pkg-a") pkg = MyPackage(s) with pytest.raises(ValueError, match="require a concrete package"): spack.install_test.PackageTest(pkg) ================================================ FILE: lib/spack/spack/test/traverse.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pytest import spack.deptypes as dt import spack.traverse as traverse from spack.spec import Spec def create_dag(nodes, edges): """ Arguments: nodes: list of package names edges: list of tuples (from, to, deptype) Returns: dict: mapping from package name to abstract Spec with proper deps. """ specs = {name: Spec(name) for name in nodes} for parent, child, deptypes in edges: depflag = deptypes if isinstance(deptypes, dt.DepFlag) else dt.canonicalize(deptypes) specs[parent].add_dependency_edge(specs[child], depflag=depflag, virtuals=()) return specs @pytest.fixture() def abstract_specs_dtuse(): return create_dag( nodes=[ "dtbuild1", "dtbuild2", "dtbuild3", "dtlink1", "dtlink2", "dtlink3", "dtlink4", "dtlink5", "dtrun1", "dtrun2", "dtrun3", "dttop", "dtuse", ], edges=[ ("dtbuild1", "dtbuild2", ("build")), ("dtbuild1", "dtlink2", ("build", "link")), ("dtbuild1", "dtrun2", ("run")), ("dtlink1", "dtlink3", ("build", "link")), ("dtlink3", "dtbuild2", ("build")), ("dtlink3", "dtlink4", ("build", "link")), ("dtrun1", "dtlink5", ("build", "link")), ("dtrun1", "dtrun3", ("run")), ("dtrun3", "dtbuild3", ("build")), ("dttop", "dtbuild1", ("build",)), ("dttop", "dtlink1", ("build", "link")), ("dttop", "dtrun1", ("run")), ("dtuse", "dttop", ("build", "link")), ], ) @pytest.fixture() def abstract_specs_dt_diamond(): return create_dag( nodes=["dt-diamond", "dt-diamond-left", "dt-diamond-right", "dt-diamond-bottom"], edges=[ ("dt-diamond", "dt-diamond-left", ("build", "link")), ("dt-diamond", "dt-diamond-right", ("build", "link")), ("dt-diamond-right", "dt-diamond-bottom", ("build", "link", "run")), ("dt-diamond-left", "dt-diamond-bottom", ("build")), ], ) @pytest.fixture() def abstract_specs_chain(): # Chain a -> b -> c -> d with skip connections # from a -> c and a -> d. return create_dag( nodes=["chain-a", "chain-b", "chain-c", "chain-d"], edges=[ ("chain-a", "chain-b", ("build", "link")), ("chain-b", "chain-c", ("build", "link")), ("chain-c", "chain-d", ("build", "link")), ("chain-a", "chain-c", ("build", "link")), ("chain-a", "chain-d", ("build", "link")), ], ) @pytest.mark.parametrize("direction", ("children", "parents")) @pytest.mark.parametrize("deptype", ("all", ("link", "build"), ("run", "link"))) def test_all_orders_traverse_the_same_nodes(direction, deptype, abstract_specs_dtuse): # Test whether all graph traversal methods visit the same set of vertices. # When testing cover=nodes, the traversal methods may reach the same vertices # through different edges, so we're using traverse_nodes here to only verify the # vertices. # # NOTE: root=False currently means "yield nodes discovered at depth > 0", # meaning that depth first search will yield dtlink5 as it is first found through # dtuse, whereas breadth first search considers dtlink5 at depth 0 and does not # yield it since it is a root. Therefore, we only use root=True. # (The inconsistency cannot be resolved by making root=False mean "don't yield # vertices without in-edges", since this is not how it's used; it's typically used # as "skip the input specs".) specs = [abstract_specs_dtuse["dtuse"], abstract_specs_dtuse["dtlink5"]] kwargs = {"root": True, "direction": direction, "deptype": deptype, "cover": "nodes"} def nodes(order): s = traverse.traverse_nodes(specs, order=order, **kwargs) return sorted(list(s)) assert nodes("pre") == nodes("post") == nodes("breadth") == nodes("topo") @pytest.mark.parametrize("direction", ("children", "parents")) @pytest.mark.parametrize("root", (True, False)) @pytest.mark.parametrize("deptype", ("all", ("link", "build"), ("run", "link"))) def test_all_orders_traverse_the_same_edges(direction, root, deptype, abstract_specs_dtuse): # Test whether all graph traversal methods visit the same set of edges. # All edges should be returned, including the artificial edges to the input # specs when root=True. specs = [abstract_specs_dtuse["dtuse"], abstract_specs_dtuse["dtlink5"]] kwargs = {"root": root, "direction": direction, "deptype": deptype, "cover": "edges"} def edges(order): s = traverse.traverse_edges(specs, order=order, **kwargs) return sorted(list(s)) assert edges("pre") == edges("post") == edges("breadth") == edges("topo") def test_breadth_first_traversal(abstract_specs_dtuse): # That that depth of discovery is non-decreasing s = abstract_specs_dtuse["dttop"] depths = [ depth for (depth, _) in traverse.traverse_nodes( [s], order="breadth", key=lambda s: s.name, depth=True ) ] assert depths == sorted(depths) def test_breadth_first_deptype_traversal(abstract_specs_dtuse): s = abstract_specs_dtuse["dtuse"] names = ["dtuse", "dttop", "dtbuild1", "dtlink1", "dtbuild2", "dtlink2", "dtlink3", "dtlink4"] traversal = traverse.traverse_nodes([s], order="breadth", key=id, deptype=("build", "link")) assert [x.name for x in traversal] == names def test_breadth_firsrt_traversal_deptype_with_builddeps(abstract_specs_dtuse): s = abstract_specs_dtuse["dttop"] names = ["dttop", "dtbuild1", "dtlink1", "dtbuild2", "dtlink2", "dtlink3", "dtlink4"] traversal = traverse.traverse_nodes([s], order="breadth", key=id, deptype=("build", "link")) assert [x.name for x in traversal] == names def test_breadth_first_traversal_deptype_full(abstract_specs_dtuse): s = abstract_specs_dtuse["dttop"] names = [ "dttop", "dtbuild1", "dtlink1", "dtrun1", "dtbuild2", "dtlink2", "dtrun2", "dtlink3", "dtlink5", "dtrun3", "dtlink4", "dtbuild3", ] traversal = traverse.traverse_nodes([s], order="breadth", key=id, deptype="all") assert [x.name for x in traversal] == names def test_breadth_first_traversal_deptype_run(abstract_specs_dtuse): s = abstract_specs_dtuse["dttop"] names = ["dttop", "dtrun1", "dtrun3"] traversal = traverse.traverse_nodes([s], order="breadth", key=id, deptype="run") assert [x.name for x in traversal] == names def test_breadth_first_traversal_reverse(abstract_specs_dt_diamond): gen = traverse.traverse_nodes( [abstract_specs_dt_diamond["dt-diamond-bottom"]], order="breadth", key=id, direction="parents", depth=True, ) assert [(depth, spec.name) for (depth, spec) in gen] == [ (0, "dt-diamond-bottom"), (1, "dt-diamond-left"), (1, "dt-diamond-right"), (2, "dt-diamond"), ] def test_breadth_first_traversal_multiple_input_specs(abstract_specs_dt_diamond): # With DFS, the branch dt-diamond -> dt-diamond-left -> dt-diamond-bottom # is followed, with BFS, dt-diamond-bottom should be traced through the second # input spec dt-diamond-right at depth 1 instead. input_specs = [ abstract_specs_dt_diamond["dt-diamond"], abstract_specs_dt_diamond["dt-diamond-right"], ] gen = traverse.traverse_edges(input_specs, order="breadth", key=id, depth=True, root=False) assert [(depth, edge.parent.name, edge.spec.name) for (depth, edge) in gen] == [ (1, "dt-diamond", "dt-diamond-left"), # edge from first input spec "to" depth 1 (1, "dt-diamond-right", "dt-diamond-bottom"), # edge from second input spec "to" depth 1 ] def test_breadth_first_versus_depth_first_tree(abstract_specs_chain): """ The packages chain-a, chain-b, chain-c, chain-d have the following DAG: a --> b --> c --> d # a chain a --> c # and "skip" connections a --> d Here we test at what depth the nodes are discovered when using BFS vs DFS. """ s = abstract_specs_chain["chain-a"] # BFS should find all nodes as direct deps assert [ (depth, edge.spec.name) for (depth, edge) in traverse.traverse_tree([s], cover="nodes", depth_first=False) ] == [(0, "chain-a"), (1, "chain-b"), (1, "chain-c"), (1, "chain-d")] # DFS will discover all nodes along the chain a -> b -> c -> d. assert [ (depth, edge.spec.name) for (depth, edge) in traverse.traverse_tree([s], cover="nodes", depth_first=True) ] == [(0, "chain-a"), (1, "chain-b"), (2, "chain-c"), (3, "chain-d")] # When covering all edges, we should never exceed depth 2 in BFS. assert [ (depth, edge.spec.name) for (depth, edge) in traverse.traverse_tree([s], cover="edges", depth_first=False) ] == [ (0, "chain-a"), (1, "chain-b"), (2, "chain-c"), (1, "chain-c"), (2, "chain-d"), (1, "chain-d"), ] # In DFS we see the chain again. assert [ (depth, edge.spec.name) for (depth, edge) in traverse.traverse_tree([s], cover="edges", depth_first=True) ] == [ (0, "chain-a"), (1, "chain-b"), (2, "chain-c"), (3, "chain-d"), (1, "chain-c"), (1, "chain-d"), ] @pytest.mark.parametrize("cover", ["nodes", "edges"]) @pytest.mark.parametrize("depth_first", [True, False]) def test_tree_traversal_with_key(cover, depth_first, abstract_specs_chain): """Compare two multisource traversals of the same DAG. In one case the DAG consists of unique Spec instances, in the second case there are identical copies of nodes and edges. Traversal should be equivalent when nodes are identified by dag_hash.""" a = abstract_specs_chain["chain-a"] c = abstract_specs_chain["chain-c"] kwargs = {"cover": cover, "depth_first": depth_first} dag_hash = lambda s: s.dag_hash() # Traverse DAG spanned by a unique set of Spec instances first = traverse.traverse_tree([a, c], key=id, **kwargs) # Traverse equivalent DAG with copies of Spec instances included, keyed by dag hash. second = traverse.traverse_tree([a, c.copy()], key=dag_hash, **kwargs) # Check that the same nodes are discovered at the same depth node_at_depth_first = [(depth, dag_hash(edge.spec)) for (depth, edge) in first] node_at_depth_second = [(depth, dag_hash(edge.spec)) for (depth, edge) in second] assert node_at_depth_first == node_at_depth_second def test_breadth_first_versus_depth_first_printing(abstract_specs_chain): """Test breadth-first versus depth-first tree printing.""" s = abstract_specs_chain["chain-a"] args = {"format": "{name}", "color": False} dfs_tree_nodes = """\ chain-a ^chain-b ^chain-c ^chain-d """ assert s.tree(depth_first=True, **args) == dfs_tree_nodes bfs_tree_nodes = """\ chain-a ^chain-b ^chain-c ^chain-d """ assert s.tree(depth_first=False, **args) == bfs_tree_nodes dfs_tree_edges = """\ chain-a ^chain-b ^chain-c ^chain-d ^chain-c ^chain-d """ assert s.tree(depth_first=True, cover="edges", **args) == dfs_tree_edges bfs_tree_edges = """\ chain-a ^chain-b ^chain-c ^chain-c ^chain-d ^chain-d """ assert s.tree(depth_first=False, cover="edges", **args) == bfs_tree_edges @pytest.fixture() def abstract_specs_toposort(): # Create a graph that both BFS and DFS would not traverse in topo order, given the # default edge ordering (by target spec name). Roots are {A, E} in forward order # and {F, G} in backward order. # forward: DFS([A, E]) traverses [A, B, F, G, C, D, E] (not topo since C < B) # forward: BFS([A, E]) traverses [A, E, B, C, D, F, G] (not topo since C < B) # reverse: DFS([F, G]) traverses [F, B, A, D, C, E, G] (not topo since D < A) # reverse: BFS([F, G]) traverses [F, G, B, A, D, C, E] (not topo since D < A) # E # | A # | | \ # | C | # \ | | # D | # | / # B # / \ # F G return create_dag( nodes=["A", "B", "C", "D", "E", "F", "G"], edges=( ("A", "B", "all"), ("A", "C", "all"), ("B", "F", "all"), ("B", "G", "all"), ("C", "D", "all"), ("D", "B", "all"), ("E", "D", "all"), ), ) def test_traverse_nodes_topo(abstract_specs_toposort): # Test whether we get topologically ordered specs when using traverse_nodes with # order=topo and cover=nodes. nodes = abstract_specs_toposort def test_topo(input_specs, direction="children"): # Ensure the invariant that all parents of specs[i] are in specs[0:i] specs = list( traverse.traverse_nodes(input_specs, order="topo", cover="nodes", direction=direction) ) reverse = "parents" if direction == "children" else "children" for i in range(len(specs)): parents = specs[i].traverse(cover="nodes", direction=reverse, root=False) assert set(list(parents)).issubset(set(specs[:i])) # Traverse forward from roots A and E and a non-root D. Notice that adding D has no # effect, it's just to make the test case a bit more complicated, as D is a starting # point for traversal, but it's also discovered as a descendant of E and A. test_topo([nodes["D"], nodes["E"], nodes["A"]], direction="children") # Traverse reverse from leafs F and G and non-leaf D test_topo([nodes["F"], nodes["D"], nodes["G"]], direction="parents") def test_traverse_edges_topo(abstract_specs_toposort): # Test the invariant that for each node in-edges precede out-edges when # using traverse_edges with order=topo. nodes = abstract_specs_toposort input_specs = [nodes["E"], nodes["A"]] # Collect pairs of (parent spec name, child spec name) edges = [ (e.parent.name, e.spec.name) for e in traverse.traverse_edges(input_specs, order="topo", cover="edges", root=False) ] # See figure above, we have 7 edges (excluding artificial ones to the root) assert set(edges) == set( [("A", "B"), ("A", "C"), ("B", "F"), ("B", "G"), ("C", "D"), ("D", "B"), ("E", "D")] ) # Verify that all in-edges precede all out-edges for node in nodes.keys(): in_edge_indices = [i for (i, (parent, child)) in enumerate(edges) if node == child] out_edge_indices = [i for (i, (parent, child)) in enumerate(edges) if node == parent] if in_edge_indices and out_edge_indices: assert max(in_edge_indices) < min(out_edge_indices) def test_traverse_nodes_no_deps(abstract_specs_dtuse): """Traversing nodes without deps should be the same as deduplicating the input specs. This may not look useful, but can be used to avoid a branch on the call site in which it's otherwise easy to forget to deduplicate input specs.""" inputs = [ abstract_specs_dtuse["dtuse"], abstract_specs_dtuse["dtlink5"], abstract_specs_dtuse["dtuse"], # <- duplicate ] outputs = [x for x in traverse.traverse_nodes(inputs, deptype=dt.NONE)] assert outputs == [abstract_specs_dtuse["dtuse"], abstract_specs_dtuse["dtlink5"]] @pytest.mark.parametrize("cover", ["nodes", "edges"]) def test_topo_is_bfs_for_trees(cover): """For trees, both DFS and BFS produce a topological order, but BFS is the most sensible for our applications, where we typically want to avoid that transitive dependencies shadow direct dependencies in global search paths, etc. This test ensures that for trees, the default topo order coincides with BFS.""" binary_tree = create_dag( nodes=["A", "B", "C", "D", "E", "F", "G"], edges=( ("A", "B", "all"), ("A", "C", "all"), ("B", "D", "all"), ("B", "E", "all"), ("C", "F", "all"), ("C", "G", "all"), ), ) assert list(traverse.traverse_nodes([binary_tree["A"]], order="topo", cover=cover)) == list( traverse.traverse_nodes([binary_tree["A"]], order="breadth", cover=cover) ) @pytest.mark.parametrize("roots", [["A"], ["A", "B"], ["B", "A"], ["A", "B", "A"]]) @pytest.mark.parametrize("order", ["breadth", "post", "pre"]) @pytest.mark.parametrize("include_root", [True, False]) def test_mixed_depth_visitor(roots, order, include_root): """Test that the MixedDepthVisitor lists unique edges that are reachable either directly from roots through build type edges, or transitively through link type edges. The tests ensures that unique edges are listed exactly once.""" my_graph = create_dag( nodes=["A", "B", "C", "D", "E", "F", "G", "H", "I"], edges=( ("A", "B", dt.LINK | dt.RUN), ("A", "C", dt.BUILD), ("A", "D", dt.BUILD | dt.RUN), ("A", "H", dt.LINK), ("A", "I", dt.RUN), ("B", "D", dt.BUILD | dt.LINK), ("C", "E", dt.BUILD | dt.LINK | dt.RUN), ("D", "F", dt.LINK), ("D", "G", dt.BUILD | dt.RUN), ("H", "B", dt.LINK), ), ) starting_points = traverse.with_artificial_edges([my_graph[root] for root in roots]) visitor = traverse.MixedDepthVisitor(direct=dt.BUILD, transitive=dt.LINK) if order == "pre": edges = traverse.traverse_depth_first_edges_generator( starting_points, visitor, post_order=False, root=include_root ) elif order == "post": edges = traverse.traverse_depth_first_edges_generator( starting_points, visitor, post_order=True, root=include_root ) elif order == "breadth": edges = traverse.traverse_breadth_first_edges_generator( starting_points, visitor, root=include_root ) artificial_edges = [(None, root) for root in roots] if include_root else [] simple_edges = [ (None if edge.parent is None else edge.parent.name, edge.spec.name) for edge in edges ] # make sure that every edge is listed exactly once and that the right edges are listed assert len(simple_edges) == len(set(simple_edges)) assert set(simple_edges) == { # the roots *artificial_edges, ("A", "B"), ("A", "C"), ("A", "D"), ("A", "H"), ("B", "D"), ("D", "F"), ("H", "B"), } ================================================ FILE: lib/spack/spack/test/url_fetch.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import filecmp import os import pathlib import sys import urllib.error import pytest import spack.concretize import spack.config import spack.error import spack.fetch_strategy as fs import spack.llnl.util.tty as tty import spack.url import spack.util.crypto as crypto import spack.util.web as web_util import spack.version from spack.llnl.util.filesystem import is_exe, working_dir from spack.stage import Stage from spack.util.executable import which @pytest.fixture def missing_curl(monkeypatch): def require_curl(): raise spack.error.FetchError("curl is required but not found") monkeypatch.setattr(web_util, "require_curl", require_curl) @pytest.fixture(params=list(crypto.hashes.keys())) def checksum_type(request): return request.param @pytest.fixture def pkg_factory(): Pkg = collections.namedtuple( "Pkg", [ "url_for_version", "all_urls_for_version", "find_valid_url_for_version", "urls", "url", "versions", "fetch_options", ], ) def factory(url, urls, fetch_options={}): def fn(v): main_url = url or urls[0] return spack.url.substitute_version(main_url, v) def fn_urls(v): urls_loc = urls or [url] return [spack.url.substitute_version(u, v) for u in urls_loc] return Pkg( find_valid_url_for_version=fn, url_for_version=fn, all_urls_for_version=fn_urls, url=url, urls=(urls,), versions=collections.defaultdict(dict), fetch_options=fetch_options, ) return factory @pytest.mark.parametrize("method", ["curl", "urllib"]) def test_urlfetchstrategy_bad_url(tmp_path: pathlib.Path, mutable_config, method): """Ensure fetch with bad URL fails as expected.""" mutable_config.set("config:url_fetch_method", method) fetcher = fs.URLFetchStrategy(url=(tmp_path / "does-not-exist").as_uri()) with Stage(fetcher, path=str(tmp_path / "stage")): with pytest.raises(fs.FailedDownloadError) as exc: fetcher.fetch() assert len(exc.value.exceptions) == 1 exception = exc.value.exceptions[0] if method == "curl": assert isinstance(exception, spack.error.FetchError) assert "Curl failed with error 37" in str(exception) # FILE_COULDNT_READ_FILE elif method == "urllib": assert isinstance(exception, urllib.error.URLError) assert isinstance(exception.reason, FileNotFoundError) def test_fetch_options(tmp_path: pathlib.Path, mock_archive): with spack.config.override("config:url_fetch_method", "curl"): fetcher = fs.URLFetchStrategy( url=mock_archive.url, fetch_options={"cookie": "True", "timeout": 10} ) with Stage(fetcher, path=str(tmp_path)): assert fetcher.archive_file is None fetcher.fetch() archive_file = fetcher.archive_file assert archive_file is not None assert filecmp.cmp(archive_file, mock_archive.archive_file) def test_fetch_curl_options(tmp_path: pathlib.Path, mock_archive, monkeypatch): with spack.config.override("config:url_fetch_method", "curl -k -q"): fetcher = fs.URLFetchStrategy( url=mock_archive.url, fetch_options={"cookie": "True", "timeout": 10} ) def check_args(*args, **kwargs): # Raise StopIteration to avoid running the rest of the fetch method # args[0] is `which curl`, next two are our config options assert args[1:3] == ("-k", "-q") raise StopIteration monkeypatch.setattr(type(fetcher.curl), "__call__", check_args) with Stage(fetcher, path=str(tmp_path)): assert fetcher.archive_file is None with pytest.raises(StopIteration): fetcher.fetch() @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) def test_archive_file_errors(tmp_path: pathlib.Path, mutable_config, mock_archive, _fetch_method): """Ensure FetchStrategy commands may only be used as intended""" with spack.config.override("config:url_fetch_method", _fetch_method): fetcher = fs.URLFetchStrategy(url=mock_archive.url) with Stage(fetcher, path=str(tmp_path)) as stage: assert fetcher.archive_file is None with pytest.raises(fs.NoArchiveFileError): fetcher.archive(str(tmp_path)) with pytest.raises(fs.NoArchiveFileError): fetcher.expand() with pytest.raises(fs.NoArchiveFileError): fetcher.reset() stage.fetch() with pytest.raises(fs.NoDigestError): fetcher.check() archive_file = fetcher.archive_file assert archive_file is not None assert filecmp.cmp(archive_file, mock_archive.archive_file) files = [(".tar.gz", "z"), (".tgz", "z")] if sys.platform != "win32": files += [(".tar.bz2", "j"), (".tbz2", "j"), (".tar.xz", "J"), (".txz", "J")] @pytest.mark.parametrize("secure", [True, False]) @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) @pytest.mark.parametrize("mock_archive", files, indirect=True) def test_fetch( mock_archive, secure, _fetch_method, checksum_type, default_mock_concretization, mutable_mock_repo, ): """Fetch an archive and make sure we can checksum it.""" algo = crypto.hash_fun_for_algo(checksum_type)() with open(mock_archive.archive_file, "rb") as f: algo.update(f.read()) checksum = algo.hexdigest() # Get a spec and tweak the test package with new checksum params s = default_mock_concretization("url-test") s.package.url = mock_archive.url s.package.versions[spack.version.Version("test")] = { checksum_type: checksum, "url": s.package.url, } # Enter the stage directory and check some properties with s.package.stage: with spack.config.override("config:verify_ssl", secure): with spack.config.override("config:url_fetch_method", _fetch_method): s.package.do_stage() with working_dir(s.package.stage.source_path): assert os.path.exists("configure") assert is_exe("configure") with open("configure", encoding="utf-8") as f: contents = f.read() assert contents.startswith("#!/bin/sh") assert "echo Building..." in contents @pytest.mark.parametrize( "spec,url,digest", [ ("url-list-test @=0.0.0", "foo-0.0.0.tar.gz", "00000000000000000000000000000000"), ("url-list-test @=1.0.0", "foo-1.0.0.tar.gz", "00000000000000000000000000000100"), ("url-list-test @=3.0", "foo-3.0.tar.gz", "00000000000000000000000000000030"), ("url-list-test @=4.5", "foo-4.5.tar.gz", "00000000000000000000000000000450"), ("url-list-test @=2.0.0b2", "foo-2.0.0b2.tar.gz", "000000000000000000000000000200b2"), ("url-list-test @=3.0a1", "foo-3.0a1.tar.gz", "000000000000000000000000000030a1"), ("url-list-test @=4.5-rc5", "foo-4.5-rc5.tar.gz", "000000000000000000000000000045c5"), ], ) @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) def test_from_list_url(mock_packages, config, spec, url, digest, _fetch_method): """ Test URLs in the url-list-test package, which means they should have checksums in the package. """ with spack.config.override("config:url_fetch_method", _fetch_method): s = spack.concretize.concretize_one(spec) fetch_strategy = fs.from_list_url(s.package) assert isinstance(fetch_strategy, fs.URLFetchStrategy) assert os.path.basename(fetch_strategy.url) == url assert fetch_strategy.digest == digest assert fetch_strategy.extra_options == {} s.package.fetch_options = {"timeout": 60} fetch_strategy = fs.from_list_url(s.package) assert fetch_strategy.extra_options == {"timeout": 60} @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) @pytest.mark.parametrize( "requested_version,tarball,digest", [ # These versions are in the web data path (test/data/web/4.html), but not in the # url-list-test package. We expect Spack to generate a URL with the new version. ("=4.5.0", "foo-4.5.0.tar.gz", None), ("=2.0.0", "foo-2.0.0.tar.gz", None), ], ) def test_new_version_from_list_url( mock_packages, config, _fetch_method, requested_version, tarball, digest ): """Test non-specific URLs from the url-list-test package.""" with spack.config.override("config:url_fetch_method", _fetch_method): s = spack.concretize.concretize_one(f"url-list-test @{requested_version}") fetch_strategy = fs.from_list_url(s.package) assert isinstance(fetch_strategy, fs.URLFetchStrategy) assert os.path.basename(fetch_strategy.url) == tarball assert fetch_strategy.digest == digest assert fetch_strategy.extra_options == {} s.package.fetch_options = {"timeout": 60} fetch_strategy = fs.from_list_url(s.package) assert fetch_strategy.extra_options == {"timeout": 60} def test_nosource_from_list_url(mock_packages, config): """This test confirms BundlePackages do not have list url.""" s = spack.concretize.concretize_one("nosource") fetch_strategy = fs.from_list_url(s.package) assert fetch_strategy is None def test_hash_detection(checksum_type): algo = crypto.hash_fun_for_algo(checksum_type)() h = "f" * (algo.digest_size * 2) # hex -> bytes checker = crypto.Checker(h) assert checker.hash_name == checksum_type def test_unknown_hash(checksum_type): with pytest.raises(ValueError): crypto.Checker("a") @pytest.mark.skipif(which("curl") is None, reason="Urllib does not have built-in status bar") def test_url_with_status_bar(tmp_path: pathlib.Path, mock_archive, monkeypatch, capfd): """Ensure fetch with status bar option succeeds.""" def is_true(): return True testpath = str(tmp_path) monkeypatch.setattr(sys.stdout, "isatty", is_true) monkeypatch.setattr(tty, "msg_enabled", is_true) with spack.config.override("config:url_fetch_method", "curl"): fetcher = fs.URLFetchStrategy(url=mock_archive.url) with Stage(fetcher, path=testpath) as stage: assert fetcher.archive_file is None stage.fetch() status = capfd.readouterr()[1] assert "##### 100" in status @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) def test_url_extra_fetch(tmp_path: pathlib.Path, mutable_config, mock_archive, _fetch_method): """Ensure a fetch after downloading is effectively a no-op.""" mutable_config.set("config:url_fetch_method", _fetch_method) fetcher = fs.URLFetchStrategy(url=mock_archive.url) with Stage(fetcher, path=str(tmp_path)) as stage: assert fetcher.archive_file is None stage.fetch() archive_file = fetcher.archive_file assert archive_file is not None assert filecmp.cmp(archive_file, mock_archive.archive_file) fetcher.fetch() @pytest.mark.parametrize( "url,urls,version,expected", [ ( None, [ "https://ftpmirror.gnu.org/autoconf/autoconf-2.69.tar.gz", "https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz", ], "2.62", [ "https://ftpmirror.gnu.org/autoconf/autoconf-2.62.tar.gz", "https://ftp.gnu.org/gnu/autoconf/autoconf-2.62.tar.gz", ], ) ], ) @pytest.mark.parametrize("_fetch_method", ["curl", "urllib"]) def test_candidate_urls(pkg_factory, url, urls, version, expected, _fetch_method): """Tests that candidate urls include mirrors and that they go through pattern matching and substitution for versions. """ with spack.config.override("config:url_fetch_method", _fetch_method): pkg = pkg_factory(url, urls) f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version) assert f.candidate_urls == expected assert f.extra_options == {} pkg = pkg_factory(url, urls, fetch_options={"timeout": 60}) f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version) assert f.extra_options == {"timeout": 60} @pytest.mark.regression("19673") def test_missing_curl(tmp_path: pathlib.Path, missing_curl, mutable_config, monkeypatch): """Ensure a fetch involving missing curl package reports the error.""" mutable_config.set("config:url_fetch_method", "curl") fetcher = fs.URLFetchStrategy(url="http://example.com/file.tar.gz") with pytest.raises(spack.error.FetchError, match="curl is required but not found"): with Stage(fetcher, path=str(tmp_path)) as stage: stage.fetch() def test_url_fetch_text_without_url(): with pytest.raises(spack.error.FetchError, match="URL is required"): web_util.fetch_url_text(None) def test_url_fetch_text_curl_failures(mutable_config, missing_curl, monkeypatch): """Check fetch_url_text if URL's curl is missing.""" mutable_config.set("config:url_fetch_method", "curl") with pytest.raises(spack.error.FetchError, match="curl is required but not found"): web_util.fetch_url_text("https://example.com/") def test_url_check_curl_errors(): """Check that standard curl error returncodes raise expected errors.""" # Check returncode 22 (i.e., 404) with pytest.raises(spack.error.FetchError, match="not found"): web_util.check_curl_code(22) # Check returncode 60 (certificate error) with pytest.raises(spack.error.FetchError, match="invalid certificate"): web_util.check_curl_code(60) def test_url_missing_curl(mutable_config, missing_curl, monkeypatch): """Check url_exists failures if URL's curl is missing.""" mutable_config.set("config:url_fetch_method", "curl") with pytest.raises(spack.error.FetchError, match="curl is required but not found"): web_util.url_exists("https://example.com/") def test_url_fetch_text_urllib_web_error(mutable_config, monkeypatch): def _raise_web_error(*args, **kwargs): raise web_util.SpackWebError("bad url") monkeypatch.setattr(web_util, "read_from_url", _raise_web_error) mutable_config.set("config:url_fetch_method", "urllib") with pytest.raises(spack.error.FetchError, match="fetch failed"): web_util.fetch_url_text("https://example.com/") ================================================ FILE: lib/spack/spack/test/url_parse.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests Spack's ability to parse the name and version of a package based on its URL. """ import os import pytest from spack.url import ( UndetectableVersionError, parse_name_and_version, parse_name_offset, parse_version_offset, strip_name_suffixes, substitute_version, ) from spack.version import Version @pytest.mark.parametrize( "url,version,expected", [ # No suffix ("rgb-1.0.6", "1.0.6", "rgb"), ("nauty26r7", "26r7", "nauty"), ("PAGIT.V1.01", "1.01", "PAGIT"), ("AmpliconNoiseV1.29", "1.29", "AmpliconNoise"), # Download type - install ("converge_install_2.3.16", "2.3.16", "converge"), # Download type - src ("jpegsrc.v9b", "9b", "jpeg"), ("blatSrc35", "35", "blat"), # Download type - open ("RepeatMasker-open-4-0-7", "4-0-7", "RepeatMasker"), # Download type - archive ("coinhsl-archive-2014.01.17", "2014.01.17", "coinhsl"), # Download type - std ("ghostscript-fonts-std-8.11", "8.11", "ghostscript-fonts"), # Download type - bin ("GapCloser-bin-v1.12-r6", "1.12-r6", "GapCloser"), # Download type - software ("orthomclSoftware-v2.0.9", "2.0.9", "orthomcl"), # Download version - release ("cbench_release_1.3.0.tar.gz", "1.3.0", "cbench"), # Download version - snapshot ("gts-snapshot-121130", "121130", "gts"), # Download version - distrib ("zoltan_distrib_v3.83", "3.83", "zoltan"), # Download version - latest ("Platypus-latest", "N/A", "Platypus"), # Download version - complex ("qt-everywhere-opensource-src-5.7.0", "5.7.0", "qt"), # Arch ("VESTA-x86_64", "3.4.6", "VESTA"), # VCS - bazaar ("libvterm-0+bzr681", "681", "libvterm"), # License - gpl ("PyQt-x11-gpl-4.11.3", "4.11.3", "PyQt"), ("PyQt4_gpl_x11-4.12.3", "4.12.3", "PyQt4"), ], ) def test_url_strip_name_suffixes(url, version, expected): stripped = strip_name_suffixes(url, version) assert stripped == expected @pytest.mark.parametrize( "name,noffset,ver,voffset,path", [ # Name in path ("antlr", 25, "2.7.7", 40, "https://github.com/antlr/antlr/tarball/v2.7.7"), # Name in stem ("gmp", 32, "6.0.0a", 36, "https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2"), # Name in suffix # Don't think I've ever seen one of these before # We don't look for it, so it would probably fail anyway # Version in path ( "nextflow", 31, "0.20.1", 59, "https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow", ), ( "hpcviewer", 30, "2024.02", 51, "https://gitlab.com/hpctoolkit/hpcviewer/-/releases/2024.02/downloads/hpcviewer.tgz", ), # Version in stem ("zlib", 24, "1.2.10", 29, "http://zlib.net/fossils/zlib-1.2.10.tar.gz"), ( "slepc", 51, "3.6.2", 57, "http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz", ), ( "cloog", 61, "0.18.1", 67, "http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz", ), ( "libxc", 58, "2.2.2", 64, "http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz", ), # Version in suffix ( "swiftsim", 36, "0.3.0", 76, "http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0", ), ( "swiftsim", 55, "0.3.0", 95, "https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0", ), ( "sionlib", 30, "1.7.1", 59, "http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1", ), # Regex in name ("voro++", 40, "0.4.6", 47, "http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz"), # SourceForge download ( "glew", 55, "2.0.0", 60, "https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download", ), ], ) def test_url_parse_offset(name, noffset, ver, voffset, path): """Tests that the name, version and offsets are computed correctly. Args: name (str): expected name noffset (int): name offset ver (str): expected version voffset (int): version offset path (str): url to be parsed """ # Make sure parse_name_offset and parse_name_version are working v, vstart, vlen, vi, vre = parse_version_offset(path) n, nstart, nlen, ni, nre = parse_name_offset(path, v) assert n == name assert v == ver assert nstart == noffset assert vstart == voffset @pytest.mark.parametrize( "name,version,url", [ # Common Repositories - github downloads # name/archive/ver.ver ("nco", "4.6.2", "https://github.com/nco/nco/archive/4.6.2.tar.gz"), # name/archive/vver.ver ("vim", "8.0.0134", "https://github.com/vim/vim/archive/v8.0.0134.tar.gz"), # name/archive/name-ver.ver ("oce", "0.18", "https://github.com/tpaviot/oce/archive/OCE-0.18.tar.gz"), # name/releases/download/vver/name-ver.ver ( "libmesh", "1.0.0", "https://github.com/libMesh/libmesh/releases/download/v1.0.0/libmesh-1.0.0.tar.bz2", ), # name/tarball/vver.ver ("git", "2.7.1", "https://github.com/git/git/tarball/v2.7.1"), # name/zipball/vver.ver ("git", "2.7.1", "https://github.com/git/git/zipball/v2.7.1"), # Common Repositories - gitlab downloads # name/repository/archive.ext?ref=vver.ver ( "swiftsim", "0.3.0", "http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0", ), # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=vver.ver ( "swiftsim", "0.3.0", "https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0", ), # name/repository/archive.ext?ref=name-ver.ver ( "icet", "1.2.3", "https://gitlab.kitware.com/icet/icet/repository/archive.tar.gz?ref=IceT-1.2.3", ), # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=name-ver.ver ( "icet", "1.2.3", "https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-1.2.3", ), # Common Repositories - bitbucket downloads # name/get/ver.ver ("eigen", "3.2.7", "https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2"), # name/get/vver.ver ("hoomd-blue", "1.3.3", "https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2"), # name/downloads/name-ver.ver ( "dolfin", "2016.1.0", "https://bitbucket.org/fenics-project/dolfin/downloads/dolfin-2016.1.0.tar.gz", ), # Common Repositories - sourceforge downloads # name-ver.ver ("libpng", "1.6.27", "http://download.sourceforge.net/libpng/libpng-1.6.27.tar.gz"), ( "lcms2", "2.6", "http://downloads.sourceforge.net/project/lcms/lcms/2.6/lcms2-2.6.tar.gz", ), ("modules", "3.2.10", "http://prdownloads.sourceforge.net/modules/modules-3.2.10.tar.gz"), # name-ver.ver.ext/download ( "glew", "2.0.0", "https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download", ), # Common Repositories - cran downloads # name.name_ver.ver-ver.ver ("TH.data", "1.0-8", "https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz"), ("knitr", "1.14", "https://cran.rstudio.com/src/contrib/knitr_1.14.tar.gz"), ("devtools", "1.12.0", "https://cloud.r-project.org/src/contrib/devtools_1.12.0.tar.gz"), # Common Repositories - pypi downloads # name.name_name-ver.ver ("3to2", "1.1.1", "https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip"), ( "mpmath", "0.19", "https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz", ), ( "pandas", "0.16.0", "https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73", ), ( "sphinx_rtd_theme", "0.1.10a0", "https://pypi.python.org/packages/da/6b/1b75f13d8aa3333f19c6cdf1f0bc9f52ea739cae464fbee050307c121857/sphinx_rtd_theme-0.1.10a0.tar.gz", ), ( "backports.ssl_match_hostname", "3.5.0.1", "https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz", ), # Common Repositories - bazaar downloads ("libvterm", "681", "http://www.leonerd.org.uk/code/libvterm/libvterm-0+bzr681.tar.gz"), # Common Tarball Formats # 1st Pass: Simplest case # Assume name contains no digits and version contains no letters # name-ver.ver ("libpng", "1.6.37", "http://download.sourceforge.net/libpng/libpng-1.6.37.tar.gz"), # 2nd Pass: Version only # Assume version contains no letters # ver.ver ("eigen", "3.2.7", "https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2"), # ver.ver-ver ( "ImageMagick", "7.0.2-7", "https://github.com/ImageMagick/ImageMagick/archive/7.0.2-7.tar.gz", ), # vver.ver ("CGNS", "3.3.0", "https://github.com/CGNS/CGNS/archive/v3.3.0.tar.gz"), # vver_ver ( "luafilesystem", "1_6_3", "https://github.com/keplerproject/luafilesystem/archive/v1_6_3.tar.gz", ), # 3rd Pass: No separator characters are used # Assume name contains no digits # namever ("turbolinux", "702", "file://{0}/turbolinux702.tar.gz".format(os.getcwd())), ("nauty", "26r7", "http://pallini.di.uniroma1.it/nauty26r7.tar.gz"), # 4th Pass: A single separator character is used # Assume name contains no digits # name-name-ver-ver ( "Trilinos", "12-10-1", "https://github.com/trilinos/Trilinos/archive/trilinos-release-12-10-1.tar.gz", ), ( "panda", "2016-03-07", "http://comopt.ifi.uni-heidelberg.de/software/PANDA/downloads/panda-2016-03-07.tar", ), ("gts", "121130", "http://gts.sourceforge.net/tarballs/gts-snapshot-121130.tar.gz"), ("cdd", "061a", "http://www.cs.mcgill.ca/~fukuda/download/cdd/cdd-061a.tar.gz"), # name_name_ver_ver ( "tinyxml", "2_6_2", "https://sourceforge.net/projects/tinyxml/files/tinyxml/2.6.2/tinyxml_2_6_2.tar.gz", ), ( "boost", "1_55_0", "http://downloads.sourceforge.net/project/boost/boost/1.55.0/boost_1_55_0.tar.bz2", ), ("yorick", "2_2_04", "https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz"), ( "tbb", "44_20160413", "https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160413oss_src.tgz", ), # name.name.ver.ver ("prank", "150803", "http://wasabiapp.org/download/prank/prank.source.150803.tgz"), ("jpeg", "9b", "http://www.ijg.org/files/jpegsrc.v9b.tar.gz"), ("openjpeg", "2.1", "https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz"), # name.namever.ver ( "atlas", "3.11.34", "http://sourceforge.net/projects/math-atlas/files/Developer%20%28unstable%29/3.11.34/atlas3.11.34.tar.bz2", ), ( "visit", "2.10.1", "http://portal.nersc.gov/project/visit/releases/2.10.1/visit2.10.1.tar.gz", ), ("geant", "4.10.01.p03", "http://geant4.cern.ch/support/source/geant4.10.01.p03.tar.gz"), ("tcl", "8.6.5", "http://prdownloads.sourceforge.net/tcl/tcl8.6.5-src.tar.gz"), # 5th Pass: Two separator characters are used # Name may contain digits, version may contain letters # name-name-ver.ver ("m4", "1.4.17", "https://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz"), ("gmp", "6.0.0a", "https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2"), ( "LaunchMON", "1.0.2", "https://github.com/LLNL/LaunchMON/releases/download/v1.0.2/launchmon-v1.0.2.tar.gz", ), # name-ver-ver.ver ("libedit", "20150325-3.1", "http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz"), # name-name-ver_ver ("icu4c", "57_1", "http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz"), # name_name_ver.ver ( "superlu_dist", "4.1", "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz", ), ("pexsi", "0.9.0", "https://math.berkeley.edu/~linlin/pexsi/download/pexsi_v0.9.0.tar.gz"), # name_name.ver.ver ("fer", "696", "ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.v696.tar.gz"), # name_name_ver-ver ( "Bridger", "2014-12-01", "https://downloads.sourceforge.net/project/rnaseqassembly/Bridger_r2014-12-01.tar.gz", ), # name-name-ver.ver-ver.ver ( "sowing", "1.1.23-p1", "http://ftp.mcs.anl.gov/pub/petsc/externalpackages/sowing-1.1.23-p1.tar.gz", ), ( "bib2xhtml", "3.0-15-gf506", "http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v3.0-15-gf506.tar.gz", ), # namever.ver-ver.ver ( "go", "1.4-bootstrap-20161024", "https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz", ), # 6th Pass: All three separator characters are used # Name may contain digits, version may contain letters # name_name-ver.ver ( "the_silver_searcher", "0.32.0", "http://geoff.greer.fm/ag/releases/the_silver_searcher-0.32.0.tar.gz", ), ( "sphinx_rtd_theme", "0.1.10a0", "https://pypi.python.org/packages/source/s/sphinx_rtd_theme/sphinx_rtd_theme-0.1.10a0.tar.gz", ), # name.name_ver.ver-ver.ver ("TH.data", "1.0-8", "https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz"), ("XML", "3.98-1.4", "https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz"), # name-name-ver.ver_ver.ver ( "pypar", "2.1.5_108", "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/pypar/pypar-2.1.5_108.tgz", ), # name-namever.ver_ver.ver ( "STAR-CCM+", "11.06.010_02", "file://{0}/STAR-CCM+11.06.010_02_linux-x86_64.tar.gz".format(os.getcwd()), ), # name-name_name-ver.ver ( "PerlIO-utf8_strict", "0.002", "http://search.cpan.org/CPAN/authors/id/L/LE/LEONT/PerlIO-utf8_strict-0.002.tar.gz", ), # Various extensions # .tar.gz ( "libXcursor", "1.1.14", "https://www.x.org/archive/individual/lib/libXcursor-1.1.14.tar.gz", ), # .tar.bz2 ("mpfr", "4.0.1", "https://ftpmirror.gnu.org/mpfr/mpfr-4.0.1.tar.bz2"), # .tar.xz ("pkgconf", "1.5.4", "http://distfiles.dereferenced.org/pkgconf/pkgconf-1.5.4.tar.xz"), # .tar.Z ( "Gblocks", "0.91b", "http://molevol.cmima.csic.es/castresana/Gblocks/Gblocks_Linux64_0.91b.tar.Z", ), # .tar.zip ( "bcl2fastq2", "2.19.1.403", "ftp://webdata2:webdata2@ussd-ftp.illumina.com/downloads/software/bcl2fastq/bcl2fastq2-v2.19.1.403-tar.zip", ), # .tar, .TAR ( "python-meep", "1.4.2", "https://launchpad.net/python-meep/1.4/1.4/+download/python-meep-1.4.2.tar", ), ( "python-meep", "1.4.2", "https://launchpad.net/python-meep/1.4/1.4/+download/python-meep-1.4.2.TAR", ), # .gz ("libXcursor", "1.1.14", "https://www.x.org/archive/individual/lib/libXcursor-1.1.14.gz"), # .bz2 ("mpfr", "4.0.1", "https://ftpmirror.gnu.org/mpfr/mpfr-4.0.1.bz2"), # .xz ("pkgconf", "1.5.4", "http://distfiles.dereferenced.org/pkgconf/pkgconf-1.5.4.xz"), # .Z ( "Gblocks", "0.91b", "http://molevol.cmima.csic.es/castresana/Gblocks/Gblocks_Linux64_0.91b.Z", ), # .zip ("bliss", "0.73", "http://www.tcs.hut.fi/Software/bliss/bliss-0.73.zip"), # .tgz ("ADOL-C", "2.6.1", "http://www.coin-or.org/download/source/ADOL-C/ADOL-C-2.6.1.tgz"), # .tbz ("mpfr", "4.0.1", "https://ftpmirror.gnu.org/mpfr/mpfr-4.0.1.tbz"), # .tbz2 ("mpfr", "4.0.1", "https://ftpmirror.gnu.org/mpfr/mpfr-4.0.1.tbz2"), # .txz ("kim-api", "2.1.0", "https://s3.openkim.org/kim-api/kim-api-2.1.0.txz"), # 8th Pass: Query strings # suffix queries ( "swiftsim", "0.3.0", "http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0", ), ( "swiftsim", "0.3.0", "https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0", ), ("sionlib", "1.7.1", "http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1"), ("jube2", "2.2.2", "https://apps.fz-juelich.de/jsc/jube/jube2/download.php?version=2.2.2"), ( "archive", "1.0.0", "https://code.ornl.gov/eck/papyrus/repository/archive.tar.bz2?ref=v1.0.0", ), ( "VecGeom", "0.3.rc", "https://gitlab.cern.ch/api/v4/projects/VecGeom%2FVecGeom/repository/archive.tar.gz?sha=v0.3.rc", ), ( "parsplice", "1.1", "https://gitlab.com/api/v4/projects/exaalt%2Fparsplice/repository/archive.tar.gz?sha=v1.1", ), ( "busco", "2.0.1", "https://gitlab.com/api/v4/projects/ezlab%2Fbusco/repository/archive.tar.gz?sha=2.0.1", ), ( "libaec", "1.0.2", "https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2", ), ( "icet", "2.1.1", "https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-2.1.1", ), ( "vtk-m", "1.3.0", "https://gitlab.kitware.com/api/v4/projects/vtk%2Fvtk-m/repository/archive.tar.gz?sha=v1.3.0", ), ( "GATK", "3.8-1-0-gf15c1c3ef", "https://software.broadinstitute.org/gatk/download/auth?package=GATK-archive&version=3.8-1-0-gf15c1c3ef", ), # stem queries ( "slepc", "3.6.2", "http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz", ), ( "otf", "1.12.5salmon", "http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz", ), ( "eospac", "6.4.0beta.1", "http://laws-green.lanl.gov/projects/data/eos/get_file.php?package=eospac&filename=eospac_v6.4.0beta.1_r20171213193219.tgz", ), ( "vampirtrace", "5.14.4", "http://wwwpub.zih.tu-dresden.de/~mlieber/dcount/dcount.php?package=vampirtrace&get=VampirTrace-5.14.4.tar.gz", ), ("EvtGen", "01.07.00", "https://evtgen.hepforge.org/downloads?f=EvtGen-01.07.00.tar.gz"), # (we don't actually look for these, they are picked up # during the preliminary stem parsing) ("octopus", "6.0", "http://octopus-code.org/down.php?file=6.0/octopus-6.0.tar.gz"), ( "cloog", "0.18.1", "http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz", ), ( "libxc", "2.2.2", "http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz", ), ( "cistem", "1.0.0-beta", "https://cistem.org/system/tdf/upload3/cistem-1.0.0-beta-source-code.tar.gz?file=1&type=cistem_details&id=37&force=0", ), ( "Magics", "4.1.0", "https://confluence.ecmwf.int/download/attachments/3473464/Magics-4.1.0-Source.tar.gz?api=v2", ), ( "grib_api", "1.17.0", "https://software.ecmwf.int/wiki/download/attachments/3473437/grib_api-1.17.0-Source.tar.gz?api=v2", ), ( "eccodes", "2.2.0", "https://software.ecmwf.int/wiki/download/attachments/45757960/eccodes-2.2.0-Source.tar.gz?api=v2", ), ( "SWFFT", "1.0", "https://xgitlab.cels.anl.gov/api/v4/projects/hacc%2FSWFFT/repository/archive.tar.gz?sha=v1.0", ), # 9th Pass: Version in path # github.com/repo/name/releases/download/name-vver/name ( "nextflow", "0.20.1", "https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow", ), # ver/name ( "ncbi", "2.2.26", "ftp://ftp.ncbi.nlm.nih.gov/blast/executables/legacy.NOTSUPPORTED/2.2.26/ncbi.tar.gz", ), # Other tests for corner cases # single character name ("R", "3.3.2", "https://cloud.r-project.org/src/base/R-3/R-3.3.2.tar.gz"), # name starts with digit ("3to2", "1.1.1", "https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip"), # plus in name ( "gtk+", "2.24.31", "http://ftp.gnome.org/pub/gnome/sources/gtk+/2.24/gtk+-2.24.31.tar.xz", ), ("voro++", "0.4.6", "http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz"), # Name comes before download.php ("sionlib", "1.7.1", "http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1"), # Ignore download.php ( "slepc", "3.6.2", "http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz", ), ( "ScientificPython", "2.8.1", "https://sourcesup.renater.fr/frs/download.php/file/4411/ScientificPython-2.8.1.tar.gz", ), # gloox beta style ("gloox", "1.0-beta7", "http://camaya.net/download/gloox-1.0-beta7.tar.bz2"), # sphinx beta style ("sphinx", "1.10-beta", "http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz"), # ruby version style ("ruby", "1.9.1-p243", "ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz"), # rc style ( "libvorbis", "1.2.2rc1", "http://downloads.xiph.org/releases/vorbis/libvorbis-1.2.2rc1.tar.bz2", ), # dash rc style ("js", "1.8.0-rc1", "http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz"), # apache version style ( "apache-cassandra", "1.2.0-rc2", "http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz", ), # xaw3d version ("Xaw3d", "1.5E", "ftp://ftp.visi.com/users/hawkeyd/X/Xaw3d-1.5E.tar.gz"), # fann version ( "fann", "2.1.0beta", "http://downloads.sourceforge.net/project/fann/fann/2.1.0beta/fann-2.1.0beta.zip", ), # imap version ("imap", "2007f", "ftp://ftp.cac.washington.edu/imap/imap-2007f.tar.gz"), # suite3270 version ( "suite3270", "3.3.12ga7", "http://sourceforge.net/projects/x3270/files/x3270/3.3.12ga7/suite3270-3.3.12ga7-src.tgz", ), # scalasca version ( "cube", "4.2.3", "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz", ), ( "cube", "4.3-TP1", "http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz", ), # github raw url ( "CLAMR", "2.0.7", "https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true", ), # luaposix version ( "luaposix", "33.4.0", "https://github.com/luaposix/luaposix/archive/release-v33.4.0.tar.gz", ), # nco version ("nco", "4.6.2-beta03", "https://github.com/nco/nco/archive/4.6.2-beta03.tar.gz"), ("nco", "4.6.3-alpha04", "https://github.com/nco/nco/archive/4.6.3-alpha04.tar.gz"), ], ) def test_url_parse_name_and_version(name, version, url): # Make sure correct name and version are extracted. parsed_name, parsed_version = parse_name_and_version(url) assert parsed_name == name assert parsed_version == Version(version) # Make sure Spack formulates the right URL when we try to # build one with a specific version. assert url == substitute_version(url, version) @pytest.mark.parametrize( "not_detectable_url", [ "http://www.netlib.org/blas/blast-forum/cblas.tgz", "http://www.netlib.org/voronoi/triangle.zip", ], ) def test_no_version(not_detectable_url): with pytest.raises(UndetectableVersionError): parse_name_and_version(not_detectable_url) ================================================ FILE: lib/spack/spack/test/url_substitution.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests Spack's ability to substitute a different version into a URL.""" import os import pytest import spack.url @pytest.mark.parametrize( "base_url,version,expected", [ # Ensures that substituting the same version results in the same URL ( "http://www.mr511.de/software/libelf-0.8.13.tar.gz", "0.8.13", "http://www.mr511.de/software/libelf-0.8.13.tar.gz", ), # Test a completely different version syntax ( "http://www.prevanders.net/libdwarf-20130729.tar.gz", "8.12", "http://www.prevanders.net/libdwarf-8.12.tar.gz", ), # Test a URL where the version appears twice # It should get substituted both times ( "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz", "2.1.3", "https://github.com/hpc/mpileaks/releases/download/v2.1.3/mpileaks-2.1.3.tar.gz", ), # Test now with a partial prefix earlier in the URL # This is hard to figure out so Spack only substitutes # the last instance of the version ( "https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2", "2.2.0", "https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.0.tar.bz2", ), ( "https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2", "2.2", "https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.tar.bz2", ), # No separator between the name and version of the package ( "file://{0}/turbolinux702.tar.gz".format(os.getcwd()), "703", "file://{0}/turbolinux703.tar.gz".format(os.getcwd()), ), ( "https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true", "2.0.7", "https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true", ), ( "https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true", "4.7", "https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v4.7.tgz?raw=true", ), # Package name contains regex characters ( "http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz", "1.2.3", "http://math.lbl.gov/voro++/download/dir/voro++-1.2.3.tar.gz", ), ], ) def test_url_substitution(base_url, version, expected): computed = spack.url.substitute_version(base_url, version) assert computed == expected ================================================ FILE: lib/spack/spack/test/util/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/test/util/archive.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import gzip import hashlib import os import shutil import tarfile from pathlib import Path, PurePath import pytest import spack.util.crypto import spack.version from spack.llnl.util.filesystem import working_dir from spack.util.archive import ( gzip_compressed_tarfile, reproducible_tarfile_from_prefix, retrieve_commit_from_archive, ) def test_gzip_compressed_tarball_is_reproducible(tmp_path: Path, monkeypatch): """Test gzip_compressed_tarfile and reproducible_tarfile_from_prefix for reproducibility""" with working_dir(str(tmp_path)): # Create a few directories root = Path("root") dir_a = root / "a" dir_b = root / "b" root.mkdir(mode=0o777) dir_a.mkdir(mode=0o777) dir_b.mkdir(mode=0o777) (root / "y").touch() (root / "x").touch() (dir_a / "executable").touch(mode=0o777) (dir_a / "data").touch(mode=0o666) (dir_a / "symlink_file").symlink_to("data") (dir_a / "symlink_dir").symlink_to(PurePath("..", "b")) try: os.link(dir_a / "executable", dir_a / "hardlink") hardlink_support = True except OSError: hardlink_support = False (dir_b / "executable").touch(mode=0o777) (dir_b / "data").touch(mode=0o666) (dir_b / "symlink_file").symlink_to("data") (dir_b / "symlink_dir").symlink_to(PurePath("..", "a")) # Create the first tarball with gzip_compressed_tarfile("fst.tar.gz") as (tar, gzip_checksum_1, tarfile_checksum_1): reproducible_tarfile_from_prefix(tar, "root") # Expected mode for non-dirs is 644 if not executable, 755 if executable. Better to compute # that as we don't know the umask of the user running the test. expected_mode = lambda name: ( 0o755 if Path(*name.split("/")).lstat().st_mode & 0o100 else 0o644 ) # Verify the tarball contents with tarfile.open("fst.tar.gz", "r:gz") as tar: # Directories (mode is always 755) for dir in ("root", "root/a", "root/b"): m = tar.getmember(dir) assert m.isdir() assert m.mode == 0o755 assert m.uid == m.gid == 0 assert m.uname == m.gname == "" # Non-executable regular files for file in ( "root/x", "root/y", "root/a/data", "root/b/data", "root/a/executable", "root/b/executable", ): m = tar.getmember(file) assert m.isreg() assert m.mode == expected_mode(file) assert m.uid == m.gid == 0 assert m.uname == m.gname == "" # Symlinks for file in ( "root/a/symlink_file", "root/a/symlink_dir", "root/b/symlink_file", "root/b/symlink_dir", ): m = tar.getmember(file) assert m.issym() assert m.mode == 0o755 assert m.uid == m.gid == m.mtime == 0 assert m.uname == m.gname == "" # Verify the symlink targets. Notice that symlink targets are copied verbatim. That # means the value is platform specific for relative symlinks within the current prefix, # as on Windows they'd be ..\a and ..\b instead of ../a and ../b. So, reproducilility # is only guaranteed per-platform currently. assert PurePath(tar.getmember("root/a/symlink_file").linkname) == PurePath("data") assert PurePath(tar.getmember("root/b/symlink_file").linkname) == PurePath("data") assert PurePath(tar.getmember("root/a/symlink_dir").linkname) == PurePath("..", "b") assert PurePath(tar.getmember("root/b/symlink_dir").linkname) == PurePath("..", "a") # Check hardlink if supported if hardlink_support: m = tar.getmember("root/a/hardlink") assert m.islnk() assert m.mode == expected_mode("root/a/hardlink") assert m.uid == m.gid == 0 assert m.uname == m.gname == "" # Hardlink targets are always in posix format, as they reference a file that exists # in the tarball. assert m.linkname == "root/a/executable" # Finally verify if entries are ordered by (is_dir, name) assert [t.name for t in tar.getmembers()] == [ "root", "root/x", "root/y", "root/a", "root/a/data", "root/a/executable", *(["root/a/hardlink"] if hardlink_support else []), "root/a/symlink_dir", "root/a/symlink_file", "root/b", "root/b/data", "root/b/executable", "root/b/symlink_dir", "root/b/symlink_file", ] # Delete the current root dir, extract the first tarball, create a second shutil.rmtree(root) with tarfile.open("fst.tar.gz", "r:gz") as tar: tar.extractall() # Create the second tarball with gzip_compressed_tarfile("snd.tar.gz") as (tar, gzip_checksum_2, tarfile_checksum_2): reproducible_tarfile_from_prefix(tar, "root") # Verify the .tar.gz checksums are identical and correct assert ( gzip_checksum_1.hexdigest() == gzip_checksum_2.hexdigest() == spack.util.crypto.checksum(hashlib.sha256, "fst.tar.gz") == spack.util.crypto.checksum(hashlib.sha256, "snd.tar.gz") ) # Verify the .tar checksums are identical and correct with gzip.open("fst.tar.gz", "rb") as f, gzip.open("snd.tar.gz", "rb") as g: assert ( tarfile_checksum_1.hexdigest() == tarfile_checksum_2.hexdigest() == spack.util.crypto.checksum_stream(hashlib.sha256, f) # type: ignore == spack.util.crypto.checksum_stream(hashlib.sha256, g) # type: ignore ) def test_reproducible_tarfile_from_prefix_path_to_name(tmp_path: Path): prefix = tmp_path / "example" prefix.mkdir() (prefix / "file1").write_bytes(b"file") (prefix / "file2").write_bytes(b"file") def map_prefix(path: str) -> str: """maps / to some/common/prefix/""" p = PurePath(path) assert p.parts[: len(prefix.parts)] == prefix.parts, f"{path} is not under {prefix}" return PurePath("some", "common", "prefix", *p.parts[len(prefix.parts) :]).as_posix() with tarfile.open(tmp_path / "example.tar", "w") as tar: reproducible_tarfile_from_prefix( tar, str(tmp_path / "example"), include_parent_directories=True, path_to_name=map_prefix, ) with tarfile.open(tmp_path / "example.tar", "r") as tar: assert [t.name for t in tar.getmembers() if t.isdir()] == [ "some", "some/common", "some/common/prefix", ] assert [t.name for t in tar.getmembers() if t.isfile()] == [ "some/common/prefix/file1", "some/common/prefix/file2", ] @pytest.mark.parametrize("ref", ("test-branch", "test-tag", "annotated-tag")) def test_get_commits_from_archive(mock_git_repository, tmp_path: Path, ref): git_exe = mock_git_repository.git_exe with working_dir(str(tmp_path)): archive_file = str(tmp_path / "archive.tar.gz") path_to_name = lambda path: PurePath(path).relative_to(mock_git_repository.path).as_posix() # round trip the git repo, the desired ref will always be checked out git_exe("-C", mock_git_repository.path, "checkout", ref) with gzip_compressed_tarfile(archive_file) as (tar, _, _): reproducible_tarfile_from_prefix( tar=tar, prefix=mock_git_repository.path, path_to_name=path_to_name ) git_exe( "-C", mock_git_repository.path, "checkout", mock_git_repository.checks["default"].revision, ) commit = retrieve_commit_from_archive(archive_file, ref) assert commit assert spack.version.is_git_commit_sha(commit) def test_can_tell_if_archive_has_git(mock_git_repository, tmp_path: Path): with working_dir(str(tmp_path)): archive_file = str(tmp_path / "archive.tar.gz") path_to_name = lambda path: PurePath(path).relative_to(mock_git_repository.path).as_posix() exclude = lambda entry: ".git" in PurePath(entry.path).parts with gzip_compressed_tarfile(archive_file) as (tar, _, _): reproducible_tarfile_from_prefix( tar=tar, prefix=mock_git_repository.path, path_to_name=path_to_name, skip=exclude ) with pytest.raises(AssertionError) as err: retrieve_commit_from_archive(archive_file, "main") assert "does not contain git data" in str(err.value) ================================================ FILE: lib/spack/spack/test/util/compression.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import os import pathlib import shutil import tarfile from itertools import product import pytest import spack.llnl.url from spack.llnl.util.filesystem import working_dir from spack.paths import spack_root from spack.util import compression from spack.util.executable import CommandNotFoundError datadir = os.path.join(spack_root, "lib", "spack", "spack", "test", "data", "compression") ext_archive = { ext: f"Foo.{ext}" for ext in spack.llnl.url.ALLOWED_ARCHIVE_TYPES if "TAR" not in ext } # Spack does not use Python native handling for tarballs or zip # Don't test tarballs or zip in native test native_archive_list = [ key for key in ext_archive.keys() if "tar" not in key and "zip" not in key and "whl" not in key ] @pytest.fixture def compr_support_check(monkeypatch): monkeypatch.setattr(compression, "LZMA_SUPPORTED", False) monkeypatch.setattr(compression, "GZIP_SUPPORTED", False) monkeypatch.setattr(compression, "BZ2_SUPPORTED", False) @pytest.fixture def archive_file_and_extension(tmp_path_factory: pytest.TempPathFactory, request): """Copy example archive to temp directory into an extension-less file for test""" archive_file_stub = os.path.join(datadir, "Foo") extension, add_extension = request.param tmpdir = tmp_path_factory.mktemp("compression") tmp_archive_file = os.path.join( str(tmpdir), "Foo" + (("." + extension) if add_extension else "") ) shutil.copy(archive_file_stub + "." + extension, tmp_archive_file) return (tmp_archive_file, extension) @pytest.mark.parametrize( "archive_file_and_extension", product(native_archive_list, [True, False]), indirect=True ) def test_native_unpacking(tmp_path_factory: pytest.TempPathFactory, archive_file_and_extension): archive_file, extension = archive_file_and_extension util = compression.decompressor_for(archive_file, extension) tmpdir = tmp_path_factory.mktemp("comp_test") with working_dir(str(tmpdir)): assert not os.listdir(os.getcwd()) util(archive_file) files = os.listdir(os.getcwd()) assert len(files) == 1 with open(files[0], "r", encoding="utf-8") as f: contents = f.read() assert "TEST" in contents @pytest.mark.not_on_windows("Only Python unpacking available on Windows") @pytest.mark.parametrize( "archive_file_and_extension", [(ext, True) for ext in ext_archive.keys() if "whl" not in ext], indirect=True, ) def test_system_unpacking( tmp_path_factory: pytest.TempPathFactory, archive_file_and_extension, compr_support_check ): # actually run test archive_file, _ = archive_file_and_extension util = compression.decompressor_for(archive_file) tmpdir = tmp_path_factory.mktemp("system_comp_test") with working_dir(str(tmpdir)): assert not os.listdir(os.getcwd()) util(archive_file) files = os.listdir(os.getcwd()) assert len(files) == 1 with open(files[0], "r", encoding="utf-8") as f: contents = f.read() assert "TEST" in contents def test_unallowed_extension(): # use a cxx file as python files included for the test # are picked up by the linter and break style checks bad_ext_archive = "Foo.cxx" with pytest.raises(CommandNotFoundError): compression.decompressor_for(bad_ext_archive) @pytest.mark.parametrize("ext", ["gz", "bz2", "xz"]) def test_file_type_check_does_not_advance_stream(tmp_path: pathlib.Path, ext): # Create a tarball compressed with the given format path = str(tmp_path / "compressed_tarball") try: if ext == "gz": tar = tarfile.open(path, "w:gz") elif ext == "bz2": tar = tarfile.open(path, "w:bz2") elif ext == "xz": tar = tarfile.open(path, "w:xz") else: assert False, f"Unsupported extension: {ext}" except tarfile.CompressionError: pytest.skip(f"Cannot create tar.{ext} files") with tar: tar.addfile(tarfile.TarInfo("test.txt"), fileobj=io.BytesIO(b"test")) # Classify the file from its magic bytes, and check that the stream is not advanced with open(path, "rb") as f: computed_ext = compression.extension_from_magic_numbers_by_stream(f, decompress=False) assert computed_ext == ext assert f.tell() == 0 computed_ext = compression.extension_from_magic_numbers_by_stream(f, decompress=True) assert computed_ext == f"tar.{ext}" assert f.tell() == 0 ================================================ FILE: lib/spack/spack/test/util/editor.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import sys import pytest import spack.util.editor as ed from spack.llnl.util.filesystem import set_executable pytestmark = [ pytest.mark.usefixtures("working_env"), pytest.mark.not_on_windows("editor not implemented on windows"), ] # env vars that control the editor EDITOR_VARS = ["SPACK_EDITOR", "VISUAL", "EDITOR"] @pytest.fixture(scope="module", autouse=True) def clean_env_vars(): """Unset all editor env vars before tests.""" for var in EDITOR_VARS: if var in os.environ: del os.environ[var] @pytest.fixture(autouse=True) def working_editor_test_env(working_env): """Don't leak environment variables between functions here.""" # parameterized fixture for editor var names @pytest.fixture(params=EDITOR_VARS) def editor_var(request): return request.param def _make_exe(tmp_path_factory: pytest.TempPathFactory, name, contents=None): if sys.platform == "win32": name += ".exe" exe_dir = tmp_path_factory.mktemp(f"{name}_exe") path = exe_dir / name if contents is not None: path.write_text(f"#!/bin/sh\n{contents}\n", encoding="utf-8") set_executable(str(path)) return str(path) @pytest.fixture(scope="session") def good_exe(tmp_path_factory: pytest.TempPathFactory): return _make_exe(tmp_path_factory, "good", "exit 0") @pytest.fixture(scope="session") def bad_exe(tmp_path_factory: pytest.TempPathFactory): return _make_exe(tmp_path_factory, "bad", "exit 1") @pytest.fixture(scope="session") def nosuch_exe(tmp_path_factory: pytest.TempPathFactory): return _make_exe(tmp_path_factory, "nosuch") @pytest.fixture(scope="session") def vim_exe(tmp_path_factory: pytest.TempPathFactory): return _make_exe(tmp_path_factory, "vim", "exit 0") @pytest.fixture(scope="session") def gvim_exe(tmp_path_factory: pytest.TempPathFactory): return _make_exe(tmp_path_factory, "gvim", "exit 0") def test_find_exe_from_env_var(good_exe): os.environ["EDITOR"] = good_exe assert ed._find_exe_from_env_var("EDITOR") == (good_exe, [good_exe]) def test_find_exe_from_env_var_with_args(good_exe): os.environ["EDITOR"] = good_exe + " a b c" assert ed._find_exe_from_env_var("EDITOR") == (good_exe, [good_exe, "a", "b", "c"]) def test_find_exe_from_env_var_bad_path(nosuch_exe): os.environ["EDITOR"] = nosuch_exe assert ed._find_exe_from_env_var("FOO") == (None, []) def test_editor_gvim_special_case(gvim_exe): os.environ["EDITOR"] = gvim_exe def assert_exec(exe, args): assert exe == gvim_exe assert args == [gvim_exe, "-f", "/path/to/file"] return 0 assert ed.editor("/path/to/file", exec_fn=assert_exec) os.environ["EDITOR"] = gvim_exe + " -f" assert ed.editor("/path/to/file", exec_fn=assert_exec) def test_editor_precedence(good_exe, gvim_exe, vim_exe, bad_exe): """Ensure we prefer editor variables in order of precedence.""" os.environ["SPACK_EDITOR"] = good_exe os.environ["VISUAL"] = gvim_exe os.environ["EDITOR"] = vim_exe correct_exe = good_exe def assert_callback(exe, args): result = ed.executable(exe, args) if result == 0: assert exe == correct_exe return result ed.editor(exec_fn=assert_callback) os.environ["SPACK_EDITOR"] = bad_exe correct_exe = gvim_exe ed.editor(exec_fn=assert_callback) os.environ["VISUAL"] = bad_exe correct_exe = vim_exe ed.editor(exec_fn=assert_callback) def test_find_exe_from_env_var_no_editor(): if "FOO" in os.environ: os.environ.unset("FOO") assert ed._find_exe_from_env_var("FOO") == (None, []) def test_editor(editor_var, good_exe): os.environ[editor_var] = good_exe def assert_exec(exe, args): assert exe == good_exe assert args == [good_exe, "/path/to/file"] return 0 ed.editor("/path/to/file", exec_fn=assert_exec) def test_editor_visual_bad(good_exe, bad_exe): os.environ["VISUAL"] = bad_exe os.environ["EDITOR"] = good_exe def assert_exec(exe, args): if exe == bad_exe: raise OSError() assert exe == good_exe assert args == [good_exe, "/path/to/file"] return 0 ed.editor("/path/to/file", exec_fn=assert_exec) def test_editor_no_visual(good_exe): os.environ["EDITOR"] = good_exe def assert_exec(exe, args): assert exe == good_exe assert args == [good_exe, "/path/to/file"] return 0 ed.editor("/path/to/file", exec_fn=assert_exec) def test_editor_no_visual_with_args(good_exe): # editor has extra args in the var (e.g., emacs -nw) os.environ["EDITOR"] = good_exe + " -nw --foo" def assert_exec(exe, args): assert exe == good_exe assert args == [good_exe, "-nw", "--foo", "/path/to/file"] return 0 ed.editor("/path/to/file", exec_fn=assert_exec) def test_editor_both_bad(nosuch_exe, vim_exe): os.environ["VISUAL"] = nosuch_exe os.environ["EDITOR"] = nosuch_exe os.environ["PATH"] = "%s%s%s" % (os.path.dirname(vim_exe), os.pathsep, os.environ["PATH"]) def assert_exec(exe, args): assert exe == vim_exe assert args == [vim_exe, "/path/to/file"] return 0 ed.editor("/path/to/file", exec_fn=assert_exec) def test_no_editor(): os.environ["PATH"] = "" def assert_exec(exe, args): assert False with pytest.raises(OSError, match=r"No text editor found.*"): ed.editor("/path/to/file", exec_fn=assert_exec) def assert_exec(exe, args): return False with pytest.raises(OSError, match=r"No text editor found.*"): ed.editor("/path/to/file", exec_fn=assert_exec) def test_exec_fn_executable(editor_var, good_exe, bad_exe): """Make sure editor() works with ``ed.executable`` as well as execv""" os.environ[editor_var] = good_exe assert ed.editor(exec_fn=ed.executable) os.environ[editor_var] = bad_exe with pytest.raises(OSError, match=r"No text editor found.*"): ed.editor(exec_fn=ed.executable) ================================================ FILE: lib/spack/spack/test/util/elf.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import pathlib import pytest import spack.llnl.util.filesystem as fs import spack.platforms import spack.util.elf as elf import spack.util.executable from spack.hooks.drop_redundant_rpaths import drop_redundant_rpaths # note that our elf parser is platform independent... but I guess creating an elf file # is slightly more difficult with system tools on non-linux. def skip_unless_linux(f): return pytest.mark.skipif( str(spack.platforms.real_host()) != "linux", reason="implementation currently requires linux", )(f) @pytest.mark.requires_executables("gcc") @skip_unless_linux @pytest.mark.parametrize( "linker_flag,is_runpath", [("-Wl,--disable-new-dtags", False), ("-Wl,--enable-new-dtags", True)], ) def test_elf_parsing_shared_linking(linker_flag, is_runpath, tmp_path: pathlib.Path): gcc = spack.util.executable.which("gcc", required=True) with fs.working_dir(str(tmp_path)): # Create a library to link to so we can force a dynamic section in an ELF file with open("foo.c", "w", encoding="utf-8") as f: f.write("int foo(){return 0;}") with open("bar.c", "w", encoding="utf-8") as f: f.write("int foo(); int _start(){return foo();}") # Create library and executable linking to it. gcc("-shared", "-o", "libfoo.so", "-Wl,-soname,libfoo.so.1", "-nostdlib", "foo.c") gcc( "-o", "bar", linker_flag, "-Wl,-rpath,/first", "-Wl,-rpath,/second", "-Wl,--no-as-needed", "-nostdlib", "libfoo.so", "bar.c", "-o", "bar", ) with open("libfoo.so", "rb") as f: foo_parsed = elf.parse_elf(f, interpreter=True, dynamic_section=True) assert not foo_parsed.has_pt_interp assert foo_parsed.has_pt_dynamic assert not foo_parsed.has_rpath assert not foo_parsed.has_needed assert foo_parsed.has_soname assert foo_parsed.dt_soname_str == b"libfoo.so.1" with open("bar", "rb") as f: bar_parsed = elf.parse_elf(f, interpreter=True, dynamic_section=True) assert bar_parsed.has_pt_interp assert bar_parsed.has_pt_dynamic assert bar_parsed.has_rpath assert bar_parsed.has_needed assert not bar_parsed.has_soname assert bar_parsed.dt_rpath_str == b"/first:/second" assert bar_parsed.dt_needed_strs == [b"libfoo.so.1"] def test_broken_elf(): # No elf magic with pytest.raises(elf.ElfParsingError, match="Not an ELF file"): elf.parse_elf(io.BytesIO(b"x")) # Incomplete ELF header with pytest.raises(elf.ElfParsingError, match="Not an ELF file"): elf.parse_elf(io.BytesIO(b"\x7fELF")) # Invalid class with pytest.raises(elf.ElfParsingError, match="Invalid class"): elf.parse_elf(io.BytesIO(b"\x7fELF\x09\x01" + b"\x00" * 10)) # Invalid data type with pytest.raises(elf.ElfParsingError, match="Invalid data type"): elf.parse_elf(io.BytesIO(b"\x7fELF\x01\x09" + b"\x00" * 10)) # 64-bit needs at least 64 bytes of header; this is only 56 bytes with pytest.raises(elf.ElfParsingError, match="ELF header malformed"): elf.parse_elf(io.BytesIO(b"\x7fELF\x02\x01" + b"\x00" * 50)) # 32-bit needs at least 52 bytes of header; this is only 46 bytes with pytest.raises(elf.ElfParsingError, match="ELF header malformed"): elf.parse_elf(io.BytesIO(b"\x7fELF\x01\x01" + b"\x00" * 40)) # Not a ET_DYN/ET_EXEC on a 32-bit LE ELF with pytest.raises(elf.ElfParsingError, match="Not an ET_DYN or ET_EXEC"): elf.parse_elf(io.BytesIO(b"\x7fELF\x01\x01" + (b"\x00" * 10) + b"\x09" + (b"\x00" * 35))) def test_parser_doesnt_deal_with_nonzero_offset(): # Currently we don't have logic to parse ELF files at nonzero offsets in a file # This could be useful when e.g. modifying an ELF file inside a tarball or so, # but currently we cannot. elf_at_offset_one = io.BytesIO(b"\x00\x7fELF\x01\x01" + b"\x00" * 10) elf_at_offset_one.read(1) with pytest.raises(elf.ElfParsingError, match="Cannot parse at a nonzero offset"): elf.parse_elf(elf_at_offset_one) def test_only_header(): # When passing only_header=True parsing a file that is literally just a header # without any sections/segments should not error. # 32 bit elf_32 = elf.parse_elf(io.BytesIO(b"\x7fELF\x01\x01" + b"\x00" * 46), only_header=True) assert not elf_32.is_64_bit assert elf_32.is_little_endian # 64 bit elf_64 = elf.parse_elf(io.BytesIO(b"\x7fELF\x02\x01" + b"\x00" * 58), only_header=True) assert elf_64.is_64_bit assert elf_64.is_little_endian @pytest.mark.requires_executables("gcc") @skip_unless_linux def test_elf_get_and_replace_rpaths_and_pt_interp(binary_with_rpaths): long_paths = ["/very/long/prefix-a/x", "/very/long/prefix-b/y"] executable = str( binary_with_rpaths(rpaths=long_paths, dynamic_linker="/very/long/prefix-b/lib/ld.so") ) # Before assert elf.get_rpaths(executable) == long_paths replacements = { b"/very/long/prefix-a": b"/short-a", b"/very/long/prefix-b": b"/short-b", b"/very/long": b"/dont", } # Replace once: should modify the file. assert elf.substitute_rpath_and_pt_interp_in_place_or_raise(executable, replacements) # Replace twice: nothing to be done. assert not elf.substitute_rpath_and_pt_interp_in_place_or_raise(executable, replacements) # Verify the rpaths were modified correctly assert elf.get_rpaths(executable) == ["/short-a/x", "/short-b/y"] assert elf.get_interpreter(executable) == "/short-b/lib/ld.so" # Going back to long rpaths should fail, since we've added trailing \0 # bytes, and replacement can't assume it can write back in repeated null # bytes -- it may correspond to zero-length strings for example. with pytest.raises(elf.ElfCStringUpdatesFailed) as info: elf.substitute_rpath_and_pt_interp_in_place_or_raise( executable, {b"/short-a": b"/very/long/prefix-a", b"/short-b": b"/very/long/prefix-b"} ) assert info.value.rpath is not None assert info.value.pt_interp is not None assert info.value.rpath.old_value == b"/short-a/x:/short-b/y" assert info.value.rpath.new_value == b"/very/long/prefix-a/x:/very/long/prefix-b/y" assert info.value.pt_interp.old_value == b"/short-b/lib/ld.so" assert info.value.pt_interp.new_value == b"/very/long/prefix-b/lib/ld.so" @pytest.mark.requires_executables("gcc") @skip_unless_linux def test_drop_redundant_rpath(tmp_path: pathlib.Path, binary_with_rpaths): """Test the post install hook that drops redundant rpath entries""" # Use existing and non-existing dirs in tmp_path non_existing_dirs = [str(tmp_path / "a"), str(tmp_path / "b")] existing_dirs = [str(tmp_path / "c"), str(tmp_path / "d")] all_dirs = non_existing_dirs + existing_dirs (tmp_path / "c").mkdir() (tmp_path / "d").mkdir() # Create a binary with rpaths to both existing and non-existing dirs binary = binary_with_rpaths(rpaths=all_dirs) # Verify that the binary has all the rpaths # sometimes compilers add extra rpaths, so we test for a subset all_rpaths = elf.get_rpaths(binary) assert all_rpaths and set(all_dirs).issubset(all_rpaths) # Test whether the right rpaths are dropped drop_redundant_rpaths(binary) new_rpaths = elf.get_rpaths(binary) assert new_rpaths and set(existing_dirs).issubset(new_rpaths) assert set(non_existing_dirs).isdisjoint(new_rpaths) def test_elf_invalid_e_shnum(tmp_path: pathlib.Path): # from llvm/test/Object/Inputs/invalid-e_shnum.elf path = tmp_path / "invalid-e_shnum.elf" with open(path, "wb") as file: file.write( b"\x7fELF\x02\x010000000000\x03\x00>\x0000000000000000000000" b"\x00\x00\x00\x00\x00\x00\x00\x000000000000@\x000000" ) with open(path, "rb") as file, pytest.raises(elf.ElfParsingError): elf.parse_elf(file) ================================================ FILE: lib/spack/spack/test/util/environment.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test Spack's environment utility functions.""" import os import pathlib import sys import pytest import spack.util.environment as envutil @pytest.fixture() def prepare_environment_for_tests(): if "TEST_ENV_VAR" in os.environ: del os.environ["TEST_ENV_VAR"] yield del os.environ["TEST_ENV_VAR"] def test_is_system_path(): sys_path = "C:\\Users" if sys.platform == "win32" else "/usr/bin" assert envutil.is_system_path(sys_path) assert not envutil.is_system_path("/nonsense_path/bin") assert not envutil.is_system_path("") assert not envutil.is_system_path(None) if sys.platform == "win32": test_paths = [ "C:\\Users", "C:\\", "C:\\ProgramData", "C:\\nonsense_path", "C:\\Program Files", "C:\\nonsense_path\\extra\\bin", ] else: test_paths = [ "/usr/bin", "/nonsense_path/lib", "/usr/local/lib", "/bin", "/nonsense_path/extra/bin", "/usr/lib64", ] def test_filter_system_paths(): nonsense_prefix = "C:\\nonsense_path" if sys.platform == "win32" else "/nonsense_path" expected = [p for p in test_paths if p.startswith(nonsense_prefix)] filtered = envutil.filter_system_paths(test_paths) assert expected == filtered def deprioritize_system_paths(): expected = [p for p in test_paths if p.startswith("/nonsense_path")] expected.extend([p for p in test_paths if not p.startswith("/nonsense_path")]) filtered = envutil.deprioritize_system_paths(test_paths) assert expected == filtered def test_prune_duplicate_paths(): test_paths = ["/a/b", "/a/c", "/a/b", "/a/a", "/a/c", "/a/a/.."] expected = ["/a/b", "/a/c", "/a/a", "/a/a/.."] assert expected == envutil.prune_duplicate_paths(test_paths) def test_get_path(prepare_environment_for_tests): os.environ["TEST_ENV_VAR"] = os.pathsep.join(["/a", "/b", "/c/d"]) expected = ["/a", "/b", "/c/d"] assert envutil.get_path("TEST_ENV_VAR") == expected def test_env_flag(prepare_environment_for_tests): assert not envutil.env_flag("TEST_NO_ENV_VAR") os.environ["TEST_ENV_VAR"] = "1" assert envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "TRUE" assert envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "True" assert envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "TRue" assert envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "true" assert envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "27" assert not envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "-2.3" assert not envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "0" assert not envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "False" assert not envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "false" assert not envutil.env_flag("TEST_ENV_VAR") os.environ["TEST_ENV_VAR"] = "garbage" assert not envutil.env_flag("TEST_ENV_VAR") def test_path_set(prepare_environment_for_tests): envutil.path_set("TEST_ENV_VAR", ["/a", "/a/b", "/a/a"]) assert os.environ["TEST_ENV_VAR"] == "/a" + os.pathsep + "/a/b" + os.pathsep + "/a/a" def test_path_put_first(prepare_environment_for_tests): envutil.path_set("TEST_ENV_VAR", test_paths) expected = ["/usr/bin", "/new_nonsense_path/a/b"] expected.extend([p for p in test_paths if p != "/usr/bin"]) envutil.path_put_first("TEST_ENV_VAR", expected) assert envutil.get_path("TEST_ENV_VAR") == expected @pytest.mark.parametrize("shell", ["pwsh", "bat"] if sys.platform == "win32" else ["bash"]) def test_dump_environment(prepare_environment_for_tests, shell_as, shell, tmp_path: pathlib.Path): test_paths = "/a:/b/x:/b/c" os.environ["TEST_ENV_VAR"] = test_paths dumpfile_path = str(tmp_path / "envdump.txt") envutil.dump_environment(dumpfile_path) with open(dumpfile_path, "r", encoding="utf-8") as dumpfile: if shell == "pwsh": assert "$Env:TEST_ENV_VAR={}\n".format(test_paths) in list(dumpfile) elif shell == "bat": assert 'set "TEST_ENV_VAR={}"\n'.format(test_paths) in list(dumpfile) else: assert "TEST_ENV_VAR={0}; export TEST_ENV_VAR\n".format(test_paths) in list(dumpfile) def test_reverse_environment_modifications(working_env): prepend_val = os.sep + os.path.join("new", "path", "prepended") append_val = os.sep + os.path.join("new", "path", "appended") start_env = { "PREPEND_PATH": prepend_val + os.pathsep + os.path.join("path", "to", "prepend", "to"), "APPEND_PATH": os.path.sep + os.path.join("path", "to", "append", "to" + os.pathsep + append_val), "UNSET": "var_to_unset", "APPEND_FLAGS": "flags to append to", } to_reverse = envutil.EnvironmentModifications() to_reverse.prepend_path("PREPEND_PATH", prepend_val) to_reverse.append_path("APPEND_PATH", append_val) to_reverse.set_path("SET_PATH", ["/one/set/path", "/two/set/path"]) to_reverse.set("SET", "a var") to_reverse.unset("UNSET") to_reverse.append_flags("APPEND_FLAGS", "more_flags") reversal = to_reverse.reversed() os.environ.clear() os.environ.update(start_env) to_reverse.apply_modifications() reversal.apply_modifications() start_env.pop("UNSET") assert os.environ == start_env def test_shell_modifications_are_properly_escaped(): """Test that variable values are properly escaped so that they can safely be eval'd.""" changes = envutil.EnvironmentModifications() changes.set("VAR", "$PATH") changes.append_path("VAR", "$ANOTHER_PATH") changes.set("RM_RF", "$(rm -rf /)") script = changes.shell_modifications(shell="sh") assert f"export VAR='$PATH{os.pathsep}$ANOTHER_PATH'" in script assert "export RM_RF='$(rm -rf /)'" in script ================================================ FILE: lib/spack/spack/test/util/executable.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import sys from typing import List import pytest import spack.llnl.util.filesystem as fs import spack.main import spack.util.executable as ex from spack.hooks.sbang import filter_shebangs_in_directory def test_read_unicode(tmp_path: pathlib.Path, working_env): with fs.working_dir(str(tmp_path)): script_name = "print_unicode.py" script_args: List[str] = [] # read the unicode back in and see whether things work if sys.platform == "win32": script = ex.Executable("%s" % (sys.executable)) script_args.append(script_name) else: script = ex.Executable("./%s" % script_name) os.environ["LD_LIBRARY_PATH"] = spack.main.spack_ld_library_path # make a script that prints some unicode with open(script_name, "w", encoding="utf-8") as f: f.write( """#!{0} print(u'\\xc3') """.format(sys.executable) ) # make it executable fs.set_executable(script_name) filter_shebangs_in_directory(".", [script_name]) assert "\xc3" == script(*script_args, output=str).strip() def test_which_relative_path_with_slash(tmp_path: pathlib.Path, working_env): (tmp_path / "exe").touch() path = str(tmp_path / "exe") os.environ["PATH"] = "" with fs.working_dir(str(tmp_path)): no_exe = ex.which(".{0}exe".format(os.path.sep)) assert no_exe is None if sys.platform == "win32": # These checks are for 'executable' files, Windows # determines this by file extension. path += ".exe" (tmp_path / "exe.exe").touch() else: fs.set_executable(path) exe = ex.which(".{0}exe".format(os.path.sep), required=True) assert exe.path == path def test_which_with_slash_ignores_path(tmp_path: pathlib.Path, working_env): (tmp_path / "exe").touch() (tmp_path / "bin").mkdir() (tmp_path / "bin" / "exe").touch() path = str(tmp_path / "exe") wrong_path = str(tmp_path / "bin" / "exe") os.environ["PATH"] = str(tmp_path / "bin") with fs.working_dir(str(tmp_path)): if sys.platform == "win32": # For Windows, need to create files with .exe after any assert is none tests (tmp_path / "exe.exe").touch() (tmp_path / "bin" / "exe.exe").touch() path = path + ".exe" wrong_path = wrong_path + ".exe" else: fs.set_executable(path) fs.set_executable(wrong_path) exe = ex.which(".{0}exe".format(os.path.sep), required=True) assert exe.path == path def test_which(tmp_path: pathlib.Path, monkeypatch): monkeypatch.setenv("PATH", str(tmp_path)) assert ex.which("spack-test-exe") is None with pytest.raises(ex.CommandNotFoundError): ex.which("spack-test-exe", required=True) path = str(tmp_path / "spack-test-exe") with fs.working_dir(str(tmp_path)): if sys.platform == "win32": # For Windows, need to create files with .exe after any assert is none tests (tmp_path / "spack-test-exe.exe").touch() path += ".exe" else: fs.touch("spack-test-exe") fs.set_executable("spack-test-exe") exe = ex.which("spack-test-exe") assert exe is not None assert exe.path == path @pytest.fixture def make_script_exe(tmp_path: pathlib.Path): if sys.platform == "win32": pytest.skip("Can't test #!/bin/sh scripts on Windows.") def make_script(name, contents): script = tmp_path / f"{name}.sh" with script.open("w", encoding="utf-8") as f: f.write("#!/bin/sh\n") f.write(contents) f.write("\n") fs.set_executable(str(script)) return ex.Executable(str(script)) return make_script def test_exe_fail(make_script_exe): fail = make_script_exe("fail", "exit 107") with pytest.raises(ex.ProcessError): fail() assert fail.returncode == 107 def test_exe_success(make_script_exe): succeed = make_script_exe("fail", "exit 0") succeed() assert succeed.returncode == 0 def test_exe_timeout(make_script_exe): timeout = make_script_exe("timeout", "sleep 100") with pytest.raises(ex.ProcessError): timeout(timeout=1) assert timeout.returncode == 1 def test_exe_not_exist(tmp_path: pathlib.Path): fail = ex.Executable(str(tmp_path / "foo")) # doesn't exist with pytest.raises(ex.ProcessError): fail() assert fail.returncode == 1 def test_construct_from_pathlib(mock_executable): """Tests that we can construct an executable from a pathlib.Path object""" expected = "Hello world!" path = mock_executable("hello", output=f"echo {expected}\n") hello = ex.Executable(path) assert expected in hello(output=str) def test_exe_disallows_str_split_as_input(mock_executable): path = mock_executable("hello", output="echo hi\n") hello = ex.Executable(path) with pytest.raises(ValueError): hello(input=str.split) def test_exe_disallows_callable_as_output(mock_executable): path = mock_executable("hello", output="echo hi\n") hello = ex.Executable(path) with pytest.raises(ValueError): hello(output=lambda line: line) ================================================ FILE: lib/spack/spack/test/util/file_cache.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test Spack's FileCache.""" import os import pathlib import pytest import spack.llnl.util.filesystem as fs from spack.util.file_cache import CacheError, FileCache @pytest.fixture() def file_cache(tmp_path: pathlib.Path): """Returns a properly initialized FileCache instance""" return FileCache(str(tmp_path)) def test_write_and_read_cache_file(file_cache): """Test writing then reading a cached file.""" with file_cache.write_transaction("test.yaml") as (old, new): assert old is None assert new is not None new.write("foobar\n") with file_cache.read_transaction("test.yaml") as stream: text = stream.read() assert text == "foobar\n" def test_read_before_init(file_cache): with file_cache.read_transaction("test.yaml") as stream: assert stream is None @pytest.mark.not_on_windows("Locks not supported on Windows") def test_failed_write_and_read_cache_file(file_cache): """Test failing to write then attempting to read a cached file.""" with pytest.raises(RuntimeError, match=r"^foobar$"): with file_cache.write_transaction("test.yaml") as (old, new): assert old is None assert new is not None raise RuntimeError("foobar") # Cache dir should have exactly one (lock) file assert os.listdir(file_cache.root) == [".lock"] # File does not exist assert not os.path.exists(file_cache.cache_path("test.yaml")) def test_write_and_remove_cache_file(file_cache): """Test two write transactions on a cached file. Then try to remove an entry from it. """ with file_cache.write_transaction("test.yaml") as (old, new): assert old is None assert new is not None new.write("foobar\n") with file_cache.write_transaction("test.yaml") as (old, new): assert old is not None text = old.read() assert text == "foobar\n" assert new is not None new.write("barbaz\n") with file_cache.read_transaction("test.yaml") as stream: text = stream.read() assert text == "barbaz\n" file_cache.remove("test.yaml") # After removal the file should not exist assert not os.path.exists(file_cache.cache_path("test.yaml")) # Whether the lock file exists is more of an implementation detail, on Linux they # continue to exist, on Windows they don't. # assert os.path.exists(file_cache._lock_path('test.yaml')) @pytest.mark.not_on_windows("Not supported on Windows (yet)") @pytest.mark.skipif(fs.getuid() == 0, reason="user is root") def test_bad_cache_permissions(file_cache, request): """Test that transactions raise CacheError on permission problems.""" relpath = fs.join_path("test-dir", "read-only-file.txt") cachefile = file_cache.cache_path(relpath) fs.touchp(cachefile) # A directory where a file is expected raises CacheError on read with pytest.raises(CacheError, match="not a file"): with file_cache.read_transaction(os.path.dirname(relpath)) as _: pass # A directory where a file is expected raises CacheError on write with pytest.raises(CacheError, match="not a file"): with file_cache.write_transaction(os.path.dirname(relpath)) as _: pass # A non-readable file raises CacheError on read os.chmod(cachefile, 0o200) request.addfinalizer(lambda c=cachefile: os.chmod(c, 0o600)) with pytest.raises(CacheError, match="Cannot access cache file"): with file_cache.read_transaction(relpath) as _: pass # A read-only parent directory raises CacheError on write relpath2 = fs.join_path("test-dir", "another-file.txxt") parent = str(file_cache.cache_path(relpath2).parent) os.chmod(parent, 0o400) request.addfinalizer(lambda p=parent: os.chmod(p, 0o700)) with pytest.raises(CacheError): with file_cache.write_transaction(relpath2) as _: pass @pytest.mark.regression("31475") def test_delete_is_idempotent(file_cache): """Deleting a non-existent key should be idempotent, to simplify life when running delete with multiple processes""" file_cache.remove("test.yaml") ================================================ FILE: lib/spack/spack/test/util/git.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import pathlib from typing import Optional import pytest import spack.util.executable as exe import spack.util.git from spack.llnl.util.filesystem import working_dir def test_git_not_found(monkeypatch): def _mock_find_git() -> Optional[str]: return None monkeypatch.setattr(spack.util.git, "_find_git", _mock_find_git) git = spack.util.git.git(required=False) assert git is None with pytest.raises(exe.CommandNotFoundError): spack.util.git.git(required=True) def test_modified_files(mock_git_package_changes): repo, filename, commits = mock_git_package_changes with working_dir(repo.packages_path): files = spack.util.git.get_modified_files(from_ref="HEAD~1", to_ref="HEAD") assert len(files) == 1 assert files[0] == filename def test_init_git_repo(git, mock_git_version_info, tmp_path: pathlib.Path): """Test that init_git_repo creates a new repo with remote but doesn't checkout.""" repo, _, _ = mock_git_version_info with working_dir(str(tmp_path / "test_git_init_repo"), create=True): spack.util.git.init_git_repo(repo) # Verify repo was initialized but no commits checked out yet assert "No commits yet" in git("status", output=str) def test_pull_checkout_commit_any_remote(git, tmp_path: pathlib.Path, mock_git_version_info): repo, _, commits = mock_git_version_info destination = str(tmp_path / "test_git_checkout_commit") with working_dir(destination, create=True): spack.util.git.init_git_repo(repo) spack.util.git.pull_checkout_commit(commits[0]) assert commits[0] in git("rev-parse", "HEAD", output=str) def test_pull_checkout_commit_specific_remote(git, tmp_path: pathlib.Path, mock_git_version_info): """Test fetching a specific commit from a specific remote.""" repo, _, commits = mock_git_version_info destination = str(tmp_path / "test_git_checkout_commit_from_remote") with working_dir(destination, create=True): spack.util.git.init_git_repo(repo) spack.util.git.pull_checkout_commit(commits[0], remote="origin", depth=1) assert commits[0] in git("rev-parse", "HEAD", output=str) def test_pull_checkout_tag(git, tmp_path: pathlib.Path, mock_git_version_info): repo, _, _ = mock_git_version_info destination = str(tmp_path / "test_git_checkout_tag") with working_dir(destination, create=True): spack.util.git.init_git_repo(repo) spack.util.git.pull_checkout_tag("v1.1") assert "v1.1" in git("describe", "--exact-match", "--tags", output=str) def test_pull_checkout_branch(git, tmp_path: pathlib.Path, mock_git_version_info): repo, _, _ = mock_git_version_info destination = str(tmp_path / "test_git_checkout_branch") with working_dir(destination, create=True): spack.util.git.init_git_repo(repo) spack.util.git.pull_checkout_branch("1.x") assert "1.x" in git("rev-parse", "--abbrev-ref", "HEAD", output=str) with open("file.txt", "w", encoding="utf-8") as f: f.write("hi harmen") with pytest.raises(exe.ProcessError): spack.util.git.pull_checkout_branch("main") @pytest.mark.parametrize( "input,answer", ( ["git version 1.7.1", (1, 7, 1)], ["git version 2.34.1.windows.2", (2, 34, 1)], ["git version 2.50.1 (Apple Git-155)", (2, 50, 1)], ["git version 1.2.3.4.150.abcd10", (1, 2, 3, 4, 150)], ), ) def test_extract_git_version(mock_util_executable, input, answer): _, _, registered_responses = mock_util_executable registered_responses["--version"] = input git = spack.util.git.GitExecutable() assert git.version == answer def test_mock_git_exe(mock_util_executable): log, should_fail, _ = mock_util_executable should_fail.append("clone") git = spack.util.git.GitExecutable() with pytest.raises(exe.ProcessError): git("clone") assert git.returncode == 1 git("status") assert git.returncode == 0 assert "clone" in "\n".join(log) assert "status" in "\n".join(log) @pytest.mark.parametrize("git_version", ("1.5.0", "1.3.0")) def test_git_exe_conditional_option(mock_util_executable, git_version): log, _, registered_responses = mock_util_executable min_version = (1, 4, 1) registered_responses["git --version"] = git_version git = spack.util.git.GitExecutable("git") mock_opt = spack.util.git.VersionConditionalOption("--maybe", min_version=min_version) args = mock_opt(git.version) if git.version >= min_version: assert "--maybe" in args else: assert not args @pytest.mark.parametrize( "git_version,ommitted_opts", (("2.18.0", ["--filter=blob:none"]), ("1.8.0", ["--filter=blob:none", "--depth"])), ) def test_git_init_fetch_ommissions(mock_util_executable, git_version, ommitted_opts): log, _, registered_responses = mock_util_executable registered_responses["git --version"] = git_version git = spack.util.git.GitExecutable("git") url = "https://foo.git" ref = "v1.2.3" spack.util.git.git_init_fetch(url, ref, git_exe=git) for opt in ommitted_opts: assert all(opt not in call for call in log) ================================================ FILE: lib/spack/spack/test/util/ld_so_conf.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import sys import pytest import spack.util.ld_so_conf as ld_so_conf @pytest.mark.skipif(sys.platform == "win32", reason="Unix path") def test_ld_so_conf_parsing(tmp_path: pathlib.Path): cwd = os.getcwd() (tmp_path / "subdir").mkdir() # Entrypoint config file with open(str(tmp_path / "main.conf"), "wb") as f: f.write(b" \n") f.write(b"include subdir/*.conf\n") f.write(b"include non-existent/file\n") f.write(b"include #nope\n") f.write(b"include \n") f.write(b"include\t\n") f.write(b"include\n") f.write(b"/main.conf/lib # and a comment\n") f.write(b"relative/path\n\n") f.write(b"#/skip/me\n") # Should be parsed: subdir/first.conf with open(str(tmp_path / "subdir" / "first.conf"), "wb") as f: f.write(b"/first.conf/lib") # Should be parsed: subdir/second.conf with open(str(tmp_path / "subdir" / "second.conf"), "wb") as f: f.write(b"/second.conf/lib") # Not matching subdir/*.conf with open(str(tmp_path / "subdir" / "third"), "wb") as f: f.write(b"/third/lib") paths = ld_so_conf.parse_ld_so_conf(str(tmp_path / "main.conf")) assert len(paths) == 3 assert "/main.conf/lib" in paths assert "/first.conf/lib" in paths assert "/second.conf/lib" in paths # Make sure globbing didn't change the working dir assert os.getcwd() == cwd def test_host_dynamic_linker_search_paths(): assert {"/usr/lib", "/usr/lib64", "/lib", "/lib64"}.issubset( ld_so_conf.host_dynamic_linker_search_paths() ) ================================================ FILE: lib/spack/spack/test/util/log_parser.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import pathlib from spack.llnl.util.tty.color import color_when from spack.util.ctest_log_parser import CTestLogParser from spack.util.log_parse import make_log_context def test_log_parser(tmp_path: pathlib.Path): log_file = tmp_path / "log.txt" with log_file.open("w") as f: f.write( """#!/bin/sh\n checking build system type... x86_64-apple-darwin16.6.0 checking host system type... x86_64-apple-darwin16.6.0 error: weird_error.c:145: something weird happened E checking for gcc... /Users/gamblin2/src/spack/lib/spack/env/clang/clang checking whether the C compiler works... yes /var/tmp/build/foo.py:60: warning: some weird warning W checking for C compiler default output file name... a.out ld: fatal: linker thing happened E checking for suffix of executables... configure: error: in /path/to/some/file: E configure: error: cannot run C compiled programs. E """ ) parser = CTestLogParser() errors, warnings = parser.parse(str(log_file)) assert len(errors) == 4 assert all(e.text.endswith("E") for e in errors) assert len(warnings) == 1 assert all(w.text.endswith("W") for w in warnings) def test_log_parser_stream(): """parse() accepts a file-like object.""" log = io.StringIO( "error: weird_error.c:145: something weird happened E\n" "checking for gcc... irrelevant line\n" "/var/tmp/build/foo.py:60: warning: some weird warning W\n" ) parser = CTestLogParser() errors, warnings = parser.parse(log) assert len(errors) == 1 assert errors[0].text.endswith("E") assert len(warnings) == 1 assert warnings[0].text.endswith("W") def test_log_parser_preserves_leading_whitespace(): """Leading whitespace (e.g. compiler caret underlines) must not be stripped.""" log = io.StringIO( "/path/to/file.c:10: error: use of undeclared identifier 'x'\n" " int y = x + 1;\n" " ^\n" ) parser = CTestLogParser() errors, _ = parser.parse(log, context=6) assert len(errors) == 1 assert errors[0].post_context[0] == " int y = x + 1;" assert errors[0].post_context[1] == " ^" def test_make_log_context_merges_overlapping_events(tmp_path: pathlib.Path): """Overlapping or adjacent context windows should produce a single merged block.""" # Two errors close together: lines 5 and 10 with context=3 means windows overlap. lines = [f"line {i}\n" for i in range(1, 21)] lines[4] = "error: first problem\n" # line 5 lines[9] = "error: second problem\n" # line 10 log_file = tmp_path / "log.txt" log_file.write_text("".join(lines)) parser = CTestLogParser() errors, warnings = parser.parse(str(log_file), context=3) log_events = sorted([*errors, *warnings], key=lambda e: e.line_no) output = make_log_context(log_events) # Should be exactly one header for the merged block, not two. assert output.count("-- lines") == 1 # The header should cover the full merged range. assert "-- lines 2 to 13 --" in output def test_make_log_context_warning_in_error_context_keeps_yellow(tmp_path: pathlib.Path): """A warning line inside an error's context window must be highlighted yellow, not red.""" # Line 5 = error, line 8 = warning, context=3 so error window covers lines 2-11 # meaning the warning at line 8 falls inside the error's context. lines = [f"line {i}\n" for i in range(1, 16)] lines[4] = "error: something broke\n" # line 5 lines[7] = "/tmp/foo.c:1: warning: something fishy\n" # line 8 log_file = tmp_path / "log.txt" log_file.write_text("".join(lines)) parser = CTestLogParser() errors, warnings = parser.parse(str(log_file), context=3) assert len(errors) == len(warnings) == 1 log_events = sorted([*errors, *warnings], key=lambda e: e.line_no) with color_when("always"): output = make_log_context(log_events) # The error line should be red (ANSI 91), the warning yellow (ANSI 93). assert "\x1b[0;91m> " in output and "something broke" in output assert "\x1b[0;93m> " in output and "something fishy" in output def test_log_parser_non_utf8_bytes(tmp_path: pathlib.Path): """parse() does not raise UnicodeDecodeError on non-UTF-8 log files.""" log_file = tmp_path / "log.bin" log_file.write_bytes(b"checking things...\nerror: \x80\xff something broke\ndone\n") parser = CTestLogParser() errors, _ = parser.parse(str(log_file)) assert len(errors) == 1 ================================================ FILE: lib/spack/spack/test/util/module_cmd.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pytest import spack.util.module_cmd @pytest.mark.not_on_windows("Module files are not supported on Windows") def test_load_module_success(monkeypatch, working_env): """Test that load_module properly handles successful module loads. This is a very lightweight test that only confirms that successful loads are not flagged as failed.""" # Mock the module function to simulate a successful module load def mock_module(*args, **kwargs): if args[0] == "show": return "" elif args[0] == "load": # Simulate successful module load by adding to LOADEDMODULES current_modules = os.environ.get("LOADEDMODULES", "") if current_modules: os.environ["LOADEDMODULES"] = f"{current_modules}:{args[1]}" else: os.environ["LOADEDMODULES"] = args[1] monkeypatch.setattr(spack.util.module_cmd, "module", mock_module) # This should succeed spack.util.module_cmd.load_module("test_module") spack.util.module_cmd.load_module("test_module_2") # Confirm LOADEDMODULES was modified assert "test_module:test_module_2" in os.environ["LOADEDMODULES"] @pytest.mark.not_on_windows("Module files are not supported on Windows") def test_load_module_failure(monkeypatch, working_env): """Test that load_module raises an exception when a module load fails.""" # Mock the module function to simulate a failed module load def mock_module(*args, **kwargs): if args[0] == "show": return "" elif args[0] == "load": # Simulate module load failure by not changing LOADEDMODULES pass monkeypatch.setattr(spack.util.module_cmd, "module", mock_module) # This should fail with ModuleLoadError with pytest.raises(spack.util.module_cmd.ModuleLoadError): spack.util.module_cmd.load_module("non_existent_module") ================================================ FILE: lib/spack/spack/test/util/package_hash.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import ast import os import pytest import spack.concretize import spack.directives_meta import spack.paths import spack.repo import spack.util.package_hash as ph from spack.spec import Spec from spack.util.unparse import unparse datadir = os.path.join(spack.paths.test_path, "data", "unparse") def compare_sans_name(eq, spec1, spec2): content1 = ph.canonical_source(spec1) content1 = content1.replace(spack.repo.PATH.get_pkg_class(spec1.name).__name__, "TestPackage") content2 = ph.canonical_source(spec2) content2 = content2.replace(spack.repo.PATH.get_pkg_class(spec2.name).__name__, "TestPackage") if eq: assert content1 == content2 else: assert content1 != content2 def compare_hash_sans_name(eq, spec1, spec2): content1 = ph.canonical_source(spec1) pkg_cls1 = spack.repo.PATH.get_pkg_class(spec1.name) content1 = content1.replace(pkg_cls1.__name__, "TestPackage") hash1 = pkg_cls1(spec1).content_hash(content=content1) content2 = ph.canonical_source(spec2) pkg_cls2 = spack.repo.PATH.get_pkg_class(spec2.name) content2 = content2.replace(pkg_cls2.__name__, "TestPackage") hash2 = pkg_cls2(spec2).content_hash(content=content2) assert (hash1 == hash2) == eq def test_hash(mock_packages, config): ph.package_hash(Spec("hash-test1@=1.2")) def test_different_variants(mock_packages, config): spec1 = Spec("hash-test1@=1.2 +variantx") spec2 = Spec("hash-test1@=1.2 +varianty") assert ph.package_hash(spec1) == ph.package_hash(spec2) def test_all_same_but_name(mock_packages, config): spec1 = Spec("hash-test1@=1.2") spec2 = Spec("hash-test2@=1.2") compare_sans_name(True, spec1, spec2) spec1 = Spec("hash-test1@=1.2 +varianty") spec2 = Spec("hash-test2@=1.2 +varianty") compare_sans_name(True, spec1, spec2) def test_all_same_but_archive_hash(mock_packages, config): """ Archive hash is not intended to be reflected in Package hash. """ spec1 = Spec("hash-test1@=1.3") spec2 = Spec("hash-test2@=1.3") compare_sans_name(True, spec1, spec2) def test_all_same_but_patch_contents(mock_packages, config): spec1 = Spec("hash-test1@=1.1") spec2 = Spec("hash-test2@=1.1") compare_sans_name(True, spec1, spec2) def test_all_same_but_patches_to_apply(mock_packages, config): spec1 = Spec("hash-test1@=1.4") spec2 = Spec("hash-test2@=1.4") compare_sans_name(True, spec1, spec2) def test_all_same_but_install(mock_packages, config): spec1 = Spec("hash-test1@=1.5") spec2 = Spec("hash-test2@=1.5") compare_sans_name(False, spec1, spec2) def test_content_hash_all_same_but_patch_contents(mock_packages, config): spec1 = spack.concretize.concretize_one("hash-test1@1.1") spec2 = spack.concretize.concretize_one("hash-test2@1.1") compare_hash_sans_name(False, spec1, spec2) def test_content_hash_not_concretized(mock_packages, config): """Check that Package.content_hash() works on abstract specs.""" # these are different due to the package hash spec1 = Spec("hash-test1@=1.1") spec2 = Spec("hash-test2@=1.3") compare_hash_sans_name(False, spec1, spec2) # at v1.1 these are actually the same package when @when's are removed # and the name isn't considered spec1 = Spec("hash-test1@=1.1") spec2 = Spec("hash-test2@=1.1") compare_hash_sans_name(True, spec1, spec2) # these end up being different b/c we can't eliminate much of the package.py # without a version. spec1 = Spec("hash-test1") spec2 = Spec("hash-test2") compare_hash_sans_name(False, spec1, spec2) def test_content_hash_different_variants(mock_packages, config): spec1 = spack.concretize.concretize_one("hash-test1@1.2 +variantx") spec2 = spack.concretize.concretize_one("hash-test2@1.2 ~variantx") compare_hash_sans_name(True, spec1, spec2) def test_content_hash_cannot_get_details_from_ast(mock_packages, config): """Packages hash-test1 and hash-test3 would be considered the same except that hash-test3 conditionally executes a phase based on a "when" directive that Spack cannot evaluate by examining the AST. This test ensures that Spack can compute a content hash for hash-test3. If Spack cannot determine when a phase applies, it adds it by default, so the test also ensures that the hashes differ where Spack includes a phase on account of AST-examination failure. """ spec3 = spack.concretize.concretize_one("hash-test1@1.7") spec4 = spack.concretize.concretize_one("hash-test3@1.7") compare_hash_sans_name(False, spec3, spec4) def test_content_hash_all_same_but_archive_hash(mock_packages, config): spec1 = spack.concretize.concretize_one("hash-test1@1.3") spec2 = spack.concretize.concretize_one("hash-test2@1.3") compare_hash_sans_name(False, spec1, spec2) def test_content_hash_parse_dynamic_function_call(mock_packages, config): spec = spack.concretize.concretize_one("hash-test4") spec.package.content_hash() many_strings = '''\ """ONE""" """TWO""" var = "THREE" # make sure this is not removed "FOUR" class ManyDocstrings: """FIVE""" """SIX""" x = "SEVEN" def method1(): """EIGHT""" print("NINE") "TEN" for i in range(10): print(i) def method2(): """ELEVEN""" return "TWELVE" def empty_func(): """THIRTEEN""" ''' many_strings_no_docstrings = """\ var = 'THREE' class ManyDocstrings: x = 'SEVEN' def method1(): print('NINE') for i in range(10): print(i) def method2(): return 'TWELVE' def empty_func(): pass """ def test_remove_docstrings(): tree = ast.parse(many_strings) tree = ph.RemoveDocstrings().visit(tree) unparsed = unparse(tree, py_ver_consistent=True) assert unparsed == many_strings_no_docstrings many_directives = """\ class HasManyDirectives: {directives} def foo(): # just a method to get in the way pass {directives} """.format( directives="\n".join(" %s()" % name for name in spack.directives_meta.directive_names) ) def test_remove_all_directives(): """Ensure all directives are removed from packages before hashing.""" for name in spack.directives_meta.directive_names: assert name in many_directives tree = ast.parse(many_directives) spec = Spec("has-many-directives") tree = ph.RemoveDirectives(spec).visit(tree) unparsed = unparse(tree, py_ver_consistent=True) for name in spack.directives_meta.directive_names: assert name not in unparsed many_attributes = """\ class HasManyMetadataAttributes: homepage = "https://example.com" url = "https://example.com/foo.tar.gz" git = "https://example.com/foo/bar.git" maintainers("alice", "bob") tags = ["foo", "bar", "baz"] depends_on("foo") conflicts("foo") """ many_attributes_canonical = """\ class HasManyMetadataAttributes: pass """ def test_remove_spack_attributes(): tree = ast.parse(many_attributes) spec = Spec("has-many-metadata-attributes") tree = ph.RemoveDirectives(spec).visit(tree) unparsed = unparse(tree, py_ver_consistent=True) assert unparsed == many_attributes_canonical complex_package_logic = """\ class ComplexPackageLogic: for variant in ["+foo", "+bar", "+baz"]: conflicts("quux" + variant) for variant in ["+foo", "+bar", "+baz"]: # logic in the loop prevents our dumb analyzer from having it removed. This # is uncommon so we don't (yet?) implement logic to detect that spec is unused. print("oops can't remove this.") conflicts("quux" + variant) # Hard to make a while loop that makes sense, so ignore the infinite loop here. # Likely nobody uses while instead of for, but we test it just in case. while x <= 10: depends_on("garply@%d.0" % x) # all of these should go away, as they only contain directives with when("@10.0"): depends_on("foo") with when("+bar"): depends_on("bar") with when("+baz"): depends_on("baz") # this whole statement should disappear if sys.platform == "linux": conflicts("baz@9.0") # the else block here should disappear if sys.platform == "linux": print("foo") else: conflicts("foo@9.0") # both blocks of this statement should disappear if sys.platform == "darwin": conflicts("baz@10.0") else: conflicts("bar@10.0") # This one is complicated as the body goes away but the else block doesn't. # Again, this could be optimized, but we're just testing removal logic here. if sys.platform() == "darwin": conflicts("baz@10.0") else: print("oops can't remove this.") conflicts("bar@10.0") """ complex_package_logic_filtered = """\ class ComplexPackageLogic: for variant in ['+foo', '+bar', '+baz']: print("oops can't remove this.") if sys.platform == 'linux': print('foo') if sys.platform() == 'darwin': pass else: print("oops can't remove this.") """ def test_remove_complex_package_logic_filtered(): tree = ast.parse(complex_package_logic) spec = Spec("has-many-metadata-attributes") tree = ph.RemoveDirectives(spec).visit(tree) unparsed = unparse(tree, py_ver_consistent=True) assert unparsed == complex_package_logic_filtered @pytest.mark.parametrize( "package_spec,expected_hash", [ ("amdfftw", "tivb752zddjgvfkogfs7cnnvp5olj6co"), ("grads", "lomrsppasfxegyamz4r33zgwiqkveftv"), ("llvm", "paicamlvy5jkgxw4xnacaxahrixe3f3i"), # has @when("@4.1.0") and raw unicode literals ("mfem", "slf5qyyyhuj66mo5lpuhkrs35akh2zck"), ("mfem@4.0.0", "slf5qyyyhuj66mo5lpuhkrs35akh2zck"), ("mfem@4.1.0", "6tjbezoh2aquz6gmvoz7jf6j6lib65m2"), # has @when("@1.5.0:") ("py-torch", "m3ucsddqr7hjevtgx4cad34nrtqgyjfg"), ("py-torch@1.0", "m3ucsddqr7hjevtgx4cad34nrtqgyjfg"), ("py-torch@1.6", "insaxs6bq34rvyhajdbyr4wddqeqb2t3"), # has a print with multiple arguments ("legion", "bq2etsik5l6pbryxmbhfhzynci56ruy4"), # has nested `with when()` blocks and loops ("trilinos", "ojbtbu3p6gpa42sbilblo2ioanvhouxu"), ], ) def test_package_hash_consistency(package_spec, expected_hash): """Ensure that that package hash is consistent python version to version. We assume these tests run across all supported Python versions in CI, and we ensure consistency with recorded hashes for some well known inputs. If this fails, then something about the way the python AST works has likely changed. If Spack is running in a new python version, we might need to modify the unparser to handle it. If not, then something has become inconsistent about the way we unparse Python code across versions. """ spec = Spec(package_spec) filename = os.path.join(datadir, "%s.txt" % spec.name) with open(filename, "rb") as f: source = f.read() h = ph.package_hash(spec, source=source) assert expected_hash == h many_multimethods = """\ class Pkg: def foo(self): print("ONE") @when("@1.0") def foo(self): print("TWO") @when("@2.0") @when(sys.platform == "darwin") def foo(self): print("THREE") @when("@3.0") def foo(self): print("FOUR") # this one should always stay @run_after("install") def some_function(self): print("FIVE") """ more_dynamic_multimethods = """\ class Pkg: @when(sys.platform == "darwin") def foo(self): print("ONE") @when("@1.0") def foo(self): print("TWO") # this one isn't dynamic, but an int fails the Spec parse, # so it's kept because it has to be evaluated at runtime. @when("@2.0") @when(1) def foo(self): print("THREE") @when("@3.0") def foo(self): print("FOUR") # this one should always stay @run_after("install") def some_function(self): print("FIVE") """ @pytest.mark.parametrize( "spec_str,source,expected,not_expected", [ # all are false but the default ("pkg@4.0", many_multimethods, ["ONE", "FIVE"], ["TWO", "THREE", "FOUR"]), # we know first @when overrides default and others are false ("pkg@1.0", many_multimethods, ["TWO", "FIVE"], ["ONE", "THREE", "FOUR"]), # we know last @when overrides default and others are false ("pkg@3.0", many_multimethods, ["FOUR", "FIVE"], ["ONE", "TWO", "THREE"]), # we don't know if default or THREE will win, include both ("pkg@2.0", many_multimethods, ["ONE", "THREE", "FIVE"], ["TWO", "FOUR"]), # we know the first one is the only one that can win. ("pkg@4.0", more_dynamic_multimethods, ["ONE", "FIVE"], ["TWO", "THREE", "FOUR"]), # now we have to include ONE and TWO because ONE may win dynamically. ("pkg@1.0", more_dynamic_multimethods, ["ONE", "TWO", "FIVE"], ["THREE", "FOUR"]), # we know FOUR is true and TWO and THREE are false, but ONE may # still win dynamically. ("pkg@3.0", more_dynamic_multimethods, ["ONE", "FOUR", "FIVE"], ["TWO", "THREE"]), # TWO and FOUR can't be satisfied, but ONE or THREE could win ("pkg@2.0", more_dynamic_multimethods, ["ONE", "THREE", "FIVE"], ["TWO", "FOUR"]), ], ) def test_multimethod_resolution(spec_str, source, expected, not_expected): filtered = ph.canonical_source(Spec(spec_str), source=source) for item in expected: assert item in filtered for item in not_expected: assert item not in filtered ================================================ FILE: lib/spack/spack/test/util/path.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import sys import pytest import spack.config import spack.llnl.util.tty as tty import spack.util.path as sup #: Some lines with lots of placeholders padded_lines = [ "==> [2021-06-23-15:59:05.020387] './configure' '--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga", # noqa: E501 "/Users/gamblin2/Workspace/spack/lib/spack/env/clang/clang -dynamiclib -install_name /Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.1.dylib -compatibility_version 1 -current_version 1.2.11 -fPIC -O2 -fPIC -DHAVE_HIDDEN -o libz.1.2.11.dylib adler32.lo crc32.lo deflate.lo infback.lo inffast.lo inflate.lo inftrees.lo trees.lo zutil.lo compress.lo uncompr.lo gzclose.lo gzlib.lo gzread.lo gzwrite.lo -lc", # noqa: E501 "rm -f /Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.a", # noqa: E501 "rm -f /Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_path_placeholder___/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.a", # noqa: E501 ] #: unpadded versions of padded_lines, with [padded-to-X-chars] replacing the padding fixed_lines = [ "==> [2021-06-23-15:59:05.020387] './configure' '--prefix=/Users/gamblin2/padding-log-test/opt/[padded-to-512-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga", # noqa: E501 "/Users/gamblin2/Workspace/spack/lib/spack/env/clang/clang -dynamiclib -install_name /Users/gamblin2/padding-log-test/opt/[padded-to-512-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.1.dylib -compatibility_version 1 -current_version 1.2.11 -fPIC -O2 -fPIC -DHAVE_HIDDEN -o libz.1.2.11.dylib adler32.lo crc32.lo deflate.lo infback.lo inffast.lo inflate.lo inftrees.lo trees.lo zutil.lo compress.lo uncompr.lo gzclose.lo gzlib.lo gzread.lo gzwrite.lo -lc", # noqa: E501 "rm -f /Users/gamblin2/padding-log-test/opt/[padded-to-512-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.a", # noqa: E501 "rm -f /Users/gamblin2/padding-log-test/opt/[padded-to-91-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.a", # noqa: E501 ] def test_sanitize_filename(): """Test filtering illegal characters out of potential filenames""" sanitized = sup.sanitize_filename("""acd/?e:f"g|h*i.\0txt""") if sys.platform == "win32": assert sanitized == "a_b_cd__e_f_g_h_i._txt" else: assert sanitized == """acd_?e:f"g|h*i._txt""" # This class pertains to path string padding manipulation specifically # which is used for binary caching. This functionality is not supported # on Windows as of yet. @pytest.mark.not_on_windows("Padding functionality unsupported on Windows") @pytest.mark.parametrize("as_bytes", [False, True]) class TestPathPadding: @pytest.fixture(autouse=True) def setup(self, as_bytes: bool): #: The filter function, either for bytes or str self.filter = sup.padding_filter_bytes if as_bytes else sup.padding_filter #: A converter of str -> bytes if we're testing the bytes filter self.convert = lambda s: s.encode("ascii") if as_bytes else s @pytest.mark.parametrize("padded,fixed", zip(padded_lines, fixed_lines)) def test_padding_substitution(self, padded, fixed): """Ensure that all padded lines are unpadded correctly.""" assert self.convert(fixed) == self.filter(self.convert(padded)) def test_no_substitution(self): """Ensure that a line not containing one full path placeholder is not modified.""" partial = "--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 p = self.convert(partial) assert self.filter(p) is p # Test fast-path identity def test_short_substitution(self): """Ensure that a single placeholder path component is replaced""" short = "--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 short_subst = "--prefix=/Users/gamblin2/padding-log-test/opt/[padded-to-63-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 assert self.convert(short_subst) == self.filter(self.convert(short)) def test_partial_substitution(self): """Ensure that a single placeholder path component is replaced""" short = "--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_p/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 short_subst = "--prefix=/Users/gamblin2/padding-log-test/opt/[padded-to-73-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'" # noqa: E501 assert self.convert(short_subst) == self.filter(self.convert(short)) def test_longest_prefix_re(self): """Test that longest_prefix_re generates correct regular expressions.""" assert "(s(?:t(?:r(?:i(?:ng?)?)?)?)?)" == sup.longest_prefix_re("string", capture=True) assert "(?:s(?:t(?:r(?:i(?:ng?)?)?)?)?)" == sup.longest_prefix_re("string", capture=False) @pytest.mark.not_on_windows("Padding functionality unsupported on Windows") def test_output_filtering(capfd, install_mockery, mutable_config): """Test filtering padding out of tty messages.""" long_path = "/" + "/".join([sup.SPACK_PATH_PADDING_CHARS] * 200) padding_string = "[padded-to-%d-chars]" % len(long_path) # test filtering when padding is enabled with spack.config.override("config:install_tree", {"padded_length": 256}): # tty.msg with filtering on the first argument with sup.filter_padding(): tty.msg("here is a long path: %s/with/a/suffix" % long_path) out, err = capfd.readouterr() assert padding_string in out # tty.msg with filtering on a laterargument with sup.filter_padding(): tty.msg("here is a long path:", "%s/with/a/suffix" % long_path) out, err = capfd.readouterr() assert padding_string in out # tty.error with filtering on the first argument with sup.filter_padding(): tty.error("here is a long path: %s/with/a/suffix" % long_path) out, err = capfd.readouterr() assert padding_string in err # tty.error with filtering on a later argument with sup.filter_padding(): tty.error("here is a long path:", "%s/with/a/suffix" % long_path) out, err = capfd.readouterr() assert padding_string in err # test no filtering tty.msg("here is a long path: %s/with/a/suffix" % long_path) out, err = capfd.readouterr() assert padding_string not in out @pytest.mark.not_on_windows("Padding functionality unsupported on Windows") def test_pad_on_path_sep_boundary(): """Ensure that padded paths do not end with path separator.""" pad_length = len(sup.SPACK_PATH_PADDING_CHARS) padded_length = 128 remainder = padded_length % (pad_length + 1) path = "a" * (remainder - 1) result = sup.add_padding(path, padded_length) assert 128 == len(result) and not result.endswith(os.path.sep) @pytest.mark.parametrize("debug", [1, 2]) def test_path_debug_padded_filter(debug, monkeypatch): """Ensure padded filter works as expected with different debug levels.""" fmt = "{0}{1}{2}{1}{3}" prefix = "[+] {0}home{0}user{0}install".format(os.sep) suffix = "mypackage" string = fmt.format(prefix, os.sep, os.sep.join([sup.SPACK_PATH_PADDING_CHARS] * 2), suffix) expected = ( fmt.format(prefix, os.sep, "[padded-to-{0}-chars]".format(72), suffix) if debug <= 1 and sys.platform != "win32" else string ) monkeypatch.setattr(tty, "_debug", debug) with spack.config.override("config:install_tree", {"padded_length": 128}): assert expected == sup.debug_padded_filter(string) @pytest.mark.not_on_windows("Unix path") def test_canonicalize_file_unix(): assert sup.canonicalize_path("/home/spack/path/to/file.txt") == "/home/spack/path/to/file.txt" assert sup.canonicalize_path("file:///home/another/config.yaml") == "/home/another/config.yaml" @pytest.mark.only_windows("Windows path") def test_canonicalize_file_windows(): assert sup.canonicalize_path(r"C:\Files (x86)\Windows\10") == r"C:\Files (x86)\Windows\10" assert sup.canonicalize_path(r"E:/spack stage") == r"E:\spack stage" def test_canonicalize_file_relative(): assert sup.canonicalize_path("path/to.txt") == os.path.join(os.getcwd(), "path", "to.txt") ================================================ FILE: lib/spack/spack/test/util/prefix.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests various features of :py:class:`spack.util.prefix.Prefix`""" import os from spack.util.prefix import Prefix def test_prefix_attributes(): """Test normal prefix attributes like ``prefix.bin``""" prefix = Prefix(os.sep + "usr") assert prefix.bin == os.sep + os.path.join("usr", "bin") assert prefix.lib == os.sep + os.path.join("usr", "lib") assert prefix.include == os.sep + os.path.join("usr", "include") def test_prefix_join(): """Test prefix join ``prefix.join(...)``""" prefix = Prefix(os.sep + "usr") a1 = prefix.join("a_{0}".format(1)).lib64 a2 = prefix.join("a-{0}".format(1)).lib64 a3 = prefix.join("a.{0}".format(1)).lib64 assert a1 == os.sep + os.path.join("usr", "a_1", "lib64") assert a2 == os.sep + os.path.join("usr", "a-1", "lib64") assert a3 == os.sep + os.path.join("usr", "a.1", "lib64") assert isinstance(a1, Prefix) assert isinstance(a2, Prefix) assert isinstance(a3, Prefix) p1 = prefix.bin.join("executable.sh") p2 = prefix.share.join("pkg-config").join("foo.pc") p3 = prefix.join("dashed-directory").foo assert p1 == os.sep + os.path.join("usr", "bin", "executable.sh") assert p2 == os.sep + os.path.join("usr", "share", "pkg-config", "foo.pc") assert p3 == os.sep + os.path.join("usr", "dashed-directory", "foo") assert isinstance(p1, Prefix) assert isinstance(p2, Prefix) assert isinstance(p3, Prefix) def test_multilevel_attributes(): """Test attributes of attributes, like ``prefix.share.man``""" prefix = Prefix(os.sep + "usr" + os.sep) assert prefix.share.man == os.sep + os.path.join("usr", "share", "man") assert prefix.man.man8 == os.sep + os.path.join("usr", "man", "man8") assert prefix.foo.bar.baz == os.sep + os.path.join("usr", "foo", "bar", "baz") share = prefix.share assert isinstance(share, Prefix) assert share.man == os.sep + os.path.join("usr", "share", "man") def test_string_like_behavior(): """Test string-like behavior of the prefix object""" prefix = Prefix("/usr") assert prefix == "/usr" assert isinstance(prefix, str) assert prefix + "/bin" == "/usr/bin" assert "--prefix=%s" % prefix == "--prefix=/usr" assert "--prefix={0}".format(prefix) == "--prefix=/usr" assert prefix.find("u", 1) assert prefix.upper() == "/USR" assert prefix.lstrip("/") == "usr" ================================================ FILE: lib/spack/spack/test/util/remote_file_cache.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os.path import pathlib import sys import pytest import spack.config import spack.llnl.util.tty as tty import spack.util.remote_file_cache as rfc_util from spack.llnl.util.filesystem import join_path github_url = "https://github.com/fake/fake/{0}/develop" gitlab_url = "https://gitlab.fake.io/user/repo/-/blob/config/defaults" @pytest.mark.parametrize( "path,err", [ ("ssh://git@github.com:spack/", "Unsupported URL scheme"), ("bad:///this/is/a/file/url/include.yaml", "Invalid URL scheme"), ], ) def test_rfc_local_path_bad_scheme(path, err): with pytest.raises(ValueError, match=err): _ = rfc_util.local_path(path, "") @pytest.mark.not_on_windows("Unix path") def test_rfc_local_file_unix(): assert rfc_util.local_path("/a/b/c/d/e/config.py", "") == "/a/b/c/d/e/config.py" assert ( rfc_util.local_path("file:///this/is/a/file/url/include.yaml", "") == "/this/is/a/file/url/include.yaml" ) @pytest.mark.only_windows("Windows path") def test_rfc_local_file_windows(): assert rfc_util.local_path(r"C:\Files (x86)\Windows\10", "") == r"C:\Files (x86)\Windows\10" assert rfc_util.local_path(r"D:/spack stage", "") == r"D:\spack stage" def test_rfc_local_file_relative(): path = "relative/packages.txt" expected = os.path.join(os.getcwd(), "relative", "packages.txt") assert rfc_util.local_path(path, "") == expected def test_rfc_remote_local_path_no_dest(): path = f"{gitlab_url}/packages.yaml" with pytest.raises(ValueError, match="Requires the destination argument"): _ = rfc_util.local_path(path, "") packages_yaml_sha256 = ( "8d428c600b215e3b4a207a08236659dfc2c9ae2782c35943a00ee4204a135702" if sys.platform != "win32" else "6c094ec3ee1eb5068860cdd97d8da965bf281be29e60ab9afc8f6e4d72d24f21" ) @pytest.mark.parametrize( "url,sha256,err,msg", [ ( f"{join_path(github_url.format('tree'), 'config.yaml')}", "", ValueError, "Requires sha256", ), # This is the packages.yaml in lib/spack/spack/test/data/config (f"{gitlab_url}/packages.yaml", packages_yaml_sha256, None, ""), (f"{gitlab_url}/packages.yaml", "abcdef", ValueError, "does not match"), (f"{github_url.format('blob')}/README.md", "", OSError, "No such"), (github_url.format("tree"), "", OSError, "No such"), ("", "", ValueError, "argument is required"), ], ) def test_rfc_remote_local_path( tmp_path: pathlib.Path, mutable_empty_config, mock_fetch_url_text, url, sha256, err, msg ): def _has_content(filename): # The first element of all configuration files for this test happen to # be the basename of the file so this check leverages that feature. If # that changes, then this check will need to change accordingly. element = f"{os.path.splitext(os.path.basename(filename))[0]}:" with open(filename, "r", encoding="utf-8") as fd: for line in fd: if element in line: return True tty.debug(f"Expected {element} in '{filename}'") return False dest_dir = join_path(str(tmp_path), "cache") if err is not None: with spack.config.override("config:url_fetch_method", "curl"): with pytest.raises(err, match=msg): rfc_util.local_path(url, sha256, dest_dir) else: with spack.config.override("config:url_fetch_method", "curl"): path = rfc_util.local_path(url, sha256, dest_dir) assert os.path.exists(path) # Ensure correct file is "fetched" assert os.path.basename(path) == os.path.basename(url) # Ensure contents of the file contains expected config element assert _has_content(path) ================================================ FILE: lib/spack/spack/test/util/spack_lock_wrapper.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for Spack's wrapper module around spack.llnl.util.lock.""" import os import pathlib import pytest import spack.error import spack.util.lock as lk from spack.llnl.util.filesystem import getuid, group_ids def test_disable_locking(tmp_path: pathlib.Path): """Ensure that locks do no real locking when disabled.""" lock_path = str(tmp_path / "lockfile") lock = lk.Lock(lock_path, enable=False) lock.acquire_read() assert not os.path.exists(lock_path) lock.acquire_write() assert not os.path.exists(lock_path) lock.release_write() assert not os.path.exists(lock_path) lock.release_read() assert not os.path.exists(lock_path) # "Disable" mock_stage fixture to avoid subdir permissions issues on cleanup. @pytest.mark.nomockstage @pytest.mark.skipif(getuid() == 0, reason="user is root") def test_lock_checks_user(tmp_path: pathlib.Path): """Ensure lock checks work with a self-owned, self-group repo.""" uid = getuid() if uid not in group_ids(): pytest.skip("user has no group with gid == uid") # self-owned, own group os.chown(tmp_path, uid, uid) # safe path = str(tmp_path) tmp_path.chmod(0o744) lk.check_lock_safety(path) # safe tmp_path.chmod(0o774) lk.check_lock_safety(path) # unsafe tmp_path.chmod(0o777) with pytest.raises(spack.error.SpackError): lk.check_lock_safety(path) # safe tmp_path.chmod(0o474) lk.check_lock_safety(path) # safe tmp_path.chmod(0o477) lk.check_lock_safety(path) # "Disable" mock_stage fixture to avoid subdir permissions issues on cleanup. @pytest.mark.nomockstage def test_lock_checks_group(tmp_path: pathlib.Path): """Ensure lock checks work with a self-owned, non-self-group repo.""" uid = getuid() gid = next((g for g in group_ids() if g != uid), None) if not gid: pytest.skip("user has no group with gid != uid") return # self-owned, another group os.chown(tmp_path, uid, gid) # safe path = str(tmp_path) tmp_path.chmod(0o744) lk.check_lock_safety(path) # unsafe tmp_path.chmod(0o774) with pytest.raises(spack.error.SpackError): lk.check_lock_safety(path) # unsafe tmp_path.chmod(0o777) with pytest.raises(spack.error.SpackError): lk.check_lock_safety(path) # safe tmp_path.chmod(0o474) lk.check_lock_safety(path) # safe tmp_path.chmod(0o477) lk.check_lock_safety(path) ================================================ FILE: lib/spack/spack/test/util/spack_yaml.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re import spack.config from spack.main import SpackCommand config_cmd = SpackCommand("config") def get_config_line(pattern, lines): """Get a configuration line that matches a particular pattern.""" line = next((x for x in lines if re.search(pattern, x)), None) assert line is not None, "no such line!" return line def check_blame(element, file_name, line=None): """Check that `config blame config` gets right file/line for an element. This runs `spack config blame config` and scrapes the output for a particular YAML key. It then checks that the requested file/line info is also on that line. Line is optional; if it is ``None`` we just check for the ``file_name``, which may just be a name for a special config scope like ``_builtin`` or ``command_line``. """ output = config_cmd("blame", "config") blame_lines = output.rstrip().split("\n") element_line = get_config_line(element + ":", blame_lines) annotation = file_name if line is not None: annotation += ":%d" % line assert file_name in element_line def test_config_blame(config): """check blame info for elements in mock configuration.""" config_file = config.get_config_filename("site", "config") check_blame("install_tree", config_file, 2) check_blame("source_cache", config_file, 11) check_blame("misc_cache", config_file, 12) check_blame("verify_ssl", config_file, 13) check_blame("checksum", config_file, 14) check_blame("dirty", config_file, 15) def test_config_blame_with_override(config): """check blame for an element from an override scope""" config_file = config.get_config_filename("site", "config") with spack.config.override("config:install_tree", {"root": "foobar"}): check_blame("install_tree", "overrides") check_blame("source_cache", config_file, 11) check_blame("misc_cache", config_file, 12) check_blame("verify_ssl", config_file, 13) check_blame("checksum", config_file, 14) check_blame("dirty", config_file, 15) def test_config_blame_defaults(): """check blame for an element from an override scope""" files = {} def get_file_lines(filename): if filename not in files: with open(filename, "r", encoding="utf-8") as f: files[filename] = [""] + f.read().split("\n") return files[filename] config_blame = config_cmd("blame", "config") for line in config_blame.split("\n"): # currently checking only simple lines with dict keys match = re.match(r"^([^:]+):(\d+)\s+([^:]+):\s+(.*)", line) # check that matches are on the lines they say they are if match: filename, line, key, val = match.groups() line = int(line) lines = get_file_lines(filename) assert key in lines[line] val = val.strip("'\"") printed_line = lines[line] if val.lower() in ("true", "false"): val = val.lower() printed_line = printed_line.lower() assert val in printed_line, filename ================================================ FILE: lib/spack/spack/test/util/timer.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import json from io import StringIO import spack.util.timer as timer class Tick: """Timer that increments the seconds passed by 1 every time tick is called.""" def __init__(self): self.time = 0.0 def tick(self): self.time += 1 return self.time def test_timer(): # 0 t = timer.Timer(now=Tick().tick) # 1 (restart) t.start() # 2 t.start("wrapped") # 3 t.start("first") # 4 t.stop("first") assert t.duration("first") == 1.0 # 5 t.start("second") # 6 t.stop("second") assert t.duration("second") == 1.0 # 7-8 with t.measure("third"): pass assert t.duration("third") == 1.0 # 9 t.stop("wrapped") assert t.duration("wrapped") == 7.0 # tick 10-13 t.start("not-stopped") assert t.duration("not-stopped") == 1.0 assert t.duration("not-stopped") == 2.0 assert t.duration("not-stopped") == 3.0 # 14 assert t.duration() == 13.0 # 15 t.stop() assert t.duration() == 14.0 def test_timer_stop_stops_all(): # Ensure that timer.stop() effectively stops all timers. # 0 t = timer.Timer(now=Tick().tick) # 1 t.start("first") # 2 t.start("second") # 3 t.start("third") # 4 t.stop() assert t.duration("first") == 3.0 assert t.duration("second") == 2.0 assert t.duration("third") == 1.0 assert t.duration() == 4.0 def test_stopping_unstarted_timer_is_no_error(): t = timer.Timer(now=Tick().tick) assert t.duration("hello") == 0.0 t.stop("hello") assert t.duration("hello") == 0.0 def test_timer_write(): text_buffer = StringIO() json_buffer = StringIO() # 0 t = timer.Timer(now=Tick().tick) # 1 t.start("timer") # 2 t.stop("timer") # 3 t.stop() t.write_tty(text_buffer) t.write_json(json_buffer) output = text_buffer.getvalue().splitlines() assert "timer" in output[0] assert "1.000s" in output[0] assert "total" in output[1] assert "3.000s" in output[1] deserialized = json.loads(json_buffer.getvalue()) assert deserialized == { "phases": [{"name": "timer", "path": "timer", "seconds": 1.0, "count": 1}], "total": 3.0, } def test_null_timer(): # Just ensure that the interface of the noop-timer doesn't break at some point buffer = StringIO() t = timer.NullTimer() t.start() t.start("first") t.stop("first") with t.measure("second"): pass t.stop() assert t.duration("first") == 0.0 assert t.duration() == 0.0 assert not t.phases t.write_json(buffer) t.write_tty(buffer) assert not buffer.getvalue() ================================================ FILE: lib/spack/spack/test/util/unparse/__init__.py ================================================ # Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers. # # SPDX-License-Identifier: Python-2.0 ================================================ FILE: lib/spack/spack/test/util/unparse/unparse.py ================================================ # Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers. # # SPDX-License-Identifier: Python-2.0 import ast import os import sys import tokenize import pytest import spack.util.unparse pytestmark = pytest.mark.not_on_windows("Test module unsupported on Windows") def read_pyfile(filename): """Read and return the contents of a Python source file (as a string), taking into account the file encoding.""" with open(filename, "rb") as pyfile: encoding = tokenize.detect_encoding(pyfile.readline)[0] with open(filename, "r", encoding=encoding) as pyfile: source = pyfile.read() return source code_parseable_in_all_parser_modes = """\ (a + b + c) * (d + e + f) """ for_else = """\ def f(): for x in range(10): break else: y = 2 z = 3 """ while_else = """\ def g(): while True: break else: y = 2 z = 3 """ relative_import = """\ from . import fred from .. import barney from .australia import shrimp as prawns """ import_many = """\ import fred, barney """ nonlocal_ex = """\ def f(): x = 1 def g(): nonlocal x x = 2 y = 7 def h(): nonlocal x, y """ # also acts as test for 'except ... as ...' raise_from = """\ try: 1 / 0 except ZeroDivisionError as e: raise ArithmeticError from e """ async_comprehensions_and_generators = """\ async def async_function(): my_set = {i async for i in aiter() if i % 2} my_list = [i async for i in aiter() if i % 2] my_dict = {i: -i async for i in aiter() if i % 2} my_gen = (i ** 2 async for i in agen()) my_other_gen = (i - 1 async for i in agen() if i % 2) """ class_decorator = """\ @f1(arg) @f2 class Foo: pass """ elif1 = """\ if cond1: suite1 elif cond2: suite2 else: suite3 """ elif2 = """\ if cond1: suite1 elif cond2: suite2 """ try_except_finally = """\ try: suite1 except ex1: suite2 except ex2: suite3 else: suite4 finally: suite5 """ with_simple = """\ with f(): suite1 """ with_as = """\ with f() as x: suite1 """ with_two_items = """\ with f() as x, g() as y: suite1 """ a_repr = """\ `{}` """ complex_f_string = '''\ f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\' ''' async_function_def = """\ async def f(): suite1 """ async_for = """\ async def f(): async for _ in reader: suite1 """ async_with = """\ async def f(): async with g(): suite1 """ async_with_as = """\ async def f(): async with g() as x: suite1 """ match_literal = """\ match status: case 400: return "Bad request" case 404 | 418: return "Not found" case _: return "Something's wrong with the internet" """ match_with_noop = """\ match status: case 400: return "Bad request" """ match_literal_and_variable = """\ match point: case (0, 0): print("Origin") case (0, y): print(f"Y={y}") case (x, 0): print(f"X={x}") case (x, y): print(f"X={x}, Y={y}") case _: raise ValueError("Not a point") """ match_classes = """\ class Point: x: int y: int def location(point): match point: case Point(x=0, y=0): print("Origin is the point's location.") case Point(x=0, y=y): print(f"Y={y} and the point is on the y-axis.") case Point(x=x, y=0): print(f"X={x} and the point is on the x-axis.") case Point(): print("The point is located somewhere else on the plane.") case _: print("Not a point") """ match_nested = """\ match points: case []: print("No points in the list.") case [Point(0, 0)]: print("The origin is the only point in the list.") case [Point(x, y)]: print(f"A single point {x}, {y} is in the list.") case [Point(0, y1), Point(0, y2)]: print(f"Two points on the Y axis at {y1}, {y2} are in the list.") case _: print("Something else is found in the list.") """ def check_ast_roundtrip(code1, filename="internal", mode="exec"): ast1 = compile(str(code1), filename, mode, ast.PyCF_ONLY_AST) code2 = spack.util.unparse.unparse(ast1) ast2 = compile(code2, filename, mode, ast.PyCF_ONLY_AST) error_msg = "Failed to roundtrip {} [mode={}]".format(filename, mode) assert ast.dump(ast1) == ast.dump(ast2), error_msg def test_core_lib_files(): """Roundtrip source files from the Python core libs.""" test_directories = [ os.path.join( getattr(sys, "real_prefix", sys.prefix), "lib", "python%s.%s" % sys.version_info[:2] ) ] names = [] for test_dir in test_directories: for n in os.listdir(test_dir): if n.endswith(".py") and not n.startswith("bad"): names.append(os.path.join(test_dir, n)) for filename in names: source = read_pyfile(filename) check_ast_roundtrip(source) @pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Only for Python 3.6 or greater") def test_simple_fstring(): check_ast_roundtrip("f'{x}'") @pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Only for Python 3.6 or greater") def test_fstrings(): # See issue 25180 check_ast_roundtrip(r"""f'{f"{0}"*3}'""") check_ast_roundtrip(r"""f'{f"{y}"*3}'""") check_ast_roundtrip("""f''""") check_ast_roundtrip('''f"""'end' "quote\\""""''') @pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Only for Python 3.6 or greater") def test_fstrings_complicated(): # See issue 28002 check_ast_roundtrip("""f'''{"'"}'''""") check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-'single quote\\'\'\'\'''') check_ast_roundtrip("f\"\"\"{'''\n'''}\"\"\"") check_ast_roundtrip("f\"\"\"{g('''\n''')}\"\"\"") check_ast_roundtrip('''f"a\\r\\nb"''') check_ast_roundtrip('''f"\\u2028{'x'}"''') def test_parser_modes(): for mode in ["exec", "single", "eval"]: check_ast_roundtrip(code_parseable_in_all_parser_modes, mode=mode) def test_del_statement(): check_ast_roundtrip("del x, y, z") def test_shifts(): check_ast_roundtrip("45 << 2") check_ast_roundtrip("13 >> 7") def test_for_else(): check_ast_roundtrip(for_else) def test_while_else(): check_ast_roundtrip(while_else) def test_unary_parens(): check_ast_roundtrip("(-1)**7") check_ast_roundtrip("(-1.)**8") check_ast_roundtrip("(-1j)**6") check_ast_roundtrip("not True or False") check_ast_roundtrip("True or not False") @pytest.mark.skipif(sys.version_info >= (3, 6), reason="Only works for Python < 3.6") def test_integer_parens(): check_ast_roundtrip("3 .__abs__()") def test_huge_float(): check_ast_roundtrip("1e1000") check_ast_roundtrip("-1e1000") check_ast_roundtrip("1e1000j") check_ast_roundtrip("-1e1000j") def test_min_int30(): check_ast_roundtrip(str(-(2**31))) check_ast_roundtrip(str(-(2**63))) def test_imaginary_literals(): check_ast_roundtrip("7j") check_ast_roundtrip("-7j") check_ast_roundtrip("0j") check_ast_roundtrip("-0j") def test_negative_zero(): check_ast_roundtrip("-0") check_ast_roundtrip("-(0)") check_ast_roundtrip("-0b0") check_ast_roundtrip("-(0b0)") check_ast_roundtrip("-0o0") check_ast_roundtrip("-(0o0)") check_ast_roundtrip("-0x0") check_ast_roundtrip("-(0x0)") def test_lambda_parentheses(): check_ast_roundtrip("(lambda: int)()") def test_chained_comparisons(): check_ast_roundtrip("1 < 4 <= 5") check_ast_roundtrip("a is b is c is not d") def test_function_arguments(): check_ast_roundtrip("def f(): pass") check_ast_roundtrip("def f(a): pass") check_ast_roundtrip("def f(b = 2): pass") check_ast_roundtrip("def f(a, b): pass") check_ast_roundtrip("def f(a, b = 2): pass") check_ast_roundtrip("def f(a = 5, b = 2): pass") check_ast_roundtrip("def f(*args, **kwargs): pass") check_ast_roundtrip("def f(*, a = 1, b = 2): pass") check_ast_roundtrip("def f(*, a = 1, b): pass") check_ast_roundtrip("def f(*, a, b = 2): pass") check_ast_roundtrip("def f(a, b = None, *, c, **kwds): pass") check_ast_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass") def test_relative_import(): check_ast_roundtrip(relative_import) def test_import_many(): check_ast_roundtrip(import_many) def test_nonlocal(): check_ast_roundtrip(nonlocal_ex) def test_raise_from(): check_ast_roundtrip(raise_from) def test_bytes(): check_ast_roundtrip("b'123'") @pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") def test_formatted_value(): check_ast_roundtrip('f"{value}"') check_ast_roundtrip('f"{value!s}"') check_ast_roundtrip('f"{value:4}"') check_ast_roundtrip('f"{value!s:4}"') @pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") def test_joined_str(): check_ast_roundtrip('f"{key}={value!s}"') check_ast_roundtrip('f"{key}={value!r}"') check_ast_roundtrip('f"{key}={value!a}"') @pytest.mark.skipif(sys.version_info != (3, 6, 0), reason="Only supported on 3.6.0") def test_joined_str_361(): check_ast_roundtrip('f"{key:4}={value!s}"') check_ast_roundtrip('f"{key:02}={value!r}"') check_ast_roundtrip('f"{key:6}={value!a}"') check_ast_roundtrip('f"{key:4}={value:#06x}"') check_ast_roundtrip('f"{key:02}={value:#06x}"') check_ast_roundtrip('f"{key:6}={value:#06x}"') check_ast_roundtrip('f"{key:4}={value!s:#06x}"') check_ast_roundtrip('f"{key:4}={value!r:#06x}"') check_ast_roundtrip('f"{key:4}={value!a:#06x}"') @pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Only for Python 3.6 or greater") def test_complex_f_string(): check_ast_roundtrip(complex_f_string) def test_annotations(): check_ast_roundtrip("def f(a : int): pass") check_ast_roundtrip("def f(a: int = 5): pass") check_ast_roundtrip("def f(*args: [int]): pass") check_ast_roundtrip("def f(**kwargs: dict): pass") check_ast_roundtrip("def f() -> None: pass") @pytest.mark.skipif(sys.version_info < (2, 7), reason="Not supported < 2.7") def test_set_literal(): check_ast_roundtrip("{'a', 'b', 'c'}") @pytest.mark.skipif(sys.version_info < (2, 7), reason="Not supported < 2.7") def test_set_comprehension(): check_ast_roundtrip("{x for x in range(5)}") @pytest.mark.skipif(sys.version_info < (2, 7), reason="Not supported < 2.7") def test_dict_comprehension(): check_ast_roundtrip("{x: x*x for x in range(10)}") @pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") def test_dict_with_unpacking(): check_ast_roundtrip("{**x}") check_ast_roundtrip("{a: b, **x}") @pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") def test_async_comp_and_gen_in_async_function(): check_ast_roundtrip(async_comprehensions_and_generators) @pytest.mark.skipif(sys.version_info < (3, 7), reason="Not supported < 3.7") def test_async_comprehension(): check_ast_roundtrip("{i async for i in aiter() if i % 2}") check_ast_roundtrip("[i async for i in aiter() if i % 2]") check_ast_roundtrip("{i: -i async for i in aiter() if i % 2}") @pytest.mark.skipif(sys.version_info < (3, 7), reason="Not supported < 3.7") def test_async_generator_expression(): check_ast_roundtrip("(i ** 2 async for i in agen())") check_ast_roundtrip("(i - 1 async for i in agen() if i % 2)") def test_class_decorators(): check_ast_roundtrip(class_decorator) def test_class_definition(): check_ast_roundtrip("class A(metaclass=type, *[], **{}): pass") def test_elifs(): check_ast_roundtrip(elif1) check_ast_roundtrip(elif2) def test_try_except_finally(): check_ast_roundtrip(try_except_finally) def test_starred_assignment(): check_ast_roundtrip("a, *b, c = seq") check_ast_roundtrip("a, (*b, c) = seq") check_ast_roundtrip("a, *b[0], c = seq") check_ast_roundtrip("a, *(b, c) = seq") @pytest.mark.skipif(sys.version_info < (3, 6), reason="Not supported < 3.6") def test_variable_annotation(): check_ast_roundtrip("a: int") check_ast_roundtrip("a: int = 0") check_ast_roundtrip("a: int = None") check_ast_roundtrip("some_list: List[int]") check_ast_roundtrip("some_list: List[int] = []") check_ast_roundtrip("t: Tuple[int, ...] = (1, 2, 3)") check_ast_roundtrip("(a): int") check_ast_roundtrip("(a): int = 0") check_ast_roundtrip("(a): int = None") def test_with_simple(): check_ast_roundtrip(with_simple) def test_with_as(): check_ast_roundtrip(with_as) @pytest.mark.skipif(sys.version_info < (2, 7), reason="Not supported < 2.7") def test_with_two_items(): check_ast_roundtrip(with_two_items) @pytest.mark.skipif(sys.version_info < (3, 5), reason="Not supported < 3.5") def test_async_function_def(): check_ast_roundtrip(async_function_def) @pytest.mark.skipif(sys.version_info < (3, 5), reason="Not supported < 3.5") def test_async_for(): check_ast_roundtrip(async_for) @pytest.mark.skipif(sys.version_info < (3, 5), reason="Not supported < 3.5") def test_async_with(): check_ast_roundtrip(async_with) @pytest.mark.skipif(sys.version_info < (3, 5), reason="Not supported < 3.5") def test_async_with_as(): check_ast_roundtrip(async_with_as) @pytest.mark.skipif(sys.version_info < (3, 10), reason="Not supported < 3.10") @pytest.mark.parametrize( "literal", [match_literal, match_with_noop, match_literal_and_variable, match_classes, match_nested], ) def test_match_literal(literal): check_ast_roundtrip(literal) @pytest.mark.skipif(sys.version_info < (3, 14), reason="Not supported < 3.14") def test_tstrings(): check_ast_roundtrip("t'foo'") check_ast_roundtrip("t'foo {bar}'") check_ast_roundtrip("t'foo {bar!s:.2f}'") check_ast_roundtrip("t'{a + b}'") check_ast_roundtrip("t'{a + b:x}'") check_ast_roundtrip("t'{a + b!s}'") check_ast_roundtrip("t'{ {a}}'") check_ast_roundtrip("t'{ {a}=}'") check_ast_roundtrip("t'{{a}}'") check_ast_roundtrip("t''") def test_subscript_with_tuple(): """Test change in visit_Subscript/visit_Index is_non_empty_tuple.""" check_ast_roundtrip("a[()]") check_ast_roundtrip("a[b]") check_ast_roundtrip("a[(*b,)]") check_ast_roundtrip("a[(1, 2)]") check_ast_roundtrip("a[(1, *b)]") @pytest.mark.skipif(sys.version_info < (3, 11), reason="Not supported < 3.11") def test_subscript_without_tuple(): """Test change in visit_Subscript/visit_Index is_non_empty_tuple.""" check_ast_roundtrip("a[*b]") check_ast_roundtrip("a[1, *b]") def test_attribute_on_int(): check_ast_roundtrip("1 .__abs__()") ================================================ FILE: lib/spack/spack/test/util/util_gpg.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.util.gpg @pytest.fixture() def has_socket_dir(): spack.util.gpg.init() return bool(spack.util.gpg.SOCKET_DIR) def test_parse_gpg_output_case_one(): # Two keys, fingerprint for primary keys, but not subkeys output = """sec::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:AAAAAAAAAA::::::::: fpr:::::::::XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX: uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) : ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:::::::::: sec::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:AAAAAAAAAA::::::::: fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY: uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) : ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:::::::::: """ keys = spack.util.gpg._parse_secret_keys_output(output) assert len(keys) == 2 assert keys[0] == "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" assert keys[1] == "YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY" def test_parse_gpg_output_case_two(): # One key, fingerprint for primary key as well as subkey output = """sec:-:2048:1:AAAAAAAAAA:AAAAAAAA:::-:::escaESCA:::+:::23::0: fpr:::::::::XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX: grp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: uid:-::::AAAAAAAAA::AAAAAAAAA::Joe (Test) ::::::::::0: ssb:-:2048:1:AAAAAAAAA::::::esa:::+:::23: fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY: grp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: """ keys = spack.util.gpg._parse_secret_keys_output(output) assert len(keys) == 1 assert keys[0] == "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" def test_parse_gpg_output_case_three(): # Two keys, fingerprint for primary keys as well as subkeys output = """sec::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:AAAAAAAAAA::::::::: fpr:::::::::WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW: uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) : ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:::::::::: fpr:::::::::XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX: sec::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:AAAAAAAAAA::::::::: fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY: uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) : ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:::::::::: fpr:::::::::ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ:""" keys = spack.util.gpg._parse_secret_keys_output(output) assert len(keys) == 2 assert keys[0] == "WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW" assert keys[1] == "YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY" @pytest.mark.requires_executables("gpg2") def test_really_long_gnupghome_dir(tmp_path: pathlib.Path, has_socket_dir): if not has_socket_dir: pytest.skip("This test requires /var/run/user/$(id -u)") N = 960 tdir = str(tmp_path) while len(tdir) < N: tdir = os.path.join(tdir, "filler") tdir = tdir[:N].rstrip(os.sep) tdir += "0" * (N - len(tdir)) with spack.util.gpg.gnupghome_override(tdir): spack.util.gpg.create( name="Spack testing 1", email="test@spack.io", comment="Spack testing key", expires="0" ) spack.util.gpg.list(True, True) ================================================ FILE: lib/spack/spack/test/util/util_url.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Test Spack's URL handling utility functions.""" import os import pathlib import urllib.parse import pytest import spack.util.path import spack.util.url as url_util from spack.llnl.util.filesystem import working_dir def test_url_local_file_path(tmp_path: pathlib.Path): # Create a file path = str(tmp_path / "hello.txt") with open(path, "wb") as f: f.write(b"hello world") assert url_util.path_to_file_url(path).startswith("file://") # Go from path -> url -> path. roundtrip = url_util.local_file_path(url_util.path_to_file_url(path)) # Verify it's the same file. assert os.path.samefile(roundtrip, path) # Test if it accepts urlparse objects parsed = urllib.parse.urlparse(url_util.path_to_file_url(path)) assert os.path.samefile(url_util.local_file_path(parsed), path) def test_url_local_file_path_no_file_scheme(): assert url_util.local_file_path("https://example.com/hello.txt") is None assert url_util.local_file_path("C:\\Program Files\\hello.txt") is None def test_relative_path_to_file_url(tmp_path: pathlib.Path): # Create a file path = str(tmp_path / "hello.txt") with open(path, "wb") as f: f.write(b"hello world") with working_dir(str(tmp_path)): roundtrip = url_util.local_file_path(url_util.path_to_file_url("hello.txt")) assert os.path.samefile(roundtrip, path) @pytest.mark.parametrize("resolve_href", [True, False]) @pytest.mark.parametrize("scheme", ["http", "s3", "gs", "file", "oci"]) def test_url_join_absolute(scheme, resolve_href): """Test that joining a URL with an absolute path works the same for schemes we care about, and whether we work in web browser mode or not.""" netloc = "" if scheme == "file" else "example.com" a1 = url_util.join(f"{scheme}://{netloc}/a/b/c", "/d/e/f", resolve_href=resolve_href) a2 = url_util.join(f"{scheme}://{netloc}/a/b/c", "/d", "e", "f", resolve_href=resolve_href) assert a1 == a2 == f"{scheme}://{netloc}/d/e/f" b1 = url_util.join(f"{scheme}://{netloc}/a", "https://b.com/b", resolve_href=resolve_href) b2 = url_util.join(f"{scheme}://{netloc}/a", "https://b.com", "b", resolve_href=resolve_href) assert b1 == b2 == "https://b.com/b" @pytest.mark.parametrize("scheme", ["http", "s3", "gs"]) def test_url_join_up(scheme): """Test that the netloc component is preserved when going .. up in the path.""" a1 = url_util.join(f"{scheme}://netloc/a/b.html", "c", resolve_href=True) assert a1 == f"{scheme}://netloc/a/c" b1 = url_util.join(f"{scheme}://netloc/a/b.html", "../c", resolve_href=True) b2 = url_util.join(f"{scheme}://netloc/a/b.html", "..", "c", resolve_href=True) assert b1 == b2 == f"{scheme}://netloc/c" c1 = url_util.join(f"{scheme}://netloc/a/b.html", "../../c", resolve_href=True) c2 = url_util.join(f"{scheme}://netloc/a/b.html", "..", "..", "c", resolve_href=True) assert c1 == c2 == f"{scheme}://netloc/c" d1 = url_util.join(f"{scheme}://netloc/a/b", "c", resolve_href=False) assert d1 == f"{scheme}://netloc/a/b/c" d2 = url_util.join(f"{scheme}://netloc/a/b", "../c", resolve_href=False) d3 = url_util.join(f"{scheme}://netloc/a/b", "..", "c", resolve_href=False) assert d2 == d3 == f"{scheme}://netloc/a/c" e1 = url_util.join(f"{scheme}://netloc/a/b", "../../c", resolve_href=False) e2 = url_util.join(f"{scheme}://netloc/a/b", "..", "..", "c", resolve_href=False) assert e1 == e2 == f"{scheme}://netloc/c" f1 = url_util.join(f"{scheme}://netloc/a/b", "../../../c", resolve_href=False) f2 = url_util.join(f"{scheme}://netloc/a/b", "..", "..", "..", "c", resolve_href=False) assert f1 == f2 == f"{scheme}://netloc/c" @pytest.mark.parametrize("scheme", ["http", "https", "ftp", "s3", "gs", "file"]) def test_url_join_resolve_href(scheme): """test that `resolve_href=True` behaves like a web browser at the base page, and `resolve_href=False` behaves like joining paths in a file system at the base directory.""" # these are equivalent because of the trailing / netloc = "" if scheme == "file" else "netloc" a1 = url_util.join(f"{scheme}://{netloc}/my/path/", "other/path", resolve_href=True) a2 = url_util.join(f"{scheme}://{netloc}/my/path/", "other", "path", resolve_href=True) assert a1 == a2 == f"{scheme}://{netloc}/my/path/other/path" b1 = url_util.join(f"{scheme}://{netloc}/my/path", "other/path", resolve_href=False) b2 = url_util.join(f"{scheme}://{netloc}/my/path", "other", "path", resolve_href=False) assert b1 == b2 == f"{scheme}://{netloc}/my/path/other/path" # this is like a web browser: relative to /my. c1 = url_util.join(f"{scheme}://{netloc}/my/path", "other/path", resolve_href=True) c2 = url_util.join(f"{scheme}://{netloc}/my/path", "other", "path", resolve_href=True) assert c1 == c2 == f"{scheme}://{netloc}/my/other/path" def test_default_download_name(): url = "https://example.com:1234/path/to/file.txt;params?abc=def#file=blob.tar" filename = url_util.default_download_filename(url) assert filename == spack.util.path.sanitize_filename(filename) def test_default_download_name_dot_dot(): """Avoid that downloaded files get names computed as ., .. or any hidden file.""" assert url_util.default_download_filename("https://example.com/.") == "_" assert url_util.default_download_filename("https://example.com/..") == "_." assert url_util.default_download_filename("https://example.com/.abcdef") == "_abcdef" def test_parse_link_rel_next(): parse = url_util.parse_link_rel_next assert parse(r'; rel="next"') == "/abc" assert parse(r'; x=y; rel="next", ; x=y; rel="prev"') == "/abc" assert parse(r'; rel="prev"; x=y, ; x=y; rel="next"') == "/def" # example from RFC5988 assert ( parse( r"""; title*=UTF-8'de'letztes%20Kapitel; rel="previous",""" r"""; title*=UTF-8'de'n%c3%a4chstes%20Kapitel; rel="next" """ ) == "/TheBook/chapter4" ) assert ( parse(r"""; key=";a=b, ; e=f"; rel="next" """) == "https://example.com/example" ) assert parse("https://example.com/example") is None assert parse(" None: # Value with invalid type a = VariantMap() with pytest.raises(TypeError): a["foo"] = 2 # Duplicate variant a["foo"] = MultiValuedVariant("foo", ("bar", "baz")) with pytest.raises(DuplicateVariantError): a["foo"] = MultiValuedVariant("foo", ("bar",)) with pytest.raises(DuplicateVariantError): a["foo"] = SingleValuedVariant("foo", "bar") with pytest.raises(DuplicateVariantError): a["foo"] = BoolValuedVariant("foo", True) # Non matching names between key and vspec.name with pytest.raises(KeyError): a["bar"] = MultiValuedVariant("foo", ("bar",)) def test_set_item(self) -> None: # Check that all the three types of variants are accepted a = VariantMap() a["foo"] = BoolValuedVariant("foo", True) a["bar"] = SingleValuedVariant("bar", "baz") a["foobar"] = MultiValuedVariant("foobar", ("a", "b", "c", "d", "e")) def test_substitute(self) -> None: # Check substitution of a key that exists a = VariantMap() a["foo"] = BoolValuedVariant("foo", True) a.substitute(SingleValuedVariant("foo", "bar")) # Trying to substitute something that is not # in the map will raise a KeyError with pytest.raises(KeyError): a.substitute(BoolValuedVariant("bar", True)) def test_satisfies_and_constrain(self) -> None: # foo=bar foobar=fee feebar=foo a = Spec() a.variants["foo"] = MultiValuedVariant("foo", ("bar",)) a.variants["foobar"] = SingleValuedVariant("foobar", "fee") a.variants["feebar"] = SingleValuedVariant("feebar", "foo") # foo=bar,baz foobar=fee shared=True b = Spec() b.variants["foo"] = MultiValuedVariant("foo", ("bar", "baz")) b.variants["foobar"] = SingleValuedVariant("foobar", "fee") b.variants["shared"] = BoolValuedVariant("shared", True) # concrete, different values do not intersect / satisfy each other assert not a.intersects(b) and not b.intersects(a) assert not a.satisfies(b) and not b.satisfies(a) # foo=bar,baz foobar=fee feebar=foo shared=True c = Spec() c.variants["foo"] = MultiValuedVariant("foo", ("bar", "baz")) c.variants["foobar"] = SingleValuedVariant("foobar", "fee") c.variants["feebar"] = SingleValuedVariant("feebar", "foo") c.variants["shared"] = BoolValuedVariant("shared", True) # concrete values cannot be constrained with pytest.raises(spack.variant.UnsatisfiableVariantSpecError): a._constrain_variants(b) def test_copy(self) -> None: a = VariantMap() a["foo"] = BoolValuedVariant("foo", True) a["bar"] = SingleValuedVariant("bar", "baz") a["foobar"] = MultiValuedVariant("foobar", ("a", "b", "c", "d", "e")) c = a.copy() assert a == c def test_str(self) -> None: c = VariantMap() c["foo"] = MultiValuedVariant("foo", ("bar", "baz")) c["foobar"] = SingleValuedVariant("foobar", "fee") c["feebar"] = SingleValuedVariant("feebar", "foo") c["shared"] = BoolValuedVariant("shared", True) assert str(c) == "+shared feebar=foo foo:=bar,baz foobar=fee" def test_disjoint_set_initialization_errors(): # Constructing from non-disjoint sets should raise an exception with pytest.raises(spack.error.SpecError) as exc_info: disjoint_sets(("a", "b"), ("b", "c")) assert "sets in input must be disjoint" in str(exc_info.value) # A set containing the reserved item 'none' along with other items # should raise an exception with pytest.raises(spack.error.SpecError) as exc_info: disjoint_sets(("a", "b"), ("none", "c")) assert "The value 'none' represents the empty set," in str(exc_info.value) def test_disjoint_set_initialization(): # Test that no error is thrown when the sets are disjoint d = disjoint_sets(("a",), ("b", "c"), ("e", "f")) assert d.default == "none" assert d.multi is True assert list(d) == ["none", "a", "b", "c", "e", "f"] def test_disjoint_set_fluent_methods(): # Construct an object without the empty set d = disjoint_sets(("a",), ("b", "c"), ("e", "f")).prohibit_empty_set() assert ("none",) not in d.sets # Call this 2 times to check that no matter whether # the empty set was allowed or not before, the state # returned is consistent. for _ in range(2): d = d.allow_empty_set() assert ("none",) in d.sets assert "none" in d assert "none" in [x for x in d] assert "none" in d.feature_values # Marking a value as 'non-feature' removes it from the # list of feature values, but not for the items returned # when iterating over the object. d = d.with_non_feature_values("none") assert "none" in d assert "none" in [x for x in d] assert "none" not in d.feature_values # Call this 2 times to check that no matter whether # the empty set was allowed or not before, the state # returned is consistent. for _ in range(2): d = d.prohibit_empty_set() assert ("none",) not in d.sets assert "none" not in d assert "none" not in [x for x in d] assert "none" not in d.feature_values @pytest.mark.regression("32694") @pytest.mark.parametrize("other", [True, False]) def test_conditional_value_comparable_to_bool(other): value = spack.variant.ConditionalValue("98", when=Spec("@1.0")) comparison = value == other assert comparison is False @pytest.mark.regression("40405") def test_wild_card_valued_variants_equivalent_to_str(): """ There was a bug prioro to PR 40406 in that variants with wildcard values "*" were being overwritten in the variant constructor. The expected/appropriate behavior is for it to behave like value=str and this test demonstrates that the two are now equivalent """ str_var = spack.variant.Variant( name="str_var", default="none", values=str, description="str variant", multi=True, validator=None, ) wild_var = spack.variant.Variant( name="wild_var", default="none", values="*", description="* variant", multi=True, validator=None, ) several_arbitrary_values = ("doe", "re", "mi") # "*" case wild_output = wild_var.make_variant(*several_arbitrary_values) wild_var.validate_or_raise(wild_output, "test-package") # str case str_output = str_var.make_variant(*several_arbitrary_values) str_var.validate_or_raise(str_output, "test-package") # equivalence each instance already validated assert str_output.value == wild_output.value def test_variant_definitions(mock_packages): pkg = spack.repo.PATH.get_pkg_class("variant-values") # two variant names assert len(pkg.variant_names()) == 2 assert "build_system" in pkg.variant_names() assert "v" in pkg.variant_names() # this name doesn't exist assert len(pkg.variant_definitions("no-such-variant")) == 0 # there are 4 definitions but one is completely shadowed by another assert len(pkg.variants) == 4 # variant_items ignores the shadowed definition assert len(list(pkg.variant_items())) == 3 # variant_definitions also ignores the shadowed definition defs = [vdef for _, vdef in pkg.variant_definitions("v")] assert len(defs) == 2 assert defs[0].default == "foo" assert defs[0].values == ("foo",) assert defs[1].default == "bar" assert defs[1].values == ("foo", "bar") @pytest.mark.parametrize( "pkg_name,value,spec,def_ids", [ ("variant-values", "foo", "", [0, 1]), ("variant-values", "bar", "", [1]), ("variant-values", "foo", "@1.0", [0]), ("variant-values", "foo", "@2.0", [1]), ("variant-values", "foo", "@3.0", [1]), ("variant-values", "foo", "@4.0", []), ("variant-values", "bar", "@2.0", [1]), ("variant-values", "bar", "@3.0", [1]), ("variant-values", "bar", "@4.0", []), # now with a global override ("variant-values-override", "bar", "", [0]), ("variant-values-override", "bar", "@1.0", [0]), ("variant-values-override", "bar", "@2.0", [0]), ("variant-values-override", "bar", "@3.0", [0]), ("variant-values-override", "bar", "@4.0", [0]), ("variant-values-override", "baz", "", [0]), ("variant-values-override", "baz", "@2.0", [0]), ("variant-values-override", "baz", "@3.0", [0]), ("variant-values-override", "baz", "@4.0", [0]), ], ) def test_prevalidate_variant_value(mock_packages, pkg_name, value, spec, def_ids): pkg = spack.repo.PATH.get_pkg_class(pkg_name) all_defs = [vdef for _, vdef in pkg.variant_definitions("v")] valid_defs = spack.variant.prevalidate_variant_value( pkg, SingleValuedVariant("v", value), spack.spec.Spec(spec) ) assert len(valid_defs) == len(def_ids) for vdef, i in zip(valid_defs, def_ids): assert vdef is all_defs[i] @pytest.mark.parametrize( "pkg_name,value,spec", [ ("variant-values", "baz", ""), ("variant-values", "bar", "@1.0"), ("variant-values", "bar", "@4.0"), ("variant-values", "baz", "@3.0"), ("variant-values", "baz", "@4.0"), # and with override ("variant-values-override", "foo", ""), ("variant-values-override", "foo", "@1.0"), ("variant-values-override", "foo", "@2.0"), ("variant-values-override", "foo", "@3.0"), ("variant-values-override", "foo", "@4.0"), ], ) def test_strict_invalid_variant_values(mock_packages, pkg_name, value, spec): pkg = spack.repo.PATH.get_pkg_class(pkg_name) with pytest.raises(spack.variant.InvalidVariantValueError): spack.variant.prevalidate_variant_value( pkg, SingleValuedVariant("v", value), spack.spec.Spec(spec), strict=True ) @pytest.mark.parametrize( "pkg_name,spec,satisfies,def_id", [ ("variant-values", "@1.0", "v=foo", 0), ("variant-values", "@2.0", "v=bar", 1), ("variant-values", "@3.0", "v=bar", 1), ("variant-values-override", "@1.0", "v=baz", 0), ("variant-values-override", "@2.0", "v=baz", 0), ("variant-values-override", "@3.0", "v=baz", 0), ], ) def test_concretize_variant_default_with_multiple_defs( mock_packages, config, pkg_name, spec, satisfies, def_id ): pkg = spack.repo.PATH.get_pkg_class(pkg_name) pkg_defs = [vdef for _, vdef in pkg.variant_definitions("v")] spec = spack.concretize.concretize_one(f"{pkg_name}{spec}") assert spec.satisfies(satisfies) assert spec.package.get_variant("v") is pkg_defs[def_id] @pytest.mark.parametrize( "spec,variant_name,narrowed_type", [ # dev_path is a special case ("foo dev_path=/path/to/source", "dev_path", spack.variant.VariantType.SINGLE), # reserved name: won't be touched ("foo patches=2349dc44", "patches", spack.variant.VariantType.MULTI), # simple case -- one definition applies ("variant-values@1.0 v=foo", "v", spack.variant.VariantType.SINGLE), # simple, but with bool valued variant ("pkg-a bvv=true", "bvv", spack.variant.VariantType.BOOL), # takes the second definition, which overrides the single-valued one ("variant-values@2.0 v=bar", "v", spack.variant.VariantType.MULTI), ], ) def test_substitute_abstract_variants_narrowing(mock_packages, spec, variant_name, narrowed_type): spec = Spec(spec) spack.spec.substitute_abstract_variants(spec) assert spec.variants[variant_name].type == narrowed_type def test_substitute_abstract_variants_failure(mock_packages): with pytest.raises(spack.spec.InvalidVariantForSpecError): # variant doesn't exist at version spack.spec.substitute_abstract_variants(Spec("variant-values@4.0 v=bar")) def test_abstract_variant_satisfies_abstract_abstract(): # rhs should be a subset of lhs assert Spec("foo=bar").satisfies("foo=bar") assert Spec("foo=bar,baz").satisfies("foo=bar") assert Spec("foo=bar,baz").satisfies("foo=bar,baz") assert not Spec("foo=bar").satisfies("foo=baz") assert not Spec("foo=bar").satisfies("foo=bar,baz") assert Spec("foo=bar").satisfies("foo=*") # rhs empty set assert Spec("foo=*").satisfies("foo=*") # lhs and rhs empty set assert not Spec("foo=*").satisfies("foo=bar") # lhs empty set, rhs not def test_abstract_variant_satisfies_concrete_abstract(): # rhs should be a subset of lhs assert Spec("foo:=bar").satisfies("foo=bar") assert Spec("foo:=bar,baz").satisfies("foo=bar") assert Spec("foo:=bar,baz").satisfies("foo=bar,baz") assert not Spec("foo:=bar").satisfies("foo=baz") assert not Spec("foo:=bar").satisfies("foo=bar,baz") assert Spec("foo:=bar").satisfies("foo=*") # rhs empty set def test_abstract_variant_satisfies_abstract_concrete(): # always false since values can be added to the lhs assert not Spec("foo=bar").satisfies("foo:=bar") assert not Spec("foo=bar,baz").satisfies("foo:=bar") assert not Spec("foo=bar,baz").satisfies("foo:=bar,baz") assert not Spec("foo=bar").satisfies("foo:=baz") assert not Spec("foo=bar").satisfies("foo:=bar,baz") assert not Spec("foo=*").satisfies("foo:=bar") # lhs empty set def test_abstract_variant_satisfies_concrete_concrete(): # concrete values only satisfy each other when equal assert Spec("foo:=bar").satisfies("foo:=bar") assert not Spec("foo:=bar,baz").satisfies("foo:=bar") assert not Spec("foo:=bar").satisfies("foo:=bar,baz") assert Spec("foo:=bar,baz").satisfies("foo:=bar,baz") def test_abstract_variant_intersects_abstract_abstract(): # always true since the union of values satisfies both assert Spec("foo=bar").intersects("foo=bar") assert Spec("foo=bar,baz").intersects("foo=bar") assert Spec("foo=bar,baz").intersects("foo=bar,baz") assert Spec("foo=bar").intersects("foo=baz") assert Spec("foo=bar").intersects("foo=bar,baz") assert Spec("foo=bar").intersects("foo=*") # rhs empty set assert Spec("foo=*").intersects("foo=*") # lhs and rhs empty set assert Spec("foo=*").intersects("foo=bar") # lhs empty set, rhs not def test_abstract_variant_intersects_concrete_abstract(): assert Spec("foo:=bar").intersects("foo=bar") assert Spec("foo:=bar,baz").intersects("foo=bar") assert Spec("foo:=bar,baz").intersects("foo=bar,baz") assert not Spec("foo:=bar").intersects("foo=baz") # rhs has at least baz, lhs has not assert not Spec("foo:=bar").intersects("foo=bar,baz") # rhs has at least baz, lhs has not assert Spec("foo:=bar").intersects("foo=*") # rhs empty set def test_abstract_variant_intersects_abstract_concrete(): assert Spec("foo=bar").intersects("foo:=bar") assert not Spec("foo=bar,baz").intersects("foo:=bar") # lhs has at least baz, rhs has not assert Spec("foo=bar,baz").intersects("foo:=bar,baz") assert not Spec("foo=bar").intersects("foo:=baz") # lhs has at least bar, rhs has not assert Spec("foo=bar").intersects("foo:=bar,baz") assert Spec("foo=*").intersects("foo:=bar") # lhs empty set def test_abstract_variant_intersects_concrete_concrete(): # concrete values only intersect each other when equal assert Spec("foo:=bar").intersects("foo:=bar") assert not Spec("foo:=bar,baz").intersects("foo:=bar") assert not Spec("foo:=bar").intersects("foo:=bar,baz") assert Spec("foo:=bar,baz").intersects("foo:=bar,baz") def test_abstract_variant_constrain_abstract_abstract(): s1 = Spec("foo=bar") s2 = Spec("foo=*") assert s1.constrain("foo=baz") assert s1 == Spec("foo=bar,baz") assert s2.constrain("foo=baz") assert s2 == Spec("foo=baz") def test_abstract_variant_constrain_abstract_concrete_fail(): with pytest.raises(UnsatisfiableVariantSpecError): Spec("foo=bar").constrain("foo:=baz") def test_abstract_variant_constrain_abstract_concrete_ok(): s1 = Spec("foo=bar") s2 = Spec("foo=*") assert s1.constrain("foo:=bar") # the change is concreteness assert s1 == Spec("foo:=bar") assert s2.constrain("foo:=bar") assert s2 == Spec("foo:=bar") def test_abstract_variant_constrain_concrete_concrete_fail(): with pytest.raises(UnsatisfiableVariantSpecError): Spec("foo:=bar").constrain("foo:=bar,baz") def test_abstract_variant_constrain_concrete_concrete_ok(): s = Spec("foo:=bar") assert not s.constrain("foo:=bar") # no change def test_abstract_variant_constrain_concrete_abstract_fail(): s = Spec("foo:=bar") with pytest.raises(UnsatisfiableVariantSpecError): s.constrain("foo=baz") def test_abstract_variant_constrain_concrete_abstract_ok(): s = Spec("foo:=bar,baz") assert not s.constrain("foo=bar") # no change in value or concreteness assert not s.constrain("foo=*") def test_patches_variant(): """patches=x,y,z is a variant with special satisfies behavior when the rhs is abstract; it allows string prefix matching of the lhs.""" assert Spec("patches:=abcdef").satisfies("patches=ab") assert Spec("patches:=abcdef").satisfies("patches=abcdef") assert not Spec("patches:=abcdef").satisfies("patches=xyz") assert Spec("patches:=abcdef,xyz").satisfies("patches=xyz") assert not Spec("patches:=abcdef").satisfies("patches=abcdefghi") # but when the rhs is concrete, it must match exactly assert Spec("patches:=abcdef").satisfies("patches:=abcdef") assert not Spec("patches:=abcdef").satisfies("patches:=ab") assert not Spec("patches:=abcdef,xyz").satisfies("patches:=abc,xyz") assert not Spec("patches:=abcdef").satisfies("patches:=abcdefghi") def test_constrain_narrowing(): s = Spec("foo=*") assert s.variants["foo"].type == spack.variant.VariantType.MULTI assert not s.variants["foo"].concrete s.constrain("+foo") assert s.variants["foo"].type == spack.variant.VariantType.BOOL assert s.variants["foo"].concrete ================================================ FILE: lib/spack/spack/test/verification.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Tests for the `spack.verify` module""" import os import pathlib import shutil import stat import pytest import spack.llnl.util.filesystem as fs import spack.spec import spack.store import spack.util.spack_json as sjson import spack.verify from spack.llnl.util.filesystem import symlink pytestmark = pytest.mark.not_on_windows("Tests fail on Win") def test_link_manifest_entry(tmp_path: pathlib.Path): # Test that symlinks are properly checked against the manifest. # Test that the appropriate errors are generated when the check fails. file = tmp_path / "file" file.touch() link = tmp_path / "link" os.symlink(str(file), str(link)) data = spack.verify.create_manifest_entry(str(link)) assert data["dest"] == str(file) assert all(x in data for x in ("mode", "owner", "group")) results = spack.verify.check_entry(str(link), data) assert not results.has_errors() file2 = tmp_path / "file2" file2.touch() os.remove(str(link)) os.symlink(str(file2), str(link)) results = spack.verify.check_entry(str(link), data) assert results.has_errors() assert str(link) in results.errors assert results.errors[str(link)] == ["link"] def test_dir_manifest_entry(tmp_path: pathlib.Path): # Test that directories are properly checked against the manifest. # Test that the appropriate errors are generated when the check fails. dirent = str(tmp_path / "dir") fs.mkdirp(dirent) data = spack.verify.create_manifest_entry(dirent) assert stat.S_ISDIR(data["mode"]) assert all(x in data for x in ("mode", "owner", "group")) results = spack.verify.check_entry(dirent, data) assert not results.has_errors() data["mode"] = "garbage" results = spack.verify.check_entry(dirent, data) assert results.has_errors() assert dirent in results.errors assert results.errors[dirent] == ["mode"] def test_file_manifest_entry(tmp_path: pathlib.Path): # Test that files are properly checked against the manifest. # Test that the appropriate errors are generated when the check fails. orig_str = "This is a file" new_str = "The file has changed" file = str(tmp_path / "dir") with open(file, "w", encoding="utf-8") as f: f.write(orig_str) data = spack.verify.create_manifest_entry(file) assert stat.S_ISREG(data["mode"]) assert data["size"] == len(orig_str) assert all(x in data for x in ("owner", "group")) results = spack.verify.check_entry(file, data) assert not results.has_errors() data["mode"] = 0x99999 results = spack.verify.check_entry(file, data) assert results.has_errors() assert file in results.errors assert results.errors[file] == ["mode"] with open(file, "w", encoding="utf-8") as f: f.write(new_str) data["mode"] = os.stat(file).st_mode results = spack.verify.check_entry(file, data) expected = ["size", "hash"] mtime = os.stat(file).st_mtime if mtime != data["time"]: expected.append("mtime") assert results.has_errors() assert file in results.errors assert sorted(results.errors[file]) == sorted(expected) def test_check_chmod_manifest_entry(tmp_path: pathlib.Path): # Check that the verification properly identifies errors for files whose # permissions have been modified. file = str(tmp_path / "dir") with open(file, "w", encoding="utf-8") as f: f.write("This is a file") data = spack.verify.create_manifest_entry(file) os.chmod(file, data["mode"] - 1) results = spack.verify.check_entry(file, data) assert results.has_errors() assert file in results.errors assert results.errors[file] == ["mode"] def test_check_prefix_manifest(tmp_path: pathlib.Path): # Test the verification of an entire prefix and its contents prefix_path = tmp_path / "prefix" prefix = str(prefix_path) spec = spack.spec.Spec("libelf") spec._mark_concrete() spec.set_prefix(prefix) results = spack.verify.check_spec_manifest(spec) assert results.has_errors() assert prefix in results.errors assert results.errors[prefix] == ["manifest missing"] metadata_dir = str(prefix_path / ".spack") bin_dir = str(prefix_path / "bin") other_dir = str(prefix_path / "other") for d in (metadata_dir, bin_dir, other_dir): fs.mkdirp(d) file = os.path.join(other_dir, "file") with open(file, "w", encoding="utf-8") as f: f.write("I'm a little file short and stout") link = os.path.join(bin_dir, "run") symlink(file, link) spack.verify.write_manifest(spec) results = spack.verify.check_spec_manifest(spec) assert not results.has_errors() os.remove(link) malware = os.path.join(metadata_dir, "hiddenmalware") with open(malware, "w", encoding="utf-8") as f: f.write("Foul evil deeds") results = spack.verify.check_spec_manifest(spec) assert results.has_errors() assert all(x in results.errors for x in (malware, link)) assert len(results.errors) == 2 assert results.errors[link] == ["deleted"] assert results.errors[malware] == ["added"] manifest_file = os.path.join( spec.prefix, spack.store.STORE.layout.metadata_dir, spack.store.STORE.layout.manifest_file_name, ) with open(manifest_file, "w", encoding="utf-8") as f: f.write("{This) string is not proper json") results = spack.verify.check_spec_manifest(spec) assert results.has_errors() assert results.errors[spec.prefix] == ["manifest corrupted"] def test_single_file_verification(tmp_path: pathlib.Path): # Test the API to verify a single file, including finding the package # to which it belongs filedir = tmp_path / "a" / "b" / "c" / "d" filepath = filedir / "file" metadir = tmp_path / spack.store.STORE.layout.metadata_dir fs.mkdirp(str(filedir)) fs.mkdirp(str(metadir)) with open(str(filepath), "w", encoding="utf-8") as f: f.write("I'm a file") data = spack.verify.create_manifest_entry(str(filepath)) manifest_file = os.path.join(metadir, spack.store.STORE.layout.manifest_file_name) with open(manifest_file, "w", encoding="utf-8") as f: sjson.dump({str(filepath): data}, f) results = spack.verify.check_file_manifest(str(filepath)) assert not results.has_errors() os.utime(str(filepath), (0, 0)) with open(str(filepath), "w", encoding="utf-8") as f: f.write("I changed.") results = spack.verify.check_file_manifest(str(filepath)) expected = ["hash"] mtime = os.stat(str(filepath)).st_mtime if mtime != data["time"]: expected.append("mtime") assert results.has_errors() assert str(filepath) in results.errors assert sorted(results.errors[str(filepath)]) == sorted(expected) shutil.rmtree(str(metadir)) results = spack.verify.check_file_manifest(filepath) assert results.has_errors() assert results.errors[filepath] == ["not owned by any package"] ================================================ FILE: lib/spack/spack/test/versions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """These version tests were taken from the RPM source code. We try to maintain compatibility with RPM's version semantics where it makes sense. """ import os import pathlib import pytest import spack.concretize import spack.package_base import spack.spec import spack.version from spack.llnl.util.filesystem import working_dir from spack.version import ( ClosedOpenRange, EmptyRangeError, GitVersion, StandardVersion, Version, VersionList, VersionLookupError, VersionRange, is_git_version, ver, ) from spack.version.git_ref_lookup import SEMVER_REGEX def assert_ver_lt(a, b): """Asserts the results of comparisons when 'a' is less than 'b'.""" a, b = ver(a), ver(b) assert a < b assert a <= b assert a != b assert not a == b assert not a > b assert not a >= b def assert_ver_gt(a, b): """Asserts the results of comparisons when 'a' is greater than 'b'.""" a, b = ver(a), ver(b) assert a > b assert a >= b assert a != b assert not a == b assert not a < b assert not a <= b def assert_ver_eq(a, b): """Asserts the results of comparisons when 'a' is equal to 'b'.""" a, b = ver(a), ver(b) assert not a > b assert a >= b assert not a != b assert a == b assert not a < b assert a <= b def assert_in(needle, haystack): """Asserts that 'needle' is in 'haystack'.""" assert ver(needle) in ver(haystack) def assert_not_in(needle, haystack): """Asserts that 'needle' is not in 'haystack'.""" assert ver(needle) not in ver(haystack) def assert_canonical(canonical_list, version_list): """Asserts that a redundant list is reduced to canonical form.""" assert ver(canonical_list) == ver(version_list) def assert_overlaps(v1, v2): """Asserts that two version ranges overlaps.""" assert ver(v1).overlaps(ver(v2)) def assert_no_overlap(v1, v2): """Asserts that two version ranges do not overlap.""" assert not ver(v1).overlaps(ver(v2)) def assert_satisfies(v1, v2): """Asserts that 'v1' satisfies 'v2'.""" assert ver(v1).satisfies(ver(v2)) def assert_does_not_satisfy(v1, v2): """Asserts that 'v1' does not satisfy 'v2'.""" assert not ver(v1).satisfies(ver(v2)) def check_intersection(expected, a, b): """Asserts that 'a' intersect 'b' == 'expected'.""" assert ver(expected) == ver(a).intersection(ver(b)) def check_union(expected, a, b): """Asserts that 'a' union 'b' == 'expected'.""" assert ver(expected) == ver(a).union(ver(b)) def test_string_prefix(): assert_ver_eq("=xsdk-0.2.0", "=xsdk-0.2.0") assert_ver_lt("=xsdk-0.2.0", "=xsdk-0.3") assert_ver_gt("=xsdk-0.3", "=xsdk-0.2.0") def test_two_segments(): assert_ver_eq("=1.0", "=1.0") assert_ver_lt("=1.0", "=2.0") assert_ver_gt("=2.0", "=1.0") def test_develop(): assert_ver_eq("=develop", "=develop") assert_ver_eq("=develop.local", "=develop.local") assert_ver_lt("=1.0", "=develop") assert_ver_gt("=develop", "=1.0") assert_ver_eq("=1.develop", "=1.develop") assert_ver_lt("=1.1", "=1.develop") assert_ver_gt("=1.develop", "=1.0") assert_ver_gt("=0.5.develop", "=0.5") assert_ver_lt("=0.5", "=0.5.develop") assert_ver_lt("=1.develop", "=2.1") assert_ver_gt("=2.1", "=1.develop") assert_ver_lt("=1.develop.1", "=1.develop.2") assert_ver_gt("=1.develop.2", "=1.develop.1") assert_ver_lt("=develop.1", "=develop.2") assert_ver_gt("=develop.2", "=develop.1") # other +infinity versions assert_ver_gt("=master", "=9.0") assert_ver_gt("=head", "=9.0") assert_ver_gt("=trunk", "=9.0") assert_ver_gt("=develop", "=9.0") # hierarchical develop-like versions assert_ver_gt("=develop", "=master") assert_ver_gt("=master", "=head") assert_ver_gt("=head", "=trunk") assert_ver_gt("=9.0", "=system") # not develop assert_ver_lt("=mydevelopmentnightmare", "=1.1") assert_ver_lt("=1.mydevelopmentnightmare", "=1.1") assert_ver_gt("=1.1", "=1.mydevelopmentnightmare") def test_isdevelop(): assert ver("=develop").isdevelop() assert ver("=develop.1").isdevelop() assert ver("=develop.local").isdevelop() assert ver("=master").isdevelop() assert ver("=head").isdevelop() assert ver("=trunk").isdevelop() assert ver("=1.develop").isdevelop() assert ver("=1.develop.2").isdevelop() assert not ver("=1.1").isdevelop() assert not ver("=1.mydevelopmentnightmare.3").isdevelop() assert not ver("=mydevelopmentnightmare.3").isdevelop() def test_three_segments(): assert_ver_eq("=2.0.1", "=2.0.1") assert_ver_lt("=2.0", "=2.0.1") assert_ver_gt("=2.0.1", "=2.0") def test_alpha(): # TODO: not sure whether I like this. 2.0.1a is *usually* # TODO: less than 2.0.1, but special-casing it makes version # TODO: comparison complicated. See version.py assert_ver_eq("=2.0.1a", "=2.0.1a") assert_ver_gt("=2.0.1a", "=2.0.1") assert_ver_lt("=2.0.1", "=2.0.1a") def test_patch(): assert_ver_eq("=5.5p1", "=5.5p1") assert_ver_lt("=5.5p1", "=5.5p2") assert_ver_gt("=5.5p2", "=5.5p1") assert_ver_eq("=5.5p10", "=5.5p10") assert_ver_lt("=5.5p1", "=5.5p10") assert_ver_gt("=5.5p10", "=5.5p1") def test_num_alpha_with_no_separator(): assert_ver_lt("=10xyz", "=10.1xyz") assert_ver_gt("=10.1xyz", "=10xyz") assert_ver_eq("=xyz10", "=xyz10") assert_ver_lt("=xyz10", "=xyz10.1") assert_ver_gt("=xyz10.1", "=xyz10") def test_alpha_with_dots(): assert_ver_eq("=xyz.4", "=xyz.4") assert_ver_lt("=xyz.4", "=8") assert_ver_gt("=8", "=xyz.4") assert_ver_lt("=xyz.4", "=2") assert_ver_gt("=2", "=xyz.4") def test_nums_and_patch(): assert_ver_lt("=5.5p2", "=5.6p1") assert_ver_gt("=5.6p1", "=5.5p2") assert_ver_lt("=5.6p1", "=6.5p1") assert_ver_gt("=6.5p1", "=5.6p1") def test_prereleases(): # pre-releases are special: they are less than final releases assert_ver_lt("=6.0alpha", "=6.0alpha0") assert_ver_lt("=6.0alpha0", "=6.0alpha1") assert_ver_lt("=6.0alpha1", "=6.0alpha2") assert_ver_lt("=6.0alpha2", "=6.0beta") assert_ver_lt("=6.0beta", "=6.0beta0") assert_ver_lt("=6.0beta0", "=6.0beta1") assert_ver_lt("=6.0beta1", "=6.0beta2") assert_ver_lt("=6.0beta2", "=6.0rc") assert_ver_lt("=6.0rc", "=6.0rc0") assert_ver_lt("=6.0rc0", "=6.0rc1") assert_ver_lt("=6.0rc1", "=6.0rc2") assert_ver_lt("=6.0rc2", "=6.0") def test_alpha_beta(): # these are not pre-releases, but ordinary string components. assert_ver_gt("=10b2", "=10a1") assert_ver_lt("=10a2", "=10b2") def test_double_alpha(): assert_ver_eq("=1.0aa", "=1.0aa") assert_ver_lt("=1.0a", "=1.0aa") assert_ver_gt("=1.0aa", "=1.0a") def test_padded_numbers(): assert_ver_eq("=10.0001", "=10.0001") assert_ver_eq("=10.0001", "=10.1") assert_ver_eq("=10.1", "=10.0001") assert_ver_lt("=10.0001", "=10.0039") assert_ver_gt("=10.0039", "=10.0001") def test_close_numbers(): assert_ver_lt("=4.999.9", "=5.0") assert_ver_gt("=5.0", "=4.999.9") def test_date_stamps(): assert_ver_eq("=20101121", "=20101121") assert_ver_lt("=20101121", "=20101122") assert_ver_gt("=20101122", "=20101121") def test_underscores(): assert_ver_eq("=2_0", "=2_0") assert_ver_eq("=2.0", "=2_0") assert_ver_eq("=2_0", "=2.0") assert_ver_eq("=2-0", "=2_0") assert_ver_eq("=2_0", "=2-0") def test_rpm_oddities(): assert_ver_eq("=1b.fc17", "=1b.fc17") assert_ver_lt("=1b.fc17", "=1.fc17") assert_ver_gt("=1.fc17", "=1b.fc17") assert_ver_eq("=1g.fc17", "=1g.fc17") assert_ver_gt("=1g.fc17", "=1.fc17") assert_ver_lt("=1.fc17", "=1g.fc17") # Stuff below here is not taken from RPM's tests and is # unique to spack def test_version_ranges(): assert_ver_lt("1.2:1.4", "1.6") assert_ver_gt("1.6", "1.2:1.4") assert_ver_eq("1.2:1.4", "1.2:1.4") assert ver("1.2:1.4") != ver("1.2:1.6") assert_ver_lt("1.2:1.4", "1.5:1.6") assert_ver_gt("1.5:1.6", "1.2:1.4") def test_version_range_with_prereleases(): # 1.2.1: means from the 1.2.1 release onwards assert_does_not_satisfy("1.2.1alpha1", "1.2.1:") assert_does_not_satisfy("1.2.1beta2", "1.2.1:") assert_does_not_satisfy("1.2.1rc3", "1.2.1:") # Pre-releases of 1.2.1 are included in the 1.2.0: range assert_satisfies("1.2.1alpha1", "1.2.0:") assert_satisfies("1.2.1beta1", "1.2.0:") assert_satisfies("1.2.1rc3", "1.2.0:") # In Spack 1.2 and 1.2.0 are distinct with 1.2 < 1.2.0. So a lowerbound on 1.2 includes # pre-releases of 1.2.0 as well. assert_satisfies("1.2.0alpha1", "1.2:") assert_satisfies("1.2.0beta2", "1.2:") assert_satisfies("1.2.0rc3", "1.2:") # An upperbound :1.1 does not include 1.2.0 pre-releases assert_does_not_satisfy("1.2.0alpha1", ":1.1") assert_does_not_satisfy("1.2.0beta2", ":1.1") assert_does_not_satisfy("1.2.0rc3", ":1.1") assert_satisfies("1.2.0alpha1", ":1.2") assert_satisfies("1.2.0beta2", ":1.2") assert_satisfies("1.2.0rc3", ":1.2") # You can also construct ranges from prereleases assert_satisfies("1.2.0alpha2:1.2.0beta1", "1.2.0alpha1:1.2.0beta2") assert_satisfies("1.2.0", "1.2.0alpha1:") assert_satisfies("=1.2.0", "1.2.0alpha1:") assert_does_not_satisfy("=1.2.0", ":1.2.0rc345") def test_contains(): assert_in("=1.3", "1.2:1.4") assert_in("=1.2.5", "1.2:1.4") assert_in("=1.3.5", "1.2:1.4") assert_in("=1.3.5-7", "1.2:1.4") assert_not_in("=1.1", "1.2:1.4") assert_not_in("=1.5", "1.2:1.4") assert_not_in("=1.5", "1.5.1:1.6") assert_not_in("=1.5", "1.5.1:") assert_in("=1.4.2", "1.2:1.4") assert_not_in("=1.4.2", "1.2:1.4.0") assert_in("=1.2.8", "1.2.7:1.4") assert_in("1.2.7:1.4", ":") assert_not_in("=1.2.5", "1.2.7:1.4") assert_in("=1.4.1", "1.2.7:1.4") assert_not_in("=1.4.1", "1.2.7:1.4.0") def test_in_list(): assert_in("1.2", ["1.5", "1.2", "1.3"]) assert_in("1.2.5", ["1.5", "1.2:1.3"]) assert_in("1.5", ["1.5", "1.2:1.3"]) assert_not_in("1.4", ["1.5", "1.2:1.3"]) assert_in("1.2.5:1.2.7", [":"]) assert_in("1.2.5:1.2.7", ["1.5", "1.2:1.3"]) assert_not_in("1.2.5:1.5", ["1.5", "1.2:1.3"]) assert_not_in("1.1:1.2.5", ["1.5", "1.2:1.3"]) def test_ranges_overlap(): assert_overlaps("1.2", "1.2") assert_overlaps("1.2.1", "1.2.1") assert_overlaps("1.2.1b", "1.2.1b") assert_overlaps("1.2:1.7", "1.6:1.9") assert_overlaps(":1.7", "1.6:1.9") assert_overlaps(":1.7", ":1.9") assert_overlaps(":1.7", "1.6:") assert_overlaps("1.2:", "1.6:1.9") assert_overlaps("1.2:", ":1.9") assert_overlaps("1.2:", "1.6:") assert_overlaps(":", ":") assert_overlaps(":", "1.6:1.9") assert_overlaps("1.6:1.9", ":") def test_overlap_with_containment(): assert_in("1.6.5", "1.6") assert_in("1.6.5", ":1.6") assert_overlaps("1.6.5", ":1.6") assert_overlaps(":1.6", "1.6.5") assert_not_in(":1.6", "1.6.5") assert_in("1.6.5", ":1.6") def test_lists_overlap(): assert_overlaps("1.2b:1.7,5", "1.6:1.9,1") assert_overlaps("1,2,3,4,5", "3,4,5,6,7") assert_overlaps("1,2,3,4,5", "5,6,7") assert_overlaps("1,2,3,4,5", "5:7") assert_overlaps("1,2,3,4,5", "3, 6:7") assert_overlaps("1, 2, 4, 6.5", "3, 6:7") assert_overlaps("1, 2, 4, 6.5", ":, 5, 8") assert_overlaps("1, 2, 4, 6.5", ":") assert_no_overlap("1, 2, 4", "3, 6:7") assert_no_overlap("1,2,3,4,5", "6,7") assert_no_overlap("1,2,3,4,5", "6:7") def test_canonicalize_list(): assert_canonical(["1.2", "1.3", "1.4"], ["1.2", "1.3", "1.3", "1.4"]) assert_canonical(["1.2", "1.3:1.4"], ["1.2", "1.3", "1.3:1.4"]) assert_canonical(["1.2", "1.3:1.4"], ["1.2", "1.3:1.4", "1.4"]) assert_canonical(["1.3:1.4"], ["1.3:1.4", "1.3", "1.3.1", "1.3.9", "1.4"]) assert_canonical(["1.3:1.4"], ["1.3", "1.3.1", "1.3.9", "1.4", "1.3:1.4"]) assert_canonical(["1.3:1.5"], ["1.3", "1.3.1", "1.3.9", "1.4:1.5", "1.3:1.4"]) assert_canonical(["1.3:1.5"], ["1.3, 1.3.1,1.3.9,1.4:1.5,1.3:1.4"]) assert_canonical(["1.3:1.5"], ["1.3, 1.3.1,1.3.9,1.4 : 1.5 , 1.3 : 1.4"]) assert_canonical([":"], [":,1.3, 1.3.1,1.3.9,1.4 : 1.5 , 1.3 : 1.4"]) def test_intersection(): check_intersection("2.5", "1.0:2.5", "2.5:3.0") check_intersection("2.5:2.7", "1.0:2.7", "2.5:3.0") check_intersection("0:1", ":", "0:1") check_intersection(["1.0", "2.5:2.7"], ["1.0:2.7"], ["2.5:3.0", "1.0"]) check_intersection(["2.5:2.7"], ["1.1:2.7"], ["2.5:3.0", "1.0"]) check_intersection(["0:1"], [":"], ["0:1"]) check_intersection(["=ref=1.0", "=1.1"], ["=ref=1.0", "1.1"], ["1:1.0", "=1.1"]) def test_intersect_with_containment(): check_intersection("1.6.5", "1.6.5", ":1.6") check_intersection("1.6.5", ":1.6", "1.6.5") check_intersection("1.6:1.6.5", ":1.6.5", "1.6") check_intersection("1.6:1.6.5", "1.6", ":1.6.5") check_intersection("11.2", "11", "11.2") check_intersection("11.2", "11.2", "11") def test_union_with_containment(): check_union(":1.6", "1.6.5", ":1.6") check_union(":1.6", ":1.6", "1.6.5") check_union(":1.6", ":1.6.5", "1.6") check_union(":1.6", "1.6", ":1.6.5") check_union(":", "1.0:", ":2.0") check_union("1:4", "1:3", "2:4") check_union("1:4", "2:4", "1:3") # Tests successor/predecessor case. check_union("1:4", "1:2", "3:4") check_union(["1:1.0", "1.1"], ["=ref=1.0", "1.1"], ["1:1.0", "=1.1"]) def test_basic_version_satisfaction(): assert_satisfies("4.7.3", "4.7.3") assert_satisfies("4.7.3", "4.7") assert_satisfies("4.7.3v2", "4.7") assert_satisfies("4.7v6", "4.7") assert_satisfies("4.7.3", "4") assert_satisfies("4.7.3v2", "4") assert_satisfies("4.7v6", "4") assert_does_not_satisfy("4.8.0", "4.9") assert_does_not_satisfy("4.8", "4.9") assert_does_not_satisfy("4", "4.9") def test_basic_version_satisfaction_in_lists(): assert_satisfies(["4.7.3"], ["4.7.3"]) assert_satisfies(["4.7.3"], ["4.7"]) assert_satisfies(["4.7.3v2"], ["4.7"]) assert_satisfies(["4.7v6"], ["4.7"]) assert_satisfies(["4.7.3"], ["4"]) assert_satisfies(["4.7.3v2"], ["4"]) assert_satisfies(["4.7v6"], ["4"]) assert_does_not_satisfy(["4.8.0"], ["4.9"]) assert_does_not_satisfy(["4.8"], ["4.9"]) assert_does_not_satisfy(["4"], ["4.9"]) def test_version_range_satisfaction(): assert_satisfies("4.7b6", "4.3:4.7") assert_satisfies("4.3.0", "4.3:4.7") assert_satisfies("4.3.2", "4.3:4.7") assert_does_not_satisfy("4.8.0", "4.3:4.7") assert_does_not_satisfy("4.3", "4.4:4.7") assert_satisfies("4.7b6", "4.3:4.7") assert_does_not_satisfy("4.8.0", "4.3:4.7") def test_version_range_satisfaction_in_lists(): assert_satisfies(["4.7b6"], ["4.3:4.7"]) assert_satisfies(["4.3.0"], ["4.3:4.7"]) assert_satisfies(["4.3.2"], ["4.3:4.7"]) assert_does_not_satisfy(["4.8.0"], ["4.3:4.7"]) assert_does_not_satisfy(["4.3"], ["4.4:4.7"]) assert_satisfies(["4.7b6"], ["4.3:4.7"]) assert_does_not_satisfy(["4.8.0"], ["4.3:4.7"]) def test_satisfaction_with_lists(): assert_satisfies("4.7", "4.3, 4.6, 4.7") assert_satisfies("4.7.3", "4.3, 4.6, 4.7") assert_satisfies("4.6.5", "4.3, 4.6, 4.7") assert_satisfies("4.6.5.2", "4.3, 4.6, 4.7") assert_does_not_satisfy("4", "4.3, 4.6, 4.7") assert_does_not_satisfy("4.8.0", "4.2, 4.3:4.7") assert_satisfies("4.8.0", "4.2, 4.3:4.8") assert_satisfies("4.8.2", "4.2, 4.3:4.8") def test_formatted_strings(): versions = ( "1.2.3b", "1_2_3b", "1-2-3b", "1.2-3b", "1.2_3b", "1-2.3b", "1-2_3b", "1_2.3b", "1_2-3b", ) for item in versions: v = Version(item) assert v.dotted.string == "1.2.3b" assert v.dashed.string == "1-2-3b" assert v.underscored.string == "1_2_3b" assert v.joined.string == "123b" assert v.dotted.dashed.string == "1-2-3b" assert v.dotted.underscored.string == "1_2_3b" assert v.dotted.dotted.string == "1.2.3b" assert v.dotted.joined.string == "123b" def test_dotted_numeric_string(): assert Version("1a2b3").dotted_numeric_string == "1.0.2.0.3" assert Version("1a2b3alpha4").dotted_numeric_string == "1.0.2.0.3.0.4" def test_up_to(): v = Version("1.23-4_5b") assert v.up_to(1).string == "1" assert v.up_to(2).string == "1.23" assert v.up_to(3).string == "1.23-4" assert v.up_to(4).string == "1.23-4_5" assert v.up_to(5).string == "1.23-4_5b" assert v.up_to(-1).string == "1.23-4_5" assert v.up_to(-2).string == "1.23-4" assert v.up_to(-3).string == "1.23" assert v.up_to(-4).string == "1" assert v.up_to(2).dotted.string == "1.23" assert v.up_to(2).dashed.string == "1-23" assert v.up_to(2).underscored.string == "1_23" assert v.up_to(2).joined.string == "123" assert v.dotted.up_to(2).string == "1.23" == v.up_to(2).dotted.string assert v.dashed.up_to(2).string == "1-23" == v.up_to(2).dashed.string assert v.underscored.up_to(2).string == "1_23" assert v.up_to(2).underscored.string == "1_23" assert v.up_to(2).up_to(1).string == "1" def test_repr_and_str(): def check_repr_and_str(vrs): a = Version(vrs) assert repr(a) == f'Version("{vrs}")' b = eval(repr(a)) assert a == b assert str(a) == vrs assert str(a) == str(b) check_repr_and_str("1.2.3") check_repr_and_str("R2016a") check_repr_and_str("R2016a.2-3_4") def test_str_and_hash_version_range(): """Test that precomputed string and hash values are consistent with computed ones.""" x = ver("1.2:3.4") assert isinstance(x, ClosedOpenRange) # Test that precomputed str() and hash() are assigned assert x._string is not None and x._hash is not None _str = str(x) _hash = hash(x) assert x._string == _str and x._hash == _hash # Ensure computed values match precomputed ones x._string = None x._hash = None assert _str == str(x) assert _hash == hash(x) @pytest.mark.parametrize( "version_str", ["1.2string3", "1.2-3xyz_4-alpha.5", "1.2beta", "1_x_rc-4"] ) def test_stringify_version(version_str): v = Version(version_str) v.string = None assert str(v) == version_str v.string = None assert v.string == version_str def test_len(): a = Version("1.2.3.4") assert len(a) == len(a.version[0]) assert len(a) == 4 b = Version("2018.0") assert len(b) == 2 def test_get_item(): a = Version("0.1_2-3") assert isinstance(a[1], int) # Test slicing b = a[0:2] assert isinstance(b, StandardVersion) assert b == Version("0.1") assert repr(b) == 'Version("0.1")' assert str(b) == "0.1" b = a[0:3] assert isinstance(b, StandardVersion) assert b == Version("0.1_2") assert repr(b) == 'Version("0.1_2")' assert str(b) == "0.1_2" b = a[1:] assert isinstance(b, StandardVersion) assert b == Version("1_2-3") assert repr(b) == 'Version("1_2-3")' assert str(b) == "1_2-3" # Raise TypeError on tuples with pytest.raises(TypeError): b.__getitem__(1, 2) def test_list_highest(): vl = VersionList(["=master", "=1.2.3", "=develop", "=3.4.5", "=foobar"]) assert vl.highest() == Version("develop") assert vl.lowest() == Version("foobar") assert vl.highest_numeric() == Version("3.4.5") vl2 = VersionList(["=master", "=develop"]) assert vl2.highest_numeric() is None assert vl2.preferred() == Version("develop") assert vl2.lowest() == Version("master") @pytest.mark.parametrize("version_str", ["foo 1.2.0", "!", "1!2", "=1.2.0"]) def test_invalid_versions(version_str): """Ensure invalid versions are rejected with a ValueError""" with pytest.raises(ValueError): Version(version_str) def test_versions_from_git(git, mock_git_version_info, monkeypatch, mock_packages): repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) for commit in commits: spec = spack.spec.Spec("git-test-commit@%s" % commit) version: GitVersion = spec.version comparator = [str(v) if not isinstance(v, int) else v for v in version.ref_version] with working_dir(repo_path): git("checkout", commit) with open(os.path.join(repo_path, filename), "r", encoding="utf-8") as f: expected = f.read() assert str(comparator) == expected @pytest.mark.parametrize( "commit_idx,expected_satisfies,expected_not_satisfies", [ # Spec based on earliest commit (-1, ("@:0",), ("@1.0",)), # Spec based on second commit (same as version 1.0) (-2, ("@1.0",), ("@1.1:",)), # Spec based on 4th commit (in timestamp order) (-4, ("@1.1", "@1.0:1.2"), tuple()), ], ) def test_git_hash_comparisons( mock_git_version_info, install_mockery, mock_packages, monkeypatch, commit_idx, expected_satisfies, expected_not_satisfies, ): """Check that hashes compare properly to versions""" repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) spec = spack.concretize.concretize_one( spack.spec.Spec(f"git-test-commit@{commits[commit_idx]}") ) for item in expected_satisfies: assert spec.satisfies(item) for item in expected_not_satisfies: assert not spec.satisfies(item) def test_git_ref_comparisons(mock_git_version_info, install_mockery, mock_packages, monkeypatch): """Check that hashes compare properly to versions""" repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) # Spec based on tag v1.0 spec_tag = spack.concretize.concretize_one("git-test-commit@git.v1.0") assert spec_tag.satisfies("@1.0") assert not spec_tag.satisfies("@1.1:") assert str(spec_tag.version) == "git.v1.0=1.0" # Spec based on branch 1.x spec_branch = spack.concretize.concretize_one("git-test-commit@git.1.x") assert spec_branch.satisfies("@1.2") assert spec_branch.satisfies("@1.1:1.3") assert str(spec_branch.version) == "git.1.x=1.2" def test_git_branch_with_slash(): class MockLookup(object): def get(self, ref): assert ref == "feature/bar" return "1.2", 0 v = spack.version.from_string("git.feature/bar") assert isinstance(v, GitVersion) v.attach_lookup(MockLookup()) # Create a version range test_number_version = spack.version.from_string("1.2") v.satisfies(test_number_version) serialized = VersionList([v]).to_dict() v_deserialized = VersionList.from_dict(serialized) assert v_deserialized[0].ref == "feature/bar" @pytest.mark.parametrize( "string,git", [ ("1.2.9", False), ("gitmain", False), ("git.foo", True), ("git.abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd", True), ("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd", True), ], ) def test_version_git_vs_base(string, git): assert is_git_version(string) == git assert isinstance(Version(string), GitVersion) == git def test_version_range_nonempty(): assert Version("1.2.9") in VersionRange("1.2.0", "1.2") assert Version("1.1.1") in ver("1.0:1") def test_empty_version_range_raises(): with pytest.raises(EmptyRangeError, match="2:1.0 is an empty range"): assert VersionRange("2", "1.0") with pytest.raises(EmptyRangeError, match="2:1.0 is an empty range"): assert ver("2:1.0") def test_version_empty_slice(): """Check an empty slice to confirm get "empty" version instead of an IndexError (#25953). """ assert Version("1.")[1:] == Version("") def test_version_wrong_idx_type(): """Ensure exception raised if attempt to use non-integer index.""" v = Version("1.1") with pytest.raises(TypeError): v["0:"] @pytest.mark.regression("29170") def test_version_range_satisfies_means_nonempty_intersection(): x = VersionRange("3.7.0", "3") y = VersionRange("3.6.0", "3.6.0") assert not x.satisfies(y) assert not y.satisfies(x) def test_version_list_with_range_and_concrete_version_is_not_concrete(): v = VersionList([Version("3.1"), VersionRange(Version("3.1.1"), Version("3.1.2"))]) assert not v.concrete @pytest.mark.parametrize( "git_ref, std_version", (("foo", "develop"), ("a" * 40, "develop"), ("a" * 40, None), ("v1.2.0", "1.2.0")), ) def test_git_versions_store_ref_requests(git_ref, std_version): """ User requested ref's should be known on creation Commit and standard version may not be known until concretization To be concrete a GitVersion must have a commit and standard version """ if std_version: vstring = f"git.{git_ref}={std_version}" else: vstring = git_ref v = Version(vstring) assert isinstance(v, GitVersion) assert v.ref == git_ref if std_version: assert v.std_version == Version(std_version) if v.is_commit: assert v.ref == v.commit_sha @pytest.mark.parametrize( "vstring, eq_vstring, is_commit", ( ("abc12" * 8 + "=develop", "develop", True), ("git." + "abc12" * 8 + "=main", "main", True), ("a" * 40 + "=develop", "develop", True), ("b" * 40 + "=3.2", "3.2", True), ("git.foo=3.2", "3.2", False), ), ) def test_git_ref_can_be_assigned_a_version(vstring, eq_vstring, is_commit): v = Version(vstring) v_equivalent = Version(eq_vstring) assert v.is_commit == is_commit assert not v._ref_lookup assert v_equivalent == v.ref_version @pytest.mark.parametrize( "lhs_str,rhs_str,expected", [ # StandardVersion ("4.7.3", "4.7.3", (True, True, True)), ("4.7.3", "4.7", (True, True, False)), ("4.7.3", "4", (True, True, False)), ("4.7.3", "4.8", (False, False, False)), # GitVersion (f"git.{'a' * 40}=develop", "develop", (True, True, False)), (f"git.{'a' * 40}=develop", f"git.{'a' * 40}=develop", (True, True, True)), (f"git.{'a' * 40}=develop", f"git.{'b' * 40}=develop", (False, False, False)), ], ) def test_version_intersects_satisfies_semantic(lhs_str, rhs_str, expected): lhs, rhs = ver(lhs_str), ver(rhs_str) intersect, lhs_sat_rhs, rhs_sat_lhs = expected assert lhs.intersects(rhs) is intersect assert lhs.intersects(rhs) is rhs.intersects(lhs) assert lhs.satisfies(rhs) is lhs_sat_rhs assert rhs.satisfies(lhs) is rhs_sat_lhs @pytest.mark.parametrize( "spec_str,tested_intersects,tested_satisfies", [ ( "git-test-commit@git.1.x", [("@:2", True), ("@:1", True), ("@:0", False), ("@1.3:", False)], [("@:2", True), ("@:1", True), ("@:0", False), ("@1.3:", False)], ), ( "git-test-commit@git.v2.0", [("@:2", True), ("@:1", False), ("@:0", False), ("@1.3:", True)], [("@:2", True), ("@:1", False), ("@:0", False), ("@1.3:", True)], ), ], ) def test_git_versions_without_explicit_reference( spec_str, tested_intersects, tested_satisfies, mock_git_version_info, mock_packages, monkeypatch, ): repo_path, filename, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", pathlib.Path(repo_path).as_uri(), raising=False ) spec = spack.spec.Spec(spec_str) for test_str, expected in tested_intersects: assert spec.intersects(test_str) is expected, test_str for test_str, expected in tested_intersects: assert spec.intersects(test_str) is expected, test_str def test_total_order_versions_and_ranges(): # The set of version ranges and individual versions are comparable, which is used in # VersionList. The comparison across types is based on default version comparison # of StandardVersion, GitVersion.ref_version, and ClosedOpenRange.lo. # StandardVersion / GitVersion (at equal ref version) assert_ver_lt("=1.2", "git.ref=1.2") assert_ver_gt("git.ref=1.2", "=1.2") # StandardVersion / GitVersion (at different ref versions) assert_ver_lt("git.ref=1.2", "=1.3") assert_ver_gt("=1.3", "git.ref=1.2") assert_ver_lt("=1.2", "git.ref=1.3") assert_ver_gt("git.ref=1.3", "=1.2") # GitVersion / ClosedOpenRange (at equal ref/lo version) assert_ver_lt("git.ref=1.2", "1.2") assert_ver_gt("1.2", "git.ref=1.2") # GitVersion / ClosedOpenRange (at different ref/lo version) assert_ver_lt("git.ref=1.2", "1.3") assert_ver_gt("1.3", "git.ref=1.2") assert_ver_lt("1.2", "git.ref=1.3") assert_ver_gt("git.ref=1.3", "1.2") # StandardVersion / ClosedOpenRange (at equal lo version) assert_ver_lt("=1.2", "1.2") assert_ver_gt("1.2", "=1.2") # StandardVersion / ClosedOpenRange (at different lo version) assert_ver_lt("=1.2", "1.3") assert_ver_gt("1.3", "=1.2") assert_ver_lt("1.2", "=1.3") assert_ver_gt("=1.3", "1.2") def test_git_version_accessors(): """Test whether iteration, indexing, slicing, dotted, dashed, and underscored works for GitVersion.""" v = GitVersion("my_branch=1.2-3") assert [x for x in v] == [1, 2, 3] assert v[0] == 1 assert v[1] == 2 assert v[2] == 3 assert v[0:2] == Version("1.2") assert v[0:10] == Version("1.2.3") assert str(v.dotted) == "1.2.3" assert str(v.dashed) == "1-2-3" assert str(v.underscored) == "1_2_3" assert v.up_to(1) == Version("1") assert v.up_to(2) == Version("1.2") assert len(v) == 3 assert not v.isdevelop() assert GitVersion("my_branch=develop").isdevelop() def test_boolness_of_versions(): # We do implement __len__, but at the end of the day versions are used as elements in # the first place, not as lists of version components. So VersionList(...).concrete # should be truthy even when there are no version components. assert bool(Version("1.2")) assert bool(Version("1.2").up_to(0)) # bool(GitVersion) shouldn't trigger a ref lookup. assert bool(GitVersion("a" * 40)) def test_version_list_normalization(): # Git versions and ordinary versions can live together in a VersionList assert len(VersionList(["=1.2", "ref=1.2"])) == 2 # But when a range is added, the only disjoint bit is the range. assert VersionList(["=1.2", "ref=1.2", "ref=1.3", "1.2:1.3"]) == VersionList(["1.2:1.3"]) # Also test normalization when using ver. assert ver("=1.0,ref=1.0,1.0:2.0") == ver(["1.0:2.0"]) assert ver("=1.0,1.0:2.0,ref=1.0") == ver(["1.0:2.0"]) assert ver("1.0:2.0,=1.0,ref=1.0") == ver(["1.0:2.0"]) def test_version_list_connected_union_of_disjoint_ranges(): # Make sure that we also simplify lists of ranges if their intersection is empty, but their # union is connected. assert ver("1.0:2.0,2.1,2.2:3,4:6") == ver(["1.0:6"]) assert ver("1.0:1.2,1.3:2") == ver("1.0:1.5,1.6:2") @pytest.mark.parametrize("version", ["=1.2", "git.ref=1.2", "1.2"]) def test_version_comparison_with_list_fails(version): vlist = VersionList(["=1.3"]) with pytest.raises(TypeError): version < vlist with pytest.raises(TypeError): vlist < version with pytest.raises(TypeError): version <= vlist with pytest.raises(TypeError): vlist <= version with pytest.raises(TypeError): version >= vlist with pytest.raises(TypeError): vlist >= version with pytest.raises(TypeError): version > vlist with pytest.raises(TypeError): vlist > version def test_inclusion_upperbound(): is_specific = spack.spec.Spec("x@=1.2") is_range = spack.spec.Spec("x@1.2") upperbound = spack.spec.Spec("x@:1.2.0") # The exact version is included in the range assert is_specific.satisfies(upperbound) # But the range 1.2:1.2 is not, since it includes for example 1.2.1 assert not is_range.satisfies(upperbound) # They do intersect of course. assert is_specific.intersects(upperbound) and is_range.intersects(upperbound) @pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_git_version_repo_attached_after_serialization( mock_git_version_info, mock_packages, config, monkeypatch ): """Test that a GitVersion instance can be serialized and deserialized without losing its repository reference. """ repo_path, _, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False ) spec = spack.concretize.concretize_one(f"git-test-commit@{commits[-2]}") # Before serialization, the repo is attached assert spec.satisfies("@1.0") # After serialization, the repo is still attached assert spack.spec.Spec.from_dict(spec.to_dict()).satisfies("@1.0") @pytest.mark.not_on_windows("Not supported on Windows (yet)") def test_resolved_git_version_is_shown_in_str( mock_git_version_info, mock_packages, config, monkeypatch ): """Test that a GitVersion from a commit without a user supplied version is printed as =, and not just .""" repo_path, _, commits = mock_git_version_info monkeypatch.setattr( spack.package_base.PackageBase, "git", "file://%s" % repo_path, raising=False ) commit = commits[-3] spec = spack.concretize.concretize_one(f"git-test-commit@{commit}") assert spec.version.satisfies(ver("1.0")) assert str(spec.version) == f"{commit}=1.0-git.1" def test_unresolvable_git_versions_error(config, mock_packages): """Test that VersionLookupError is raised when a git prop is not set on a package.""" with pytest.raises(VersionLookupError): # The package exists, but does not have a git property set. When dereferencing # the version, we should get VersionLookupError, not a generic AttributeError. spack.spec.Spec(f"git-test-commit@{'a' * 40}").version.ref_version @pytest.mark.parametrize( "tag,expected", [ ("v100.2.3", "100.2.3"), ("v1.2.3", "1.2.3"), ("v1.2.3-pre.release+build.1", "1.2.3-pre.release+build.1"), ("v1.2.3+build.1", "1.2.3+build.1"), ("v1.2.3+build_1", None), ("v1.2.3-pre.release", "1.2.3-pre.release"), ("v1.2.3-pre_release", None), ("1.2.3", "1.2.3"), ("1.2.3.", None), ], ) def test_semver_regex(tag, expected): result = SEMVER_REGEX.search(tag) if expected is None: assert result is None else: assert result.group() == expected ================================================ FILE: lib/spack/spack/test/views.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import pathlib import pytest import spack.concretize from spack.directory_layout import DirectoryLayout from spack.filesystem_view import SimpleFilesystemView, YamlFilesystemView from spack.installer import PackageInstaller from spack.spec import Spec def test_remove_extensions_ordered(install_mockery, mock_fetch, tmp_path: pathlib.Path): view_dir = str(tmp_path / "view") layout = DirectoryLayout(view_dir) view = YamlFilesystemView(view_dir, layout) e2 = spack.concretize.concretize_one("extension2") PackageInstaller([e2.package], explicit=True).install() view.add_specs(e2) e1 = e2["extension1"] view.remove_specs(e1, e2) @pytest.mark.regression("32456") def test_view_with_spec_not_contributing_files(mock_packages, tmp_path: pathlib.Path): view_dir = str(tmp_path / "view") os.mkdir(view_dir) layout = DirectoryLayout(view_dir) view = SimpleFilesystemView(view_dir, layout) a = Spec("pkg-a") b = Spec("pkg-b") a.set_prefix(str(tmp_path / "a")) b.set_prefix(str(tmp_path / "b")) a._mark_concrete() b._mark_concrete() # Create directory structure for a and b, and view os.makedirs(a.prefix.subdir) os.makedirs(b.prefix.subdir) os.makedirs(os.path.join(a.prefix, ".spack")) os.makedirs(os.path.join(b.prefix, ".spack")) # Add files to b's prefix, but not to a's with open(b.prefix.file, "w", encoding="utf-8") as f: f.write("file 1") with open(b.prefix.subdir.file, "w", encoding="utf-8") as f: f.write("file 2") # In previous versions of Spack we incorrectly called add_files_to_view # with b's merge map. It shouldn't be called at all, since a has no # files to add to the view. def pkg_a_add_files_to_view(view, merge_map, skip_if_exists=True): assert False, "There shouldn't be files to add" a.package.add_files_to_view = pkg_a_add_files_to_view # Create view and see if files are linked. view.add_specs(a, b) assert os.path.lexists(os.path.join(view_dir, "file")) assert os.path.lexists(os.path.join(view_dir, "subdir", "file")) ================================================ FILE: lib/spack/spack/test/web.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections import email.message import os import pathlib import pickle import ssl import urllib.request from typing import Dict import pytest import spack.config import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.paths import spack.url import spack.util.s3 import spack.util.url as url_util import spack.util.web from spack.llnl.util.filesystem import working_dir from spack.version import Version def _create_url(relative_url): web_data_path = os.path.join(spack.paths.test_path, "data", "web") return url_util.path_to_file_url(os.path.join(web_data_path, relative_url)) root = _create_url("index.html") root_tarball = _create_url("foo-0.0.0.tar.gz") page_1 = _create_url("1.html") page_2 = _create_url("2.html") page_3 = _create_url("3.html") page_4 = _create_url("4.html") root_with_fragment = _create_url("index_with_fragment.html") root_with_javascript = _create_url("index_with_javascript.html") @pytest.mark.parametrize( "depth,expected_found,expected_not_found,expected_text", [ ( 0, {"pages": [root], "links": [page_1]}, {"pages": [page_1, page_2, page_3, page_4], "links": [root, page_2, page_3, page_4]}, {root: "This is the root page."}, ), ( 1, {"pages": [root, page_1], "links": [page_1, page_2]}, {"pages": [page_2, page_3, page_4], "links": [root, page_3, page_4]}, {root: "This is the root page.", page_1: "This is page 1."}, ), ( 2, {"pages": [root, page_1, page_2], "links": [page_1, page_2, page_3, page_4]}, {"pages": [page_3, page_4], "links": [root]}, {root: "This is the root page.", page_1: "This is page 1.", page_2: "This is page 2."}, ), ( 3, { "pages": [root, page_1, page_2, page_3, page_4], "links": [root, page_1, page_2, page_3, page_4], }, {"pages": [], "links": []}, { root: "This is the root page.", page_1: "This is page 1.", page_2: "This is page 2.", page_3: "This is page 3.", page_4: "This is page 4.", }, ), ], ) def test_spider(depth, expected_found, expected_not_found, expected_text): pages, links = spack.util.web.spider(root, depth=depth) for page in expected_found["pages"]: assert page in pages for page in expected_not_found["pages"]: assert page not in pages for link in expected_found["links"]: assert link in links for link in expected_not_found["links"]: assert link not in links for page, text in expected_text.items(): assert text in pages[page] def test_spider_no_response(monkeypatch): # Mock the absence of a response monkeypatch.setattr(spack.util.web, "read_from_url", lambda x, y: (None, None, None)) pages, links, _, _ = spack.util.web._spider(root, collect_nested=False, _visited=set()) assert not pages and not links def test_find_versions_of_archive_0(): versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=0) assert Version("0.0.0") in versions def test_find_versions_of_archive_1(): versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=1) assert Version("0.0.0") in versions assert Version("1.0.0") in versions def test_find_versions_of_archive_2(): versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=2) assert Version("0.0.0") in versions assert Version("1.0.0") in versions assert Version("2.0.0") in versions def test_find_exotic_versions_of_archive_2(): versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=2) # up for grabs to make this better. assert Version("2.0.0b2") in versions def test_find_versions_of_archive_3(): versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=3) assert Version("0.0.0") in versions assert Version("1.0.0") in versions assert Version("2.0.0") in versions assert Version("3.0") in versions assert Version("4.5") in versions def test_find_exotic_versions_of_archive_3(): versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=3) assert Version("2.0.0b2") in versions assert Version("3.0a1") in versions assert Version("4.5-rc5") in versions def test_find_versions_of_archive_with_fragment(): versions = spack.url.find_versions_of_archive(root_tarball, root_with_fragment, list_depth=0) assert Version("5.0.0") in versions def test_find_versions_of_archive_with_javascript(): versions = spack.url.find_versions_of_archive(root_tarball, root_with_javascript, list_depth=0) assert Version("5.0.0") in versions def test_get_header(): headers = {"Content-type": "text/plain"} # looking up headers should just work like a plain dict # lookup when there is an entry with the right key assert spack.util.web.get_header(headers, "Content-type") == "text/plain" # looking up headers should still work if there is a fuzzy match assert spack.util.web.get_header(headers, "contentType") == "text/plain" # ...unless there is an exact match for the "fuzzy" spelling. headers["contentType"] = "text/html" assert spack.util.web.get_header(headers, "contentType") == "text/html" # If lookup has to fallback to fuzzy matching and there are more than one # fuzzy match, the result depends on the internal ordering of the given # mapping headers = collections.OrderedDict() headers["Content-type"] = "text/plain" headers["contentType"] = "text/html" assert spack.util.web.get_header(headers, "CONTENT_TYPE") == "text/plain" del headers["Content-type"] assert spack.util.web.get_header(headers, "CONTENT_TYPE") == "text/html" # Same as above, but different ordering headers = collections.OrderedDict() headers["contentType"] = "text/html" headers["Content-type"] = "text/plain" assert spack.util.web.get_header(headers, "CONTENT_TYPE") == "text/html" del headers["contentType"] assert spack.util.web.get_header(headers, "CONTENT_TYPE") == "text/plain" # If there isn't even a fuzzy match, raise KeyError with pytest.raises(KeyError): spack.util.web.get_header(headers, "ContentLength") def test_etag_parser(): # This follows rfc7232 to some extent, relaxing the quote requirement. assert spack.util.web.parse_etag('"abcdef"') == "abcdef" assert spack.util.web.parse_etag("abcdef") == "abcdef" # No empty tags assert spack.util.web.parse_etag("") is None # No quotes or spaces allowed assert spack.util.web.parse_etag('"abcdef"ghi"') is None assert spack.util.web.parse_etag('"abc def"') is None assert spack.util.web.parse_etag("abc def") is None def test_list_url(tmp_path: pathlib.Path): testpath = str(tmp_path) testpath_url = url_util.path_to_file_url(testpath) os.mkdir(os.path.join(testpath, "dir")) with open(os.path.join(testpath, "file-0.txt"), "w", encoding="utf-8"): pass with open(os.path.join(testpath, "file-1.txt"), "w", encoding="utf-8"): pass with open(os.path.join(testpath, "file-2.txt"), "w", encoding="utf-8"): pass with open(os.path.join(testpath, "dir", "another-file.txt"), "w", encoding="utf-8"): pass list_url = lambda recursive: list( sorted(spack.util.web.list_url(testpath_url, recursive=recursive)) ) assert list_url(False) == ["file-0.txt", "file-1.txt", "file-2.txt"] assert list_url(True) == ["dir/another-file.txt", "file-0.txt", "file-1.txt", "file-2.txt"] class MockPages: def search(self, *args, **kwargs): return [{"Key": "keyone"}, {"Key": "keytwo"}, {"Key": "keythree"}] class MockPaginator: def paginate(self, *args, **kwargs): return MockPages() class MockClientError(Exception): def __init__(self): self.response = { "Error": {"Code": "NoSuchKey"}, "ResponseMetadata": {"HTTPStatusCode": 404}, } class MockS3Client: def get_paginator(self, *args, **kwargs): return MockPaginator() def delete_objects(self, *args, **kwargs): return { "Errors": [{"Key": "keyone", "Message": "Access Denied"}], "Deleted": [{"Key": "keytwo"}, {"Key": "keythree"}], } def delete_object(self, *args, **kwargs): pass def get_object(self, Bucket=None, Key=None): self.ClientError = MockClientError if Bucket == "my-bucket" and Key == "subdirectory/my-file": return {"ResponseMetadata": {"HTTPHeaders": {}}} raise self.ClientError def head_object(self, Bucket=None, Key=None): self.ClientError = MockClientError if Bucket == "my-bucket" and Key == "subdirectory/my-file": return {"ResponseMetadata": {"HTTPHeaders": {}}} raise self.ClientError def test_gather_s3_information(monkeypatch): mirror = spack.mirrors.mirror.Mirror( { "fetch": { "access_token": "AAAAAAA", "profile": "SPacKDeV", "access_pair": ("SPA", "CK"), "endpoint_url": "https://127.0.0.1:8888", }, "push": { "access_token": "AAAAAAA", "profile": "SPacKDeV", "access_pair": ("SPA", "CK"), "endpoint_url": "https://127.0.0.1:8888", }, } ) session_args, client_args = spack.util.s3.get_mirror_s3_connection_info(mirror, "push") # Session args are used to create the S3 Session object assert "aws_session_token" in session_args assert session_args.get("aws_session_token") == "AAAAAAA" assert "aws_access_key_id" in session_args assert session_args.get("aws_access_key_id") == "SPA" assert "aws_secret_access_key" in session_args assert session_args.get("aws_secret_access_key") == "CK" assert "profile_name" in session_args assert session_args.get("profile_name") == "SPacKDeV" # In addition to the session object, use the client_args to create the s3 # Client object assert "endpoint_url" in client_args def test_remove_s3_url(monkeypatch, capfd): fake_s3_url = "s3://my-bucket/subdirectory/mirror" def get_s3_session(url, method="fetch"): return MockS3Client() monkeypatch.setattr(spack.util.web, "get_s3_session", get_s3_session) current_debug_level = tty.debug_level() tty.set_debug(1) spack.util.web.remove_url(fake_s3_url, recursive=True) err = capfd.readouterr()[1] tty.set_debug(current_debug_level) assert "Failed to delete keyone (Access Denied)" in err assert "Deleted keythree" in err assert "Deleted keytwo" in err def test_s3_url_exists(monkeypatch): def get_s3_session(url, method="fetch"): return MockS3Client() monkeypatch.setattr(spack.util.s3, "get_s3_session", get_s3_session) fake_s3_url_exists = "s3://my-bucket/subdirectory/my-file" assert spack.util.web.url_exists(fake_s3_url_exists) fake_s3_url_does_not_exist = "s3://my-bucket/subdirectory/my-notfound-file" assert not spack.util.web.url_exists(fake_s3_url_does_not_exist) def test_s3_url_parsing(): assert spack.util.s3._parse_s3_endpoint_url("example.com") == "https://example.com" assert spack.util.s3._parse_s3_endpoint_url("http://example.com") == "http://example.com" def test_detailed_http_error_pickle(tmp_path: pathlib.Path): (tmp_path / "response").write_text("response") headers = email.message.Message() headers.add_header("Content-Type", "text/plain") # Use a temporary file object as a response body with open(str(tmp_path / "response"), "rb") as f: error = spack.util.web.DetailedHTTPError( urllib.request.Request("http://example.com"), 404, "Not Found", headers, f ) deserialized = pickle.loads(pickle.dumps(error)) assert isinstance(deserialized, spack.util.web.DetailedHTTPError) assert deserialized.code == 404 assert deserialized.filename == "http://example.com" assert deserialized.reason == "Not Found" assert str(deserialized.info()) == str(headers) assert str(deserialized) == str(error) @pytest.fixture() def ssl_scrubbed_env(mutable_config, monkeypatch): """clear out environment variables that could give false positives for SSL Cert tests""" monkeypatch.delenv("SSL_CERT_FILE", raising=False) monkeypatch.delenv("SSL_CERT_DIR", raising=False) monkeypatch.delenv("CURL_CA_BUNDLE", raising=False) spack.config.set("config:verify_ssl", True) @pytest.mark.parametrize( "cert_path,cert_creator", [ pytest.param( lambda base_path: os.path.join(base_path, "mock_cert.crt"), lambda cert_path: open(cert_path, "w", encoding="utf-8").close(), id="cert_file", ), pytest.param( lambda base_path: os.path.join(base_path, "mock_cert"), lambda cert_path: os.mkdir(cert_path), id="cert_directory", ), ], ) def test_ssl_urllib( cert_path, cert_creator, tmp_path: pathlib.Path, ssl_scrubbed_env, mutable_config, monkeypatch ): """ create a proposed cert type and then verify that they exist inside ssl's checks """ spack.config.set("config:url_fetch_method", "urllib") def mock_verify_locations(self, cafile, capath, cadata): """overwrite ssl's verification to simply check for valid file/path""" assert cafile or capath if cafile: assert os.path.isfile(cafile) if capath: assert os.path.isdir(capath) monkeypatch.setattr(ssl.SSLContext, "load_verify_locations", mock_verify_locations) with working_dir(str(tmp_path)): mock_cert = cert_path(str(tmp_path)) cert_creator(mock_cert) spack.config.set("config:ssl_certs", mock_cert) assert mock_cert == spack.config.get("config:ssl_certs", None) ssl_context = spack.util.web.ssl_create_default_context() assert ssl_context.verify_mode == ssl.CERT_REQUIRED @pytest.mark.parametrize("cert_exists", [True, False], ids=["exists", "missing"]) def test_ssl_curl_cert_file( cert_exists, tmp_path: pathlib.Path, ssl_scrubbed_env, mutable_config, monkeypatch ): """ Assure that if a valid cert file is specified curl executes with CURL_CA_BUNDLE in the env """ spack.config.set("config:url_fetch_method", "curl") with working_dir(str(tmp_path)): mock_cert = str(tmp_path / "mock_cert.crt") spack.config.set("config:ssl_certs", mock_cert) if cert_exists: open(mock_cert, "w", encoding="utf-8").close() assert os.path.isfile(mock_cert) curl = spack.util.web.require_curl() # arbitrary call to query the run env dump_env: Dict[str, str] = {} curl("--help", output=str, _dump_env=dump_env) if cert_exists: assert dump_env["CURL_CA_BUNDLE"] == mock_cert else: assert "CURL_CA_BUNDLE" not in dump_env @pytest.mark.parametrize( "error_code,num_errors,max_retries,expect_failure", [ (500, 2, 5, False), # transient, enough retries (500, 2, 2, True), # transient, not enough retries (429, 2, 5, False), # rate limit, enough retries (404, 1, 5, True), # not transient, never retried ], ) def test_retry_on_transient_error(error_code, num_errors, max_retries, expect_failure): import urllib.error call_count = 0 sleep_times = [] def flaky_func(): nonlocal call_count call_count += 1 if call_count <= num_errors: raise urllib.error.HTTPError( url="https://example.com", code=error_code, msg="err", hdrs={}, fp=None ) return "ok" retrying = spack.util.web.retry_on_transient_error( flaky_func, retries=max_retries, sleep=sleep_times.append ) if expect_failure: with pytest.raises(urllib.error.HTTPError): retrying() else: assert retrying() == "ok" assert sleep_times == [2**i for i in range(num_errors)] def test_retry_on_transient_error_non_oserror(): """Non-OSError exceptions with transient names (e.g. botocore) should be retried.""" class ResponseStreamingError(Exception): pass call_count = 0 sleep_times = [] def flaky_func(): nonlocal call_count call_count += 1 if call_count <= 2: raise ResponseStreamingError("IncompleteRead") return "ok" retrying = spack.util.web.retry_on_transient_error( flaky_func, retries=5, sleep=sleep_times.append ) assert retrying() == "ok" assert call_count == 3 assert sleep_times == [1, 2] ================================================ FILE: lib/spack/spack/tokenize.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """This module provides building blocks for tokenizing strings. Users can define tokens by inheriting from TokenBase and defining tokens as ordered enum members. The Tokenizer class can then be used to iterate over tokens in a string.""" import enum import re from typing import Generator, Match, Optional, Type class TokenBase(enum.Enum): """Base class for an enum type with a regex value""" def __new__(cls, *args, **kwargs): value = len(cls.__members__) + 1 obj = object.__new__(cls) obj._value_ = value return obj def __init__(self, regex): self.regex = regex def __str__(self): return f"{self._name_}" class Token: """Represents tokens; generated from input by lexer and fed to parse().""" __slots__ = "kind", "value", "start", "end", "subvalues" def __init__(self, kind: TokenBase, value: str, start: int = 0, end: int = 0, **kwargs): self.kind = kind self.value = value self.start = start self.end = end self.subvalues = kwargs if kwargs else None def __repr__(self): return str(self) def __str__(self): parts = [self.kind, self.value] if self.subvalues: parts += [self.subvalues] return f"({', '.join(f'`{p}`' for p in parts)})" def __eq__(self, other): return ( self.kind == other.kind and self.value == other.value and self.subvalues == other.subvalues ) def token_match_regex(token: TokenBase): """Generate a regular expression that matches the provided token and its subvalues. This will extract named capture groups from the provided regex and prefix them with token name, so they can coexist together in a larger, joined regular expression. Returns: A regex with a capture group for the token and rewritten capture groups for any subvalues. """ pairs = [] def replace(m): subvalue_name = m.group(1) token_prefixed_subvalue_name = f"{token.name}_{subvalue_name}" pairs.append((subvalue_name, token_prefixed_subvalue_name)) return f"(?P<{token_prefixed_subvalue_name}>" # rewrite all subvalue capture groups so they're prefixed with the token name rewritten_token_regex = re.sub(r"\(\?P<([^>]+)>", replace, token.regex) # construct a regex that matches the token as a whole *and* the subvalue capture groups token_regex = f"(?P<{token}>{rewritten_token_regex})" return token_regex, pairs class Tokenizer: def __init__(self, tokens: Type[TokenBase]): self.tokens = tokens # tokens can have named subexpressions, if their regexes define named capture groups. # record this so we can associate them with the token self.token_subvalues = {} parts = [] for token in tokens: token_regex, pairs = token_match_regex(token) parts.append(token_regex) if pairs: self.token_subvalues[token.name] = pairs self.regex = re.compile("|".join(parts)) def tokenize(self, text: str) -> Generator[Token, None, None]: if not text: return scanner = self.regex.scanner(text) # type: ignore[attr-defined] m: Optional[Match] = None for m in iter(scanner.match, None): # The following two assertions are to help mypy msg = ( "unexpected value encountered during parsing. Please submit a bug report " "at https://github.com/spack/spack/issues/new/choose" ) assert m is not None, msg assert m.lastgroup is not None, msg token = Token(self.tokens.__members__[m.lastgroup], m.group(), m.start(), m.end()) # add any subvalues to the token subvalues = self.token_subvalues.get(m.lastgroup) if subvalues: if any(m.group(rewritten) for subval, rewritten in subvalues): token.subvalues = { subval: m.group(rewritten) for subval, rewritten in subvalues } yield token ================================================ FILE: lib/spack/spack/traverse.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from collections import defaultdict, deque from typing import ( TYPE_CHECKING, Any, Callable, Iterable, List, NamedTuple, Optional, Sequence, Set, Tuple, Union, overload, ) from spack.vendor.typing_extensions import Literal import spack.deptypes as dt if TYPE_CHECKING: import spack.spec # Export only the high-level API. __all__ = ["traverse_edges", "traverse_nodes", "traverse_tree"] #: Data class that stores a directed edge together with depth at #: which the target vertex was found. It is passed to ``accept`` #: and ``neighbors`` of visitors, so they can decide whether to #: follow the edge or not. class EdgeAndDepth(NamedTuple): edge: "spack.spec.DependencySpec" depth: int # Sort edges by name first, then abstract hash, then full edge comparison to break ties def sort_edges(edges): edges.sort(key=lambda edge: (edge.spec.name or "", edge.spec.abstract_hash or "", edge)) return edges class BaseVisitor: """A simple visitor that accepts all edges unconditionally and follows all edges to dependencies of a given ``deptype``.""" def __init__(self, depflag: dt.DepFlag = dt.ALL): self.depflag = depflag def accept(self, item): """ Arguments: item (EdgeAndDepth): Provides the depth and the edge through which the node was discovered Returns: bool: Returns ``True`` if the node is accepted. When ``False``, this indicates that the node won't be yielded by iterators and dependencies are not followed. """ return True def neighbors(self, item): return sort_edges(item.edge.spec.edges_to_dependencies(depflag=self.depflag)) class ReverseVisitor: """A visitor that reverses the arrows in the DAG, following dependents.""" def __init__(self, visitor, depflag: dt.DepFlag = dt.ALL): self.visitor = visitor self.depflag = depflag def accept(self, item): return self.visitor.accept(item) def neighbors(self, item): """Return dependents, note that we actually flip the edge direction to allow generic programming""" spec = item.edge.spec return sort_edges( [edge.flip() for edge in spec.edges_from_dependents(depflag=self.depflag)] ) class CoverNodesVisitor: """A visitor that traverses each node once.""" def __init__(self, visitor, key=id, visited=None): self.visitor = visitor self.key = key self.visited = set() if visited is None else visited def accept(self, item): # Covering nodes means: visit nodes once and only once. key = self.key(item.edge.spec) if key in self.visited: return False accept = self.visitor.accept(item) self.visited.add(key) return accept def neighbors(self, item): return self.visitor.neighbors(item) class CoverEdgesVisitor: """A visitor that traverses all edges once.""" def __init__(self, visitor, key=id, visited=None): self.visitor = visitor self.visited = set() if visited is None else visited self.key = key def accept(self, item): return self.visitor.accept(item) def neighbors(self, item): # Covering edges means: drop dependencies of visited nodes. key = self.key(item.edge.spec) if key in self.visited: return [] self.visited.add(key) return self.visitor.neighbors(item) class MixedDepthVisitor: """Visits all unique edges of the sub-DAG induced by direct dependencies of type ``direct`` and transitive dependencies of type ``transitive``. An example use for this is traversing build type dependencies non-recursively, and link dependencies recursively.""" def __init__( self, *, direct: dt.DepFlag, transitive: dt.DepFlag, key: Callable[["spack.spec.Spec"], Any] = id, ) -> None: self.direct_type = direct self.transitive_type = transitive self.key = key self.seen: Set[Any] = set() self.seen_roots: Set[Any] = set() def accept(self, item: EdgeAndDepth) -> bool: # Do not accept duplicate root nodes. This only happens if the user starts iterating from # multiple roots and lists one of the roots multiple times. if item.edge.parent is None: node_id = self.key(item.edge.spec) if node_id in self.seen_roots: return False self.seen_roots.add(node_id) return True def neighbors(self, item: EdgeAndDepth) -> List[EdgeAndDepth]: # If we're here through an artificial source node, it's a root, and we return all # direct_type and transitive_type edges. If we're here through a transitive_type edge, we # return all transitive_type edges. To avoid returning the same edge twice: # 1. If we had already encountered the current node through a transitive_type edge, we # don't need to return transitive_type edges again. # 2. If we encounter the current node through a direct_type edge, and we had already seen # it through a transitive_type edge, only return the non-transitive_type, direct_type # edges. node_id = self.key(item.edge.spec) seen = node_id in self.seen is_root = item.edge.parent is None follow_transitive = is_root or bool(item.edge.depflag & self.transitive_type) follow = self.direct_type if is_root else dt.NONE if follow_transitive and not seen: follow |= self.transitive_type self.seen.add(node_id) elif follow == dt.NONE: return [] edges = item.edge.spec.edges_to_dependencies(depflag=follow) # filter direct_type edges already followed before because they were also transitive_type. if seen: edges = [edge for edge in edges if not edge.depflag & self.transitive_type] return sort_edges(edges) def get_visitor_from_args( cover, direction, depflag: Union[dt.DepFlag, dt.DepTypes], key=id, visited=None, visitor=None ): """ Create a visitor object from common keyword arguments. Arguments: cover (str): Determines how extensively to cover the dag. Possible values: ``nodes`` -- Visit each unique node in the dag only once. ``edges`` -- If a node has been visited once but is reached along a new path, it's accepted, but not recursively followed. This traverses each 'edge' in the DAG once. ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple times if they're reachable by multiple paths. direction (str): ``children`` or ``parents``. If ``children``, does a traversal of this spec's children. If ``parents``, traverses upwards in the DAG towards the root. deptype: allowed dependency types key: function that takes a spec and outputs a key for uniqueness test. visited (set or None): a set of nodes not to follow (when using cover=nodes/edges) visitor: An initial visitor that is used for composition. Returns: A visitor """ if not isinstance(depflag, dt.DepFlag): depflag = dt.canonicalize(depflag) visitor = visitor or BaseVisitor(depflag) if cover == "nodes": visitor = CoverNodesVisitor(visitor, key, visited) elif cover == "edges": visitor = CoverEdgesVisitor(visitor, key, visited) if direction == "parents": visitor = ReverseVisitor(visitor, depflag) return visitor def with_artificial_edges(specs): """Initialize a deque of edges from an artificial root node to the root specs.""" from spack.spec import DependencySpec return deque( EdgeAndDepth(edge=DependencySpec(parent=None, spec=s, depflag=0, virtuals=()), depth=0) for s in specs ) def traverse_depth_first_edges_generator(edges, visitor, post_order=False, root=True, depth=False): """Generator that takes explores a DAG in depth-first fashion starting from a list of edges. Note that typically DFS would take a vertex not a list of edges, but the API is like this so we don't have to create an artificial root node when traversing from multiple roots in a DAG. Arguments: edges (list): List of EdgeAndDepth instances visitor: class instance implementing accept() and neighbors() post_order (bool): Whether to yield nodes when backtracking root (bool): whether to yield at depth 0 depth (bool): when ``True`` yield a tuple of depth and edge, otherwise only the edge. """ for edge in edges: if not visitor.accept(edge): continue yield_me = root or edge.depth > 0 # Pre if yield_me and not post_order: yield (edge.depth, edge.edge) if depth else edge.edge neighbors = [EdgeAndDepth(edge=n, depth=edge.depth + 1) for n in visitor.neighbors(edge)] # This extra branch is just for efficiency. if len(neighbors) > 0: for item in traverse_depth_first_edges_generator( neighbors, visitor, post_order, root, depth ): yield item # Post if yield_me and post_order: yield (edge.depth, edge.edge) if depth else edge.edge def traverse_breadth_first_edges_generator(queue: deque, visitor, root=True, depth=False): while len(queue) > 0: edge = queue.popleft() # If the visitor doesn't accept the node, we don't yield it nor follow its edges. if not visitor.accept(edge): continue if root or edge.depth > 0: yield (edge.depth, edge.edge) if depth else edge.edge for e in visitor.neighbors(edge): queue.append(EdgeAndDepth(e, edge.depth + 1)) def traverse_breadth_first_with_visitor(specs, visitor): """Performs breadth first traversal for a list of specs (not a generator). Arguments: specs (list): List of Spec instances. visitor: object that implements accept and neighbors interface, see for example BaseVisitor. """ queue = with_artificial_edges(specs) while len(queue) > 0: edge = queue.popleft() # If the visitor doesn't accept the node, we don't traverse it further. if not visitor.accept(edge): continue for e in visitor.neighbors(edge): queue.append(EdgeAndDepth(e, edge.depth + 1)) def traverse_depth_first_with_visitor(edges, visitor): """Traverse a DAG in depth-first fashion using a visitor, starting from a list of edges. Note that typically DFS would take a vertex not a list of edges, but the API is like this so we don't have to create an artificial root node when traversing from multiple roots in a DAG. Arguments: edges (list): List of EdgeAndDepth instances visitor: class instance implementing accept(), pre(), post() and neighbors() """ for edge in edges: if not visitor.accept(edge): continue visitor.pre(edge) neighbors = [EdgeAndDepth(edge=e, depth=edge.depth + 1) for e in visitor.neighbors(edge)] traverse_depth_first_with_visitor(neighbors, visitor) visitor.post(edge) # Helper functions for generating a tree using breadth-first traversal def breadth_first_to_tree_edges(roots, deptype="all", key=id): """This produces an adjacency list (with edges) and a map of parents. There may be nodes that are reached through multiple edges. To print as a tree, one should use the parents dict to verify if the path leading to the node is through the correct parent. If not, the branch should be truncated.""" edges = defaultdict(list) parents = dict() for edge in traverse_edges(roots, order="breadth", cover="edges", deptype=deptype, key=key): parent_id = None if edge.parent is None else key(edge.parent) child_id = key(edge.spec) edges[parent_id].append(edge) if child_id not in parents: parents[child_id] = parent_id return edges, parents def breadth_first_to_tree_nodes(roots, deptype="all", key=id): """This produces a list of edges that forms a tree; every node has no more that one incoming edge.""" edges = defaultdict(list) for edge in traverse_edges(roots, order="breadth", cover="nodes", deptype=deptype, key=key): parent_id = None if edge.parent is None else key(edge.parent) edges[parent_id].append(edge) return edges def traverse_breadth_first_tree_edges(parent_id, edges, parents, key=id, depth=0): """Do a depth-first search on edges generated by bread-first traversal, which can be used to produce a tree.""" for edge in edges[parent_id]: yield (depth, edge) child_id = key(edge.spec) # Don't follow further if we're not the parent if parents[child_id] != parent_id: continue yield from traverse_breadth_first_tree_edges(child_id, edges, parents, key, depth + 1) def traverse_breadth_first_tree_nodes(parent_id, edges, key=id, depth=0): for edge in edges[parent_id]: yield (depth, edge) for item in traverse_breadth_first_tree_nodes(key(edge.spec), edges, key, depth + 1): yield item def traverse_topo_edges_generator(edges, visitor, key=id, root=True, all_edges=False): """ Returns a list of edges in topological order, in the sense that all in-edges of a vertex appear before all out-edges. Arguments: edges (list): List of EdgeAndDepth instances visitor: visitor that produces unique edges defining the (sub)DAG of interest. key: function that takes a spec and outputs a key for uniqueness test. root (bool): Yield the root nodes themselves all_edges (bool): When ``False`` only one in-edge per node is returned, when ``True`` all reachable edges are returned. """ # Topo order used to be implemented using a DFS visitor, which was relatively efficient in that # it would visit nodes only once, and it was composable. In practice however it would yield a # DFS order on DAGs that are trees, which is undesirable in many cases. For example, a list of # search paths for trees is better in BFS order, so that direct dependencies are listed first. # That way a transitive dependency cannot shadow a direct one. So, here we collect the sub-DAG # of interest and then compute a topological order that is the most breadth-first possible. # maps node identifier to the number of remaining in-edges in_edge_count = defaultdict(int) # maps parent identifier to a list of edges, where None is a special identifier # for the artificial root/source. node_to_edges = defaultdict(list) for edge in traverse_breadth_first_edges_generator(edges, visitor, root=True, depth=False): in_edge_count[key(edge.spec)] += 1 parent_id = key(edge.parent) if edge.parent is not None else None node_to_edges[parent_id].append(edge) queue = deque((None,)) while queue: for edge in node_to_edges[queue.popleft()]: child_id = key(edge.spec) in_edge_count[child_id] -= 1 should_yield = root or edge.parent is not None if all_edges and should_yield: yield edge if in_edge_count[child_id] == 0: if not all_edges and should_yield: yield edge queue.append(key(edge.spec)) # High-level API: traverse_edges, traverse_nodes, traverse_tree. OrderType = Literal["pre", "post", "breadth", "topo"] CoverType = Literal["nodes", "edges", "paths"] DirectionType = Literal["children", "parents"] @overload def traverse_edges( specs: Sequence["spack.spec.Spec"], *, root: bool = ..., order: OrderType = ..., cover: CoverType = ..., direction: DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: Literal[False] = False, key: Callable[["spack.spec.Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable["spack.spec.DependencySpec"]: ... @overload def traverse_edges( specs: Sequence["spack.spec.Spec"], *, root: bool = ..., order: OrderType = ..., cover: CoverType = ..., direction: DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: Literal[True], key: Callable[["spack.spec.Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable[Tuple[int, "spack.spec.DependencySpec"]]: ... @overload def traverse_edges( specs: Sequence["spack.spec.Spec"], *, root: bool = ..., order: OrderType = ..., cover: CoverType = ..., direction: DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: bool, key: Callable[["spack.spec.Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable[Union["spack.spec.DependencySpec", Tuple[int, "spack.spec.DependencySpec"]]]: ... def traverse_edges( specs: Sequence["spack.spec.Spec"], root: bool = True, order: OrderType = "pre", cover: CoverType = "nodes", direction: DirectionType = "children", deptype: Union[dt.DepFlag, dt.DepTypes] = "all", depth: bool = False, key: Callable[["spack.spec.Spec"], Any] = id, visited: Optional[Set[Any]] = None, ) -> Iterable[Union["spack.spec.DependencySpec", Tuple[int, "spack.spec.DependencySpec"]]]: """ Iterable of edges from the DAG, starting from a list of root specs. Arguments: specs: List of root specs (considered to be depth 0) root: Yield the root nodes themselves order: What order of traversal to use in the DAG. For depth-first search this can be ``pre`` or ``post``. For BFS this should be ``breadth``. For topological order use ``topo`` cover: Determines how extensively to cover the dag. Possible values: ``nodes`` -- Visit each unique node in the dag only once. ``edges`` -- If a node has been visited once but is reached along a new path, it's accepted, but not recursively followed. This traverses each 'edge' in the DAG once. ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple times if they're reachable by multiple paths. direction: ``children`` or ``parents``. If ``children``, does a traversal of this spec's children. If ``parents``, traverses upwards in the DAG towards the root. deptype: allowed dependency types depth: When ``False``, yield just edges. When ``True`` yield the tuple (depth, edge), where depth corresponds to the depth at which edge.spec was discovered. key: function that takes a spec and outputs a key for uniqueness test. visited: a set of nodes not to follow Returns: An iterable of ``DependencySpec`` if depth is ``False`` or a tuple of ``(depth, DependencySpec)`` if depth is ``True``. """ # validate input if order == "topo": if cover == "paths": raise ValueError("cover=paths not supported for order=topo") if visited is not None: raise ValueError("visited set not implemented for order=topo") elif order not in ("post", "pre", "breadth"): raise ValueError(f"Unknown order {order}") # In topo traversal we need to construct a sub-DAG including all unique edges even if we are # yielding a subset of them, hence "edges". _cover = "edges" if order == "topo" else cover visitor = get_visitor_from_args(_cover, direction, deptype, key, visited) root_edges = with_artificial_edges(specs) # Depth-first if order == "pre" or order == "post": return traverse_depth_first_edges_generator( root_edges, visitor, order == "post", root, depth ) elif order == "breadth": return traverse_breadth_first_edges_generator(root_edges, visitor, root, depth) elif order == "topo": return traverse_topo_edges_generator( root_edges, visitor, key, root, all_edges=cover == "edges" ) @overload def traverse_nodes( specs: Sequence["spack.spec.Spec"], *, root: bool = ..., order: OrderType = ..., cover: CoverType = ..., direction: DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: Literal[False] = False, key: Callable[["spack.spec.Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable["spack.spec.Spec"]: ... @overload def traverse_nodes( specs: Sequence["spack.spec.Spec"], *, root: bool = ..., order: OrderType = ..., cover: CoverType = ..., direction: DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: Literal[True], key: Callable[["spack.spec.Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable[Tuple[int, "spack.spec.Spec"]]: ... @overload def traverse_nodes( specs: Sequence["spack.spec.Spec"], *, root: bool = ..., order: OrderType = ..., cover: CoverType = ..., direction: DirectionType = ..., deptype: Union[dt.DepFlag, dt.DepTypes] = ..., depth: bool, key: Callable[["spack.spec.Spec"], Any] = ..., visited: Optional[Set[Any]] = ..., ) -> Iterable[Union["spack.spec.Spec", Tuple[int, "spack.spec.Spec"]]]: ... def traverse_nodes( specs: Sequence["spack.spec.Spec"], *, root: bool = True, order: OrderType = "pre", cover: CoverType = "nodes", direction: DirectionType = "children", deptype: Union[dt.DepFlag, dt.DepTypes] = "all", depth: bool = False, key: Callable[["spack.spec.Spec"], Any] = id, visited: Optional[Set[Any]] = None, ) -> Iterable[Union["spack.spec.Spec", Tuple[int, "spack.spec.Spec"]]]: """ Iterable of specs from the DAG, starting from a list of root specs. Arguments: specs: List of root specs (considered to be depth 0) root: Yield the root nodes themselves order: What order of traversal to use in the DAG. For depth-first search this can be ``pre`` or ``post``. For BFS this should be ``breadth``. cover: Determines how extensively to cover the dag. Possible values: ``nodes`` -- Visit each unique node in the dag only once. ``edges`` -- If a node has been visited once but is reached along a new path, it's accepted, but not recursively followed. This traverses each 'edge' in the DAG once. ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple times if they're reachable by multiple paths. direction: ``children`` or ``parents``. If ``children``, does a traversal of this spec's children. If ``parents``, traverses upwards in the DAG towards the root. deptype: allowed dependency types depth: When ``False``, yield just edges. When ``True`` yield the tuple ``(depth, edge)``, where depth corresponds to the depth at which ``edge.spec`` was discovered. key: function that takes a spec and outputs a key for uniqueness test. visited: a set of nodes not to follow Yields: By default :class:`~spack.spec.Spec`, or a tuple ``(depth, Spec)`` if depth is set to ``True``. """ for item in traverse_edges( specs, root=root, order=order, cover=cover, direction=direction, deptype=deptype, depth=depth, key=key, visited=visited, ): yield (item[0], item[1].spec) if depth else item.spec # type: ignore def traverse_tree( specs: Sequence["spack.spec.Spec"], cover: CoverType = "nodes", deptype: Union[dt.DepFlag, dt.DepTypes] = "all", key: Callable[["spack.spec.Spec"], Any] = id, depth_first: bool = True, ) -> Iterable[Tuple[int, "spack.spec.DependencySpec"]]: """ Generator that yields ``(depth, DependencySpec)`` tuples in the depth-first pre-order, so that a tree can be printed from it. Arguments: specs: List of root specs (considered to be depth 0) cover: Determines how extensively to cover the dag. Possible values: ``nodes`` -- Visit each unique node in the dag only once. ``edges`` -- If a node has been visited once but is reached along a new path, it's accepted, but not recursively followed. This traverses each 'edge' in the DAG once. ``paths`` -- Explore every unique path reachable from the root. This descends into visited subtrees and will accept nodes multiple times if they're reachable by multiple paths. deptype: allowed dependency types key: function that takes a spec and outputs a key for uniqueness test. depth_first: Explore the tree in depth-first or breadth-first order. When setting ``depth_first=True`` and ``cover=nodes``, each spec only occurs once at the shallowest level, which is useful when rendering the tree in a terminal. Returns: A generator that yields ``(depth, DependencySpec)`` tuples in such an order that a tree can be printed. """ # BFS only makes sense when going over edges and nodes, for paths the tree is # identical to DFS, which is much more efficient then. if not depth_first and cover == "edges": edges, parents = breadth_first_to_tree_edges(specs, deptype, key) return traverse_breadth_first_tree_edges(None, edges, parents, key) elif not depth_first and cover == "nodes": edges = breadth_first_to_tree_nodes(specs, deptype, key) return traverse_breadth_first_tree_nodes(None, edges, key) return traverse_edges(specs, order="pre", cover=cover, deptype=deptype, key=key, depth=True) def by_dag_hash(s: "spack.spec.Spec") -> str: """Used very often as a key function for traversals.""" return s.dag_hash() ================================================ FILE: lib/spack/spack/url.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This module has methods for parsing names and versions of packages from URLs. The idea is to allow package creators to supply nothing more than the download location of the package, and figure out version and name information from there. **Example:** when spack is given the following URL: .. code-block:: https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.12/src/hdf-4.2.12.tar.gz It can figure out that the package name is ``hdf``, and that it is at version ``4.2.12``. This is useful for making the creation of packages simple: a user just supplies a URL and skeleton code is generated automatically. Spack can also figure out that it can most likely download 4.2.6 at this URL: .. code-block:: https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.6/src/hdf-4.2.6.tar.gz This is useful if a user asks for a package at a particular version number; spack doesn't need anyone to tell it where to get the tarball even though it's never been told about that version before. """ import io import os import pathlib import re from typing import Any, Dict, Optional, Sequence, Tuple, Union import spack.error import spack.llnl.url import spack.util.web import spack.version from spack.llnl.path import convert_to_posix_path from spack.llnl.util.tty.color import cescape, colorize # # Note: We call the input to most of these functions a "path" but the functions # work on paths and URLs. There's not a good word for both of these, but # "path" seemed like the most generic term. # def strip_name_suffixes(path: str, version: Union[str, spack.version.StandardVersion]) -> str: """Most tarballs contain a package name followed by a version number. However, some also contain extraneous information in-between the name and version: * ``rgb-1.0.6`` * ``converge_install_2.3.16`` * ``jpegsrc.v9b`` These strings are not part of the package name and should be ignored. This function strips the version number and any extraneous suffixes off and returns the remaining string. The goal is that the name is always the last thing in ``path``: * ``rgb`` * ``converge`` * ``jpeg`` Args: path: The filename or URL for the package version: The version detected for this URL Returns: str: The ``path`` with any extraneous suffixes removed """ # NOTE: This could be done with complicated regexes in parse_name_offset # NOTE: The problem is that we would have to add these regexes to every # NOTE: single name regex. Easier to just strip them off permanently suffix_regexes = [ # Strip off the version and anything after it # name-ver # name_ver # name.ver r"[._-][rvV]?" + str(version) + ".*", # namever r"V?" + str(version) + ".*", # Download type r"install", r"[Ss]rc", r"(open)?[Ss]ources?", r"[._-]open", r"[._-]archive", r"[._-]std", r"[._-]bin", r"Software", # Download version r"release", r"snapshot", r"distrib", r"everywhere", r"latest", # Arch r"Linux(64)?", r"x86_64", # VCS r"0\+bzr", # License r"gpl", # Needs to come before and after gpl, appears in both orders r"[._-]x11", r"gpl", ] for regex in suffix_regexes: # Remove the suffix from the end of the path # This may be done multiple times path = re.sub("[._-]?" + regex + "$", "", path) return path def parse_version_offset(path: str) -> Tuple[str, int, int, int, str]: """Try to extract a version string from a filename or URL. Args: path (str): The filename or URL for the package Returns: A tuple containing * version of the package * first index of version * length of version string * the index of the matching regex * the matching regex Raises: UndetectableVersionError: If the URL does not match any regexes """ original_path = path # path: The prefix of the URL, everything before the ext and suffix # ext: The file extension # suffix: Any kind of query string that begins with a '?' path, ext, suffix = spack.llnl.url.split_url_extension(path) # stem: Everything from path after the final '/' original_stem = os.path.basename(path) # Try to strip off anything after the version number stem = spack.llnl.url.strip_version_suffixes(original_stem) # Assumptions: # # 1. version always comes after the name # 2. separators include '-', '_', and '.' # 3. names can contain A-Z, a-z, 0-9, '+', separators # 4. versions can contain A-Z, a-z, 0-9, separators # 5. versions always start with a digit # 6. versions are often prefixed by a 'v' or 'r' character # 7. separators are most reliable to determine name/version boundaries # List of the following format: # # [ # (regex, string), # ... # ] # # The first regex that matches string will be used to determine # the version of the package. Therefore, hyperspecific regexes should # come first while generic, catch-all regexes should come last. # With that said, regular expressions are slow, so if possible, put # ones that only catch one or two URLs at the bottom. version_regexes = [ # 1st Pass: Simplest case # Assume name contains no digits and version contains no letters # e.g. libpng-1.6.27 (r"^[a-zA-Z+._-]+[._-]v?(\d[\d._-]*)$", stem), # 2nd Pass: Version only # Assume version contains no letters # ver # e.g. 3.2.7, 7.0.2-7, v3.3.0, v1_6_3 (r"^v?(\d[\d._-]*)$", stem), # 3rd Pass: No separator characters are used # Assume name contains no digits # namever # e.g. turbolinux702, nauty26r7 (r"^[a-zA-Z+]*(\d[\da-zA-Z]*)$", stem), # 4th Pass: A single separator character is used # Assume name contains no digits # name-name-ver-ver # e.g. panda-2016-03-07, gts-snapshot-121130, cdd-061a (r"^[a-zA-Z+-]*(\d[\da-zA-Z-]*)$", stem), # name_name_ver_ver # e.g. tinyxml_2_6_2, boost_1_55_0, tbb2017_20161128 (r"^[a-zA-Z+_]*(\d[\da-zA-Z_]*)$", stem), # name.name.ver.ver # e.g. prank.source.150803, jpegsrc.v9b, atlas3.11.34, geant4.10.01.p03 (r"^[a-zA-Z+.]*(\d[\da-zA-Z.]*)$", stem), # 5th Pass: Two separator characters are used # Name may contain digits, version may contain letters # name-name-ver.ver # e.g. m4-1.4.17, gmp-6.0.0a, launchmon-v1.0.2 (r"^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$", stem), # name-name-ver_ver # e.g. icu4c-57_1 (r"^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z_]*)$", stem), # name_name_ver.ver # e.g. superlu_dist_4.1, pexsi_v0.9.0 (r"^[a-zA-Z\d+_]+_v?(\d[\da-zA-Z.]*)$", stem), # name_name.ver.ver # e.g. fer_source.v696 (r"^[a-zA-Z\d+_]+\.v?(\d[\da-zA-Z.]*)$", stem), # name_ver-ver # e.g. Bridger_r2014-12-01 (r"^[a-zA-Z\d+]+_r?(\d[\da-zA-Z-]*)$", stem), # name-name-ver.ver-ver.ver # e.g. sowing-1.1.23-p1, bib2xhtml-v3.0-15-gf506, 4.6.3-alpha04 (r"^(?:[a-zA-Z\d+-]+-)?v?(\d[\da-zA-Z.-]*)$", stem), # namever.ver-ver.ver # e.g. go1.4-bootstrap-20161024 (r"^[a-zA-Z+]+v?(\d[\da-zA-Z.-]*)$", stem), # 6th Pass: All three separator characters are used # Name may contain digits, version may contain letters # name_name-ver.ver # e.g. the_silver_searcher-0.32.0, sphinx_rtd_theme-0.1.10a0 (r"^[a-zA-Z\d+_]+-v?(\d[\da-zA-Z.]*)$", stem), # name.name_ver.ver-ver.ver # e.g. TH.data_1.0-8, XML_3.98-1.4 (r"^[a-zA-Z\d+.]+_v?(\d[\da-zA-Z.-]*)$", stem), # name-name-ver.ver_ver.ver # e.g. pypar-2.1.5_108 (r"^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z._]*)$", stem), # name.name_name-ver.ver # e.g. tap.py-1.6, backports.ssl_match_hostname-3.5.0.1 (r"^[a-zA-Z\d+._]+-v?(\d[\da-zA-Z.]*)$", stem), # name-namever.ver_ver.ver # e.g. STAR-CCM+11.06.010_02 (r"^[a-zA-Z+-]+(\d[\da-zA-Z._]*)$", stem), # name-name_name-ver.ver # e.g. PerlIO-utf8_strict-0.002 (r"^[a-zA-Z\d+_-]+-v?(\d[\da-zA-Z.]*)$", stem), # 7th Pass: Specific VCS # bazaar # e.g. libvterm-0+bzr681 (r"bzr(\d[\da-zA-Z._-]*)$", stem), # 8th Pass: Query strings # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0 # e.g. https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-2.1.1 # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0 # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1 # e.g. https://software.broadinstitute.org/gatk/download/auth?package=GATK-archive&version=3.8-1-0-gf15c1c3ef (r"[?&](?:sha|ref|version)=[a-zA-Z\d+-]*[_-]?v?(\d[\da-zA-Z._-]*)$", suffix), # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz # e.g. http://laws-green.lanl.gov/projects/data/eos/get_file.php?package=eospac&filename=eospac_v6.4.0beta.1_r20171213193219.tgz # e.g. https://evtgen.hepforge.org/downloads?f=EvtGen-01.07.00.tar.gz # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz (r"[?&](?:filename|f|get)=[a-zA-Z\d+-]+[_-]v?(\d[\da-zA-Z.]*)", stem), # 9th Pass: Version in path # github.com/repo/name/releases/download/vver/name # e.g. https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow # e.g. https://gitlab.com/hpctoolkit/hpcviewer/-/releases/2024.02/downloads/hpcviewer.tgz (r"github\.com/[^/]+/[^/]+/releases/download/[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)/", path), (r"gitlab\.com/[^/]+/.+/-/releases/[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)/downloads/", path), # e.g. ftp://ftp.ncbi.nlm.nih.gov/blast/executables/legacy.NOTSUPPORTED/2.2.26/ncbi.tar.gz (r"(\d[\da-zA-Z._-]*)/[^/]+$", path), ] for i, version_regex in enumerate(version_regexes): regex, match_string = version_regex match = re.search(regex, match_string) if match and match.group(1) is not None: version = match.group(1) start = match.start(1) # If we matched from the stem or suffix, we need to add offset offset = 0 if match_string is stem: offset = len(path) - len(original_stem) elif match_string is suffix: offset = len(path) if ext: offset += len(ext) + 1 # .tar.gz is converted to tar.gz start += offset return version, start, len(version), i, regex raise UndetectableVersionError(original_path) def parse_version(path: str) -> spack.version.StandardVersion: """Try to extract a version string from a filename or URL. Args: path: The filename or URL for the package Returns: The version of the package Raises: UndetectableVersionError: If the URL does not match any regexes """ version, start, length, i, regex = parse_version_offset(path) return spack.version.StandardVersion.from_string(version) def parse_name_offset( path: str, v: Optional[Union[str, spack.version.StandardVersion]] = None ) -> Tuple[str, int, int, int, str]: """Try to determine the name of a package from its filename or URL. Args: path: The filename or URL for the package v: The version of the package Returns: A tuple containing * name of the package * first index of name * length of name * the index of the matching regex * the matching regex Raises: UndetectableNameError: If the URL does not match any regexes """ original_path = path # We really need to know the version of the package # This helps us prevent collisions between the name and version if v is None: try: v = parse_version(path) except UndetectableVersionError: # Not all URLs contain a version. We still want to be able # to determine a name if possible. v = "unknown" # path: The prefix of the URL, everything before the ext and suffix # ext: The file extension # suffix: Any kind of query string that begins with a '?' path, ext, suffix = spack.llnl.url.split_url_extension(path) # stem: Everything from path after the final '/' original_stem = os.path.basename(path) # Try to strip off anything after the package name stem = strip_name_suffixes(original_stem, v) # List of the following format: # # [ # (regex, string), # ... # ] # # The first regex that matches string will be used to determine # the name of the package. Therefore, hyperspecific regexes should # come first while generic, catch-all regexes should come last. # With that said, regular expressions are slow, so if possible, put # ones that only catch one or two URLs at the bottom. name_regexes = [ # 1st Pass: Common repositories # GitHub: github.com/repo/name/ # e.g. https://github.com/nco/nco/archive/4.6.2.tar.gz (r"github\.com/[^/]+/([^/]+)", path), # GitLab API endpoint: gitlab.*/api/v4/projects/NAMESPACE%2Fname/ # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0 (r"gitlab[^/]+/api/v4/projects/[^/]+%2F([^/]+)", path), # GitLab non-API endpoint: gitlab.*/repo/name/ # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0 (r"gitlab[^/]+/(?!api/v4/projects)[^/]+/([^/]+)", path), # Bitbucket: bitbucket.org/repo/name/ # e.g. https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2 (r"bitbucket\.org/[^/]+/([^/]+)", path), # PyPI: pypi.(python.org|io)/packages/source/first-letter/name/ # e.g. https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz # e.g. https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz (r"pypi\.(?:python\.org|io)/packages/source/[A-Za-z\d]/([^/]+)", path), # 2nd Pass: Query strings # ?filename=name-ver.ver # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz (r"\?filename=([A-Za-z\d+-]+)$", stem), # ?f=name-ver.ver # e.g. https://evtgen.hepforge.org/downloads?f=EvtGen-01.07.00.tar.gz (r"\?f=([A-Za-z\d+-]+)$", stem), # ?package=name # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz (r"\?package=([A-Za-z\d+-]+)", stem), # ?package=name-version (r"\?package=([A-Za-z\d]+)", suffix), # download.php # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1 (r"([^/]+)/download.php$", path), # 3rd Pass: Name followed by version in archive (r"^([A-Za-z\d+\._-]+)$", stem), ] for i, name_regex in enumerate(name_regexes): regex, match_string = name_regex match = re.search(regex, match_string) if match: name = match.group(1) start = match.start(1) # If we matched from the stem or suffix, we need to add offset offset = 0 if match_string is stem: offset = len(path) - len(original_stem) elif match_string is suffix: offset = len(path) if ext: offset += len(ext) + 1 # .tar.gz is converted to tar.gz start += offset return name, start, len(name), i, regex raise UndetectableNameError(original_path) def parse_name(path, ver=None): """Try to determine the name of a package from its filename or URL. Args: path (str): The filename or URL for the package ver (str): The version of the package Returns: str: The name of the package Raises: UndetectableNameError: If the URL does not match any regexes """ name, start, length, i, regex = parse_name_offset(path, ver) return name def parse_name_and_version(path: str) -> Tuple[str, spack.version.StandardVersion]: """Try to determine the name of a package and extract its version from its filename or URL. Args: path: The filename or URL for the package Returns: tuple: a tuple containing the package (name, version) Raises: UndetectableVersionError: If the URL does not match any regexes UndetectableNameError: If the URL does not match any regexes """ ver = parse_version(path) name = parse_name(path, ver) return (name, ver) def find_all(substring, string): """Returns a list containing the indices of every occurrence of substring in string.""" occurrences = [] index = 0 while index < len(string): index = string.find(substring, index) if index == -1: break occurrences.append(index) index += len(substring) return occurrences def substitution_offsets(path): """This returns offsets for substituting versions and names in the provided path. It is a helper for :func:`substitute_version`. """ # Get name and version offsets try: ver, vs, vl, vi, vregex = parse_version_offset(path) name, ns, nl, ni, nregex = parse_name_offset(path, ver) except UndetectableNameError: return (None, -1, -1, (), ver, vs, vl, (vs,)) except UndetectableVersionError: try: name, ns, nl, ni, nregex = parse_name_offset(path) return (name, ns, nl, (ns,), None, -1, -1, ()) except UndetectableNameError: return (None, -1, -1, (), None, -1, -1, ()) # Find the index of every occurrence of name and ver in path name_offsets = find_all(name, path) ver_offsets = find_all(ver, path) return (name, ns, nl, name_offsets, ver, vs, vl, ver_offsets) def wildcard_version(path): """Find the version in the supplied path, and return a regular expression that will match this path with any version in its place. """ # Get version so we can replace it with a wildcard version = parse_version(path) # Split path by versions vparts = path.split(str(version)) # Replace each version with a generic capture group to find versions # and escape everything else so it's not interpreted as a regex result = r"(\d.*)".join(re.escape(vp) for vp in vparts) return result def substitute_version(path: str, new_version) -> str: """Given a URL or archive name, find the version in the path and substitute the new version for it. Replace all occurrences of the version *if* they don't overlap with the package name. Simple example: .. code-block:: pycon >>> substitute_version("http://www.mr511.de/software/libelf-0.8.13.tar.gz", "2.9.3") "http://www.mr511.de/software/libelf-2.9.3.tar.gz" Complex example: .. code-block:: pycon >>> substitute_version("https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.12/src/hdf-4.2.12.tar.gz", "2.3") "https://www.hdfgroup.org/ftp/HDF/releases/HDF2.3/src/hdf-2.3.tar.gz" """ # noqa: E501 (name, ns, nl, noffs, ver, vs, vl, voffs) = substitution_offsets(path) new_path = "" last = 0 for vo in voffs: new_path += path[last:vo] new_path += str(new_version) last = vo + vl new_path += path[last:] return new_path def color_url(path, **kwargs): """Color the parts of the url according to Spack's parsing. Colors are: * Cyan: The version found by :func:`parse_version_offset`. * Red: The name found by :func:`parse_name_offset`. * Green: Instances of version string from :func:`substitute_version`. * Magenta: Instances of the name (protected from substitution). Args: path (str): The filename or URL for the package errors (bool): Append parse errors at end of string. subs (bool): Color substitutions as well as parsed name/version. """ # Allow URLs containing @ and } path = cescape(path) errors = kwargs.get("errors", False) subs = kwargs.get("subs", False) (name, ns, nl, noffs, ver, vs, vl, voffs) = substitution_offsets(path) nends = [no + nl - 1 for no in noffs] vends = [vo + vl - 1 for vo in voffs] nerr = verr = 0 out = io.StringIO() for i in range(len(path)): if i == vs: out.write("@c") verr += 1 elif i == ns: out.write("@r") nerr += 1 elif subs: if i in voffs: out.write("@g") elif i in noffs: out.write("@m") out.write(path[i]) if i == vs + vl - 1: out.write("@.") verr += 1 elif i == ns + nl - 1: out.write("@.") nerr += 1 elif subs: if i in vends or i in nends: out.write("@.") if errors: if nerr == 0: out.write(" @r{[no name]}") if verr == 0: out.write(" @r{[no version]}") if nerr == 1: out.write(" @r{[incomplete name]}") if verr == 1: out.write(" @r{[incomplete version]}") return colorize(out.getvalue()) def find_versions_of_archive( archive_urls: Union[str, Sequence[str]], list_url: Optional[str] = None, list_depth: int = 0, concurrency: Optional[int] = 32, reference_package: Optional[Any] = None, ) -> Dict[spack.version.StandardVersion, str]: """Scrape web pages for new versions of a tarball. This function prefers URLs in the following order: links found on the scraped page that match a url generated by the reference package, found and in the archive_urls list, found and derived from those in the archive_urls list, and if none are found for a version then the item in the archive_urls list is included for the version. Args: archive_urls: URL or sequence of URLs for different versions of a package. Typically these are just the tarballs from the package file itself. By default, this searches the parent directories of archives. list_url: URL for a listing of archives. Spack will scrape these pages for download links that look like the archive URL. list_depth: max depth to follow links on list_url pages. Defaults to 0. concurrency: maximum number of concurrent requests reference_package: a spack package used as a reference for url detection. Uses the url_for_version method on the package to produce reference urls which, if found, are preferred. """ if isinstance(archive_urls, str): archive_urls = [archive_urls] # Generate a list of list_urls based on archive urls and any # explicitly listed list_url in the package list_urls = set() if list_url is not None: list_urls.add(list_url) for aurl in archive_urls: list_urls |= spack.llnl.url.find_list_urls(aurl) # Add '/' to the end of the URL. Some web servers require this. additional_list_urls = set() for lurl in list_urls: if not lurl.endswith("/"): additional_list_urls.add(lurl + "/") list_urls |= additional_list_urls # Grab some web pages to scrape. _, links = spack.util.web.spider(list_urls, depth=list_depth, concurrency=concurrency) # Scrape them for archive URLs regexes = [] for aurl in archive_urls: # This creates a regex from the URL with a capture group for # the version part of the URL. The capture group is converted # to a generic wildcard, so we can use this to extract things # on a page that look like archive URLs. url_regex = wildcard_version(aurl) # We'll be a bit more liberal and just look for the archive # part, not the full path. # this is a URL so it is a posixpath even on Windows url_regex = pathlib.PurePosixPath(url_regex).name # We need to add a / to the beginning of the regex to prevent # Spack from picking up similarly named packages like: # https://cran.r-project.org/src/contrib/pls_2.6-0.tar.gz # https://cran.r-project.org/src/contrib/enpls_5.7.tar.gz # https://cran.r-project.org/src/contrib/autopls_1.3.tar.gz # https://cran.r-project.org/src/contrib/matrixpls_1.0.4.tar.gz url_regex = "/" + url_regex # We need to add a $ anchor to the end of the regex to prevent # Spack from picking up signature files like: # .asc # .md5 # .sha256 # .sig # However, SourceForge downloads still need to end in '/download'. url_regex += r"(\/download)?" # PyPI adds #sha256=... to the end of the URL url_regex += "(#sha256=.*)?" url_regex += "$" regexes.append(url_regex) regexes = [re.compile(r) for r in regexes] # Build a dict version -> URL from any links that match the wildcards. # Walk through archive_url links first. # Any conflicting versions will be overwritten by the list_url links. versions: Dict[spack.version.StandardVersion, str] = {} matched = set() for url in sorted(links): url = convert_to_posix_path(url) if any(r.search(url) for r in regexes): try: ver = parse_version(url) if ver in matched: continue versions[ver] = url # prevent this version from getting overwritten if reference_package is not None: if url == reference_package.url_for_version(ver): matched.add(ver) else: extrapolated_urls = [substitute_version(u, ver) for u in archive_urls] if url in extrapolated_urls: matched.add(ver) except UndetectableVersionError: continue for url in archive_urls: url = convert_to_posix_path(url) ver = parse_version(url) if ver not in versions: versions[ver] = url return versions class UrlParseError(spack.error.SpackError): """Raised when the URL module can't parse something correctly.""" def __init__(self, msg, path): super().__init__(msg) self.path = path class UndetectableVersionError(UrlParseError): """Raised when we can't parse a version from a string.""" def __init__(self, path): super().__init__("Couldn't detect version in: " + path, path) class UndetectableNameError(UrlParseError): """Raised when we can't parse a package name from a string.""" def __init__(self, path): super().__init__("Couldn't parse package name in: " + path, path) ================================================ FILE: lib/spack/spack/url_buildcache.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import enum import fnmatch import gzip import io import json import os import re import shutil import urllib.parse from contextlib import closing, contextmanager from datetime import datetime from pathlib import Path from tempfile import TemporaryDirectory from typing import Any, Callable, Dict, List, Optional, Tuple, Type import spack.vendor.jsonschema import spack.config as config import spack.database import spack.error import spack.hash_types as ht import spack.llnl.util.filesystem as fsys import spack.llnl.util.tty as tty import spack.mirrors.mirror import spack.spec import spack.stage import spack.util.crypto import spack.util.gpg import spack.util.url as url_util import spack.util.web as web_util from spack.schema.url_buildcache_manifest import schema as buildcache_manifest_schema from spack.util.archive import ChecksumWriter from spack.util.crypto import hash_fun_for_algo from spack.util.executable import which #: The build cache layout version that this version of Spack creates. #: Version 3: Introduces content-addressable tarballs CURRENT_BUILD_CACHE_LAYOUT_VERSION = 3 #: The name of the default buildcache index manifest file INDEX_MANIFEST_FILE = "index.manifest.json" class BuildcacheComponent(enum.Enum): """Enumeration of the kinds of things that live in a URL buildcache These enums serve two purposes: They allow different buildcache layout versions to specify different relative location of these entities, and they're used to map buildcache objects to their respective media types. """ # manifest files MANIFEST = enum.auto() # metadata file for a binary package SPEC = enum.auto() # things that live in the blobs directory BLOB = enum.auto() # binary mirror index INDEX = enum.auto() # public key used for verifying signed binary packages KEY = enum.auto() # index of all public keys found in the mirror KEY_INDEX = enum.auto() # compressed archive of spec installation directory TARBALL = enum.auto() # binary mirror descriptor file LAYOUT_JSON = enum.auto() class BlobRecord: """Class to describe a single data element (blob) from a manifest""" def __init__( self, content_length: int, media_type: str, compression_alg: str, checksum_alg: str, checksum: str, ) -> None: self.content_length = content_length self.media_type = media_type self.compression_alg = compression_alg self.checksum_alg = checksum_alg self.checksum = checksum @classmethod def from_dict(cls, record_dict): return BlobRecord( record_dict["contentLength"], record_dict["mediaType"], record_dict["compression"], record_dict["checksumAlgorithm"], record_dict["checksum"], ) def to_dict(self): return { "contentLength": self.content_length, "mediaType": self.media_type, "compression": self.compression_alg, "checksumAlgorithm": self.checksum_alg, "checksum": self.checksum, } class BuildcacheManifest: """A class to represent a buildcache manifest, which consists of a version number and an array of data blobs, each of which is represented by a BlobRecord.""" def __init__(self, layout_version: int, data: Optional[List[BlobRecord]] = None): self.version: int = layout_version if data: self.data: List[BlobRecord] = [ BlobRecord( rec.content_length, rec.media_type, rec.compression_alg, rec.checksum_alg, rec.checksum, ) for rec in data ] else: self.data = [] def to_dict(self): return {"version": self.version, "data": [rec.to_dict() for rec in self.data]} @classmethod def from_dict(cls, manifest_json: Dict[str, Any]) -> "BuildcacheManifest": spack.vendor.jsonschema.validate(manifest_json, buildcache_manifest_schema) return BuildcacheManifest( layout_version=manifest_json["version"], data=[BlobRecord.from_dict(blob_json) for blob_json in manifest_json["data"]], ) def get_blob_records(self, media_type: str) -> List[BlobRecord]: """Return any blob records from the manifest matching the given media type""" matches: List[BlobRecord] = [] for record in self.data: if record.media_type == media_type: matches.append(record) if matches: return matches raise NoSuchBlobException(f"Manifest has no blobs of type {media_type}") class URLBuildcacheEntry: """A class for managing URL-style buildcache entries This class manages access to a versioned buildcache entry by providing a means to download both the metadata (spec file) and compressed archive. It also provides methods for accessing the paths/urls associated with buildcache entries. Starting with buildcache layout version 3, it is not possible to know the full path to a compressed archive without either building it locally, or else fetching and reading the metadata first. This class provides api for fetching the metadata, as well as fetching the archive, and it enforces the need to fetch the metadata first. To help with downloading, this class manages two spack.spec.Stage objects internally, which must be destroyed when finished. Specifically, if you call either of the following methods on an instance, you must eventually also call destroy():: fetch_metadata() fetch_archive() This class also provides generic manifest and blob management api, and it can be used to fetch and push other kinds of buildcache entries aside from just binary packages. It can be used to work with public keys, buildcache indices, and any other type of data represented as a manifest which refers to blobs of data. """ SPEC_URL_REGEX = re.compile(r"(.+)/v([\d]+)/manifests/.+") LAYOUT_VERSION = 3 BUILDCACHE_INDEX_MEDIATYPE = f"application/vnd.spack.db.v{spack.database._DB_VERSION}+json" SPEC_MEDIATYPE = f"application/vnd.spack.spec.v{spack.spec.SPECFILE_FORMAT_VERSION}+json" TARBALL_MEDIATYPE = "application/vnd.spack.install.v2.tar+gzip" PUBLIC_KEY_MEDIATYPE = "application/pgp-keys" PUBLIC_KEY_INDEX_MEDIATYPE = "application/vnd.spack.keyindex.v1+json" BUILDCACHE_INDEX_FILE = "index.manifest.json" COMPONENT_PATHS = { BuildcacheComponent.MANIFEST: [f"v{LAYOUT_VERSION}", "manifests"], BuildcacheComponent.BLOB: ["blobs"], BuildcacheComponent.INDEX: [f"v{LAYOUT_VERSION}", "manifests", "index"], BuildcacheComponent.KEY: [f"v{LAYOUT_VERSION}", "manifests", "key"], BuildcacheComponent.SPEC: [f"v{LAYOUT_VERSION}", "manifests", "spec"], BuildcacheComponent.KEY_INDEX: [f"v{LAYOUT_VERSION}", "manifests", "key"], BuildcacheComponent.TARBALL: ["blobs"], BuildcacheComponent.LAYOUT_JSON: [f"v{LAYOUT_VERSION}", "layout.json"], } def __init__( self, mirror_url: str, spec: Optional[spack.spec.Spec] = None, allow_unsigned: bool = False ): """Lazily initialize the object""" self.mirror_url: str = mirror_url self.spec: Optional[spack.spec.Spec] = spec self.allow_unsigned: bool = allow_unsigned self.manifest: Optional[BuildcacheManifest] = None self.remote_manifest_url: str = "" self.stages: Dict[BlobRecord, spack.stage.Stage] = {} @classmethod def get_layout_version(cls) -> int: """Returns the layout version of this class""" return cls.LAYOUT_VERSION @classmethod def check_layout_json_exists(cls, mirror_url: str) -> bool: """Return True if layout.json exists in the expected location, False otherwise""" layout_json_url = url_util.join( mirror_url, *cls.get_relative_path_components(BuildcacheComponent.LAYOUT_JSON) ) return web_util.url_exists(layout_json_url) @classmethod def maybe_push_layout_json(cls, mirror_url: str) -> None: """This function does nothing if layout.json already exists, otherwise it pushes layout.json to the expected location in the mirror""" if cls.check_layout_json_exists(mirror_url): return layout_contents = {"signing": "gpg"} with TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: local_layout_path = os.path.join(tmpdir, "layout.json") with open(local_layout_path, "w", encoding="utf-8") as fd: json.dump(layout_contents, fd) remote_layout_url = url_util.join( mirror_url, *cls.get_relative_path_components(BuildcacheComponent.LAYOUT_JSON) ) web_util.push_to_url(local_layout_path, remote_layout_url, keep_original=False) @classmethod def get_base_url(cls, manifest_url: str) -> str: """Given any manifest url (i.e. one containing ``v3/manifests/``) return the base part of the url""" rematch = cls.SPEC_URL_REGEX.match(manifest_url) if not rematch: raise BuildcacheEntryError(f"Unable to parse spec url: {manifest_url}") return rematch.group(1) @classmethod def get_index_url(cls, mirror_url: str, view: Optional[str] = None): return url_util.join( mirror_url, *cls.get_relative_path_components(BuildcacheComponent.INDEX), url_util.join(view or "", cls.BUILDCACHE_INDEX_FILE), ) @classmethod def get_relative_path_components(cls, component: BuildcacheComponent) -> List[str]: """Given any type of buildcache component, return its relative location within a mirror as a list path elements""" return cls.COMPONENT_PATHS[component] @classmethod def get_manifest_filename(cls, spec: spack.spec.Spec) -> str: """Given a concrete spec, compute and return the name (i.e. basename) of the manifest file representing it""" spec_formatted = spec.format_path("{name}-{version}-{hash}") return f"{spec_formatted}.spec.manifest.json" @classmethod def get_manifest_url(cls, spec: spack.spec.Spec, mirror_url: str) -> str: """Given a concrete spec and a base url, return the full url where the spec manifest should be found""" path_components = cls.get_relative_path_components(BuildcacheComponent.SPEC) return url_util.join( mirror_url, *path_components, spec.name, cls.get_manifest_filename(spec) ) @classmethod def get_buildcache_component_include_pattern( cls, buildcache_component: BuildcacheComponent ) -> str: """Given a buildcache component, return the glob pattern that can be used to match it in a directory listing. If None is provided, return a catch-all pattern that will match all buildcache components.""" if buildcache_component is BuildcacheComponent.MANIFEST: return "*.manifest.json" elif buildcache_component == BuildcacheComponent.SPEC: return "*.spec.manifest.json" elif buildcache_component == BuildcacheComponent.INDEX: return ".*index.manifest.json" elif buildcache_component == BuildcacheComponent.KEY: return "*.key.manifest.json" elif buildcache_component == BuildcacheComponent.KEY_INDEX: return "keys.manifest.json" raise BuildcacheEntryError(f"Not a manifest component: {buildcache_component}") @classmethod def component_to_media_type(cls, component: BuildcacheComponent) -> str: """Mapping from buildcache component to media type""" if component == BuildcacheComponent.SPEC: return cls.SPEC_MEDIATYPE elif component == BuildcacheComponent.TARBALL: return cls.TARBALL_MEDIATYPE elif component == BuildcacheComponent.INDEX: return cls.BUILDCACHE_INDEX_MEDIATYPE elif component == BuildcacheComponent.KEY: return cls.PUBLIC_KEY_MEDIATYPE elif component == BuildcacheComponent.KEY_INDEX: return cls.PUBLIC_KEY_INDEX_MEDIATYPE raise BuildcacheEntryError(f"Not a blob component: {component}") def get_local_spec_path(self) -> str: """Convenience method to return the local path of a fetched spec file""" return self.get_staged_blob_path(self.get_blob_record(BuildcacheComponent.SPEC)) def get_local_archive_path(self) -> str: """Convenience method to return the local path of a fetched tarball""" return self.get_staged_blob_path(self.get_blob_record(BuildcacheComponent.TARBALL)) def get_blob_record(self, blob_type: BuildcacheComponent) -> BlobRecord: """Return the first blob record of the given type. Assumes the manifest has already been fetched.""" if not self.manifest: raise BuildcacheEntryError("Read manifest before accessing blob records") records = self.manifest.get_blob_records(self.component_to_media_type(blob_type)) if len(records) == 0: raise BuildcacheEntryError(f"Manifest has no blob record of type {blob_type}") return records[0] def check_blob_exists(self, record: BlobRecord) -> bool: """Return True if the blob given by record exists on the mirror, False otherwise""" blob_url = self.get_blob_url(self.mirror_url, record) return web_util.url_exists(blob_url) @classmethod def get_blob_path_components(cls, record: BlobRecord) -> List[str]: """Given a BlobRecord, return the relative path of the blob within a mirror as a list of path components""" return [ *cls.get_relative_path_components(BuildcacheComponent.BLOB), record.checksum_alg, record.checksum[:2], record.checksum, ] @classmethod def get_blob_url(cls, mirror_url: str, record: BlobRecord) -> str: """Return the full url of the blob given by record""" return url_util.join(mirror_url, *cls.get_blob_path_components(record)) def fetch_blob(self, record: BlobRecord) -> str: """Given a blob record, find associated blob in the manifest and stage it Returns the local path to the staged blob """ if record not in self.stages: blob_url = self.get_blob_url(self.mirror_url, record) blob_stage = spack.stage.Stage(blob_url) # Fetch the blob, or else cleanup and exit early try: blob_stage.create() blob_stage.fetch() except spack.error.FetchError as e: self.destroy() raise BuildcacheEntryError(f"Unable to fetch blob from {blob_url}") from e # Raises if checksum does not match expectation validate_checksum(blob_stage.save_filename, record.checksum_alg, record.checksum) self.stages[record] = blob_stage return self.get_staged_blob_path(record) def get_staged_blob_path(self, record: BlobRecord) -> str: """Convenience method to return the local path of a staged blob""" if record not in self.stages: raise BuildcacheEntryError(f"Blob not staged: {record}") return self.stages[record].save_filename def exists(self, components: List[BuildcacheComponent]) -> bool: """Check whether blobs exist for all specified components Returns True if there is a blob present in the mirror for every given component type. """ try: self.read_manifest() except BuildcacheEntryError: return False if not self.manifest: return False for component in components: component_blobs = self.manifest.get_blob_records( self.component_to_media_type(component) ) if len(component_blobs) == 0: return False if not self.check_blob_exists(component_blobs[0]): return False return True @classmethod def verify_and_extract_manifest(cls, manifest_contents: str, verify: bool = False) -> dict: """Possibly verify clearsig, then extract contents and return as json""" magic_string = "-----BEGIN PGP SIGNED MESSAGE-----" if manifest_contents.startswith(magic_string): if verify: # Try to verify and raise if we fail with TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: manifest_path = os.path.join(tmpdir, "manifest.json.sig") with open(manifest_path, "w", encoding="utf-8") as fd: fd.write(manifest_contents) if not try_verify(manifest_path): raise NoVerifyException("Signature could not be verified") return spack.spec.Spec.extract_json_from_clearsig(manifest_contents) elif verify: raise NoVerifyException("Required signature was not found on manifest") return json.loads(manifest_contents) def read_manifest(self, manifest_url: Optional[str] = None) -> BuildcacheManifest: """Read and process the the buildcache entry manifest. If no manifest url is provided, build the url from the internal spec and base push url.""" if self.manifest: if not manifest_url or manifest_url == self.remote_manifest_url: # We already have a manifest, so now calling this method without a specific # manifest url, or with the same one we have internally, then skip reading # again, and just return the manifest we already read. return self.manifest self.manifest = None if not manifest_url: if not self.spec or not self.mirror_url: raise BuildcacheEntryError( "Either manifest url or spec and mirror are required to read manifest" ) manifest_url = self.get_manifest_url(self.spec, self.mirror_url) self.remote_manifest_url = manifest_url manifest_contents = "" try: manifest_contents = web_util.read_text(manifest_url) except (web_util.SpackWebError, OSError) as e: raise BuildcacheEntryError(f"Error reading manifest at {manifest_url}") from e if not manifest_contents: raise BuildcacheEntryError("Unable to read manifest or manifest empty") manifest_contents = self.verify_and_extract_manifest( manifest_contents, verify=not self.allow_unsigned ) self.manifest = BuildcacheManifest.from_dict(manifest_contents) if self.manifest.version != 3: raise BuildcacheEntryError("Layout version mismatch in fetched manifest") return self.manifest def fetch_metadata(self) -> dict: """Retrieve metadata for the spec, returns the validated spec dict""" if not self.manifest: # Reading the manifest will either successfully compute the remote # spec url, or else raise an exception self.read_manifest() local_specfile_path = self.fetch_blob(self.get_blob_record(BuildcacheComponent.SPEC)) # Check spec file for validity and read it, or else cleanup and exit early try: spec_dict, _ = get_valid_spec_file(local_specfile_path, self.get_layout_version()) except InvalidMetadataFile as e: self.destroy() raise BuildcacheEntryError("Buildcache entry does not have valid metadata file") from e return spec_dict def fetch_archive(self) -> str: """Retrieve the archive file and return the local archive file path""" if not self.manifest: # Raises if problems encountered, including not being able to verify signagure self.read_manifest() return self.fetch_blob(self.get_blob_record(BuildcacheComponent.TARBALL)) def get_archive_stage(self) -> Optional[spack.stage.Stage]: return self.stages[self.get_blob_record(BuildcacheComponent.TARBALL)] def remove(self): """Remove a binary package (spec file and tarball) and the associated manifest from the mirror.""" if self.manifest: try: web_util.remove_url(self.remote_manifest_url) except Exception as e: tty.debug(f"Failed to remove previous manfifest: {e}") try: web_util.remove_url( self.get_blob_url( self.mirror_url, self.get_blob_record(BuildcacheComponent.TARBALL) ) ) except Exception as e: tty.debug(f"Failed to remove previous archive: {e}") try: web_util.remove_url( self.get_blob_url( self.mirror_url, self.get_blob_record(BuildcacheComponent.SPEC) ) ) except Exception as e: tty.debug(f"Failed to remove previous metadata: {e}") self.manifest = None @classmethod def push_blob(cls, mirror_url: str, blob_path: str, record: BlobRecord) -> None: """Push the blob_path file to mirror as a blob represented by the given record""" blob_destination_url = cls.get_blob_url(mirror_url, record) web_util.push_to_url(blob_path, blob_destination_url, keep_original=False) @classmethod def push_manifest( cls, mirror_url: str, manifest_name: str, manifest: BuildcacheManifest, tmpdir: str, component_type: BuildcacheComponent = BuildcacheComponent.SPEC, signing_key: Optional[str] = None, ) -> None: """Given a BuildcacheManifest, push it to the mirror using the given manifest name. The component_type is used to indicate what type of thing the manifest represents, so it can be placed in the correct relative path within the mirror. If a signing_key is provided, it will be used to clearsign the manifest before pushing it.""" # write the manifest to a temporary location manifest_file_name = f"{manifest_name}.manifest.json" manifest_path = os.path.join(tmpdir, manifest_file_name) os.makedirs(os.path.dirname(manifest_path), exist_ok=True) with open(manifest_path, "w", encoding="utf-8") as f: json.dump(manifest.to_dict(), f, indent=0, separators=(",", ":")) # Note: when using gpg clear sign, we need to avoid long lines (19995 # chars). If lines are longer, they are truncated without error. So, # here we still add newlines, but no indent, so save on file size and # line length. if signing_key: manifest_path = sign_file(signing_key, manifest_path) manifest_destination_url = url_util.join( mirror_url, *cls.get_relative_path_components(component_type), manifest_file_name ) web_util.push_to_url(manifest_path, manifest_destination_url, keep_original=False) @classmethod def push_local_file_as_blob( cls, local_file_path: str, mirror_url: str, manifest_name: str, component_type: BuildcacheComponent, compression: str = "none", ) -> None: """Convenience method to push a local file to a mirror as a blob. Both manifest and blob are pushed as a component of the given component_type. If ``compression`` is ``"gzip"`` the blob will be compressed before pushing, otherwise it will be pushed uncompressed.""" cache_class = get_url_buildcache_class() checksum_algo = "sha256" blob_to_push = local_file_path with TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir: blob_to_push = os.path.join(tmpdir, os.path.basename(local_file_path)) with compression_writer(blob_to_push, compression, checksum_algo) as ( fout, checker, ), open(local_file_path, "rb") as fin: shutil.copyfileobj(fin, fout) record = BlobRecord( checker.length, cache_class.component_to_media_type(component_type), compression, checksum_algo, checker.hexdigest(), ) manifest = BuildcacheManifest( layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION, data=[record] ) cls.push_blob(mirror_url, blob_to_push, record) cls.push_manifest( mirror_url, manifest_name, manifest, tmpdir, component_type=component_type ) def push_binary_package( self, spec: spack.spec.Spec, tarball_path: str, checksum_algorithm: str, tarball_checksum: str, tmpdir: str, signing_key: Optional[str], ) -> None: """Convenience method to push tarball, specfile, and manifest to the remote mirror Pushing should only be done after checking for the pre-existence of a buildcache entry for this spec, and represents a force push if one is found. Thus, any pre-existing files are first removed. """ spec_dict = spec.to_dict(hash=ht.dag_hash) # TODO: Remove this key once oci buildcache no longer uses it spec_dict["buildcache_layout_version"] = 2 tarball_content_length = os.stat(tarball_path).st_size compression = "gzip" # Delete the previously existing version self.remove() if not self.remote_manifest_url: self.remote_manifest_url = self.get_manifest_url(spec, self.mirror_url) # Any previous archive/tarball is gone, compute the path to the new one remote_archive_url = url_util.join( self.mirror_url, *self.get_relative_path_components(BuildcacheComponent.BLOB), checksum_algorithm, tarball_checksum[:2], tarball_checksum, ) # push the archive/tarball blob to the remote web_util.push_to_url(tarball_path, remote_archive_url, keep_original=False) # Clear out the previous data, then add a record for the new blob blobs: List[BlobRecord] = [] blobs.append( BlobRecord( tarball_content_length, self.TARBALL_MEDIATYPE, compression, checksum_algorithm, tarball_checksum, ) ) # compress the spec dict and compute its checksum specfile = os.path.join(tmpdir, f"{spec.dag_hash()}.spec.json") metadata_checksum, metadata_size = compressed_json_from_dict( specfile, spec_dict, checksum_algorithm ) # Any previous metadata blob is gone, compute the path to the new one remote_spec_url = url_util.join( self.mirror_url, *self.get_relative_path_components(BuildcacheComponent.BLOB), checksum_algorithm, metadata_checksum[:2], metadata_checksum, ) # push the metadata/spec blob to the remote web_util.push_to_url(specfile, remote_spec_url, keep_original=False) blobs.append( BlobRecord( metadata_size, self.SPEC_MEDIATYPE, compression, checksum_algorithm, metadata_checksum, ) ) # generate the manifest manifest = { "version": self.get_layout_version(), "data": [record.to_dict() for record in blobs], } # write the manifest to a temporary location manifest_path = os.path.join(tmpdir, f"{spec.dag_hash()}.manifest.json") with open(manifest_path, "w", encoding="utf-8") as f: json.dump(manifest, f, indent=0, separators=(",", ":")) # Note: when using gpg clear sign, we need to avoid long lines (19995 # chars). If lines are longer, they are truncated without error. So, # here we still add newlines, but no indent, so save on file size and # line length. # possibly sign the manifest if signing_key: manifest_path = sign_file(signing_key, manifest_path) # Push the manifest file to the remote. The remote manifest url for # a given concrete spec is fixed, so we don't have to recompute it, # even if we deleted the pre-existing one. web_util.push_to_url(manifest_path, self.remote_manifest_url, keep_original=False) def destroy(self): """Destroy any existing stages""" for blob_stage in self.stages.values(): blob_stage.destroy() self.stages = {} class URLBuildcacheEntryV2(URLBuildcacheEntry): """This class exists to provide read-only support for reading older buildcache layouts in a way that is transparent to binary_distribution code responsible for downloading and extracting binary packages. Since support for layout v2 is read-only, and since v2 did not have support for manifests and blobs, many class and instance methods are overridden simply to raise, hopefully making the intended use and limitations of the class clear to developers.""" SPEC_URL_REGEX = re.compile(r"(.+)/build_cache/.+") LAYOUT_VERSION = 2 BUILDCACHE_INDEX_FILE = "index.json" COMPONENT_PATHS = { BuildcacheComponent.MANIFEST: ["build_cache"], BuildcacheComponent.BLOB: ["build_cache"], BuildcacheComponent.INDEX: ["build_cache"], BuildcacheComponent.KEY: ["build_cache", "_pgp"], BuildcacheComponent.SPEC: ["build_cache"], BuildcacheComponent.KEY_INDEX: ["build_cache", "_pgp"], BuildcacheComponent.TARBALL: ["build_cache"], BuildcacheComponent.LAYOUT_JSON: ["build_cache", "layout.json"], } def __init__( self, push_url_base: str, spec: Optional[spack.spec.Spec] = None, allow_unsigned: bool = False, ): """Lazily initialize the object""" self.mirror_url: str = push_url_base self.spec: Optional[spack.spec.Spec] = spec self.allow_unsigned: bool = allow_unsigned self.has_metadata: bool = False self.has_tarball: bool = False self.has_signed: bool = False self.has_unsigned: bool = False self.spec_stage: Optional[spack.stage.Stage] = None self.local_specfile_path: str = "" self.archive_stage: Optional[spack.stage.Stage] = None self.local_archive_path: str = "" self.remote_spec_url: str = "" self.remote_archive_url: str = "" self.remote_archive_checksum_algorithm: str = "" self.remote_archive_checksum_hash: str = "" self.spec_dict: Dict[Any, Any] = {} self._checked_signed = False self._checked_unsigned = False self._checked_exists = False @classmethod def get_layout_version(cls) -> int: return cls.LAYOUT_VERSION @classmethod def maybe_push_layout_json(cls, mirror_url: str) -> None: raise BuildcacheEntryError("spack can no longer write to v2 buildcaches") def _get_spec_url( self, spec: spack.spec.Spec, mirror_url: str, ext: str = ".spec.json.sig" ) -> str: spec_formatted = spec.format_path( "{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}" ) path_components = self.get_relative_path_components(BuildcacheComponent.SPEC) return url_util.join(mirror_url, *path_components, f"{spec_formatted}{ext}") def _get_tarball_url(self, spec: spack.spec.Spec, mirror_url: str) -> str: directory_name = spec.format_path( "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}" ) spec_formatted = spec.format_path( "{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}" ) filename = f"{spec_formatted}.spack" return url_util.join( mirror_url, *self.get_relative_path_components(BuildcacheComponent.BLOB), directory_name, filename, ) def _check_metadata_exists(self): if not self.spec: return if not self._checked_signed: signed_url = self._get_spec_url(self.spec, self.mirror_url, ext=".spec.json.sig") if web_util.url_exists(signed_url): self.remote_spec_url = signed_url self.has_signed = True self._checked_signed = True if not self.has_signed and not self._checked_unsigned: unsigned_url = self._get_spec_url(self.spec, self.mirror_url, ext=".spec.json") if web_util.url_exists(unsigned_url): self.remote_spec_url = unsigned_url self.has_unsigned = True self._checked_unsigned = True def exists(self, components: List[BuildcacheComponent]) -> bool: if not self.spec: return False if ( len(components) != 2 or BuildcacheComponent.SPEC not in components or BuildcacheComponent.TARBALL not in components ): return False self._check_metadata_exists() if not self.has_signed and not self.has_unsigned: return False if not web_util.url_exists(self._get_tarball_url(self.spec, self.mirror_url)): return False return True def fetch_metadata(self) -> dict: """Retrieve the v2 specfile for the spec, yields the validated spec+ dict""" if self.spec_dict: # Only fetch the metadata once return self.spec_dict self._check_metadata_exists() if not self.remote_spec_url: raise BuildcacheEntryError(f"Mirror {self.mirror_url} does not have metadata for spec") if not self.allow_unsigned and self.has_unsigned: raise BuildcacheEntryError( f"Mirror {self.mirror_url} does not have signed metadata for spec" ) self.spec_stage = spack.stage.Stage(self.remote_spec_url) # Fetch the spec file, or else cleanup and exit early try: self.spec_stage.create() self.spec_stage.fetch() except spack.error.FetchError as e: self.destroy() raise BuildcacheEntryError( f"Unable to fetch metadata from {self.remote_spec_url}" ) from e self.local_specfile_path = self.spec_stage.save_filename if not self.allow_unsigned and not try_verify(self.local_specfile_path): raise NoVerifyException(f"Signature on {self.remote_spec_url} could not be verified") # Check spec file for validity and read it, or else cleanup and exit early try: spec_dict, _ = get_valid_spec_file(self.local_specfile_path, self.get_layout_version()) except InvalidMetadataFile as e: self.destroy() raise BuildcacheEntryError("Buildcache entry does not have valid metadata file") from e try: self.spec = spack.spec.Spec.from_dict(spec_dict) except Exception as err: raise BuildcacheEntryError("Fetched spec dict does not contain valid spec") from err self.spec_dict = spec_dict # Retrieve the alg and hash from the spec dict, use them to build the path to # the tarball. if "binary_cache_checksum" not in self.spec_dict: raise BuildcacheEntryError("Provided spec dict must contain 'binary_cache_checksum'") bchecksum = self.spec_dict["binary_cache_checksum"] if "hash_algorithm" not in bchecksum or "hash" not in bchecksum: raise BuildcacheEntryError( "Provided spec dict contains invalid 'binary_cache_checksum'" ) self.remote_archive_checksum_algorithm = bchecksum["hash_algorithm"] self.remote_archive_checksum_hash = bchecksum["hash"] self.remote_archive_url = self._get_tarball_url(self.spec, self.mirror_url) return self.spec_dict def fetch_archive(self) -> str: self.fetch_metadata() # Adding this, we can avoid passing a dictionary of stages around the # install logic, and in fact completely avoid fetching the metadata in # the new (v3) approach. if self.spec_stage: self.spec_stage.destroy() self.spec_stage = None self.archive_stage = spack.stage.Stage(self.remote_archive_url) # Fetch the archive file, or else cleanup and exit early try: self.archive_stage.create() self.archive_stage.fetch() except spack.error.FetchError as e: self.destroy() raise BuildcacheEntryError( f"Unable to fetch archive from {self.remote_archive_url}" ) from e self.local_archive_path = self.archive_stage.save_filename # Raises if checksum does not match expected validate_checksum( self.local_archive_path, self.remote_archive_checksum_algorithm, self.remote_archive_checksum_hash, ) return self.local_archive_path def get_archive_stage(self) -> Optional[spack.stage.Stage]: return self.archive_stage @classmethod def get_manifest_filename(cls, spec: spack.spec.Spec) -> str: raise BuildcacheEntryError("v2 buildcache entries do not have a manifest file") @classmethod def get_manifest_url(cls, spec: spack.spec.Spec, mirror_url: str) -> str: raise BuildcacheEntryError("v2 buildcache entries do not have a manifest url") @classmethod def get_buildcache_component_include_pattern( cls, buildcache_component: BuildcacheComponent ) -> str: raise BuildcacheEntryError("v2 buildcache entries do not have a manifest file") def read_manifest(self, manifest_url: Optional[str] = None) -> BuildcacheManifest: raise BuildcacheEntryError("v2 buildcache entries do not have a manifest file") def remove(self): raise BuildcacheEntryError("Spack cannot delete v2 buildcache entries") def get_blob_record(self, blob_type: BuildcacheComponent) -> BlobRecord: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") def check_blob_exists(self, record: BlobRecord) -> bool: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") @classmethod def get_blob_path_components(cls, record: BlobRecord) -> List[str]: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") @classmethod def get_blob_url(cls, mirror_url: str, record: BlobRecord) -> str: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") def fetch_blob(self, record: BlobRecord) -> str: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") def get_staged_blob_path(self, record: BlobRecord) -> str: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") @classmethod def verify_and_extract_manifest(cls, manifest_contents: str, verify: bool = False) -> dict: raise BuildcacheEntryError("v2 buildcache entries do not have a manifest file") @classmethod def push_blob(cls, mirror_url: str, blob_path: str, record: BlobRecord) -> None: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") @classmethod def push_manifest( cls, mirror_url: str, manifest_name: str, manifest: BuildcacheManifest, tmpdir: str, component_type: BuildcacheComponent = BuildcacheComponent.SPEC, signing_key: Optional[str] = None, ) -> None: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") @classmethod def push_local_file_as_blob( cls, local_file_path: str, mirror_url: str, manifest_name: str, component_type: BuildcacheComponent, compression: str = "none", ) -> None: raise BuildcacheEntryError("v2 buildcache layout is unaware of manifests and blobs") def push_binary_package( self, spec: spack.spec.Spec, tarball_path: str, checksum_algorithm: str, tarball_checksum: str, tmpdir: str, signing_key: Optional[str], ) -> None: raise BuildcacheEntryError("Spack can no longer push v2 buildcache entries") def destroy(self): if self.archive_stage: self.archive_stage.destroy() self.archive_stage = None if self.spec_stage: self.spec_stage.destroy() self.spec_stage = None def get_url_buildcache_class( layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION, ) -> Type[URLBuildcacheEntry]: """Given a layout version, return the class responsible for managing access to buildcache entries of that version""" if layout_version == 2: return URLBuildcacheEntryV2 elif layout_version == 3: return URLBuildcacheEntry else: raise UnknownBuildcacheLayoutError( f"Cannot create buildcache class for unknown layout version {layout_version}" ) def check_mirror_for_layout(mirror: spack.mirrors.mirror.Mirror): """Check specified mirror, and warn if missing layout.json""" cache_class = get_url_buildcache_class() if not cache_class.check_layout_json_exists(mirror.fetch_url): msg = ( f"Configured mirror {mirror.name} is missing layout.json and has either \n" " never been pushed or is of an old layout version. If it's the latter, \n" " consider running 'spack buildcache migrate' or rebuilding the specs in \n" " in this mirror." ) tty.warn(msg) def _entries_from_cache_aws_cli(url: str, tmpspecsdir: str, component_type: BuildcacheComponent): """Use aws cli to sync all manifests into a local temporary directory. Args: url: prefix of the build cache on s3 tmpspecsdir: path to temporary directory to use for writing files component_type: type of buildcache component to sync (spec, index, key, etc.) Return: A tuple where the first item is a list of local file paths pointing to the manifests that should be read from the mirror, and the second item is a function taking a url or file path and returning a :class:`URLBuildcacheEntry` for that manifest. """ read_fn = None file_list = None aws = which("aws") cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION) if not aws: tty.warn("Failed to use aws s3 sync to retrieve specs, falling back to parallel fetch") return file_list, read_fn def file_read_method(manifest_path: str) -> URLBuildcacheEntry: cache_entry = cache_class(mirror_url=url, allow_unsigned=True) cache_entry.read_manifest(manifest_url=manifest_path) return cache_entry include_pattern = cache_class.get_buildcache_component_include_pattern(component_type) component_prefix = cache_class.get_relative_path_components(component_type) sync_command_args = [ "s3", "sync", "--exclude", "*", "--include", include_pattern, url_util.join(url, *component_prefix), tmpspecsdir, ] # Use aws s3 ls to get mtimes of manifests ls_command_args = ["s3", "ls", "--recursive", url] s3_ls_regex = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+\d+\s+(.+)$") filename_to_mtime: Dict[str, float] = {} tty.debug(f"Using aws s3 sync to download manifests from {url} to {tmpspecsdir}") try: aws(*sync_command_args, output=os.devnull, error=os.devnull) file_list = fsys.find(tmpspecsdir, [include_pattern]) read_fn = file_read_method # Use `aws s3 ls` to get mtimes of manifests for line in aws(*ls_command_args, output=str, error=os.devnull).splitlines(): match = s3_ls_regex.match(line) if match: # Parse the url and use the S3 path of the file to derive the # local path of the file (i.e. where `aws s3 sync` put it). parsed_url = urllib.parse.urlparse(url) s3_path = parsed_url.path.lstrip("/") filename = match.group(2) if s3_path and filename.startswith(s3_path): filename = filename[len(s3_path) :].lstrip("/") local_path = url_util.join(tmpspecsdir, filename) if Path(local_path).exists(): filename_to_mtime[url_util.path_to_file_url(local_path)] = datetime.strptime( match.group(1), "%Y-%m-%d %H:%M:%S" ).timestamp() except Exception as e: tty.warn("Failed to use aws s3 sync to retrieve specs, falling back to parallel fetch") raise e return filename_to_mtime, read_fn def _entries_from_cache_fallback(url: str, tmpspecsdir: str, component_type: BuildcacheComponent): """Use spack.util.web module to get a list of all the manifests at the remote url. Args: url: Base url of mirror (location of manifest files) tmpspecsdir: path to temporary directory to use for writing files component_type: type of buildcache component to sync (spec, index, key, etc.) Return: A tuple where the first item is a list of absolute file paths or urls pointing to the manifests that should be read from the mirror, and the second item is a function taking a url or file path of a manifest and returning a :class:`URLBuildcacheEntry` for that manifest. """ read_fn = None filename_to_mtime = None cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION) def url_read_method(manifest_url: str) -> URLBuildcacheEntry: cache_entry = cache_class(mirror_url=url, allow_unsigned=True) cache_entry.read_manifest(manifest_url) return cache_entry try: filename_to_mtime = {} component_path_parts = cache_class.get_relative_path_components(component_type) component_prefix: str = url_util.join(url, *component_path_parts) component_pattern = cache_class.get_buildcache_component_include_pattern(component_type) for entry in web_util.list_url(component_prefix, recursive=True): if fnmatch.fnmatch(entry, component_pattern): entry_url = url_util.join(component_prefix, entry) stat_result = web_util.stat_url(entry_url) if stat_result is not None: filename_to_mtime[entry_url] = stat_result[1] # mtime is second element read_fn = url_read_method except Exception as err: # If we got some kind of S3 (access denied or other connection error), the first non # boto-specific class in the exception is Exception. Just print a warning and return tty.warn(f"Encountered problem listing packages at {url}: {err}") return filename_to_mtime, read_fn def get_entries_from_cache(url: str, tmpspecsdir: str, component_type: BuildcacheComponent): """Get a list of all the manifests in the mirror and a function to read them. Args: url: Base url of mirror (location of spec files) tmpspecsdir: Temporary location for writing files component_type: type of buildcache component to sync (spec, index, key, etc.) Return: A tuple where the first item is a list of absolute file paths or urls pointing to the manifests that should be read from the mirror, and the second item is a function taking a url or file path and returning a :class:`URLBuildcacheEntry` for that manifest. """ callbacks: List[Callable] = [] if url.startswith("s3://"): callbacks.append(_entries_from_cache_aws_cli) callbacks.append(_entries_from_cache_fallback) for specs_from_cache_fn in callbacks: file_to_mtime_mapping, read_fn = specs_from_cache_fn(url, tmpspecsdir, component_type) if file_to_mtime_mapping: return file_to_mtime_mapping, read_fn raise ListMirrorSpecsError("Failed to get list of entries from {0}".format(url)) def validate_checksum(file_path, checksum_algorithm, expected_checksum) -> None: """Compute the checksum of the given file and raise if invalid""" local_checksum = spack.util.crypto.checksum(hash_fun_for_algo(checksum_algorithm), file_path) if local_checksum != expected_checksum: size, contents = fsys.filesummary(file_path) raise spack.error.NoChecksumException( file_path, size, contents, checksum_algorithm, expected_checksum, local_checksum ) def _get_compressor(compression: str, writable: io.BufferedIOBase) -> io.BufferedIOBase: if compression == "gzip": return gzip.GzipFile(filename="", mode="wb", compresslevel=6, mtime=0, fileobj=writable) elif compression == "none": return writable else: raise BuildcacheEntryError(f"Unknown compression type: {compression}") @contextmanager def compression_writer(output_path: str, compression: str, checksum_algo: str): """Create and return a writer capable of writing compressed data. Available options for ``compression`` are ``"gzip"`` or ``"none"``, ``checksum_algo`` is used to pick the checksum algorithm used by the :class:`~spack.util.archive.ChecksumWriter`. Yields: A tuple containing * An :class:`io.BufferedIOBase` writer that can compress (or not) as it writes * A :class:`~spack.util.archive.ChecksumWriter` that provides checksum and length of written data """ with open(output_path, "wb") as writer, ChecksumWriter( fileobj=writer, algorithm=hash_fun_for_algo(checksum_algo) ) as checksum_writer, closing( _get_compressor(compression, checksum_writer) ) as compress_writer: yield compress_writer, checksum_writer def compressed_json_from_dict( output_path: str, spec_dict: dict, checksum_algo: str ) -> Tuple[str, int]: """Compress the spec dict and write it to the given path Return the checksum (using the given algorithm) and size on disk of the file """ with compression_writer(output_path, "gzip", checksum_algo) as ( f_bin, checker, ), io.TextIOWrapper(f_bin, encoding="utf-8") as f_txt: json.dump(spec_dict, f_txt, separators=(",", ":")) return checker.hexdigest(), checker.length def get_valid_spec_file(path: str, max_supported_layout: int) -> Tuple[Dict, int]: """Read and validate a spec file, returning the spec dict with its layout version, or raising InvalidMetadataFile if invalid.""" try: with open(path, "rb") as f: binary_content = f.read() except OSError as e: raise InvalidMetadataFile(f"No such file: {path}") from e # Decompress spec file if necessary if binary_content[:2] == b"\x1f\x8b": binary_content = gzip.decompress(binary_content) try: as_string = binary_content.decode("utf-8") if path.endswith(".json.sig"): spec_dict = spack.spec.Spec.extract_json_from_clearsig(as_string) else: spec_dict = json.loads(as_string) except Exception as e: raise InvalidMetadataFile(f"Could not parse {path} due to: {e}") from e # Ensure this version is not too new. try: layout_version = int(spec_dict.get("buildcache_layout_version", 0)) except ValueError as e: raise InvalidMetadataFile("Could not parse layout version") from e if layout_version > max_supported_layout: raise InvalidMetadataFile( f"Layout version {layout_version} is too new for this version of Spack" ) return spec_dict, layout_version def sign_file(key: str, file_path: str) -> str: """sign and return the path to the signed file""" signed_file_path = f"{file_path}.sig" spack.util.gpg.sign(key, file_path, signed_file_path, clearsign=True) return signed_file_path def try_verify(specfile_path): """Utility function to attempt to verify a local file. Assumes the file is a clearsigned signature file. Args: specfile_path (str): Path to file to be verified. Returns: ``True`` if the signature could be verified, ``False`` otherwise. """ suppress = config.get("config:suppress_gpg_warnings", False) try: spack.util.gpg.verify(specfile_path, suppress_warnings=suppress) except Exception: return False return True class MirrorMetadata: """Simple class to hold a mirror url and a buildcache layout version This class is used by BinaryCacheIndex to produce a key used to keep track of downloaded/processed buildcache index files from remote mirrors in some layout version.""" __slots__ = ("url", "version", "view") def __init__(self, url: str, version: int, view: Optional[str] = None): self.url = url self.version = version self.view = view def __str__(self): s = f"{self.url}__v{self.version}" if self.view: s += f"__{self.view}" return s def __eq__(self, other): if not isinstance(other, MirrorMetadata): return NotImplemented return self.url == other.url and self.version == other.version and self.view == other.view def __hash__(self): return hash((self.url, self.version, self.view)) @classmethod def from_string(cls, s: str): m = re.match(r"^(.*)__v([0-9]+)(?:__(.*))?$", s) if not m: raise MirrorMetadataError(f"Malformed string {s}") url, version, view = m.groups() return cls(url, int(version), view) def strip_view(self) -> "MirrorMetadata": return MirrorMetadata(self.url, self.version) class InvalidMetadataFile(spack.error.SpackError): """Raised when spack encounters a spec file it cannot understand or process""" class BuildcacheEntryError(spack.error.SpackError): """Raised for problems finding or accessing binary cache entry on mirror""" class NoSuchBlobException(spack.error.SpackError): """Raised when manifest does have some requested type of requested type""" class NoVerifyException(BuildcacheEntryError): """Raised if file fails signature verification""" class UnknownBuildcacheLayoutError(BuildcacheEntryError): """Raised when unrecognized buildcache layout version is encountered""" class ListMirrorSpecsError(spack.error.SpackError): """Raised when unable to retrieve list of specs from the mirror""" class MirrorMetadataError(spack.error.SpackError): """Raised when unable to interpret a MirrorMetadata string""" ================================================ FILE: lib/spack/spack/user_environment.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import re import sys import spack.build_environment import spack.config import spack.spec import spack.util.environment as environment from spack import traverse from spack.context import Context #: Environment variable name Spack uses to track individually loaded packages spack_loaded_hashes_var = "SPACK_LOADED_HASHES" def prefix_inspections(platform: str) -> dict: """Get list of prefix inspections for platform Arguments: platform: the name of the platform to consider. The platform determines what environment variables Spack will use for some inspections. Returns: A dictionary mapping subdirectory names to lists of environment variables to modify with that directory if it exists. """ inspections = spack.config.get("modules:prefix_inspections") if isinstance(inspections, dict): return inspections inspections = { "bin": ["PATH"], "man": ["MANPATH"], "share/man": ["MANPATH"], "share/aclocal": ["ACLOCAL_PATH"], "lib/pkgconfig": ["PKG_CONFIG_PATH"], "lib64/pkgconfig": ["PKG_CONFIG_PATH"], "share/pkgconfig": ["PKG_CONFIG_PATH"], "": ["CMAKE_PREFIX_PATH"], } if platform == "darwin": inspections["lib"] = ["DYLD_FALLBACK_LIBRARY_PATH"] inspections["lib64"] = ["DYLD_FALLBACK_LIBRARY_PATH"] return inspections def unconditional_environment_modifications(view): """List of environment (shell) modifications to be processed for view. This list does not depend on the specs in this environment""" env = environment.EnvironmentModifications() for subdir, vars in prefix_inspections(sys.platform).items(): full_subdir = os.path.join(view.root, subdir) for var in vars: env.prepend_path(var, full_subdir) return env def project_env_mods( *specs: spack.spec.Spec, view, env: environment.EnvironmentModifications ) -> None: """Given a list of environment modifications, project paths changes to the view.""" prefix_to_prefix = { str(s.prefix): view.get_projection_for_spec(s) for s in specs if not s.external } # Avoid empty regex if all external if not prefix_to_prefix: return prefix_regex = re.compile("|".join(re.escape(p) for p in prefix_to_prefix.keys())) for mod in env.env_modifications: if isinstance(mod, environment.NameValueModifier): mod.value = prefix_regex.sub(lambda m: prefix_to_prefix[m.group(0)], mod.value) def environment_modifications_for_specs( *specs: spack.spec.Spec, view=None, set_package_py_globals: bool = True ): """List of environment (shell) modifications to be processed for spec. This list is specific to the location of the spec or its projection in the view. Args: specs: spec(s) for which to list the environment modifications view: view associated with the spec passed as first argument set_package_py_globals: whether or not to set the global variables in the package.py files (this may be problematic when using buildcaches that have been built on a different but compatible OS) """ env = environment.EnvironmentModifications() topo_ordered = list( traverse.traverse_nodes(specs, root=True, deptype=("run", "link"), order="topo") ) # Static environment changes (prefix inspections) for s in reversed(topo_ordered): static = environment.inspect_path( s.prefix, prefix_inspections(s.platform), exclude=environment.is_system_path ) env.extend(static) # Dynamic environment changes (setup_run_environment etc) setup_context = spack.build_environment.SetupContext(*specs, context=Context.RUN) if set_package_py_globals: setup_context.set_all_package_py_globals() env.extend(setup_context.get_env_modifications()) # Apply view projections if any. if view: project_env_mods(*topo_ordered, view=view, env=env) return env ================================================ FILE: lib/spack/spack/util/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/util/archive.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import errno import hashlib import io import os import pathlib import tarfile from contextlib import closing, contextmanager from gzip import GzipFile from typing import Callable, Dict, Generator, List, Tuple from spack.llnl.util import tty from spack.llnl.util.filesystem import readlink from spack.util.git import is_git_commit_sha class ChecksumWriter(io.BufferedIOBase): """Checksum writer computes a checksum while writing to a file.""" myfileobj = None def __init__(self, fileobj, algorithm=hashlib.sha256): self.fileobj = fileobj self.hasher = algorithm() self.length = 0 def hexdigest(self): return self.hasher.hexdigest() def write(self, data): if isinstance(data, (bytes, bytearray)): length = len(data) else: data = memoryview(data) length = data.nbytes if length > 0: self.fileobj.write(data) self.hasher.update(data) self.length += length return length def read(self, size=-1): raise OSError(errno.EBADF, "read() on write-only object") def read1(self, size=-1): raise OSError(errno.EBADF, "read1() on write-only object") def peek(self, n): raise OSError(errno.EBADF, "peek() on write-only object") @property def closed(self): return self.fileobj is None def close(self): fileobj = self.fileobj if fileobj is None: return self.fileobj.close() self.fileobj = None def flush(self): self.fileobj.flush() def fileno(self): return self.fileobj.fileno() def rewind(self): raise OSError("Can't rewind while computing checksum") def readable(self): return False def writable(self): return True def seekable(self): return True def tell(self): return self.fileobj.tell() def seek(self, offset, whence=io.SEEK_SET): # In principle forward seek is possible with b"0" padding, # but this is not implemented. if offset == 0 and whence == io.SEEK_CUR: return raise OSError("Can't seek while computing checksum") def readline(self, size=-1): raise OSError(errno.EBADF, "readline() on write-only object") @contextmanager def gzip_compressed_tarfile( path: str, ) -> Generator[Tuple[tarfile.TarFile, ChecksumWriter, ChecksumWriter], None, None]: """Create a reproducible, gzip compressed tarfile, and keep track of shasums of both the compressed and uncompressed tarfile. Reproducibility is achieved by normalizing the gzip header (no file name and zero mtime). Yields: A tuple of three elements * :class:`tarfile.TarFile`: tarfile object * :class:`ChecksumWriter`: checksum of the gzip compressed tarfile * :class:`ChecksumWriter`: checksum of the uncompressed tarfile """ # Create gzip compressed tarball of the install prefix # 1) Use explicit empty filename and mtime 0 for gzip header reproducibility. # If the filename="" is dropped, Python will use fileobj.name instead. # This should effectively mimic `gzip --no-name`. # 2) On AMD Ryzen 3700X and an SSD disk, we have the following on compression speed: # compresslevel=6 gzip default: llvm takes 4mins, roughly 2.1GB # compresslevel=9 python default: llvm takes 12mins, roughly 2.1GB # So we follow gzip. with open(path, "wb") as f, ChecksumWriter(f) as gzip_checksum, closing( GzipFile(filename="", mode="wb", compresslevel=6, mtime=0, fileobj=gzip_checksum) ) as gzip_file, ChecksumWriter(gzip_file) as tarfile_checksum, tarfile.TarFile( name="", mode="w", fileobj=tarfile_checksum ) as tar: yield tar, gzip_checksum, tarfile_checksum def default_path_to_name(path: str) -> str: """Converts a path to a tarfile name, which uses posix path separators.""" p = pathlib.PurePath(path) # Drop the leading slash on posix and the drive letter on windows, and always format as a # posix path. return pathlib.PurePath(*p.parts[1:]).as_posix() if p.is_absolute() else p.as_posix() def default_add_file(tar: tarfile.TarFile, file_info: tarfile.TarInfo, path: str) -> None: with open(path, "rb") as f: tar.addfile(file_info, f) def default_add_link(tar: tarfile.TarFile, file_info: tarfile.TarInfo, path: str) -> None: tar.addfile(file_info) def reproducible_tarfile_from_prefix( tar: tarfile.TarFile, prefix: str, *, include_parent_directories: bool = False, skip: Callable[[os.DirEntry], bool] = lambda entry: False, path_to_name: Callable[[str], str] = default_path_to_name, add_file: Callable[[tarfile.TarFile, tarfile.TarInfo, str], None] = default_add_file, add_symlink: Callable[[tarfile.TarFile, tarfile.TarInfo, str], None] = default_add_link, add_hardlink: Callable[[tarfile.TarFile, tarfile.TarInfo, str], None] = default_add_link, ) -> None: """Create a tarball from a given directory. Only adds regular files, symlinks and dirs. Skips devices, fifos. Preserves hardlinks. Normalizes permissions like git. Tar entries are added in depth-first pre-order, with dir entries partitioned by file | dir, and sorted lexicographically, for reproducibility. Partitioning ensures only one dir is in memory at a time, and sorting improves compression. Args: tar: tarfile object opened in write mode prefix: path to directory to tar (either absolute or relative) include_parent_directories: whether to include every directory leading up to ``prefix`` in the tarball skip: function that receives a DirEntry and returns True if the entry should be skipped, whether it is a file or directory. Default implementation does not skip anything. path_to_name: function that converts a path string to a tarfile entry name, which should be in posix format. Not only is it necessary to transform paths in certain cases, such as windows path to posix format, but it can also be used to prepend a directory to each entry even if it does not exist on the filesystem. The default implementation drops the leading slash on posix and the drive letter on windows for absolute paths, and formats as a posix.""" hardlink_to_tarinfo_name: Dict[Tuple[int, int], str] = dict() if include_parent_directories: parent_dirs = reversed(pathlib.PurePosixPath(path_to_name(prefix)).parents) next(parent_dirs) # skip the root: slices are supported from python 3.10 for parent_dir in parent_dirs: dir_info = tarfile.TarInfo(str(parent_dir)) dir_info.type = tarfile.DIRTYPE dir_info.mode = 0o755 tar.addfile(dir_info) dir_stack = [prefix] new_dirs: List[str] = [] while dir_stack: dir = dir_stack.pop() new_dirs.clear() # Add the dir before its contents dir_info = tarfile.TarInfo(path_to_name(dir)) dir_info.type = tarfile.DIRTYPE dir_info.mode = 0o755 tar.addfile(dir_info) # Sort by name: reproducible & improves compression with os.scandir(dir) as it: entries = sorted(it, key=lambda entry: entry.name) for entry in entries: if skip(entry): continue if entry.is_dir(follow_symlinks=False): new_dirs.append(entry.path) continue file_info = tarfile.TarInfo(path_to_name(entry.path)) if entry.is_symlink(): file_info.type = tarfile.SYMTYPE file_info.linkname = readlink(entry.path) # According to POSIX: "the value of the file mode bits returned in the # st_mode field of the stat structure is unspecified." So we set it to # something sensible without lstat'ing the link. file_info.mode = 0o755 add_symlink(tar, file_info, entry.path) elif entry.is_file(follow_symlinks=False): # entry.stat has zero (st_ino, st_dev, st_nlink) on Windows: use lstat. s = os.lstat(entry.path) # Normalize permissions like git file_info.mode = 0o755 if s.st_mode & 0o100 else 0o644 # Deduplicate hardlinks if s.st_nlink > 1: ident = (s.st_dev, s.st_ino) if ident in hardlink_to_tarinfo_name: file_info.type = tarfile.LNKTYPE file_info.linkname = hardlink_to_tarinfo_name[ident] add_hardlink(tar, file_info, entry.path) continue hardlink_to_tarinfo_name[ident] = file_info.name # If file not yet seen, copy it file_info.type = tarfile.REGTYPE file_info.size = s.st_size add_file(tar, file_info, entry.path) dir_stack.extend(reversed(new_dirs)) # we pop, so reverse to stay alphabetical def retrieve_commit_from_archive(archive_path, ref): """Extract git data from an archive with out expanding it Open the archive and searches for .git/HEAD. Return if HEAD is a commit (detached head or tag) Otherwise attempt to read the ref that .git/HEAD is pointing to and return the commit associated with it. """ if not os.path.isfile(archive_path): raise FileNotFoundError(f"The file {archive_path} does not exist") try: with tarfile.open(archive_path, "r") as tar: names = tar.getnames() # since we always have a prefix and can't guarantee the value we need this lookup. prefix = "" for name in names: if name.endswith(".git"): prefix = name[:-4] break if f"{prefix}.git/HEAD" in names: head = tar.extractfile(f"{prefix}.git/HEAD").read().decode("utf-8").strip() if is_git_commit_sha(head): # detached HEAD/ lightweight tag return head else: # refs in had have the format "ref " ref = head.split()[1] contents = ( tar.extractfile(f"{prefix}.git/{ref}").read().decode("utf-8").strip() ) if is_git_commit_sha(contents): return contents except tarfile.ReadError: tty.warn(f"Archive {archive_path} does not appear to contain git data") return ================================================ FILE: lib/spack/spack/util/compression.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import errno import inspect import io import os import shutil import sys from typing import Any, BinaryIO, Callable, Dict, List, Optional import spack.llnl.url from spack.error import SpackError from spack.llnl.util import tty from spack.util.executable import CommandNotFoundError, which try: import bz2 # noqa BZ2_SUPPORTED = True except ImportError: BZ2_SUPPORTED = False try: import gzip # noqa GZIP_SUPPORTED = True except ImportError: GZIP_SUPPORTED = False try: import lzma # noqa # novermin LZMA_SUPPORTED = True except ImportError: LZMA_SUPPORTED = False def _system_untar(archive_file: str, remove_archive_file: bool = False) -> str: """Returns path to unarchived tar file. Untars archive via system tar. Args: archive_file (str): absolute path to the archive to be extracted. Can be one of .tar(.[gz|bz2|xz|Z]) or .(tgz|tbz|tbz2|txz). """ archive_file_no_ext = spack.llnl.url.strip_extension(archive_file) outfile = os.path.basename(archive_file_no_ext) if archive_file_no_ext == archive_file: # the archive file has no extension. Tar on windows cannot untar onto itself # archive_file can be a tar file (which causes the problem on windows) but it can # also have other extensions (on Unix) such as tgz, tbz2, ... archive_file = archive_file_no_ext + "-input" shutil.move(archive_file_no_ext, archive_file) tar = which("tar", required=True) # GNU tar's --no-same-owner is not as portable, -o works for BSD tar too. This flag is relevant # when extracting archives as root, where tar attempts to set original ownership of files. This # is redundant when distributing tarballs, as the tarballs are created on different systems # than where they are extracted. In certain cases like rootless containers, setting original # ownership is known to fail, so we need to disable it. tar.add_default_arg("-oxf") tar(archive_file) if remove_archive_file: # remove input file to prevent two stage # extractions from being treated as exploding # archives by the fetcher os.remove(archive_file) return outfile def _bunzip2(archive_file: str) -> str: """Returns path to decompressed file. Uses Python's bz2 module to decompress bz2 compressed archives Fall back to system utility failing to find Python module `bz2` Args: archive_file: absolute path to the bz2 archive to be decompressed """ if BZ2_SUPPORTED: return _py_bunzip(archive_file) else: return _system_bunzip(archive_file) def _py_bunzip(archive_file: str) -> str: """Returns path to decompressed file. Decompresses bz2 compressed archives/files via python's bz2 module""" decompressed_file = os.path.basename( spack.llnl.url.strip_compression_extension(archive_file, "bz2") ) working_dir = os.getcwd() archive_out = os.path.join(working_dir, decompressed_file) f_bz = bz2.BZ2File(archive_file, mode="rb") with open(archive_out, "wb") as ar: shutil.copyfileobj(f_bz, ar) f_bz.close() return archive_out def _system_bunzip(archive_file: str) -> str: """Returns path to decompressed file. Decompresses bz2 compressed archives/files via system bzip2 utility""" compressed_file_name = os.path.basename(archive_file) decompressed_file = os.path.basename( spack.llnl.url.strip_compression_extension(archive_file, "bz2") ) working_dir = os.getcwd() archive_out = os.path.join(working_dir, decompressed_file) copy_path = os.path.join(working_dir, compressed_file_name) shutil.copy(archive_file, copy_path) bunzip2 = which("bunzip2", required=True) bunzip2.add_default_arg("-q") bunzip2(copy_path) return archive_out def _gunzip(archive_file: str) -> str: """Returns path to gunzip'd file. Decompresses `.gz` extensions. Prefer native Python `gzip` module. Falling back to system utility gunzip. Like gunzip, but extracts in the current working directory instead of in-place. Args: archive_file: absolute path of the file to be decompressed """ return _py_gunzip(archive_file) if GZIP_SUPPORTED else _system_gunzip(archive_file) def _py_gunzip(archive_file: str) -> str: """Returns path to gunzip'd file. Decompresses `.gz` compressed archives via python gzip module""" decompressed_file = os.path.basename( spack.llnl.url.strip_compression_extension(archive_file, "gz") ) working_dir = os.getcwd() destination_abspath = os.path.join(working_dir, decompressed_file) f_in = gzip.open(archive_file, "rb") with open(destination_abspath, "wb") as f_out: shutil.copyfileobj(f_in, f_out) f_in.close() return destination_abspath def _system_gunzip(archive_file: str) -> str: """Returns path to gunzip'd file. Decompresses `.gz` compressed files via system gzip""" archive_file_no_ext = spack.llnl.url.strip_compression_extension(archive_file) if archive_file_no_ext == archive_file: # the zip file has no extension. On Unix gunzip cannot unzip onto itself archive_file = archive_file + ".gz" shutil.move(archive_file_no_ext, archive_file) decompressed_file = os.path.basename(archive_file_no_ext) working_dir = os.getcwd() destination_abspath = os.path.join(working_dir, decompressed_file) compressed_file = os.path.basename(archive_file) copy_path = os.path.join(working_dir, compressed_file) shutil.copy(archive_file, copy_path) gzip = which("gzip", required=True) gzip.add_default_arg("-d") gzip(copy_path) return destination_abspath def _do_nothing(archive_file: str) -> None: return None def _unzip(archive_file: str) -> str: """Returns path to extracted zip archive. Extract Zipfile, searching for unzip system executable. If unavailable, search for 'tar' executable on system and use instead. Args: archive_file: absolute path of the file to be decompressed """ if sys.platform == "win32": return _system_untar(archive_file) unzip = which("unzip", required=True) unzip.add_default_arg("-q") unzip(archive_file) return os.path.basename(spack.llnl.url.strip_extension(archive_file, extension="zip")) def _system_unZ(archive_file: str) -> str: """Returns path to decompressed file Decompress UNIX compress style compression Utilizes gunzip on unix and 7zip on Windows """ if sys.platform == "win32": return _system_7zip(archive_file) return _system_gunzip(archive_file) def _lzma_decomp(archive_file): """Returns path to decompressed xz file. Decompress lzma compressed files. Prefer Python native lzma module, but fall back on command line xz tooling to find available Python support.""" return _py_lzma(archive_file) if LZMA_SUPPORTED else _xz(archive_file) def _win_compressed_tarball_handler(decompressor: Callable[[str], str]) -> Callable[[str], str]: """Returns function pointer to two stage decompression and extraction method Decompress and extract compressed tarballs on Windows. This method uses a decompression method in conjunction with the tar utility to perform decompression and extraction in a two step process first using decompressor to decompress, and tar to extract. The motivation for this method is Windows tar utility's lack of access to the xz tool (unsupported natively on Windows) but can be installed manually or via spack """ def unarchive(archive_file: str): # perform intermediate extraction step # record name of new archive so we can extract decomped_tarball = decompressor(archive_file) # run tar on newly decomped archive outfile = _system_untar(decomped_tarball, remove_archive_file=True) return outfile return unarchive def _py_lzma(archive_file: str) -> str: """Returns path to decompressed .xz files. Decompress lzma compressed .xz files via Python lzma module.""" decompressed_file = os.path.basename( spack.llnl.url.strip_compression_extension(archive_file, "xz") ) archive_out = os.path.join(os.getcwd(), decompressed_file) with open(archive_out, "wb") as ar: with lzma.open(archive_file) as lar: shutil.copyfileobj(lar, ar) return archive_out def _xz(archive_file): """Returns path to decompressed xz files. Decompress lzma compressed .xz files via xz command line tool.""" decompressed_file = os.path.basename( spack.llnl.url.strip_extension(archive_file, extension="xz") ) working_dir = os.getcwd() destination_abspath = os.path.join(working_dir, decompressed_file) compressed_file = os.path.basename(archive_file) copy_path = os.path.join(working_dir, compressed_file) shutil.copy(archive_file, copy_path) xz = which("xz", required=True) xz.add_default_arg("-d") xz(copy_path) return destination_abspath def _system_7zip(archive_file): """Returns path to decompressed file Unpack/decompress with 7z executable 7z is able to handle a number file extensions however it may not be available on system. Without 7z, Windows users with certain versions of Python may be unable to extract .xz files, and all Windows users will be unable to extract .Z files. If we cannot find 7z either externally or a Spack installed copy, we fail, but inform the user that 7z can be installed via `spack install 7zip` Args: archive_file (str): absolute path of file to be unarchived """ outfile = os.path.basename(spack.llnl.url.strip_compression_extension(archive_file)) _7z = which("7z") if not _7z: raise CommandNotFoundError( "7z unavailable, unable to extract %s files. 7z can be installed via Spack" % spack.llnl.url.extension_from_path(archive_file) ) _7z.add_default_arg("e") _7z(archive_file) return outfile def decompressor_for(path: str, extension: Optional[str] = None): """Returns appropriate decompression/extraction algorithm function pointer for provided extension. If extension is none, it is computed from the ``path`` and the decompression function is derived from that information.""" if not extension: extension = extension_from_magic_numbers(path, decompress=True) if not extension or not spack.llnl.url.allowed_archive(extension): raise CommandNotFoundError( f"Cannot extract {path}, unrecognized file extension: '{extension}'" ) if sys.platform == "win32": return decompressor_for_win(extension) else: return decompressor_for_nix(extension) def decompressor_for_nix(extension: str) -> Callable[[str], Any]: """Returns a function pointer to appropriate decompression algorithm based on extension type and unix specific considerations i.e. a reasonable expectation system utils like gzip, bzip2, and xz are available Args: extension: path of the archive file requiring decompression """ extension_to_decompressor: Dict[str, Callable[[str], Any]] = { "zip": _unzip, "gz": _gunzip, "bz2": _bunzip2, "Z": _system_unZ, # no builtin support for .Z files "xz": _lzma_decomp, "whl": _do_nothing, } return extension_to_decompressor.get(extension, _system_untar) def _determine_py_decomp_archive_strategy(extension: str) -> Optional[Callable[[str], Any]]: """Returns appropriate python based decompression strategy based on extension type""" extension_to_decompressor: Dict[str, Callable[[str], str]] = { "gz": _py_gunzip, "bz2": _py_bunzip, "xz": _py_lzma, } return extension_to_decompressor.get(extension, None) def decompressor_for_win(extension: str) -> Callable[[str], Any]: """Returns a function pointer to appropriate decompression algorithm based on extension type and Windows specific considerations Windows natively vendors *only* tar, no other archive/compression utilities So we must rely exclusively on Python module support for all compression operations, tar for tarballs and zip files, and 7zip for Z compressed archives and files as Python does not provide support for the UNIX compress algorithm """ extension = spack.llnl.url.expand_contracted_extension(extension) extension_to_decompressor: Dict[str, Callable[[str], Any]] = { # Windows native tar can handle .zip extensions, use standard unzip method "zip": _unzip, # if extension is standard tarball, invoke Windows native tar "tar": _system_untar, # Python does not have native support of any kind for .Z files. In these cases, we rely on # 7zip, which must be installed outside of Spack and added to the PATH or externally # detected "Z": _system_unZ, "xz": _lzma_decomp, "whl": _do_nothing, } decompressor = extension_to_decompressor.get(extension) if decompressor: return decompressor # Windows vendors no native decompression tools, attempt to derive Python based decompression # strategy. Expand extension from abbreviated ones, i.e. tar.gz from .tgz compression_extension = spack.llnl.url.compression_ext_from_compressed_archive(extension) decompressor = ( _determine_py_decomp_archive_strategy(compression_extension) if compression_extension else None ) if not decompressor: raise SpackError( "Spack was unable to determine a proper decompression strategy for" f"valid extension: {extension}" "This is a bug, please file an issue at https://github.com/spack/spack/issues" ) if "tar" not in extension: return decompressor return _win_compressed_tarball_handler(decompressor) class FileTypeInterface: """Base interface class for describing and querying file type information. FileType describes information about a single file type such as typical extension and byte header properties, and provides an interface to check a given file against said type based on magic number. This class should be subclassed each time a new type is to be described. Subclasses should each describe a different type of file. In order to do so, they must define the extension string, magic number, and header offset (if non zero). If a class has multiple magic numbers, it will need to override the method describing that file type's magic numbers and the method that checks a types magic numbers against a given file's.""" OFFSET = 0 extension: str name: str @classmethod def magic_numbers(cls) -> List[bytes]: """Return a list of all potential magic numbers for a filetype""" return [ value for name, value in inspect.getmembers(cls) if name.startswith("_MAGIC_NUMBER") ] @classmethod def header_size(cls) -> int: """Return size of largest magic number associated with file type""" return max(len(x) for x in cls.magic_numbers()) def matches_magic(self, stream: BinaryIO) -> bool: """Returns true if the stream matches the current file type by any of its magic numbers. Resets stream to original position. Args: stream: file byte stream """ # move to location of magic bytes offset = stream.tell() stream.seek(self.OFFSET) magic_bytes = stream.read(self.header_size()) stream.seek(offset) return any(magic_bytes.startswith(magic) for magic in self.magic_numbers()) class CompressedFileTypeInterface(FileTypeInterface): """Interface class for FileTypes that include compression information""" def peek(self, stream: BinaryIO, num_bytes: int) -> Optional[io.BytesIO]: """This method returns the first num_bytes of a decompressed stream. Returns None if no builtin support for decompression.""" return None def _decompressed_peek( decompressed_stream: io.BufferedIOBase, stream: BinaryIO, num_bytes: int ) -> io.BytesIO: # Read the first num_bytes of the decompressed stream, do not advance the stream position. pos = stream.tell() data = decompressed_stream.read(num_bytes) stream.seek(pos) return io.BytesIO(data) class BZipFileType(CompressedFileTypeInterface): _MAGIC_NUMBER = b"\x42\x5a\x68" extension = "bz2" name = "bzip2 compressed data" def peek(self, stream: BinaryIO, num_bytes: int) -> Optional[io.BytesIO]: if BZ2_SUPPORTED: return _decompressed_peek(bz2.BZ2File(stream), stream, num_bytes) return None class ZCompressedFileType(CompressedFileTypeInterface): _MAGIC_NUMBER_LZW = b"\x1f\x9d" _MAGIC_NUMBER_LZH = b"\x1f\xa0" extension = "Z" name = "compress'd data" class GZipFileType(CompressedFileTypeInterface): _MAGIC_NUMBER = b"\x1f\x8b\x08" extension = "gz" name = "gzip compressed data" def peek(self, stream: BinaryIO, num_bytes: int) -> Optional[io.BytesIO]: if GZIP_SUPPORTED: return _decompressed_peek(gzip.GzipFile(fileobj=stream), stream, num_bytes) return None class LzmaFileType(CompressedFileTypeInterface): _MAGIC_NUMBER = b"\xfd7zXZ" extension = "xz" name = "xz compressed data" def peek(self, stream: BinaryIO, num_bytes: int) -> Optional[io.BytesIO]: if LZMA_SUPPORTED: return _decompressed_peek(lzma.LZMAFile(stream), stream, num_bytes) return None class TarFileType(FileTypeInterface): OFFSET = 257 _MAGIC_NUMBER_GNU = b"ustar \0" _MAGIC_NUMBER_POSIX = b"ustar\x0000" extension = "tar" name = "tar archive" class ZipFleType(FileTypeInterface): _MAGIC_NUMBER = b"PK\003\004" extension = "zip" name = "Zip archive data" #: Maximum number of bytes to read from a file to determine any archive type. Tar is the largest. MAX_BYTES_ARCHIVE_HEADER = TarFileType.OFFSET + TarFileType.header_size() #: Collection of supported archive and compression file type identifier classes. SUPPORTED_FILETYPES: List[FileTypeInterface] = [ BZipFileType(), ZCompressedFileType(), GZipFileType(), LzmaFileType(), TarFileType(), ZipFleType(), ] def _extension_of_compressed_file( file_type: CompressedFileTypeInterface, stream: BinaryIO ) -> Optional[str]: """Retrieves the extension of a file after decompression from its magic numbers, if it can be decompressed.""" # To classify the file we only need to decompress the first so many bytes. decompressed_magic = file_type.peek(stream, MAX_BYTES_ARCHIVE_HEADER) if not decompressed_magic: return None return extension_from_magic_numbers_by_stream(decompressed_magic, decompress=False) def extension_from_magic_numbers_by_stream( stream: BinaryIO, decompress: bool = False ) -> Optional[str]: """Returns the typical extension for the opened file, without leading ``.``, based on its magic numbers. If the stream does not represent file type recognized by Spack (see :py:data:`SUPPORTED_FILETYPES`), the method will return None Args: stream: stream representing a file on system decompress: if True, compressed files are checked for archive types beneath compression. For example tar.gz if True versus only gz if False.""" for file_type in SUPPORTED_FILETYPES: if not file_type.matches_magic(stream): continue ext = file_type.extension if decompress and isinstance(file_type, CompressedFileTypeInterface): uncompressed_ext = _extension_of_compressed_file(file_type, stream) if not uncompressed_ext: tty.debug( "Cannot derive file extension from magic number;" " falling back to original file name." ) return spack.llnl.url.extension_from_path(stream.name) ext = f"{uncompressed_ext}.{ext}" tty.debug(f"File extension {ext} successfully derived by magic number.") return ext return None def _maybe_abbreviate_extension(path: str, extension: str) -> str: """If the file is a compressed tar archive, return the abbreviated extension t[xz|gz|bz2|bz] instead of tar.[xz|gz|bz2|bz] if the file's original name also has an abbreviated extension.""" if not extension.startswith("tar."): return extension abbr = f"t{extension[4:]}" return abbr if spack.llnl.url.has_extension(path, abbr) else extension def extension_from_magic_numbers(path: str, decompress: bool = False) -> Optional[str]: """Return typical extension without leading ``.`` of a compressed file or archive at the given path, based on its magic numbers, similar to the ``file`` utility. Notice that the extension returned from this function may not coincide with the file's given extension. Args: path: file to determine extension of decompress: If True, method will peek into decompressed file to check for archive file types. If False, the method will return only the top-level extension (for example ``gz`` and not ``tar.gz``). Returns: Spack recognized archive file extension as determined by file's magic number and file name. If file is not on system or is of a type not recognized by Spack as an archive or compression type, None is returned. If the file is classified as a compressed tarball, the extension is abbreviated (for instance ``tgz`` not ``tar.gz``) if that matches the file's given extension. """ try: with open(path, "rb") as f: ext = extension_from_magic_numbers_by_stream(f, decompress) except OSError as e: if e.errno == errno.ENOENT: return None raise # Return the extension derived from the magic number if possible. if ext: return _maybe_abbreviate_extension(path, ext) # Otherwise, use the extension from the file name. return spack.llnl.url.extension_from_path(path) ================================================ FILE: lib/spack/spack/util/cpus.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import multiprocessing import os def cpus_available(): """ Returns the number of CPUs available for the current process, or the number of physical CPUs when that information cannot be retrieved. The number of available CPUs might differ from the number of physical CPUs when using spack through Slurm or container runtimes. """ try: return len(os.sched_getaffinity(0)) # novermin except Exception: return multiprocessing.cpu_count() ================================================ FILE: lib/spack/spack/util/crypto.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import hashlib from typing import BinaryIO, Callable, Dict, Optional import spack.llnl.util.tty as tty HashFactory = Callable[[], "hashlib._Hash"] #: Set of hash algorithms that Spack can use, mapped to digest size in bytes hashes = {"sha256": 32, "md5": 16, "sha1": 20, "sha224": 28, "sha384": 48, "sha512": 64} # Note: keys are ordered by popularity for earliest return in ``hash_key in version_dict`` checks. #: size of hash digests in bytes, mapped to algorithm names _size_to_hash = dict((v, k) for k, v in hashes.items()) #: List of deprecated hash functions. On some systems, these cannot be #: used without special options to hashlib. _deprecated_hash_algorithms = ["md5"] #: cache of hash functions generated _hash_functions: Dict[str, HashFactory] = {} class DeprecatedHash: def __init__(self, hash_alg, alert_fn, disable_security_check): self.hash_alg = hash_alg self.alert_fn = alert_fn self.disable_security_check = disable_security_check def __call__(self, disable_alert=False): if not disable_alert: self.alert_fn( "Deprecation warning: {0} checksums will not be" " supported in future Spack releases.".format(self.hash_alg) ) if self.disable_security_check: return hashlib.new(self.hash_alg, usedforsecurity=False) # novermin else: return hashlib.new(self.hash_alg) def hash_fun_for_algo(algo: str) -> HashFactory: """Get a function that can perform the specified hash algorithm.""" fun = _hash_functions.get(algo) if fun: return fun elif algo not in _deprecated_hash_algorithms: _hash_functions[algo] = getattr(hashlib, algo) else: try: deprecated_fun = DeprecatedHash(algo, tty.debug, disable_security_check=False) # call once to get a ValueError if usedforsecurity is needed deprecated_fun(disable_alert=True) except ValueError: # Some systems may support the 'usedforsecurity' option # so try with that (but display a warning when it is used) deprecated_fun = DeprecatedHash(algo, tty.warn, disable_security_check=True) _hash_functions[algo] = deprecated_fun return _hash_functions[algo] def hash_algo_for_digest(hexdigest: str) -> str: """Gets name of the hash algorithm for a hex digest.""" algo = _size_to_hash.get(len(hexdigest) // 2) if algo is None: raise ValueError(f"Spack knows no hash algorithm for this digest: {hexdigest}") return algo def hash_fun_for_digest(hexdigest: str) -> HashFactory: """Gets a hash function corresponding to a hex digest.""" return hash_fun_for_algo(hash_algo_for_digest(hexdigest)) def checksum_stream(hashlib_algo: HashFactory, fp: BinaryIO, *, block_size: int = 2**20) -> str: """Returns a hex digest of the stream generated using given algorithm from hashlib.""" hasher = hashlib_algo() while True: data = fp.read(block_size) if not data: break hasher.update(data) return hasher.hexdigest() def checksum(hashlib_algo: HashFactory, filename: str, *, block_size: int = 2**20) -> str: """Returns a hex digest of the filename generated using an algorithm from hashlib.""" with open(filename, "rb") as f: return checksum_stream(hashlib_algo, f, block_size=block_size) class Checker: """A checker checks files against one particular hex digest. It will automatically determine what hashing algorithm to used based on the length of the digest it's initialized with. e.g., if the digest is 32 hex characters long this will use md5. Example: know your tarball should hash to ``abc123``. You want to check files against this. You would use this class like so:: hexdigest = 'abc123' checker = Checker(hexdigest) success = checker.check('downloaded.tar.gz') After the call to check, the actual checksum is available in checker.sum, in case it's needed for error output. You can trade read performance and memory usage by adjusting the block_size optional arg. By default it's a 1MB (2**20 bytes) buffer. """ def __init__(self, hexdigest: str, **kwargs) -> None: self.block_size = kwargs.get("block_size", 2**20) self.hexdigest = hexdigest self.sum: Optional[str] = None self.hash_fun = hash_fun_for_digest(hexdigest) @property def hash_name(self) -> str: """Get the name of the hash function this Checker is using.""" return self.hash_fun().name.lower() def check(self, filename: str) -> bool: """Read the file with the specified name and check its checksum against self.hexdigest. Return True if they match, False otherwise. Actual checksum is stored in self.sum. """ self.sum = checksum(self.hash_fun, filename, block_size=self.block_size) return self.sum == self.hexdigest def prefix_bits(byte_array, bits): """Return the first bits of a byte array as an integer.""" b2i = lambda b: b # In Python 3, indexing byte_array gives int result = 0 n = 0 for i, b in enumerate(byte_array): n += 8 result = (result << 8) | b2i(b) if n >= bits: break result >>= n - bits return result def bit_length(num): """Number of bits required to represent an integer in binary.""" s = bin(num) s = s.lstrip("-0b") return len(s) ================================================ FILE: lib/spack/spack/util/ctest_log_parser.py ================================================ # pylint: skip-file # ----------------------------------------------------------------------------- # CMake - Cross Platform Makefile Generator # Copyright 2000-2017 Kitware, Inc. and Contributors # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # * Neither the name of Kitware, Inc. nor the names of Contributors # may be used to endorse or promote products derived from this # software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # ----------------------------------------------------------------------------- # # The above copyright and license notice applies to distributions of # CMake in source and binary form. Third-party software packages supplied # with CMake under compatible licenses provide their own copyright notices # documented in corresponding subdirectories or source files. # # ----------------------------------------------------------------------------- # # CMake was initially developed by Kitware with the following sponsorship: # # * National Library of Medicine at the National Institutes of Health # as part of the Insight Segmentation and Registration Toolkit (ITK). # # * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel # Visualization Initiative. # # * National Alliance for Medical Image Computing (NAMIC) is funded by the # National Institutes of Health through the NIH Roadmap for Medical # Research, Grant U54 EB005149. # # * Kitware, Inc. # ----------------------------------------------------------------------------- """Functions to parse build logs and extract error messages. This is a python port of the regular expressions CTest uses to parse log files here: .. code-block:: https://github.com/Kitware/CMake/blob/master/Source/CTest/cmCTestBuildHandler.cxx This file takes the regexes verbatim from there and adds some parsing algorithms that duplicate the way CTest scrapes log files. To keep this up to date with CTest, just make sure the ``*_matches`` and ``*_exceptions`` lists are kept up to date with CTest's build handler. """ import io import math import re import time from collections import deque from contextlib import contextmanager from typing import List, TextIO, Tuple, Union _error_matches = [ "^FAIL: ", "^FATAL: ", "^failed ", "FAILED", "Failed test", "^[Bb]us [Ee]rror", "^[Ss]egmentation [Vv]iolation", "^[Ss]egmentation [Ff]ault", "Permission [Dd]enied", "permission [Dd]enied", ":[0-9]+: [^ \\t]", ": error[ \\t]*[0-9]+[ \\t]*:", "^Error ([0-9]+):", "^Fatal", "^[Ee]rror: ", "^Error ", " ERROR: ", '^"[^"]+", line [0-9]+: [^Ww]', "^cc[^C]*CC: ERROR File = ([^,]+), Line = ([0-9]+)", "^ld([^:])*:([ \\t])*ERROR([^:])*:", "^ild:([ \\t])*\\(undefined symbol\\)", ": (error|fatal error|catastrophic error)", ": (Error:|error|undefined reference|multiply defined)", "\\([^\\)]+\\) ?: (error|fatal error|catastrophic error)", "^fatal error C[0-9]+:", ": syntax error ", "^collect2: ld returned 1 exit status", "ld terminated with signal", "Unsatisfied symbol", "^Unresolved:", "Undefined symbol", "^Undefined[ \\t]+first referenced", "^CMake Error", ":[ \\t]cannot find", ":[ \\t]can't find", ": \\*\\*\\* No rule to make target [`'].*\\'. Stop", ": \\*\\*\\* No targets specified and no makefile found", ": Invalid loader fixup for symbol", ": Invalid fixups exist", ": Can't find library for", ": internal link edit command failed", ": Unrecognized option [`'].*\\'", '", line [0-9]+\\.[0-9]+: [0-9]+-[0-9]+ \\([^WI]\\)', "ld: 0706-006 Cannot find or open library file: -l ", "ild: \\(argument error\\) can't find library argument ::", "^could not be found and will not be loaded.", "^WARNING: '.*' is missing on your system", "s:616 string too big", "make: Fatal error: ", "ld: 0711-993 Error occurred while writing to the output file:", "ld: fatal: ", "final link failed:", "make: \\*\\*\\*.*Error", "make\\[.*\\]: \\*\\*\\*.*Error", "\\*\\*\\* Error code", "nternal error:", "Makefile:[0-9]+: \\*\\*\\* .* Stop\\.", ": No such file or directory", ": Invalid argument", "^The project cannot be built\\.", "^\\[ERROR\\]", "^Command .* failed with exit code", ] _error_exceptions = [ "instantiated from ", "candidates are:", ": warning", ": WARNING", ": \\(Warning\\)", ": note", " ok", "Note:", ":[ \\t]+Where:", ":[0-9]+: Warning", "------ Build started: .* ------", ] #: Regexes to match file/line numbers in error/warning messages _warning_matches = [ ":[0-9]+: warning:", ":[0-9]+: note:", "^cc[^C]*CC: WARNING File = ([^,]+), Line = ([0-9]+)", "^ld([^:])*:([ \\t])*WARNING([^:])*:", ": warning [0-9]+:", '^"[^"]+", line [0-9]+: [Ww](arning|arnung)', ": warning[ \\t]*[0-9]+[ \\t]*:", "^(Warning|Warnung) ([0-9]+):", "^(Warning|Warnung)[ :]", "WARNING: ", ": warning", '", line [0-9]+\\.[0-9]+: [0-9]+-[0-9]+ \\([WI]\\)', "^cxx: Warning:", "file: .* has no symbols", ":[0-9]+: (Warning|Warnung)", "\\([0-9]*\\): remark #[0-9]*", '".*", line [0-9]+: remark\\([0-9]*\\):', "cc-[0-9]* CC: REMARK File = .*, Line = [0-9]*", "^CMake Warning", "^\\[WARNING\\]", ] #: Regexes to match file/line numbers in error/warning messages _warning_exceptions = [ "/usr/.*/X11/Xlib\\.h:[0-9]+: war.*: ANSI C\\+\\+ forbids declaration", "/usr/.*/X11/Xutil\\.h:[0-9]+: war.*: ANSI C\\+\\+ forbids declaration", "/usr/.*/X11/XResource\\.h:[0-9]+: war.*: ANSI C\\+\\+ forbids declaration", "WARNING 84 :", "WARNING 47 :", "warning: Clock skew detected. Your build may be incomplete.", "/usr/openwin/include/GL/[^:]+:", "bind_at_load", "XrmQGetResource", "IceFlush", "warning LNK4089: all references to [^ \\t]+ discarded by .OPT:REF", "ld32: WARNING 85: definition of dataKey in", 'cc: warning 422: Unknown option "\\+b', "_with_warning_C", ] #: Regexes to match file/line numbers in error/warning messages _file_line_matches = [ "^Warning W[0-9]+ ([a-zA-Z.\\:/0-9_+ ~-]+) ([0-9]+):", "^([a-zA-Z./0-9_+ ~-]+):([0-9]+):", "^([a-zA-Z.\\:/0-9_+ ~-]+)\\(([0-9]+)\\)", "^[0-9]+>([a-zA-Z.\\:/0-9_+ ~-]+)\\(([0-9]+)\\)", "^([a-zA-Z./0-9_+ ~-]+)\\(([0-9]+)\\)", '"([a-zA-Z./0-9_+ ~-]+)", line ([0-9]+)', "File = ([a-zA-Z./0-9_+ ~-]+), Line = ([0-9]+)", ] class LogEvent: """Class representing interesting events (e.g., errors) in a build log.""" #: color name when rendering in the terminal color = "" def __init__( self, text, line_no, source_file=None, source_line_no=None, pre_context=None, post_context=None, ): self.text = text self.line_no = line_no self.source_file = (source_file,) self.source_line_no = (source_line_no,) self.pre_context = pre_context if pre_context is not None else [] self.post_context = post_context if post_context is not None else [] self.repeat_count = 0 @property def start(self): """First line in the log with text for the event or its context.""" return self.line_no - len(self.pre_context) @property def end(self): """Last line in the log with text for event or its context.""" return self.line_no + len(self.post_context) + 1 def __getitem__(self, line_no): """Index event text and context by actual line number in file.""" if line_no == self.line_no: return self.text elif line_no < self.line_no: return self.pre_context[line_no - self.line_no] elif line_no > self.line_no: return self.post_context[line_no - self.line_no - 1] def __str__(self): """Returns event lines and context.""" out = io.StringIO() for i in range(self.start, self.end): if i == self.line_no: out.write(" >> %-6d%s" % (i, self[i])) else: out.write(" %-6d%s" % (i, self[i])) return out.getvalue() class BuildError(LogEvent): """LogEvent subclass for build errors.""" color = "R" class BuildWarning(LogEvent): """LogEvent subclass for build warnings.""" color = "Y" def chunks(xs, n): """Divide xs into n approximately-even chunks.""" chunksize = int(math.ceil(len(xs) / n)) return [xs[i : i + chunksize] for i in range(0, len(xs), chunksize)] @contextmanager def _time(times, i): start = time.time() yield end = time.time() times[i] += end - start def _match(matches, exceptions, line): """True if line matches a regex in matches and none in exceptions.""" return any(m.search(line) for m in matches) and not any(e.search(line) for e in exceptions) def _profile_match(matches, exceptions, line, match_times, exc_times): """Profiled version of match(). Timing is expensive so we have two whole functions. This is much longer because we have to break up the ``any()`` calls. """ for i, m in enumerate(matches): with _time(match_times, i): if m.search(line): break else: return False for i, m in enumerate(exceptions): with _time(exc_times, i): if m.search(line): return False else: return True def _parse(stream, profile, context): def compile(regex_array): return [re.compile(regex) for regex in regex_array] error_matches = compile(_error_matches) error_exceptions = compile(_error_exceptions) warning_matches = compile(_warning_matches) warning_exceptions = compile(_warning_exceptions) file_line_matches = compile(_file_line_matches) matcher, _ = _match, [] timings = [] if profile: matcher = _profile_match timings = [ [0.0] * len(error_matches), [0.0] * len(error_exceptions), [0.0] * len(warning_matches), [0.0] * len(warning_exceptions), ] errors = [] warnings = [] # rolling window of recent lines pre_context = deque(maxlen=context) # list of (event, remaining_post_context_lines) pending_events: List[Tuple[LogEvent, int]] = [] for i, line in enumerate(stream): rstripped_line = line.rstrip() # feed this line into every event still collecting post_context if pending_events: active_events = [] for event, remaining in pending_events: event.post_context.append(rstripped_line) if remaining > 1: active_events.append((event, remaining - 1)) elif isinstance(event, BuildError): errors.append(event) else: warnings.append(event) pending_events = active_events # use CTest's regular expressions to scrape the log for events if matcher(error_matches, error_exceptions, line, *timings[:2]): event = BuildError(rstripped_line, i + 1) elif matcher(warning_matches, warning_exceptions, line, *timings[2:]): event = BuildWarning(rstripped_line, i + 1) else: pre_context.append(rstripped_line) continue event.pre_context = list(pre_context) event.post_context = [] # get file/line number for the event, if possible for flm in file_line_matches: match = flm.search(line) if match: event.source_file, event.source_line_no = match.groups() break if context > 0: pending_events.append((event, context)) elif isinstance(event, BuildError): errors.append(event) else: warnings.append(event) pre_context.append(rstripped_line) # flush events whose post_context window extends past EOF for event, _ in pending_events: if isinstance(event, BuildError): errors.append(event) else: warnings.append(event) return errors, warnings, timings class CTestLogParser: """Log file parser that extracts errors and warnings.""" def __init__(self, profile=False): # whether to record timing information self.timings = [] self.profile = profile def print_timings(self): """Print out profile of time spent in different regular expressions.""" def stringify(elt): return elt if isinstance(elt, str) else elt.pattern index = 0 for name, arr in [ ("error_matches", _error_matches), ("error_exceptions", _error_exceptions), ("warning_matches", _warning_matches), ("warning_exceptions", _warning_exceptions), ]: print() print(name) for i, elt in enumerate(arr): print("%16.2f %s" % (self.timings[index][i] * 1e6, stringify(elt))) index += 1 def parse( self, stream: Union[str, TextIO], context: int = 6 ) -> Tuple[List[BuildError], List[BuildWarning]]: """Parse a log file by searching each line for errors and warnings. Args: stream: filename or stream to read from context: lines of context to extract around each log event Returns: two lists containing :class:`BuildError` and :class:`BuildWarning` objects. """ if isinstance(stream, str): with open(stream, encoding="utf-8", errors="replace") as f: return self.parse(f, context) errors, warnings, self.timings = _parse(stream, self.profile, context) return errors, warnings ================================================ FILE: lib/spack/spack/util/editor.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Module for finding the user's preferred text editor. Defines one function, editor(), which invokes the editor defined by the user's VISUAL environment variable if set. We fall back to the editor defined by the EDITOR environment variable if VISUAL is not set or the specified editor fails (e.g. no DISPLAY for a graphical editor). If neither variable is set, we fall back to one of several common editors, raising an OSError if we are unable to find one. """ import os import shlex from typing import Callable, List import spack.config import spack.llnl.util.tty as tty import spack.util.executable #: editors to try if VISUAL and EDITOR are not set _default_editors = ["vim", "vi", "emacs", "nano", "notepad"] def _find_exe_from_env_var(var: str): """Find an executable from an environment variable. Args: var (str): environment variable name Returns: (str or None, list): executable string (or None if not found) and arguments parsed from the env var """ # try to get the environment variable exe = os.environ.get(var) if not exe: return None, [] # split env var into executable and args if needed args = shlex.split(str(exe)) if not args: return None, [] exe = spack.util.executable.which_string(args[0]) args = [exe] + args[1:] return exe, args def executable(exe: str, args: List[str]) -> int: """Wrapper that makes ``spack.util.executable.Executable`` look like ``os.execv()``. Use this with ``editor()`` if you want it to return instead of running ``execv``. """ cmd = spack.util.executable.Executable(exe) cmd(*args[1:], fail_on_error=False) return cmd.returncode def editor(*args: str, exec_fn: Callable[[str, List[str]], int] = os.execv) -> bool: """Invoke the user's editor. This will try to execute the following, in order: 1. ``$VISUAL ``: the "visual" editor (per POSIX) 2. ``$EDITOR ``: the regular editor (per POSIX) 3. some default editor (see ``_default_editors``) with If an environment variable isn't defined, it is skipped. If it points to something that can't be executed, we'll print a warning. And if we can't find anything that can be executed after searching the full list above, we'll raise an error. Arguments: args: args to pass to editor exec_fn: invoke this function to run; use ``spack.util.editor.executable`` if you want something that returns, instead of the default ``os.execv()``. """ def try_exec(exe, args, var=None): """Try to execute an editor with execv, and warn if it fails. Returns: (bool) False if the editor failed, ideally does not return if ``execv`` succeeds, and ``True`` if the ``exec`` does return successfully. """ # gvim runs in the background by default so we force it to run # in the foreground to ensure it gets attention. if "gvim" in exe and "-f" not in args: exe, *rest = args args = [exe, "-f"] + rest try: return exec_fn(exe, args) == 0 except (OSError, spack.util.executable.ProcessError) as e: if spack.config.get("config:debug"): raise # Show variable we were trying to use, if it's from one if var: exe = "$%s (%s)" % (var, exe) tty.warn("Could not execute %s due to error:" % exe, str(e)) return False def try_env_var(var): """Find an editor from an environment variable and try to exec it. This will warn if the variable points to something is not executable, or if there is an error when trying to exec it. """ if var not in os.environ: return False exe, editor_args = _find_exe_from_env_var(var) if not exe: tty.warn("$%s is not an executable:" % var, os.environ[var]) return False full_args = editor_args + list(args) return try_exec(exe, full_args, var) # try standard environment variables if try_env_var("SPACK_EDITOR"): return True if try_env_var("VISUAL"): return True if try_env_var("EDITOR"): return True # nothing worked -- try the first default we can find don't bother # trying them all -- if we get here and one fails, something is # probably much more deeply wrong with the environment. exe = spack.util.executable.which_string(*_default_editors) if exe and try_exec(exe, [exe] + list(args)): return True # Fail if nothing could be found raise OSError( "No text editor found! Please set the VISUAL and/or EDITOR " "environment variable(s) to your preferred text editor." ) ================================================ FILE: lib/spack/spack/util/elf.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import bisect import re import struct from struct import calcsize, unpack, unpack_from from typing import BinaryIO, Callable, Dict, List, NamedTuple, Optional, Pattern, Tuple class ElfHeader(NamedTuple): e_type: int e_machine: int e_version: int e_entry: int e_phoff: int e_shoff: int e_flags: int e_ehsize: int e_phentsize: int e_phnum: int e_shentsize: int e_shnum: int e_shstrndx: int class SectionHeader(NamedTuple): sh_name: int sh_type: int sh_flags: int sh_addr: int sh_offset: int sh_size: int sh_link: int sh_info: int sh_addralign: int sh_entsize: int class ProgramHeader32(NamedTuple): p_type: int p_offset: int p_vaddr: int p_paddr: int p_filesz: int p_memsz: int p_flags: int p_align: int class ProgramHeader64(NamedTuple): p_type: int p_flags: int p_offset: int p_vaddr: int p_paddr: int p_filesz: int p_memsz: int p_align: int class ELF_CONSTANTS: MAGIC = b"\x7fELF" CLASS32 = 1 CLASS64 = 2 DATA2LSB = 1 DATA2MSB = 2 ET_EXEC = 2 ET_DYN = 3 PT_LOAD = 1 PT_DYNAMIC = 2 PT_INTERP = 3 DT_NULL = 0 DT_NEEDED = 1 DT_STRTAB = 5 DT_SONAME = 14 DT_RPATH = 15 DT_RUNPATH = 29 SHT_STRTAB = 3 class ElfFile: """Parsed ELF file.""" is_64_bit: bool is_little_endian: bool byte_order: str elf_hdr: ElfHeader pt_load: List[Tuple[int, int]] has_pt_interp: bool pt_interp_p_offset: int pt_interp_p_filesz: int pt_interp_str: bytes has_pt_dynamic: bool pt_dynamic_p_offset: int pt_dynamic_p_filesz: int pt_dynamic_strtab_offset: int has_rpath: bool dt_rpath_offset: int dt_rpath_str: bytes rpath_strtab_offset: int is_runpath: bool has_needed: bool dt_needed_strtab_offsets: List[int] dt_needed_strs: List[bytes] has_soname: bool dt_soname_strtab_offset: int dt_soname_str: bytes __slots__ = [ "is_64_bit", "is_little_endian", "byte_order", "elf_hdr", "pt_load", # pt_interp "has_pt_interp", "pt_interp_p_offset", "pt_interp_p_filesz", "pt_interp_str", # pt_dynamic "has_pt_dynamic", "pt_dynamic_p_offset", "pt_dynamic_p_filesz", "pt_dynamic_strtab_offset", # string table for dynamic section # rpath "has_rpath", "dt_rpath_offset", "dt_rpath_str", "rpath_strtab_offset", "is_runpath", # dt needed "has_needed", "dt_needed_strtab_offsets", "dt_needed_strs", # dt soname "has_soname", "dt_soname_strtab_offset", "dt_soname_str", ] def __init__(self): self.dt_needed_strtab_offsets = [] self.has_soname = False self.has_rpath = False self.has_needed = False self.pt_load = [] self.has_pt_dynamic = False self.has_pt_interp = False def parse_c_string(byte_string: bytes, start: int = 0) -> bytes: """ Retrieve a C-string at a given offset in a byte string Arguments: byte_string: String start: Offset into the string Returns: bytes: A copy of the C-string excluding the terminating null byte """ str_end = byte_string.find(b"\0", start) if str_end == -1: raise ElfParsingError("C-string is not null terminated") return byte_string[start:str_end] def read_exactly(f: BinaryIO, num_bytes: int, msg: str) -> bytes: """ Read exactly num_bytes at the current offset, otherwise raise a parsing error with the given error message. Arguments: f: file handle num_bytes: Number of bytes to read msg: Error to show when bytes cannot be read Returns: bytes: the ``num_bytes`` bytes that were read. """ data = f.read(num_bytes) if len(data) != num_bytes: raise ElfParsingError(msg) return data def parse_program_headers(f: BinaryIO, elf: ElfFile) -> None: """ Parse program headers Arguments: f: file handle elf: ELF file parser data """ # Forward to the program header try: f.seek(elf.elf_hdr.e_phoff) except OSError: raise ElfParsingError("Could not seek to program header") # Here we have to make a mapping from virtual address to offset in the file. ph_fmt = elf.byte_order + ("LLQQQQQQ" if elf.is_64_bit else "LLLLLLLL") ph_size = calcsize(ph_fmt) ph_num = elf.elf_hdr.e_phnum # Read all program headers in one go data = read_exactly(f, ph_num * ph_size, "Malformed program header") ProgramHeader = ProgramHeader64 if elf.is_64_bit else ProgramHeader32 for i in range(ph_num): # mypy currently does not understand the union of two named tuples with equal fields ph = ProgramHeader(*unpack_from(ph_fmt, data, i * ph_size)) # Skip segments of size 0; we don't distinguish between missing segment and # empty segments. I've see an empty PT_DYNAMIC section for an ELF file that # contained debug data. if ph.p_filesz == 0: # type: ignore continue # For PT_LOAD entries: Save offsets and virtual addrs of the loaded ELF segments # This way we can map offsets by virtual address to offsets in the file. if ph.p_type == ELF_CONSTANTS.PT_LOAD: # type: ignore elf.pt_load.append((ph.p_offset, ph.p_vaddr)) # type: ignore elif ph.p_type == ELF_CONSTANTS.PT_INTERP: # type: ignore elf.pt_interp_p_offset = ph.p_offset # type: ignore elf.pt_interp_p_filesz = ph.p_filesz # type: ignore elf.has_pt_interp = True elif ph.p_type == ELF_CONSTANTS.PT_DYNAMIC: # type: ignore elf.pt_dynamic_p_offset = ph.p_offset # type: ignore elf.pt_dynamic_p_filesz = ph.p_filesz # type: ignore elf.has_pt_dynamic = True # The linker sorts PT_LOAD segments by vaddr, but let's do it just to be sure, since # patchelf for example has a flag to leave them in an arbitrary order. elf.pt_load.sort(key=lambda x: x[1]) def parse_pt_interp(f: BinaryIO, elf: ElfFile) -> None: """ Parse the interpreter (i.e. absolute path to the dynamic linker) Arguments: f: file handle elf: ELF file parser data """ try: f.seek(elf.pt_interp_p_offset) except OSError: raise ElfParsingError("Could not seek to PT_INTERP entry") data = read_exactly(f, elf.pt_interp_p_filesz, "Malformed PT_INTERP entry") elf.pt_interp_str = parse_c_string(data) def find_strtab_size_at_offset(f: BinaryIO, elf: ElfFile, offset: int) -> int: """ Retrieve the size of a string table section at a particular known offset Arguments: f: file handle elf: ELF file parser data offset: offset of the section in the file (i.e. ``sh_offset``) Returns: int: the size of the string table in bytes """ section_hdr_fmt = elf.byte_order + ("LLQQQQLLQQ" if elf.is_64_bit else "LLLLLLLLLL") section_hdr_size = calcsize(section_hdr_fmt) try: f.seek(elf.elf_hdr.e_shoff) except OSError: raise ElfParsingError("Could not seek to section header table") for _ in range(elf.elf_hdr.e_shnum): data = read_exactly(f, section_hdr_size, "Malformed section header") sh = SectionHeader(*unpack(section_hdr_fmt, data)) if sh.sh_type == ELF_CONSTANTS.SHT_STRTAB and sh.sh_offset == offset: return sh.sh_size raise ElfParsingError("Could not determine strtab size") def retrieve_strtab(f: BinaryIO, elf: ElfFile, offset: int) -> bytes: """ Read a full string table at the given offset, which requires looking it up in the section headers. Arguments: elf: ELF file parser data vaddr: virtual address Returns: file offset """ size = find_strtab_size_at_offset(f, elf, offset) try: f.seek(offset) except OSError: raise ElfParsingError("Could not seek to string table") return read_exactly(f, size, "Could not read string table") def vaddr_to_offset(elf: ElfFile, vaddr: int) -> int: """ Given a virtual address, find the corresponding offset in the ELF file itself. Arguments: elf: ELF file parser data vaddr: virtual address """ idx = bisect.bisect_right([p_vaddr for (p_offset, p_vaddr) in elf.pt_load], vaddr) - 1 p_offset, p_vaddr = elf.pt_load[idx] return p_offset - p_vaddr + vaddr def parse_pt_dynamic(f: BinaryIO, elf: ElfFile) -> None: """ Parse the dynamic section of an ELF file Arguments: f: file handle elf: ELF file parse data """ dynamic_array_fmt = elf.byte_order + ("qQ" if elf.is_64_bit else "lL") dynamic_array_size = calcsize(dynamic_array_fmt) current_offset = elf.pt_dynamic_p_offset count_rpath = 0 count_runpath = 0 count_strtab = 0 try: f.seek(elf.pt_dynamic_p_offset) except OSError: raise ElfParsingError("Could not seek to PT_DYNAMIC entry") # In case of broken ELF files, don't read beyond the advertised size. for _ in range(elf.pt_dynamic_p_filesz // dynamic_array_size): data = read_exactly(f, dynamic_array_size, "Malformed dynamic array entry") tag, val = unpack(dynamic_array_fmt, data) if tag == ELF_CONSTANTS.DT_NULL: break elif tag == ELF_CONSTANTS.DT_RPATH: count_rpath += 1 elf.rpath_strtab_offset = val elf.dt_rpath_offset = current_offset elf.is_runpath = False elf.has_rpath = True elif tag == ELF_CONSTANTS.DT_RUNPATH: count_runpath += 1 elf.rpath_strtab_offset = val elf.dt_rpath_offset = current_offset elf.is_runpath = True elf.has_rpath = True elif tag == ELF_CONSTANTS.DT_STRTAB: count_strtab += 1 strtab_vaddr = val elif tag == ELF_CONSTANTS.DT_NEEDED: elf.has_needed = True elf.dt_needed_strtab_offsets.append(val) elif tag == ELF_CONSTANTS.DT_SONAME: elf.has_soname = True elf.dt_soname_strtab_offset = val current_offset += dynamic_array_size # No rpath/runpath, that happens. if count_rpath == count_runpath == 0: elf.has_rpath = False elif count_rpath + count_runpath != 1: raise ElfParsingError("Could not find a unique rpath/runpath.") if count_strtab != 1: raise ElfParsingError("Could not find a unique strtab of for the dynamic section strings") # Nothing to retrieve, so don't bother getting the string table. if not (elf.has_rpath or elf.has_soname or elf.has_needed): return elf.pt_dynamic_strtab_offset = vaddr_to_offset(elf, strtab_vaddr) string_table = retrieve_strtab(f, elf, elf.pt_dynamic_strtab_offset) if elf.has_needed: elf.dt_needed_strs = list( parse_c_string(string_table, offset) for offset in elf.dt_needed_strtab_offsets ) if elf.has_soname: elf.dt_soname_str = parse_c_string(string_table, elf.dt_soname_strtab_offset) if elf.has_rpath: elf.dt_rpath_str = parse_c_string(string_table, elf.rpath_strtab_offset) def parse_header(f: BinaryIO, elf: ElfFile) -> None: # Read the 32/64 bit class independent part of the header and validate e_ident = f.read(16) # Require ELF magic bytes. if len(e_ident) != 16 or e_ident[:4] != ELF_CONSTANTS.MAGIC: raise ElfParsingError("Not an ELF file") # Defensively require a valid class and data. e_ident_class, e_ident_data = e_ident[4], e_ident[5] if e_ident_class not in (ELF_CONSTANTS.CLASS32, ELF_CONSTANTS.CLASS64): raise ElfParsingError("Invalid class found") if e_ident_data not in (ELF_CONSTANTS.DATA2LSB, ELF_CONSTANTS.DATA2MSB): raise ElfParsingError("Invalid data type") elf.is_64_bit = e_ident_class == ELF_CONSTANTS.CLASS64 elf.is_little_endian = e_ident_data == ELF_CONSTANTS.DATA2LSB # Set up byte order and types for unpacking elf.byte_order = "<" if elf.is_little_endian else ">" # Parse the rest of the header elf_header_fmt = elf.byte_order + ("HHLQQQLHHHHHH" if elf.is_64_bit else "HHLLLLLHHHHHH") hdr_size = calcsize(elf_header_fmt) data = read_exactly(f, hdr_size, "ELF header malformed") elf.elf_hdr = ElfHeader(*unpack(elf_header_fmt, data)) def _do_parse_elf( f: BinaryIO, interpreter: bool = True, dynamic_section: bool = True, only_header: bool = False ) -> ElfFile: # We don't (yet?) allow parsing ELF files at a nonzero offset, we just # jump to absolute offsets as they are specified in the ELF file. if f.tell() != 0: raise ElfParsingError("Cannot parse at a nonzero offset") elf = ElfFile() parse_header(f, elf) if only_header: return elf # We don't handle anything but executables and shared libraries now. if elf.elf_hdr.e_type not in (ELF_CONSTANTS.ET_EXEC, ELF_CONSTANTS.ET_DYN): raise ElfParsingError("Not an ET_DYN or ET_EXEC type") parse_program_headers(f, elf) # Parse PT_INTERP section if interpreter and elf.has_pt_interp: parse_pt_interp(f, elf) # Parse PT_DYNAMIC section. if dynamic_section and elf.has_pt_dynamic and len(elf.pt_load) > 0: parse_pt_dynamic(f, elf) return elf def parse_elf( f: BinaryIO, interpreter: bool = False, dynamic_section: bool = False, only_header: bool = False, ) -> ElfFile: """Given a file handle ``f`` for an ELF file opened in binary mode, return an :class:`~spack.util.elf.ElfFile` object with the parsed contents.""" try: return _do_parse_elf(f, interpreter, dynamic_section, only_header) except (DeprecationWarning, struct.error): # According to the docs old versions of Python can throw DeprecationWarning # instead of struct.error. raise ElfParsingError("Malformed ELF file") def get_rpaths(path: str) -> Optional[List[str]]: """Returns list of rpaths of the given file as UTF-8 strings, or None if not set.""" try: with open(path, "rb") as f: elf = parse_elf(f, interpreter=False, dynamic_section=True) return elf.dt_rpath_str.decode("utf-8").split(":") if elf.has_rpath else None except ElfParsingError: return None def get_interpreter(path: str) -> Optional[str]: """Returns the interpreter of the given file as UTF-8 string, or None if not set.""" try: with open(path, "rb") as f: elf = parse_elf(f, interpreter=True, dynamic_section=False) return elf.pt_interp_str.decode("utf-8") if elf.has_pt_interp else None except ElfParsingError: return None def _delete_dynamic_array_entry( f: BinaryIO, elf: ElfFile, should_delete: Callable[[int, int], bool] ) -> None: try: f.seek(elf.pt_dynamic_p_offset) except OSError: raise ElfParsingError("Could not seek to PT_DYNAMIC entry") dynamic_array_fmt = elf.byte_order + ("qQ" if elf.is_64_bit else "lL") dynamic_array_size = calcsize(dynamic_array_fmt) new_offset = elf.pt_dynamic_p_offset # points to the new dynamic array old_offset = elf.pt_dynamic_p_offset # points to the current dynamic array for _ in range(elf.pt_dynamic_p_filesz // dynamic_array_size): data = read_exactly(f, dynamic_array_size, "Malformed dynamic array entry") tag, val = unpack(dynamic_array_fmt, data) if tag == ELF_CONSTANTS.DT_NULL or not should_delete(tag, val): if new_offset != old_offset: f.seek(new_offset) f.write(data) f.seek(old_offset + dynamic_array_size) new_offset += dynamic_array_size if tag == ELF_CONSTANTS.DT_NULL: break old_offset += dynamic_array_size def delete_rpath(path: str) -> None: """Modifies a binary to remove the rpath. It zeros out the rpath string and also drops the ``DT_RPATH`` / ``DT_RUNPATH`` entry from the dynamic section, so it doesn't show up in ``readelf -d file``, nor in ``strings file``.""" with open(path, "rb+") as f: elf = parse_elf(f, interpreter=False, dynamic_section=True) if not elf.has_rpath: return # Zero out the rpath *string* in the binary new_rpath_string = b"\x00" * len(elf.dt_rpath_str) rpath_offset = elf.pt_dynamic_strtab_offset + elf.rpath_strtab_offset f.seek(rpath_offset) f.write(new_rpath_string) # Delete DT_RPATH / DT_RUNPATH entries from the dynamic section _delete_dynamic_array_entry( f, elf, lambda tag, _: tag == ELF_CONSTANTS.DT_RPATH or tag == ELF_CONSTANTS.DT_RUNPATH ) def delete_needed_from_elf(f: BinaryIO, elf: ElfFile, needed: bytes) -> None: """Delete a needed library from the dynamic section of an ELF file""" if not elf.has_needed or needed not in elf.dt_needed_strs: return offset = elf.dt_needed_strtab_offsets[elf.dt_needed_strs.index(needed)] _delete_dynamic_array_entry( f, elf, lambda tag, val: tag == ELF_CONSTANTS.DT_NEEDED and val == offset ) class CStringType: PT_INTERP = 1 RPATH = 2 class UpdateCStringAction: def __init__(self, old_value: bytes, new_value: bytes, offset: int): self.old_value = old_value self.new_value = new_value self.offset = offset @property def inplace(self) -> bool: return len(self.new_value) <= len(self.old_value) def apply(self, f: BinaryIO) -> None: assert self.inplace f.seek(self.offset) f.write(self.new_value) # We zero out the bits we shortened because (a) it should be a # C-string and (b) it's nice not to have spurious parts of old # paths in the output of `strings file`. Note that we're all # good when pad == 0; the original terminating null is used. f.write(b"\x00" * (len(self.old_value) - len(self.new_value))) def _get_rpath_substitution( elf: ElfFile, regex: Pattern, substitutions: Dict[bytes, bytes] ) -> Optional[UpdateCStringAction]: """Make rpath substitutions in-place.""" # If there's no RPATH, then there's no need to replace anything. if not elf.has_rpath: return None # Get the non-empty rpaths. Sometimes there's a bunch of trailing # colons ::::: used for padding, we don't add them back to make it # more likely that the string doesn't grow. rpaths = list(filter(len, elf.dt_rpath_str.split(b":"))) num_rpaths = len(rpaths) if num_rpaths == 0: return None changed = False for i in range(num_rpaths): old_rpath = rpaths[i] match = regex.match(old_rpath) if match: changed = True rpaths[i] = substitutions[match.group()] + old_rpath[match.end() :] # Nothing to replace! if not changed: return None return UpdateCStringAction( old_value=elf.dt_rpath_str, new_value=b":".join(rpaths), # The rpath is at a given offset in the string table used by the dynamic section. offset=elf.pt_dynamic_strtab_offset + elf.rpath_strtab_offset, ) def _get_pt_interp_substitution( elf: ElfFile, regex: Pattern, substitutions: Dict[bytes, bytes] ) -> Optional[UpdateCStringAction]: """Make interpreter substitutions in-place.""" if not elf.has_pt_interp: return None match = regex.match(elf.pt_interp_str) if not match: return None return UpdateCStringAction( old_value=elf.pt_interp_str, new_value=substitutions[match.group()] + elf.pt_interp_str[match.end() :], offset=elf.pt_interp_p_offset, ) def substitute_rpath_and_pt_interp_in_place_or_raise( path: str, substitutions: Dict[bytes, bytes] ) -> bool: """Returns true if the rpath and interpreter were modified, false if there was nothing to do. Raises ElfCStringUpdatesFailed if the ELF file cannot be updated in-place. This exception contains a list of actions to perform with other tools. The file is left untouched in this case.""" regex = re.compile(b"|".join(re.escape(p) for p in substitutions.keys())) try: with open(path, "rb+") as f: elf = parse_elf(f, interpreter=True, dynamic_section=True) # Get the actions to perform. rpath = _get_rpath_substitution(elf, regex, substitutions) pt_interp = _get_pt_interp_substitution(elf, regex, substitutions) # Nothing to do. if not rpath and not pt_interp: return False # If we can't update in-place, leave it to other tools, don't do partial updates. if rpath and not rpath.inplace or pt_interp and not pt_interp.inplace: raise ElfCStringUpdatesFailed(rpath, pt_interp) # Otherwise, apply the updates. if rpath: rpath.apply(f) if pt_interp: pt_interp.apply(f) return True except ElfParsingError: # This just means the file wasn't an elf file, so there's no point # in updating its rpath anyways; ignore this problem. return False def pt_interp(path: str) -> Optional[str]: """Retrieve the interpreter of an executable at ``path``.""" try: with open(path, "rb") as f: elf = parse_elf(f, interpreter=True) except (OSError, ElfParsingError): return None if not elf.has_pt_interp: return None return elf.pt_interp_str.decode("utf-8") def get_elf_compat(path: str) -> Tuple[bool, bool, int]: """Get a triplet (is_64_bit, is_little_endian, e_machine) from an ELF file, which can be used to see if two ELF files are compatible.""" # On ELF platforms supporting, we try to be a bit smarter when it comes to shared # libraries, by dropping those that are not host compatible. with open(path, "rb") as f: elf = parse_elf(f, only_header=True) return (elf.is_64_bit, elf.is_little_endian, elf.elf_hdr.e_machine) class ElfCStringUpdatesFailed(Exception): def __init__( self, rpath: Optional[UpdateCStringAction], pt_interp: Optional[UpdateCStringAction] ): self.rpath = rpath self.pt_interp = pt_interp class ElfParsingError(Exception): pass ================================================ FILE: lib/spack/spack/util/environment.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Set, unset or modify environment variables.""" import collections import contextlib import inspect import json import os import pathlib import pickle import re import shlex import subprocess import sys import warnings from typing import Any, Callable, Dict, Iterable, List, MutableMapping, Optional, Tuple, Union import spack.error from spack.llnl.path import path_to_os_path, system_path_filter from spack.llnl.util import tty from spack.llnl.util.lang import dedupe # List is invariant, so List[str] is not a subtype of List[Union[str, pathlib.PurePath]]. # Sequence is covariant, but because str itself is a subtype of Sequence[str], we cannot exclude it # in the type hint. So, use an awkward union type to allow (mixed) str and PurePath items. ListOfPaths = Union[List[str], List[pathlib.PurePath], List[Union[str, pathlib.PurePath]]] if sys.platform == "win32": SYSTEM_PATHS = [ "C:\\", "C:\\Program Files", "C:\\Program Files (x86)", "C:\\Users", "C:\\ProgramData", ] SUFFIXES = [] DEFAULT_SHELL = os.environ.get("SPACK_SHELL", "bat") else: SYSTEM_PATHS = ["/", "/usr", "/usr/local"] SUFFIXES = ["bin", "bin64", "include", "lib", "lib64"] DEFAULT_SHELL = "sh" SYSTEM_DIRS = [os.path.join(p, s) for s in SUFFIXES for p in SYSTEM_PATHS] + SYSTEM_PATHS #: used in the compiler wrapper's ``/usr/lib|/usr/lib64|...)`` case entry SYSTEM_DIR_CASE_ENTRY = "|".join(sorted(f'"{d}{suff}"' for d in SYSTEM_DIRS for suff in ("", "/"))) _SHELL_SET_STRINGS = { "sh": "export {0}={1};\n", "csh": "setenv {0} {1};\n", "fish": "set -gx {0} {1};\n", "bat": 'set "{0}={1}"\n', "pwsh": "$Env:{0}='{1}'\n", } _SHELL_UNSET_STRINGS = { "sh": "unset {0};\n", "csh": "unsetenv {0};\n", "fish": "set -e {0};\n", "bat": 'set "{0}="\n', "pwsh": "Set-Item -Path Env:{0}\n", } TRACING_ENABLED = False Path = str ModificationList = List[Union["NameModifier", "NameValueModifier"]] def is_system_path(path: Path) -> bool: """Returns True if the argument is a system path, False otherwise.""" return bool(path) and (os.path.normpath(path) in SYSTEM_DIRS) def filter_system_paths(paths: Iterable[Path]) -> List[Path]: """Returns a copy of the input where system paths are filtered out.""" return [p for p in paths if not is_system_path(p)] def deprioritize_system_paths(paths: List[Path]) -> List[Path]: """Reorders input paths by putting system paths at the end of the list, otherwise preserving order. """ return list(sorted(paths, key=is_system_path)) def prune_duplicate_paths(paths: List[Path]) -> List[Path]: """Returns the input list with duplicates removed, otherwise preserving order.""" return list(dedupe(paths)) def get_path(name: str) -> List[Path]: """Given the name of an environment variable containing multiple paths separated by :data:`os.pathsep`, returns a list of the paths. """ path = os.environ.get(name, "").strip() if path: return path.split(os.pathsep) return [] def env_flag(name: str) -> bool: """Given the name of an environment variable, returns True if the lowercase value is set to ``true`` or to ``1``, False otherwise. """ if name in os.environ: value = os.environ[name].lower() return value in ("true", "1") return False def path_set(var_name: str, directories: List[Path]): """Sets the variable passed as input to the :data:`os.pathsep` joined list of directories.""" path_str = os.pathsep.join(str(dir) for dir in directories) os.environ[var_name] = path_str def path_put_first(var_name: str, directories: List[Path]): """Puts the provided directories first in the path, adding them if they're not already there. """ path = os.environ.get(var_name, "").split(os.pathsep) for directory in directories: if directory in path: path.remove(directory) new_path = list(directories) + list(path) path_set(var_name, new_path) BASH_FUNCTION_FINDER = re.compile(r"BASH_FUNC_(.*?)\(\)") def _win_env_var_to_set_line(var: str, val: str) -> str: is_pwsh = os.environ.get("SPACK_SHELL", None) == "pwsh" env_set_phrase = f"$Env:{var}={val}" if is_pwsh else f'set "{var}={val}"' return env_set_phrase def _nix_env_var_to_source_line(var: str, val: str) -> str: if var.startswith("BASH_FUNC"): source_line = "function {fname}{decl}; export -f {fname}".format( fname=BASH_FUNCTION_FINDER.sub(r"\1", var), decl=val ) else: source_line = f"{var}={shlex.quote(val)}; export {var}" return source_line def _env_var_to_source_line(var: str, val: str) -> str: if sys.platform == "win32": return _win_env_var_to_set_line(var, val) else: return _nix_env_var_to_source_line(var, val) @system_path_filter(arg_slice=slice(1)) def dump_environment(path: Path, environment: Optional[MutableMapping[str, str]] = None): """Dump an environment dictionary to a source-able file. Args: path: path of the file to write environment: environment to be written. If None os.environ is used. """ use_env = environment or os.environ hidden_vars = {"PS1", "PWD", "OLDPWD", "TERM_SESSION_ID"} file_descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(file_descriptor, "w", encoding="utf-8") as env_file: for var, val in sorted(use_env.items()): env_file.write( "".join( ["#" if var in hidden_vars else "", _env_var_to_source_line(var, val), "\n"] ) ) @system_path_filter(arg_slice=slice(1)) def pickle_environment(path: Path, environment: Optional[Dict[str, str]] = None): """Pickle an environment dictionary to a file.""" with open(path, "wb") as pickle_file: pickle.dump(dict(environment if environment else os.environ), pickle_file, protocol=2) @contextlib.contextmanager def set_env(**kwargs): """Temporarily sets and restores environment variables. Variables can be set as keyword arguments to this function. .. note:: If the goal is to set environment variables for a subprocess, it is strongly recommended to use the ``extra_env`` argument of :func:`spack.util.executable.Executable.__call__` instead of this function. This function is intended to modify the *current* process's environment (which is an unsafe operation in general). """ saved = {} for var, value in kwargs.items(): if var in os.environ: saved[var] = os.environ[var] if value is None: if var in os.environ: del os.environ[var] else: os.environ[var] = value yield for var, value in kwargs.items(): if var in saved: os.environ[var] = saved[var] else: if var in os.environ: del os.environ[var] class Trace: """Trace information on a function call""" __slots__ = ("filename", "lineno", "context") def __init__(self, *, filename: str, lineno: int, context: str): self.filename = filename self.lineno = lineno self.context = context def __str__(self): return f"{self.context} at {self.filename}:{self.lineno}" def __repr__(self): return f"Trace(filename={self.filename}, lineno={self.lineno}, context={self.context})" class NameModifier: """Base class for modifiers that act on the environment variable as a whole, and thus store just its name """ __slots__ = ("name", "separator", "trace") def __init__(self, name: str, *, separator: str = os.pathsep, trace: Optional[Trace] = None): self.name = name.upper() if sys.platform == "win32" else name self.separator = separator self.trace = trace def __eq__(self, other: object): if not isinstance(other, NameModifier): return NotImplemented return self.name == other.name def execute(self, env: MutableMapping[str, str]): """Apply the modification to the mapping passed as input""" raise NotImplementedError("must be implemented by derived classes") class NameValueModifier: """Base class for modifiers that modify the value of an environment variable.""" __slots__ = ("name", "value", "separator", "trace") def __init__( self, name: str, value: str, *, separator: str = os.pathsep, trace: Optional[Trace] = None ): self.name = name.upper() if sys.platform == "win32" else name self.value = value self.separator = separator self.trace = trace def __eq__(self, other: object): if not isinstance(other, NameValueModifier): return NotImplemented return ( self.name == other.name and self.value == other.value and self.separator == other.separator ) def execute(self, env: MutableMapping[str, str]): """Apply the modification to the mapping passed as input""" raise NotImplementedError("must be implemented by derived classes") class NamePathModifier(NameValueModifier): """Base class for modifiers that modify the value of an environment variable that is a path.""" __slots__ = () def __init__( self, name: str, value: Union[str, pathlib.PurePath], *, separator: str = os.pathsep, trace: Optional[Trace] = None, ): super().__init__(name, str(value), separator=separator, trace=trace) class SetEnv(NameValueModifier): __slots__ = ("force", "raw") def __init__( self, name: str, value: str, *, trace: Optional[Trace] = None, force: bool = False, raw: bool = False, ): super().__init__(name, value, trace=trace) self.force = force self.raw = raw def execute(self, env: MutableMapping[str, str]): tty.debug(f"SetEnv: {self.name}={self.value}", level=3) env[self.name] = self.value class AppendFlagsEnv(NameValueModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"AppendFlagsEnv: {self.name}={self.value}", level=3) if self.name in env and env[self.name]: env[self.name] += self.separator + self.value else: env[self.name] = self.value class UnsetEnv(NameModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"UnsetEnv: {self.name}", level=3) # Avoid throwing if the variable was not set env.pop(self.name, None) class RemoveFlagsEnv(NameValueModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"RemoveFlagsEnv: {self.name}-{self.value}", level=3) environment_value = env.get(self.name, "") flags = environment_value.split(self.separator) if environment_value else [] flags = [f for f in flags if f != self.value] env[self.name] = self.separator.join(flags) class SetPath(NameValueModifier): def __init__( self, name: str, value: ListOfPaths, *, separator: str = os.pathsep, trace: Optional[Trace] = None, ): super().__init__( name, separator.join(str(x) for x in value), separator=separator, trace=trace ) def execute(self, env: MutableMapping[str, str]): tty.debug(f"SetPath: {self.name}={self.value}", level=3) env[self.name] = self.value class AppendPath(NamePathModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"AppendPath: {self.name}+{self.value}", level=3) environment_value = env.get(self.name, "") directories = environment_value.split(self.separator) if environment_value else [] directories.append(path_to_os_path(os.path.normpath(self.value)).pop()) env[self.name] = self.separator.join(directories) class PrependPath(NamePathModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"PrependPath: {self.name}+{self.value}", level=3) environment_value = env.get(self.name, "") directories = environment_value.split(self.separator) if environment_value else [] directories = [path_to_os_path(os.path.normpath(self.value)).pop()] + directories env[self.name] = self.separator.join(directories) class RemoveFirstPath(NamePathModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"RemoveFirstPath: {self.name}-{self.value}", level=3) environment_value = env.get(self.name, "") directories = environment_value.split(self.separator) directories = [path_to_os_path(os.path.normpath(x)).pop() for x in directories] val = path_to_os_path(os.path.normpath(self.value)).pop() if val in directories: directories.remove(val) env[self.name] = self.separator.join(directories) class RemoveLastPath(NamePathModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"RemoveLastPath: {self.name}-{self.value}", level=3) environment_value = env.get(self.name, "") directories = environment_value.split(self.separator)[::-1] directories = [path_to_os_path(os.path.normpath(x)).pop() for x in directories] val = path_to_os_path(os.path.normpath(self.value)).pop() if val in directories: directories.remove(val) env[self.name] = self.separator.join(directories[::-1]) class RemovePath(NamePathModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"RemovePath: {self.name}-{self.value}", level=3) environment_value = env.get(self.name, "") directories = environment_value.split(self.separator) directories = [ path_to_os_path(os.path.normpath(x)).pop() for x in directories if x != path_to_os_path(os.path.normpath(self.value)).pop() ] env[self.name] = self.separator.join(directories) class DeprioritizeSystemPaths(NameModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"DeprioritizeSystemPaths: {self.name}", level=3) environment_value = env.get(self.name, "") directories = environment_value.split(self.separator) if environment_value else [] directories = deprioritize_system_paths( [path_to_os_path(os.path.normpath(x)).pop() for x in directories] ) env[self.name] = self.separator.join(directories) class PruneDuplicatePaths(NameModifier): def execute(self, env: MutableMapping[str, str]): tty.debug(f"PruneDuplicatePaths: {self.name}", level=3) environment_value = env.get(self.name, "") directories = environment_value.split(self.separator) if environment_value else [] directories = prune_duplicate_paths( [path_to_os_path(os.path.normpath(x)).pop() for x in directories] ) env[self.name] = self.separator.join(directories) def _validate_path_value(name: str, value: Any) -> Union[str, pathlib.PurePath]: """Ensure the value for an env variable is string or path""" types = (str, pathlib.PurePath) if isinstance(value, types): return value types_str = " or ".join([f"`{t.__name__}`" for t in types]) warnings.warn( f"when setting environment variable {name}={value}: value is of type " f"`{type(value).__name__}`, but {types_str} was expected. This is deprecated and will be " f"an error in Spack v1.0", spack.error.SpackAPIWarning, stacklevel=3, ) return str(value) def _validate_value(name: str, value: Any) -> str: """Ensure the value for an env variable is a string""" if isinstance(value, str): return value warnings.warn( f"when setting environment variable {name}={value}: value is of type " f"`{type(value).__name__}`, but `str` was expected. This is deprecated and will be an " "error in Spack v1.0", spack.error.SpackAPIWarning, stacklevel=3, ) return str(value) class EnvironmentModifications: """ Tracks and applies a sequence of environment variable modifications. This class provides a high-level interface for building up a list of environment changes, such as setting, unsetting, appending, prepending, or removing values from environment variables. Modifications are stored and can be applied to a given environment dictionary, or rendered as shell code. Package authors typically receive an instance of this class and call :meth:`set`, :meth:`unset`, :meth:`prepend_path`, :meth:`remove_path`, etc., to queue up modifications. Spack runs :meth:`apply_modifications` to apply these modifications to the environment when needed. Modifications can be grouped by variable name, reversed (where possible), validated for suspicious patterns, and extended from other instances. The class also supports tracing the origin of modifications for debugging. Example: .. code-block:: python env = EnvironmentModifications() env.set("FOO", "bar") env.prepend_path("PATH", "/custom/bin") env.apply_modifications() # applies changes to os.environ """ def __init__( self, other: Optional["EnvironmentModifications"] = None, traced: Union[None, bool] = None ): """Initializes a new instance, copying commands from 'other' if it is not None. Args: other: list of environment modifications to be extended (optional) traced: enable or disable stack trace inspection to log the origin of the environment modifications """ self.traced = TRACING_ENABLED if traced is None else bool(traced) self.env_modifications: List[Union[NameModifier, NameValueModifier]] = [] if other is not None: self.extend(other) def __iter__(self): return iter(self.env_modifications) def __len__(self): return len(self.env_modifications) def extend(self, other: "EnvironmentModifications"): """Extends the current instance with modifications from another instance.""" self._check_other(other) self.env_modifications.extend(other.env_modifications) @staticmethod def _check_other(other: "EnvironmentModifications"): if not isinstance(other, EnvironmentModifications): raise TypeError("other must be an instance of EnvironmentModifications") def _trace(self) -> Optional[Trace]: """Returns a trace object if tracing is enabled, else None.""" if not self.traced: return None stack = inspect.stack() try: _, filename, lineno, _, context, index = stack[2] assert index is not None, "index must be an integer" current_context = context[index].strip() if context is not None else "unknown context" except Exception: filename = "unknown file" lineno = -1 current_context = "unknown context" return Trace(filename=filename, lineno=lineno, context=current_context) def set(self, name: str, value: str, *, force: bool = False, raw: bool = False) -> None: """Stores a request to set an environment variable. Args: name: name of the environment variable value: value of the environment variable force: if True, audit will not consider this modification a warning raw: if True, format of value string is skipped """ value = _validate_value(name, value) item = SetEnv(name, value, trace=self._trace(), force=force, raw=raw) self.env_modifications.append(item) def append_flags(self, name: str, value: str, sep: str = " ") -> None: """Stores a request to append flags to an environment variable. Args: name: name of the environment variable value: flags to be appended sep: separator for the flags (default: ``" "``) """ value = _validate_value(name, value) item = AppendFlagsEnv(name, value, separator=sep, trace=self._trace()) self.env_modifications.append(item) def unset(self, name: str) -> None: """Stores a request to unset an environment variable. Args: name: name of the environment variable """ item = UnsetEnv(name, trace=self._trace()) self.env_modifications.append(item) def remove_flags(self, name: str, value: str, sep: str = " ") -> None: """Stores a request to remove flags from an environment variable Args: name: name of the environment variable value: flags to be removed sep: separator for the flags (default: ``" "``) """ value = _validate_value(name, value) item = RemoveFlagsEnv(name, value, separator=sep, trace=self._trace()) self.env_modifications.append(item) def set_path(self, name: str, elements: ListOfPaths, separator: str = os.pathsep) -> None: """Stores a request to set an environment variable to a list of paths, separated by a character defined in input. Args: name: name of the environment variable elements: ordered list paths separator: separator for the paths (default: :data:`os.pathsep`) """ elements = [_validate_path_value(name, x) for x in elements] item = SetPath(name, elements, separator=separator, trace=self._trace()) self.env_modifications.append(item) def append_path( self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep ) -> None: """Stores a request to append a path to list of paths. Args: name: name of the environment variable path: path to be appended separator: separator for the paths (default: :data:`os.pathsep`) """ path = _validate_path_value(name, path) item = AppendPath(name, path, separator=separator, trace=self._trace()) self.env_modifications.append(item) def prepend_path( self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep ) -> None: """Stores a request to prepend a path to list of paths. Args: name: name of the environment variable path: path to be prepended separator: separator for the paths (default: :data:`os.pathsep`) """ path = _validate_path_value(name, path) item = PrependPath(name, path, separator=separator, trace=self._trace()) self.env_modifications.append(item) def remove_first_path( self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep ) -> None: """Stores a request to remove first instance of path from a list of paths. Args: name: name of the environment variable path: path to be removed separator: separator for the paths (default: :data:`os.pathsep`) """ path = _validate_path_value(name, path) item = RemoveFirstPath(name, path, separator=separator, trace=self._trace()) self.env_modifications.append(item) def remove_last_path( self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep ) -> None: """Stores a request to remove last instance of path from a list of paths. Args: name: name of the environment variable path: path to be removed separator: separator for the paths (default: :data:`os.pathsep`) """ path = _validate_path_value(name, path) item = RemoveLastPath(name, path, separator=separator, trace=self._trace()) self.env_modifications.append(item) def remove_path( self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep ) -> None: """Stores a request to remove a path from a list of paths. Args: name: name of the environment variable path: path to be removed separator: separator for the paths (default: :data:`os.pathsep`) """ path = _validate_path_value(name, path) item = RemovePath(name, path, separator=separator, trace=self._trace()) self.env_modifications.append(item) def deprioritize_system_paths(self, name: str, separator: str = os.pathsep) -> None: """Stores a request to deprioritize system paths in a path list, otherwise preserving the order. Args: name: name of the environment variable separator: separator for the paths (default: :data:`os.pathsep`) """ item = DeprioritizeSystemPaths(name, separator=separator, trace=self._trace()) self.env_modifications.append(item) def prune_duplicate_paths(self, name: str, separator: str = os.pathsep) -> None: """Stores a request to remove duplicates from a path list, otherwise preserving the order. Args: name: name of the environment variable separator: separator for the paths (default: :data:`os.pathsep`) """ item = PruneDuplicatePaths(name, separator=separator, trace=self._trace()) self.env_modifications.append(item) def group_by_name(self) -> Dict[str, ModificationList]: """Returns a dict of the current modifications keyed by variable name.""" modifications = collections.defaultdict(list) for item in self: modifications[item.name].append(item) return modifications def drop(self, *name) -> bool: """Drop all modifications to the variable with the given name.""" old_mods = self.env_modifications new_mods = [x for x in self.env_modifications if x.name not in name] self.env_modifications = new_mods return len(old_mods) != len(new_mods) def is_unset(self, variable_name: str) -> bool: """Returns :data:`True` if the last modification to a variable is to unset it, :data:`False` otherwise.""" modifications = self.group_by_name() if variable_name not in modifications: return False # The last modification must unset the variable for it to be considered unset return isinstance(modifications[variable_name][-1], UnsetEnv) def clear(self): """Clears the current list of modifications.""" self.env_modifications = [] def reversed(self) -> "EnvironmentModifications": """Returns the EnvironmentModifications object that will reverse self Only creates reversals for additions to the environment, as reversing :meth:`unset` and :meth:`remove_path` modifications is impossible. Reversible operations are :meth:`set`, :meth:`prepend_path`, :meth:`append_path`, :meth:`set_path`, and :meth:`append_flags`. """ rev = EnvironmentModifications() for envmod in reversed(self.env_modifications): if isinstance(envmod, SetEnv): tty.debug("Reversing `Set` environment operation may lose the original value") rev.unset(envmod.name) elif isinstance(envmod, AppendPath): rev.remove_last_path(envmod.name, envmod.value) elif isinstance(envmod, PrependPath): rev.remove_first_path(envmod.name, envmod.value) elif isinstance(envmod, SetPath): tty.debug("Reversing `SetPath` environment operation may lose the original value") rev.unset(envmod.name) elif isinstance(envmod, AppendFlagsEnv): rev.remove_flags(envmod.name, envmod.value) else: tty.debug( f"Skipping reversal of irreversible operation {type(envmod)} {envmod.name}" ) return rev def apply_modifications(self, env: Optional[MutableMapping[str, str]] = None): """Applies the modifications to the environment. Args: env: environment to be modified. If None, :obj:`os.environ` will be used. """ env = os.environ if env is None else env modifications = self.group_by_name() for _, actions in sorted(modifications.items()): for modifier in actions: modifier.execute(env) def shell_modifications( self, shell: str = DEFAULT_SHELL, explicit: bool = False, env: Optional[MutableMapping[str, str]] = None, ) -> str: """Return shell code to apply the modifications.""" modifications = self.group_by_name() env = os.environ if env is None else env new_env = dict(env.items()) for _, actions in sorted(modifications.items()): for modifier in actions: modifier.execute(new_env) if "MANPATH" in new_env and not new_env["MANPATH"].endswith(os.pathsep): new_env["MANPATH"] += os.pathsep cmds = "" for name in sorted(set(modifications)): new = new_env.get(name, None) old = env.get(name, None) if explicit or new != old: if new is None: cmds += _SHELL_UNSET_STRINGS[shell].format(name) else: value = new_env[name] if shell not in ("bat", "pwsh"): value = shlex.quote(value) cmd = _SHELL_SET_STRINGS[shell].format(name, value) cmds += cmd return cmds @staticmethod def from_sourcing_file( filename: Path, *arguments: str, **kwargs: Any ) -> "EnvironmentModifications": """Returns the environment modifications that have the same effect as sourcing the input file in a shell. Args: filename: the file to be sourced *arguments: arguments to pass on the command line Keyword Args: shell (str): the shell to use (default: ``bash``) shell_options (str): options passed to the shell (default: ``-c``) source_command (str): the command to run (default: ``source``) suppress_output (str): redirect used to suppress output of command (default: ``&> /dev/null``) concatenate_on_success (str): operator used to execute a command only when the previous command succeeds (default: ``&&``) exclude ([str or re.Pattern[str]]): ignore any modifications of these variables (default: []) include ([str or re.Pattern[str]]): always respect modifications of these variables (default: []). Supersedes any excluded variables. clean (bool): in addition to removing empty entries, also remove duplicate entries (default: False). """ tty.debug(f"EnvironmentModifications.from_sourcing_file: {filename}") # Check if the file actually exists if not os.path.isfile(filename): msg = f"Trying to source non-existing file: {filename}" raise RuntimeError(msg) # Prepare include and exclude lists of environment variable names exclude = kwargs.get("exclude", []) include = kwargs.get("include", []) clean = kwargs.get("clean", False) # Other variables unrelated to sourcing a file exclude.extend( [ # Bash internals "SHLVL", "_", "PWD", "OLDPWD", "PS1", "PS2", "ENV", # Environment Modules or Lmod "LOADEDMODULES", "_LMFILES_", "MODULEPATH", "MODULERCFILE", "BASH_FUNC_ml()", "BASH_FUNC_module()", # Environment Modules-specific configuration "MODULESHOME", "BASH_FUNC__module_raw()", r"MODULES_(.*)", r"__MODULES_(.*)", r"(\w*)_mod(quar|share)", # Lmod-specific configuration r"LMOD_(.*)", ] ) before_kwargs = {**kwargs} if sys.platform == "win32": # Windows cannot source os.devnull, but it can echo from it # so we override the "source" action in the method that # extracts the env (environment_after_sourcing_files) if "source_command" not in kwargs: before_kwargs["source_command"] = "echo" # Compute the environments before and after sourcing # First look at the environment after doing nothing to # establish baseline before = sanitize( environment_after_sourcing_files(os.devnull, **before_kwargs), exclude=exclude, include=include, ) file_and_args = (filename,) + arguments after = sanitize( environment_after_sourcing_files(file_and_args, **kwargs), exclude=exclude, include=include, ) # Delegate to the other factory return EnvironmentModifications.from_environment_diff(before, after, clean) @staticmethod def from_environment_diff( before: MutableMapping[str, str], after: MutableMapping[str, str], clean: bool = False ) -> "EnvironmentModifications": """Constructs the environment modifications from the diff of two environments. Args: before: environment before the modifications are applied after: environment after the modifications are applied clean: in addition to removing empty entries, also remove duplicate entries """ # Fill the EnvironmentModifications instance env = EnvironmentModifications() # New variables new_variables = list(set(after) - set(before)) # Variables that have been unset unset_variables = list(set(before) - set(after)) # Variables that have been modified common_variables = set(before).intersection(set(after)) modified_variables = [x for x in common_variables if before[x] != after[x]] # Consistent output order - looks nicer, easier comparison... new_variables.sort() unset_variables.sort() modified_variables.sort() def return_separator_if_any(*args): separators = [os.pathsep] if sys.platform == "win32" else [":", ";"] for separator in separators: for arg in args: if separator in arg: return separator return None # Add variables to env. # Assume that variables with 'PATH' in the name or that contain # separators like ':' or ';' are more likely to be paths for variable_name in new_variables: sep = return_separator_if_any(after[variable_name]) if sep: env.prepend_path(variable_name, after[variable_name], separator=sep) elif "PATH" in variable_name: env.prepend_path(variable_name, after[variable_name]) else: # We just need to set the variable to the new value env.set(variable_name, after[variable_name]) for variable_name in unset_variables: env.unset(variable_name) for variable_name in modified_variables: value_before = before[variable_name] value_after = after[variable_name] sep = return_separator_if_any(value_before, value_after) if sep: before_list = value_before.split(sep) after_list = value_after.split(sep) # Filter out empty strings before_list = list(filter(None, before_list)) after_list = list(filter(None, after_list)) # Remove duplicate entries (worse matching, bloats env) if clean: before_list = list(dedupe(before_list)) after_list = list(dedupe(after_list)) # The reassembled cleaned entries value_before = sep.join(before_list) value_after = sep.join(after_list) # Paths that have been removed remove_list = [ii for ii in before_list if ii not in after_list] # Check that nothing has been added in the middle of # before_list remaining_list = [ii for ii in before_list if ii in after_list] try: start = after_list.index(remaining_list[0]) end = after_list.index(remaining_list[-1]) search = sep.join(after_list[start : end + 1]) except IndexError: env.prepend_path(variable_name, value_after) continue if search not in value_before: # We just need to set the variable to the new value env.prepend_path(variable_name, value_after) else: try: prepend_list = after_list[:start] prepend_list.reverse() # Preserve order after prepend except KeyError: prepend_list = [] try: append_list = after_list[end + 1 :] except KeyError: append_list = [] for item in remove_list: env.remove_path(variable_name, item) for item in append_list: env.append_path(variable_name, item) for item in prepend_list: env.prepend_path(variable_name, item) else: # We just need to set the variable to the new value env.set(variable_name, value_after) return env def _set_or_unset_not_first( variable: str, changes: ModificationList, errstream: Callable[[str], None] ): """Check if we are going to set or unset something after other modifications have already been requested. """ indexes = [ ii for ii, item in enumerate(changes) if ii != 0 and isinstance(item, (SetEnv, UnsetEnv)) and not getattr(item, "force", False) ] if indexes: good = "\t \t{}" nogood = "\t--->\t{}" errstream(f"Different requests to set/unset '{variable}' have been found") for idx, item in enumerate(changes): print_format = nogood if idx in indexes else good errstream(print_format.format(item.trace)) def validate(env: EnvironmentModifications, errstream: Callable[[str], None]): """Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for everything that was found. Current checks: - set or unset variables after other changes on the same variable Args: env: list of environment modifications errstream: callable to log error messages """ if not env.traced: return modifications = env.group_by_name() for variable, list_of_changes in sorted(modifications.items()): _set_or_unset_not_first(variable, list_of_changes, errstream) def inspect_path( root: Path, inspections: MutableMapping[str, List[str]], exclude: Optional[Callable[[Path], bool]] = None, ) -> EnvironmentModifications: """Inspects ``root`` to search for the subdirectories in ``inspections``. Adds every path found to a list of prepend-path commands and returns it. Args: root: absolute path where to search for subdirectories inspections: maps relative paths to a list of environment variables that will be modified if the path exists. The modifications are not performed immediately, but stored in a command object that is returned to client exclude: optional callable. If present it must accept an absolute path and return True if it should be excluded from the inspection Examples: The following lines execute an inspection in ``/usr`` to search for ``/usr/include`` and ``/usr/lib64``. If found we want to prepend ``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``. .. code-block:: python # Set up the dictionary containing the inspection inspections = { "include": ["CPATH"], "lib64": ["MY_LIB64_PATH"] } # Get back the list of command needed to modify the environment env = inspect_path("/usr", inspections) # Eventually execute the commands env.apply_modifications() """ if exclude is None: exclude = lambda x: False env = EnvironmentModifications() # Inspect the prefix to check for the existence of common directories for relative_path, variables in inspections.items(): expected = os.path.join(root, os.path.normpath(relative_path)) if os.path.isdir(expected) and not exclude(expected): for variable in variables: env.prepend_path(variable, expected) return env @contextlib.contextmanager def preserve_environment(*variables: str): """Ensures that the value of the environment variables passed as arguments is the same before entering to the context manager and after exiting it. Variables that are unset before entering the context manager will be explicitly unset on exit. Args: variables: list of environment variables to be preserved """ cache = {} for var in variables: # The environment variable to be preserved might not be there. # In that case store None as a placeholder. cache[var] = os.environ.get(var, None) yield for var in variables: value = cache[var] msg = "[PRESERVE_ENVIRONMENT]" if value is not None: # Print a debug statement if the value changed if var not in os.environ: msg += ' {0} was unset, will be reset to "{1}"' tty.debug(msg.format(var, value)) elif os.environ[var] != value: msg += ' {0} was set to "{1}", will be reset to "{2}"' tty.debug(msg.format(var, os.environ[var], value)) os.environ[var] = value elif var in os.environ: msg += ' {0} was set to "{1}", will be unset' tty.debug(msg.format(var, os.environ[var])) del os.environ[var] def environment_after_sourcing_files( *files: Union[Path, Tuple[str, ...]], **kwargs ) -> Dict[str, str]: """Returns a dictionary with the environment that one would have after sourcing the files passed as argument. Args: *files: each item can either be a string containing the path of the file to be sourced or a sequence, where the first element is the file to be sourced and the remaining are arguments to be passed to the command line Keyword Args: env (dict): the initial environment (default: current environment) shell (str): the shell to use (default: ``/bin/bash`` or ``cmd.exe`` (Windows)) shell_options (str): options passed to the shell (default: ``-c`` or ``/C`` (Windows)) source_command (str): the command to run (default: ``source``) suppress_output (str): redirect used to suppress output of command (default: ``&> /dev/null``) concatenate_on_success (str): operator used to execute a command only when the previous command succeeds (default: ``&&``) """ # Set the shell executable that will be used to source files if sys.platform == "win32": shell_cmd = kwargs.get("shell", "cmd.exe") shell_options = kwargs.get("shell_options", "/C") suppress_output = kwargs.get("suppress_output", "> nul") source_command = kwargs.get("source_command", "") else: shell_cmd = kwargs.get("shell", "/bin/bash") shell_options = kwargs.get("shell_options", "-c") suppress_output = kwargs.get("suppress_output", "&> /dev/null") source_command = kwargs.get("source_command", "source") concatenate_on_success = kwargs.get("concatenate_on_success", "&&") def _source_single_file(file_and_args, environment): shell_options_list = shell_options.split() source_file = [source_command] source_file.extend(x for x in file_and_args) source_file = " ".join(source_file) dump_cmd = "import os, json; print(json.dumps(dict(os.environ)))" dump_environment_cmd = sys.executable + f' -E -c "{dump_cmd}"' # Try to source the file source_file_arguments = " ".join( [source_file, suppress_output, concatenate_on_success, dump_environment_cmd] ) # Popens argument processing can break command invocations # on Windows, compose to a string to avoid said processing cmd = [shell_cmd, *shell_options_list, source_file_arguments] cmd = " ".join(cmd) if sys.platform == "win32" else cmd with subprocess.Popen( cmd, env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) as shell: output, _ = shell.communicate() return json.loads(output) current_environment = kwargs.get("env", dict(os.environ)) for file in files: # Normalize the input to the helper function if isinstance(file, str): file = (file,) current_environment = _source_single_file(file, environment=current_environment) return current_environment def sanitize( environment: MutableMapping[str, str], exclude: List[str], include: List[str] ) -> Dict[str, str]: """Returns a copy of the input dictionary where all the keys that match an excluded pattern and don't match an included pattern are removed. Args: environment (dict): input dictionary exclude (list): literals or regex patterns to be excluded include (list): literals or regex patterns to be included """ def set_intersection(fullset, *args): # A set intersection using string literals and regexs meta = "[" + re.escape("[$()*?[]^{|}") + "]" subset = fullset & set(args) # As literal for name in args: if re.search(meta, name): pattern = re.compile(name) for k in fullset: if re.match(pattern, k): subset.add(k) return subset # Don't modify input, make a copy instead environment = dict(environment) # include supersedes any excluded items prune = set_intersection(set(environment), *exclude) prune -= set_intersection(prune, *include) for k in prune: environment.pop(k, None) return environment ================================================ FILE: lib/spack/spack/util/executable.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import os import re import shlex import subprocess import sys from pathlib import Path, PurePath from typing import BinaryIO, Callable, Dict, List, Optional, Sequence, Tuple, Type, Union, overload from spack.vendor.typing_extensions import Literal import spack.error import spack.llnl.util.tty as tty from spack.util.environment import EnvironmentModifications __all__ = ["Executable", "which", "which_string", "ProcessError"] OutType = Union[Optional[BinaryIO], str, Type[str], Callable] def _process_cmd_output( out: bytes, err: bytes, output: OutType, error: OutType, encoding: str = "ISO-8859-1" if sys.platform == "win32" else "utf-8", ) -> Optional[str]: if output is str or output is str.split or error is str or error is str.split: result = "" if output is str or output is str.split: outstr = out.decode(encoding) result += outstr if output is str.split: sys.stdout.write(outstr) if error is str or error is str.split: errstr = err.decode(encoding) result += errstr if error is str.split: sys.stderr.write(errstr) return result else: return None def _streamify_output(arg: OutType, name: str) -> Tuple[Union[int, BinaryIO, None], bool]: if isinstance(arg, str): return open(arg, "wb"), True elif arg is str or arg is str.split: return subprocess.PIPE, False elif callable(arg): raise ValueError(f"`{name}` must be a stream, a filename, or `str`/`str.split`") else: return arg, False class Executable: """ Represent an executable file that can be run as a subprocess. This class provides a simple interface for running executables with custom arguments and environment variables. It supports setting default arguments and environment modifications, copying instances, and running commands with various options for input/output/error handling. Example usage: .. code-block:: python ls = Executable("ls") ls.add_default_arg("-l") ls.add_default_env("LC_ALL", "C") output = ls("-a", output=str) # Run 'ls -l -a' and capture output as string """ def __init__(self, name: Union[str, Path]) -> None: file_path = str(Path(name)) if sys.platform != "win32" and isinstance(name, str) and name.startswith("."): # pathlib strips the ./ from relative paths so it must be added back file_path = os.path.join(".", file_path) self.exe = [file_path] self._default_env: Dict[str, str] = {} self._default_envmod = EnvironmentModifications() #: Return code of the last executed command. self.returncode: int = 1 # 1 until proven successful #: Whether to warn users that quotes are not needed, as Spack does not use a shell. self.ignore_quotes: bool = False def add_default_arg(self, *args: str) -> None: """Add default argument(s) to the command.""" self.exe.extend(args) def with_default_args(self, *args: str) -> "Executable": """Same as add_default_arg, but returns a copy of the executable.""" new = self.copy() new.add_default_arg(*args) return new def copy(self) -> "Executable": """Return a copy of this Executable.""" new = Executable(self.exe[0]) new.exe[:] = self.exe new._default_env.update(self._default_env) new._default_envmod.extend(self._default_envmod) return new def add_default_env(self, key: str, value: str) -> None: """Set an environment variable when the command is run. Parameters: key: The environment variable to set value: The value to set it to """ self._default_env[key] = value def add_default_envmod(self, envmod: EnvironmentModifications) -> None: """Set an :class:`spack.util.environment.EnvironmentModifications` to use when the command is run.""" self._default_envmod.extend(envmod) @property def command(self) -> str: """Returns the entire command-line string""" return " ".join(self.exe) @property def name(self) -> str: """Returns the executable name""" return PurePath(self.path).name @property def path(self) -> str: """Returns the executable path""" return str(PurePath(self.exe[0])) @overload def __call__( self, *args: str, fail_on_error: bool = ..., ignore_errors: Union[int, Sequence[int]] = ..., ignore_quotes: Optional[bool] = ..., timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., input: Optional[BinaryIO] = ..., output: Union[Optional[BinaryIO], str] = ..., error: Union[Optional[BinaryIO], str] = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> None: ... @overload def __call__( self, *args: str, fail_on_error: bool = ..., ignore_errors: Union[int, Sequence[int]] = ..., ignore_quotes: Optional[bool] = ..., timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., input: Optional[BinaryIO] = ..., output: Union[Type[str], Callable], # str or str.split error: OutType = ..., _dump_env: Optional[Dict[str, str]] = ..., ) -> str: ... @overload def __call__( self, *args: str, fail_on_error: bool = ..., ignore_errors: Union[int, Sequence[int]] = ..., ignore_quotes: Optional[bool] = ..., timeout: Optional[int] = ..., env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ..., input: Optional[BinaryIO] = ..., output: OutType = ..., error: Union[Type[str], Callable], # str or str.split _dump_env: Optional[Dict[str, str]] = ..., ) -> str: ... def __call__( self, *args: str, fail_on_error: bool = True, ignore_errors: Union[int, Sequence[int]] = (), ignore_quotes: Optional[bool] = None, timeout: Optional[int] = None, env: Optional[Union[Dict[str, str], EnvironmentModifications]] = None, extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = None, input: Optional[BinaryIO] = None, output: OutType = None, error: OutType = None, _dump_env: Optional[Dict[str, str]] = None, ) -> Optional[str]: """Runs this executable in a subprocess. Parameters: *args: command-line arguments to the executable to run fail_on_error: if True, raises an exception if the subprocess returns an error The return code is available as :attr:`returncode` ignore_errors: a sequence of error codes to ignore. If these codes are returned, this process will not raise an exception, even if ``fail_on_error`` is set to ``True`` ignore_quotes: if False, warn users that quotes are not needed, as Spack does not use a shell. If None, use :attr:`ignore_quotes`. timeout: the number of seconds to wait before killing the child process env: the environment with which to run the executable extra_env: extra items to add to the environment (neither requires nor precludes env) input: where to read stdin from output: where to send stdout error: where to send stderr _dump_env: dict to be set to the environment actually used (envisaged for testing purposes only) Accepted values for ``input``, ``output``, and ``error``: * Python streams: open Python file objects or ``os.devnull`` * :obj:`str`: the Python string **type**. If you set these to :obj:`str`, output and error will be written to pipes and returned as a string. If both ``output`` and ``error`` are set to :obj:`str`, then one string is returned containing output concatenated with error. Not valid for ``input``. * :obj:`str.split`: the split method of the Python string type. Behaves the same as :obj:`str`, except that value is also written to ``stdout`` or ``stderr``. For ``output`` and ``error`` it's also accepted to pass a string with a filename, which will be automatically opened for writing. By default, the subprocess inherits the parent's file descriptors. """ # Setup default environment current_environment = os.environ.copy() if env is None else {} self._default_envmod.apply_modifications(current_environment) current_environment.update(self._default_env) # Apply env argument if isinstance(env, EnvironmentModifications): env.apply_modifications(current_environment) elif env: current_environment.update(env) # Apply extra env if isinstance(extra_env, EnvironmentModifications): extra_env.apply_modifications(current_environment) elif extra_env is not None: current_environment.update(extra_env) if _dump_env is not None: _dump_env.clear() _dump_env.update(current_environment) if ignore_quotes is None: ignore_quotes = self.ignore_quotes # If they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): ignore_errors = (ignore_errors,) if input is str or input is str.split: raise ValueError("Cannot use `str` or `str.split` as input stream.") elif isinstance(input, str): istream, close_istream = open(input, "rb"), True else: istream, close_istream = input, False ostream, close_ostream = _streamify_output(output, "output") estream, close_estream = _streamify_output(error, "error") if not ignore_quotes: quoted_args = [arg for arg in args if re.search(r'^".*"$|^\'.*\'$', arg)] if quoted_args: tty.warn( "Quotes in command arguments can confuse scripts like configure.", "The following arguments may cause problems when executed:", str("\n".join([" " + arg for arg in quoted_args])), "Quotes aren't needed because spack doesn't use a shell. " "Consider removing them.", "If multiple levels of quotation are required, use `ignore_quotes=True`.", ) cmd = self.exe + list(args) cmd_line_string = " ".join(shlex.quote(arg) for arg in cmd) tty.debug(cmd_line_string) result = None try: proc = subprocess.Popen( cmd, stdin=istream, stderr=estream, stdout=ostream, env=current_environment, close_fds=False, ) except OSError as e: message = "Command: " + cmd_line_string if " " in self.exe[0]: message += "\nDid you mean to add a space to the command?" raise ProcessError(f"{self.exe[0]}: {e.strerror}", message) try: out, err = proc.communicate(timeout=timeout) result = _process_cmd_output(out, err, output, error) rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): long_msg = cmd_line_string if result: # If the output is not captured in the result, it will have # been stored either in the specified files (e.g. if # 'output' specifies a file) or written to the parent's # stdout/stderr (e.g. if 'output' is not specified) long_msg += "\n" + result raise ProcessError(f"Command exited with status {proc.returncode}:", long_msg) except subprocess.TimeoutExpired as te: proc.kill() out, err = proc.communicate() result = _process_cmd_output(out, err, output, error) long_msg = cmd_line_string + f"\n{result}" if fail_on_error: raise ProcessTimeoutError( f"\nProcess timed out after {timeout}s. " "We expected the following command to run quickly but it did not, " f"please report this as an issue: {long_msg}", long_message=long_msg, ) from te finally: # The isinstance checks are only needed for type checking. if close_ostream and isinstance(ostream, io.IOBase): ostream.close() if close_estream and isinstance(estream, io.IOBase): estream.close() if close_istream and isinstance(istream, io.IOBase): istream.close() return result def __eq__(self, other): return hasattr(other, "exe") and self.exe == other.exe def __hash__(self): return hash((type(self),) + tuple(self.exe)) def __repr__(self): return f"" def __str__(self): return " ".join(self.exe) @overload def which_string( *args: str, path: Optional[Union[List[str], str]] = ..., required: Literal[True] ) -> str: ... @overload def which_string( *args: str, path: Optional[Union[List[str], str]] = ..., required: bool = ... ) -> Optional[str]: ... def which_string( *args: str, path: Optional[Union[List[str], str]] = None, required: bool = False ) -> Optional[str]: """Like :func:`which`, but returns a string instead of an :class:`Executable`.""" if path is None: path = os.environ.get("PATH", "") if isinstance(path, list): paths = [Path(str(x)) for x in path] if isinstance(path, str): paths = [Path(x) for x in path.split(os.pathsep)] def get_candidate_items(search_item): if sys.platform == "win32" and not search_item.suffix: return [search_item.parent / (search_item.name + ext) for ext in [".exe", ".bat"]] return [Path(search_item)] def add_extra_search_paths(paths): with_parents = [] with_parents.extend(paths) if sys.platform == "win32": for p in paths: if p.name == "bin": with_parents.append(p.parent) return with_parents for search_item in args: search_paths = [] search_paths.extend(paths) if search_item.startswith("."): # we do this because pathlib will strip any leading ./ search_paths.insert(0, Path.cwd()) search_paths = add_extra_search_paths(search_paths) candidate_items = get_candidate_items(Path(search_item)) for candidate_item in candidate_items: for directory in search_paths: exe = directory / candidate_item try: if exe.is_file() and os.access(str(exe), os.X_OK): return str(exe) except OSError: pass if required: raise CommandNotFoundError(f"spack requires '{args[0]}'. Make sure it is in your path.") return None @overload def which( *args: str, path: Optional[Union[List[str], str]] = ..., required: Literal[True] ) -> Executable: ... @overload def which( *args: str, path: Optional[Union[List[str], str]] = ..., required: bool = ... ) -> Optional[Executable]: ... def which( *args: str, path: Optional[Union[List[str], str]] = None, required: bool = False ) -> Optional[Executable]: """Finds an executable in the path like command-line which. If given multiple executables, returns the first one that is found. If no executables are found, returns None. Parameters: *args: one or more executables to search for path: the path to search. Defaults to ``PATH`` required: if set to :data:`True`, raise an error if executable not found Returns: The first executable that is found in the path or :data:`None` if not found. """ exe = which_string(*args, path=path, required=required) return Executable(exe) if exe is not None else None class ProcessError(spack.error.SpackError): """Raised when :class:`Executable` exits with an error code.""" class ProcessTimeoutError(ProcessError): """Raised when :class:`Executable` calls with a specified timeout exceed that time.""" class CommandNotFoundError(spack.error.SpackError): """Raised when :func:`which()` can't find a required executable.""" ================================================ FILE: lib/spack/spack/util/file_cache.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import hashlib import os import pathlib import shutil import tempfile from contextlib import contextmanager from typing import IO, Dict, Iterator, Optional, Tuple, Union from spack.error import SpackError from spack.llnl.util.filesystem import rename from spack.util.lock import Lock def _maybe_open(path: Union[str, pathlib.Path]) -> Optional[IO[str]]: try: return open(path, "r", encoding="utf-8") except IsADirectoryError: raise CacheError("Cache file is not a file: %s" % path) except PermissionError: raise CacheError("Cannot access cache file: %s" % path) except FileNotFoundError: return None def _open_temp(context_dir: Union[str, pathlib.Path]) -> Tuple[IO[str], str]: """Open a temporary file in a directory This implementation minimizes the number of system calls for the case the target directory already exists. """ try: fd, path = tempfile.mkstemp(dir=context_dir) except FileNotFoundError: os.makedirs(context_dir, exist_ok=True) fd, path = tempfile.mkstemp(dir=context_dir) stream = os.fdopen(fd, "w", encoding="utf-8") return stream, path class ReadContextManager: def __init__(self, path: Union[str, pathlib.Path]) -> None: self.path = path def __enter__(self) -> Optional[IO[str]]: """Return a file object for the cache if it exists.""" self.cache_file = _maybe_open(self.path) return self.cache_file def __exit__(self, type, value, traceback): if self.cache_file: self.cache_file.close() class WriteContextManager: def __init__(self, path: Union[str, pathlib.Path]) -> None: self.path = path def __enter__(self) -> Tuple[Optional[IO[str]], IO[str]]: """Return (old_file, new_file) file objects, where old_file is optional.""" try: self.old_file = _maybe_open(self.path) self.new_file, self.tmp_path = _open_temp(os.path.dirname(self.path)) except PermissionError: if self.old_file: self.old_file.close() raise CacheError(f"Insufficient permissions to write to file cache at {self.path}") return self.old_file, self.new_file def __exit__(self, type, value, traceback): if self.old_file: self.old_file.close() self.new_file.close() if value: try: os.remove(self.tmp_path) except OSError: pass else: rename(self.tmp_path, self.path) class FileCache: """This class manages cached data in the filesystem. - Cache files are fetched and stored by unique keys. Keys can be relative paths, so that there can be some hierarchy in the cache. - The FileCache handles locking cache files for reading and writing, so client code need not manage locks for cache entries. """ def __init__(self, root: Union[str, pathlib.Path], timeout=120): """Create a file cache object. This will create the cache directory if it does not exist yet. Args: root: specifies the root directory where the cache stores files timeout: when there is contention among multiple Spack processes for cache files, this specifies how long Spack should wait before assuming that there is a deadlock. """ if isinstance(root, str): root = pathlib.Path(root) self.root = root self.root.mkdir(parents=True, exist_ok=True) self.lock_path = self.root / ".lock" self._locks: Dict[Union[pathlib.Path, str], Lock] = {} self.lock_timeout = timeout def destroy(self): """Remove all files under the cache root.""" for f in self.root.iterdir(): if f.is_dir(): shutil.rmtree(f, True) else: f.unlink() def cache_path(self, key: Union[str, pathlib.Path]): """Path to the file in the cache for a particular key.""" return self.root / key def _get_lock_offsets(self, key: str) -> Tuple[int, int]: """Hash function to determine byte-range offsets for a key. Returns (start, length) for the lock.""" hasher = hashlib.sha256(key.encode("utf-8")) hash_int = int.from_bytes(hasher.digest()[:8], "little") start_offset = hash_int % (2**63 - 1) return start_offset, 1 def _get_lock(self, key: Union[str, pathlib.Path]): """Create a lock for a key using byte-range offsets.""" key_str = str(key) if key_str not in self._locks: start, length = self._get_lock_offsets(key_str) self._locks[key_str] = Lock( str(self.lock_path), start=start, length=length, default_timeout=self.lock_timeout, desc=f"key:{key_str}", ) return self._locks[key_str] @contextmanager def read_transaction(self, key: Union[str, pathlib.Path]) -> Iterator[Optional[IO[str]]]: """Get a read transaction on a file cache item. Returns a context manager that yields an open file object for reading, or None if the cache file does not exist. You can use it like this:: with file_cache_object.read_transaction(key) as cache_file: if cache_file is not None: cache_file.read() """ lock = self._get_lock(key) lock.acquire_read() try: with ReadContextManager(self.cache_path(key)) as f: yield f finally: lock.release_read() @contextmanager def write_transaction( self, key: Union[str, pathlib.Path] ) -> Iterator[Tuple[Optional[IO[str]], IO[str]]]: """Get a write transaction on a file cache item. Returns a context manager that yields (old_file, new_file) where old_file is the existing cache file (or None), and new_file is a writable temporary file. Once the context manager exits cleanly, moves the temporary file into place atomically. """ path = self.cache_path(key) lock = self._get_lock(key) try: lock.acquire_write() except PermissionError: raise CacheError(f"Insufficient permissions to write to file cache at {path}") try: with WriteContextManager(str(path)) as (old, new): yield old, new finally: lock.release_write() def remove(self, key: Union[str, pathlib.Path]): file = self.cache_path(key) lock = self._get_lock(key) lock.acquire_write() try: file.unlink() except FileNotFoundError: pass finally: lock.release_write() class CacheError(SpackError): pass ================================================ FILE: lib/spack/spack/util/file_permissions.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import stat as st import spack.llnl.util.filesystem as fs import spack.package_prefs as pp from spack.error import SpackError def set_permissions_by_spec(path, spec): # Get permissions for spec if os.path.isdir(path): perms = pp.get_package_dir_permissions(spec) else: perms = pp.get_package_permissions(spec) group = pp.get_package_group(spec) set_permissions(path, perms, group) def set_permissions(path, perms, group=None): # Preserve higher-order bits of file permissions perms |= os.stat(path).st_mode & (st.S_ISUID | st.S_ISGID | st.S_ISVTX) # Do not let users create world/group writable suid binaries if perms & st.S_ISUID: if perms & st.S_IWOTH: raise InvalidPermissionsError("Attempting to set suid with world writable") if perms & st.S_IWGRP: raise InvalidPermissionsError("Attempting to set suid with group writable") # Or world writable sgid binaries if perms & st.S_ISGID: if perms & st.S_IWOTH: raise InvalidPermissionsError("Attempting to set sgid with world writable") fs.chmod_x(path, perms) if group: fs.chgrp(path, group, follow_symlinks=False) class InvalidPermissionsError(SpackError): """Error class for invalid permission setters""" ================================================ FILE: lib/spack/spack/util/filesystem.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ Utilities for interacting with files, like those in spack.llnl.util.filesystem, but which require logic from spack.util """ import glob import os from spack.llnl.util.filesystem import edit_in_place_through_temporary_file from spack.util.executable import Executable def fix_darwin_install_name(path: str) -> None: """Fix install name of dynamic libraries on Darwin to have full path. There are two parts of this task: 1. Use ``install_name -id ...`` to change install name of a single lib 2. Use ``install_name -change ...`` to change the cross linking between libs. The function assumes that all libraries are in one folder and currently won't follow subfolders. Parameters: path: directory in which ``.dylib`` files are located """ libs = glob.glob(os.path.join(path, "*.dylib")) install_name_tool = Executable("install_name_tool") otool = Executable("otool") for lib in libs: args = ["-id", lib] long_deps = otool("-L", lib, output=str).split("\n") deps = [dep.partition(" ")[0][1::] for dep in long_deps[2:-1]] # fix all dependencies: for dep in deps: for loc in libs: # We really want to check for either # dep == os.path.basename(loc) or # dep == join_path(builddir, os.path.basename(loc)), # but we don't know builddir (nor how symbolic links look # in builddir). We thus only compare the basenames. if os.path.basename(dep) == os.path.basename(loc): args.extend(("-change", dep, loc)) break with edit_in_place_through_temporary_file(lib) as tmp: install_name_tool(*args, tmp) ================================================ FILE: lib/spack/spack/util/format.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) def get_version_lines(version_hashes_dict: dict) -> str: """ Renders out a set of versions like those found in a package's package.py file for a given set of versions and hashes. Args: version_hashes_dict: A dictionary of the form: version -> checksum. Returns: Rendered version lines. """ version_lines = [] for v, h in version_hashes_dict.items(): version_lines.append(f' version("{v}", sha256="{h}")') return "\n".join(version_lines) ================================================ FILE: lib/spack/spack/util/gcs.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This file contains the definition of the GCS Blob storage Class used to integrate GCS Blob storage with spack buildcache. """ import os import sys import urllib.parse import urllib.response from typing import List from urllib.error import URLError from urllib.request import BaseHandler import spack.llnl.util.tty as tty def gcs_client(): """Create a GCS client Creates an authenticated GCS client to access GCS buckets and blobs """ try: import google.auth from google.cloud import storage except ImportError as ex: tty.error( "{0}, google-cloud-storage python module is missing.".format(ex) + " Please install to use the gs:// backend." ) sys.exit(1) storage_credentials, storage_project = google.auth.default() storage_client = storage.Client(storage_project, storage_credentials) return storage_client class GCSBucket: """GCS Bucket Object Create a wrapper object for a GCS Bucket. Provides methods to wrap spack related tasks, such as destroy. """ def __init__(self, url, client=None): """Constructor for GCSBucket objects Args: url (str): The url pointing to the GCS bucket to build an object out of client (google.cloud.storage.client.Client): A pre-defined storage client that will be used to access the GCS bucket. """ if url.scheme != "gs": raise ValueError( "Can not create GCS bucket connection with scheme {SCHEME}".format( SCHEME=url.scheme ) ) self.url = url self.name = self.url.netloc if self.url.path[0] == "/": self.prefix = self.url.path[1:] else: self.prefix = self.url.path self.client = client or gcs_client() self.bucket = None tty.debug("New GCS bucket:") tty.debug(" name: {0}".format(self.name)) tty.debug(" prefix: {0}".format(self.prefix)) def exists(self): from google.cloud.exceptions import NotFound if not self.bucket: try: self.bucket = self.client.bucket(self.name) except NotFound as ex: tty.error("{0}, Failed check for bucket existence".format(ex)) sys.exit(1) return self.bucket is not None def create(self): if not self.bucket: self.bucket = self.client.create_bucket(self.name) def get_blob(self, blob_path): if self.exists(): return self.bucket.get_blob(blob_path) return None def blob(self, blob_path): if self.exists(): return self.bucket.blob(blob_path) return None def get_all_blobs(self, recursive: bool = True, relative: bool = True) -> List[str]: """Get a list of all blobs Returns: a list of all blobs within this bucket. Args: relative: If true (default), print blob paths relative to 'build_cache' directory. If false, print absolute blob paths (useful for destruction of bucket) """ tty.debug("Getting GCS blobs... Recurse {0} -- Rel: {1}".format(recursive, relative)) converter = self._relative_blob_name if relative else str blob_list: List[str] = [] if self.exists(): all_blobs = self.bucket.list_blobs(prefix=self.prefix) base_dirs = len(self.prefix.split("/")) + 1 for blob in all_blobs: if not recursive: num_dirs = len(blob.name.split("/")) if num_dirs <= base_dirs: blob_list.append(converter(blob.name)) else: blob_list.append(converter(blob.name)) return blob_list def _relative_blob_name(self, blob_name): return os.path.relpath(blob_name, self.prefix) def destroy(self, recursive=False, **kwargs): """Bucket destruction method Deletes all blobs within the bucket, and then deletes the bucket itself. Uses GCS Batch operations to bundle several delete operations together. """ from google.cloud.exceptions import NotFound tty.debug("Bucket.destroy(recursive={0})".format(recursive)) try: bucket_blobs = self.get_all_blobs(recursive=recursive, relative=False) batch_size = 1000 num_blobs = len(bucket_blobs) for i in range(0, num_blobs, batch_size): with self.client.batch(): for j in range(i, min(i + batch_size, num_blobs)): blob = self.blob(bucket_blobs[j]) blob.delete() except NotFound as ex: tty.error("{0}, Could not delete a blob in bucket {1}.".format(ex, self.name)) sys.exit(1) class GCSBlob: """GCS Blob object Wraps some blob methods for spack functionality """ def __init__(self, url, client=None): self.url = url if url.scheme != "gs": raise ValueError( "Can not create GCS blob connection with scheme: {SCHEME}".format( SCHEME=url.scheme ) ) self.client = client or gcs_client() self.bucket = GCSBucket(url) self.blob_path = self.url.path.lstrip("/") tty.debug("New GCSBlob") tty.debug(" blob_path = {0}".format(self.blob_path)) if not self.bucket.exists(): tty.warn("The bucket {0} does not exist, it will be created".format(self.bucket.name)) self.bucket.create() def get(self): return self.bucket.get_blob(self.blob_path) def exists(self): from google.cloud.exceptions import NotFound try: blob = self.bucket.blob(self.blob_path) exists = blob.exists() except NotFound: return False return exists def delete_blob(self): from google.cloud.exceptions import NotFound try: blob = self.bucket.blob(self.blob_path) blob.delete() except NotFound as ex: tty.error("{0}, Could not delete gcs blob {1}".format(ex, self.blob_path)) def upload_to_blob(self, local_file_path): blob = self.bucket.blob(self.blob_path) blob.upload_from_filename(local_file_path) def get_blob_byte_stream(self): return self.bucket.get_blob(self.blob_path).open(mode="rb") def get_blob_headers(self): blob = self.bucket.get_blob(self.blob_path) headers = { "Content-type": blob.content_type, "Content-encoding": blob.content_encoding, "Content-language": blob.content_language, "MD5Hash": blob.md5_hash, } return headers def gcs_open(req, *args, **kwargs): """Open a reader stream to a blob object on GCS""" url = urllib.parse.urlparse(req.get_full_url()) gcsblob = GCSBlob(url) if not gcsblob.exists(): raise URLError("GCS blob {0} does not exist".format(gcsblob.blob_path)) stream = gcsblob.get_blob_byte_stream() headers = gcsblob.get_blob_headers() return urllib.response.addinfourl(stream, headers, url) class GCSHandler(BaseHandler): def gs_open(self, req): return gcs_open(req) ================================================ FILE: lib/spack/spack/util/git.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Single util module where Spack should get a git executable.""" import os import re import shutil import sys from typing import List, Optional, overload from spack.vendor.typing_extensions import Literal import spack.llnl.util.filesystem as fs import spack.llnl.util.lang import spack.util.executable as exe from spack.util.environment import EnvironmentModifications # regex for a commit version COMMIT_VERSION = re.compile(r"^[a-f0-9]{40}$") # regex for a git version to extract only the numeric parts GIT_VERSION = re.compile(r"(\d+(?:\.\d+)*)") def is_git_commit_sha(string: str) -> bool: return len(string) == 40 and bool(COMMIT_VERSION.match(string)) @spack.llnl.util.lang.memoized def _find_git() -> Optional[str]: """Find the git executable in the system path.""" return exe.which_string("git", required=False) def extract_git_version_str(git_exe: exe.Executable) -> str: match = re.search(GIT_VERSION, git_exe("--version", output=str)) return match.group(1) if match else "" class GitExecutable(exe.Executable): """Specialized executable that encodes the git version for optimized option selection""" def __init__(self, name=None): if not name: name = _find_git() super().__init__(name) self._version = None @property def version(self): # lazy init git version if not self._version: v_string = extract_git_version_str(self) self._version = tuple(int(i) for i in v_string.split(".")) return self._version class VersionConditionalOption: def __init__(self, key, value=None, min_version=(0, 0, 0), max_version=(99, 99, 99)): self.key = key self.value = value self.min_version = min_version self.max_version = max_version def __call__(self, exe_version, value=None) -> List: if (self.min_version <= exe_version) and (self.max_version >= exe_version): option = [self.key] if value: option.append(value) elif self.value: option.append(self.value) return option else: return [] # The earliest git version where we start trying to optimize clones # git@1.8.5 is when branch could also accept tag so we don't have to track ref types as closely # This also corresponds to system git on RHEL7 MIN_OPT_VERSION = (1, 8, 5, 2) MIN_DIRECT_COMMIT_FETCH = (2, 5, 0) # Technically the flags existed earlier but we are pruning our logic to 1.8.5 or greater BRANCH = VersionConditionalOption("--branch", min_version=MIN_OPT_VERSION) SINGLE_BRANCH = VersionConditionalOption("--single-branch", min_version=MIN_OPT_VERSION) NO_SINGLE_BRANCH = VersionConditionalOption("--no-single-branch", min_version=MIN_OPT_VERSION) # Depth was introduced in 1.7.11 but isn't worth much without the --branch options DEPTH = VersionConditionalOption("--depth", 1, min_version=MIN_OPT_VERSION) FILTER_BLOB_NONE = VersionConditionalOption("--filter=blob:none", min_version=(2, 19, 0)) NO_CHECKOUT = VersionConditionalOption("--no-checkout", min_version=(2, 34, 0)) # technically sparse-checkout was added in 2.25, but we go forward since the model we use only # works with the `--cone` option SPARSE_CHECKOUT = VersionConditionalOption("sparse-checkout", "set", min_version=(2, 34, 0)) @overload def git(required: Literal[True]) -> GitExecutable: ... @overload def git(required: bool = ...) -> Optional[GitExecutable]: ... def git(required: bool = False) -> Optional[GitExecutable]: """Get a git executable. The returned executable automatically unsets ``GIT_EXTERNAL_DIFF`` and ``GIT_DIFF_OPTS`` environment variables that can interfere with spack git diff operations. Args: required (bool): if True, raises CommandNotFoundError when git is not found Returns: GitExecutable, or None if git is not found and required is False """ git_path = _find_git() if not git_path: if required: raise exe.CommandNotFoundError("spack requires 'git'. Make sure it is in your path.") return None git = GitExecutable(git_path) # If we're running under pytest, add this to ignore the fix for CVE-2022-39253 in # git 2.38.1+. Do this in one place; we need git to do this in all parts of Spack. if "pytest" in sys.modules: git.add_default_arg("-c", "protocol.file.allow=always") # Block environment variables that can interfere with git diff operations # this can cause problems for spack ci verify-versions and spack repo show-version-updates env_blocklist = EnvironmentModifications() env_blocklist.unset("GIT_EXTERNAL_DIFF") env_blocklist.unset("GIT_DIFF_OPTS") git.add_default_envmod(env_blocklist) return git def init_git_repo( repository: str, remote: str = "origin", git_exe: Optional[exe.Executable] = None ): """Initialize a new Git repository and configure it with a remote.""" git_exe = git_exe or git(required=True) git_exe("init", "--quiet", output=str) git_exe("remote", "add", remote, repository) # versions of git prior to v2.24 may not have the manyFiles feature # so we should ignore errors here on older versions of git git_exe("config", "feature.manyFiles", "true", ignore_errors=True) def pull_checkout_commit( commit: str, remote: Optional[str] = None, depth: Optional[int] = None, git_exe: Optional[exe.Executable] = None, ): """Checkout the specified commit (fetched if necessary).""" git_exe = git_exe or git(required=True) # Do not do any fetching if the commit is already present. try: git_exe("checkout", "--quiet", commit, error=os.devnull) return except exe.ProcessError: pass # First try to fetch the specific commit from a specific remote. This allows fixed depth, but # the server needs to support it. if remote is not None: try: flags = [] if depth is None else [f"--depth={depth}"] git_exe("fetch", "--quiet", "--progress", *flags, remote, commit, error=os.devnull) git_exe("checkout", "--quiet", commit) return except exe.ProcessError: pass # Fall back to fetching all while unshallowing, to guarantee we get the commit. The depth flag # is equivalent to --unshallow, and needed cause git can pedantically error with # "--unshallow on a complete repository does not make sense". remote_flag = "--all" if remote is None else remote git_exe("fetch", "--quiet", "--progress", "--depth=2147483647", remote_flag) git_exe("checkout", "--quiet", commit) def pull_checkout_tag( tag: str, remote: str = "origin", depth: Optional[int] = None, git_exe: Optional[exe.Executable] = None, ): """Fetch tags with specified depth and checkout the given tag.""" git_exe = git_exe or git(required=True) fetch_args = ["--quiet", "--progress", "--tags"] if depth is not None: if depth <= 0: raise ValueError("depth must be a positive integer") fetch_args.append(f"--depth={depth}") git_exe("fetch", *fetch_args, remote) git_exe("checkout", tag) def pull_checkout_branch( branch: str, remote: str = "origin", depth: Optional[int] = None, git_exe: Optional[exe.Executable] = None, ): """Fetch and checkout branch, then rebase with remote tracking branch.""" git_exe = git_exe or git(required=True) fetch_args = ["--quiet", "--progress"] if depth: if depth <= 0: raise ValueError("depth must be a positive integer") fetch_args.append(f"--depth={depth}") git_exe("fetch", *fetch_args, remote, f"refs/heads/{branch}:refs/remotes/{remote}/{branch}") git_exe("checkout", "--quiet", branch) try: git_exe("rebase", "--quiet", f"{remote}/{branch}") except exe.ProcessError: git_exe("rebase", "--abort", fail_on_error=False, error=str, output=str) raise def get_modified_files( from_ref: str = "HEAD~1", to_ref: str = "HEAD", git_exe: Optional[exe.Executable] = None ) -> List[str]: """Get a list of files modified between ``from_ref`` and ``to_ref`` Args: from_ref (str): oldest git ref, defaults to ``HEAD~1`` to_ref (str): newer git ref, defaults to ``HEAD`` Returns: list of file paths """ git_exe = git_exe or git(required=True) stdout = git_exe("diff", "--name-only", from_ref, to_ref, output=str) return stdout.split() def get_commit_sha(path: str, ref: str) -> Optional[str]: """Get a commit sha for an arbitrary ref using ls-remote""" # search for matching branch, annotated tag's commit, then lightweight tag ref_list = [f"refs/heads/{ref}", f"refs/tags/{ref}^{{}}", f"refs/tags/{ref}"] if os.path.isdir(path): # for the filesystem an unpacked mirror could be in a detached state from a depth 1 clone # only reference there will be HEAD ref_list.append("HEAD") for try_ref in ref_list: # this command enabled in git@1.7 so no version checking supplied (1.7 released in 2009) try: query = git(required=True)( "ls-remote", path, try_ref, output=str, error=os.devnull, extra_env={"GIT_TERMINAL_PROMPT": "0"}, ) if query: return query.strip().split()[0] except exe.ProcessError: continue return None def _exec_git_commands(git_exe, cmds, debug, dest=None): dest_args = ["-C", dest] if dest else [] error_stream = sys.stdout if debug else os.devnull # swallow extra output for non-debug for cmd in cmds: git_exe(*dest_args, *cmd, error=error_stream) def _exec_git_commands_unique_dir(git_exe, cmds, debug, dest=None): if dest: # mimic creating a dir and clean up if there is a failure like git clone assert not os.path.isdir(dest) os.mkdir(dest) try: _exec_git_commands(git_exe, cmds, debug, dest) except exe.ProcessError: shutil.rmtree( dest, ignore_errors=False, onerror=fs.readonly_file_handler(ignore_errors=True) ) raise else: _exec_git_commands(git_exe, cmds, debug, dest) def protocol_supports_shallow_clone(url): """Shallow clone operations (``--depth #``) are not supported by the basic HTTP protocol or by no-protocol file specifications. Use (e.g.) ``https://`` or ``file://`` instead.""" return not (url.startswith("http://") or url.startswith("/")) def git_init_fetch(url, ref, depth=None, debug=False, dest=None, git_exe=None): """Utilize ``git init`` and then ``git fetch`` for a minimal clone of a single git ref This method runs git init, repo add, fetch to get a minimal set of source data. Profiling has shown this method can be 10-20% less storage than purely using sparse-checkout, and is even smaller than git clone --depth 1. This makes it the preferred method for single commit checkouts and source mirror population. There is a trade off since less git data means less flexibility with additional git operations. Technically adding the remote is not necessary, but we do it since there are test cases where we may want to fetch additional data. Checkout is explicitly deferred to a second method so we can intercept and add sparse-checkout options uniformly whether we use `git clone` or `init fetch` """ git_exe = git_exe or git(required=True) version = git_exe.version # minimum criteria for fetching a single commit, but also requires server to be configured # fall-back to a process error so an old git version or a fetch failure from an nonsupporting # server can be caught the same way. if ref and is_git_commit_sha(ref) and version < MIN_DIRECT_COMMIT_FETCH: raise exe.ProcessError("Git older than 2.5 detected, can't fetch commit directly") init = ["init"] remote = ["remote", "add", "origin", url] config = ["config", "remote.origin.fetch", "+refs/heads/*:origin/refs/*"] fetch = ["fetch"] if not debug: fetch.append("--quiet") if depth and protocol_supports_shallow_clone(url): fetch.extend(DEPTH(version, str(depth))) filter_args = FILTER_BLOB_NONE(version) if filter_args: fetch.extend(filter_args) fetch.extend([url, ref]) partial_clone = ["config", "extensions.partialClone", "true"] if filter_args else None if partial_clone is not None: cmds = [init, partial_clone, remote, config, fetch] else: cmds = [init, remote, config, fetch] _exec_git_commands_unique_dir(git_exe, cmds, debug, dest) def git_checkout( ref: Optional[str] = None, sparse_paths: List[str] = [], debug: bool = False, dest: Optional[str] = None, git_exe: Optional[GitExecutable] = None, ): """A generic method for running ``git checkout`` that integrates sparse-checkout Several methods in this module explicitly delay checkout so sparse-checkout can be called. It is intended to be used with ``git clone --no-checkout`` or ``git init && git fetch``. There is minimal impact to performance since the initial clone operation filters blobs and has to download a minimal subset of git data. """ git_exe = git_exe or git(required=True) checkout = ["checkout"] sparse_checkout = SPARSE_CHECKOUT(git_exe.version) if not debug: checkout.append("--quiet") if ref: checkout.append(ref) cmds = [] if sparse_paths and sparse_checkout: sparse_checkout.extend([*sparse_paths, "--cone"]) cmds.append(sparse_checkout) cmds.append(checkout) _exec_git_commands(git_exe, cmds, debug, dest) def git_clone( url: str, ref: Optional[str] = None, full_repo: bool = False, depth: Optional[int] = None, debug: bool = False, dest: Optional[str] = None, git_exe: Optional[GitExecutable] = None, ): """A git clone that prefers deferring expensive blob fetching for modern git installations This is our fallback method for capturing more git data than the ``init && fetch`` model. It is still optimized to capture a minimal set of ``./.git`` data and expects to be paired with a call to ``git checkout`` to fully download the source code. """ git_exe = git_exe or git(required=True) version = git_exe.version clone = ["clone"] # only need fetch if it's a really old git so we don't fail a checkout old = version < MIN_OPT_VERSION fetch = ["fetch"] if not debug: clone.append("--quiet") fetch.append("--quiet") if not old and depth and not full_repo and protocol_supports_shallow_clone(url): clone.extend(DEPTH(version, str(depth))) if full_repo: if old: fetch.extend(["--all"]) else: clone.extend(NO_SINGLE_BRANCH(version)) elif ref and not is_git_commit_sha(ref): if old: fetch.extend(["origin", ref]) else: clone.extend([*SINGLE_BRANCH(version), *BRANCH(version, ref)]) clone.extend([*FILTER_BLOB_NONE(version), *NO_CHECKOUT(version), url]) if dest: clone.append(dest) _exec_git_commands(git_exe, [clone], debug) if old: _exec_git_commands(git_exe, [fetch], debug, dest) ================================================ FILE: lib/spack/spack/util/gpg.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import contextlib import errno import functools import os import re from typing import List import spack.error import spack.llnl.util.filesystem import spack.paths import spack.util.executable import spack.version #: Executable instance for "gpg", initialized lazily GPG = None #: Executable instance for "gpgconf", initialized lazily GPGCONF = None #: Socket directory required if a non default home directory is used SOCKET_DIR = None #: GNUPGHOME environment variable in the context of this Python module GNUPGHOME = None def clear(): """Reset the global state to uninitialized.""" global GPG, GPGCONF, SOCKET_DIR, GNUPGHOME GPG, GPGCONF, SOCKET_DIR, GNUPGHOME = None, None, None, None def init(gnupghome=None, force=False): """Initialize the global objects in the module, if not set. When calling any gpg executable, the GNUPGHOME environment variable is set to: 1. The value of the ``gnupghome`` argument, if not None 2. The value of the "SPACK_GNUPGHOME" environment variable, if set 3. The default gpg path for Spack otherwise Args: gnupghome (str): value to be used for GNUPGHOME when calling GnuPG executables force (bool): if True forces the re-initialization even if the global objects are set already """ global GPG, GPGCONF, SOCKET_DIR, GNUPGHOME import spack.bootstrap if force: clear() # If the executables are already set, there's nothing to do if GPG and GNUPGHOME: return # Set the value of GNUPGHOME to be used in this module GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") or spack.paths.gpg_path # Set the executable objects for "gpg" and "gpgconf" with spack.bootstrap.ensure_bootstrap_configuration(): spack.bootstrap.ensure_gpg_in_path_or_raise() GPG, GPGCONF = _gpg(), _gpgconf() GPG.add_default_env("GNUPGHOME", GNUPGHOME) if GPGCONF: GPGCONF.add_default_env("GNUPGHOME", GNUPGHOME) # Set the socket dir if not using GnuPG defaults SOCKET_DIR = _socket_dir(GPGCONF) # Make sure that the GNUPGHOME exists if not os.path.exists(GNUPGHOME): os.makedirs(GNUPGHOME) os.chmod(GNUPGHOME, 0o700) if not os.path.isdir(GNUPGHOME): msg = 'GNUPGHOME "{0}" exists and is not a directory'.format(GNUPGHOME) raise SpackGPGError(msg) if SOCKET_DIR is not None: GPGCONF("--create-socketdir") def _autoinit(func): """Decorator to ensure that global variables have been initialized before running the decorated function. Args: func (callable): decorated function """ @functools.wraps(func) def _wrapped(*args, **kwargs): init() return func(*args, **kwargs) return _wrapped @contextlib.contextmanager def gnupghome_override(dir): """Set the GNUPGHOME to a new location for this context. Args: dir (str): new value for GNUPGHOME """ global GPG, GPGCONF, SOCKET_DIR, GNUPGHOME # Store backup values _GPG, _GPGCONF = GPG, GPGCONF _SOCKET_DIR, _GNUPGHOME = SOCKET_DIR, GNUPGHOME clear() # Clear global state init(gnupghome=dir, force=True) yield clear() GPG, GPGCONF = _GPG, _GPGCONF SOCKET_DIR, GNUPGHOME = _SOCKET_DIR, _GNUPGHOME def _parse_secret_keys_output(output: str) -> List[str]: keys: List[str] = [] found_sec = False for line in output.split("\n"): if found_sec: if line.startswith("fpr"): keys.append(line.split(":")[9]) found_sec = False elif line.startswith("ssb"): found_sec = False elif line.startswith("sec"): found_sec = True return keys def _parse_public_keys_output(output): """ Returns a list of public keys with their fingerprints """ keys = [] found_pub = False current_pub_key = "" for line in output.split("\n"): if found_pub: if line.startswith("fpr"): keys.append((current_pub_key, line.split(":")[9])) found_pub = False elif line.startswith("ssb"): found_pub = False elif line.startswith("pub"): current_pub_key = line.split(":")[4] found_pub = True return keys def _get_unimported_public_keys(output): keys = [] for line in output.split("\n"): if line.startswith("pub"): keys.append(line.split(":")[4]) return keys class SpackGPGError(spack.error.SpackError): """Class raised when GPG errors are detected.""" @_autoinit def create(**kwargs): """Create a new key pair.""" r, w = os.pipe() with contextlib.closing(os.fdopen(r, "r")) as r: with contextlib.closing(os.fdopen(w, "w")) as w: w.write( """ Key-Type: rsa Key-Length: 4096 Key-Usage: sign Name-Real: %(name)s Name-Email: %(email)s Name-Comment: %(comment)s Expire-Date: %(expires)s %%no-protection %%commit """ % kwargs ) GPG("--gen-key", "--batch", input=r) @_autoinit def signing_keys(*args) -> List[str]: """Return the keys that can be used to sign binaries.""" assert GPG output: str = GPG("--list-secret-keys", "--with-colons", "--fingerprint", *args, output=str) return _parse_secret_keys_output(output) @_autoinit def public_keys_to_fingerprint(*args): """Return the keys that can be used to verify binaries.""" output = GPG("--list-public-keys", "--with-colons", "--fingerprint", *args, output=str) return _parse_public_keys_output(output) @_autoinit def public_keys(*args): """Return a list of fingerprints""" keys_and_fpr = public_keys_to_fingerprint(*args) return [key_and_fpr[1] for key_and_fpr in keys_and_fpr] @_autoinit def export_keys(location, keys, secret=False): """Export public keys to a location passed as argument. Args: location (str): where to export the keys keys (list): keys to be exported secret (bool): whether to export secret keys or not """ if secret: GPG("--export-secret-keys", "--armor", "--output", location, *keys) else: GPG("--batch", "--yes", "--armor", "--export", "--output", location, *keys) @_autoinit def trust(keyfile): """Import a public key from a file and trust it. Args: keyfile (str): file with the public key """ # Get the public keys we are about to import output = GPG("--with-colons", keyfile, output=str, error=str) keys = _get_unimported_public_keys(output) # Import them GPG("--batch", "--import", keyfile) # Set trust to ultimate key_to_fpr = dict(public_keys_to_fingerprint()) for key in keys: # Skip over keys we cannot find a fingerprint for. if key not in key_to_fpr: continue fpr = key_to_fpr[key] r, w = os.pipe() with contextlib.closing(os.fdopen(r, "r")) as r: with contextlib.closing(os.fdopen(w, "w")) as w: w.write("{0}:6:\n".format(fpr)) GPG("--import-ownertrust", input=r) @_autoinit def untrust(signing, *keys): """Delete known keys. Args: signing (bool): if True deletes the secret keys *keys: keys to be deleted """ if signing: skeys = signing_keys(*keys) GPG("--batch", "--yes", "--delete-secret-keys", *skeys) pkeys = public_keys(*keys) GPG("--batch", "--yes", "--delete-keys", *pkeys) @_autoinit def sign(key, file, output, clearsign=False): """Sign a file with a key. Args: key: key to be used to sign file (str): file to be signed output (str): output file (either the clearsigned file or the detached signature) clearsign (bool): if True wraps the document in an ASCII-armored signature, if False creates a detached signature """ signopt = "--clearsign" if clearsign else "--detach-sign" GPG(signopt, "--armor", "--local-user", key, "--output", output, file) @_autoinit def verify(signature, file=None, suppress_warnings=False): """Verify the signature on a file. Args: signature (str): signature of the file (or clearsigned file) file (str): file to be verified. If None, then signature is assumed to be a clearsigned file. suppress_warnings (bool): whether or not to suppress warnings from GnuPG """ args = [signature] if file: args.append(file) kwargs = {"error": str} if suppress_warnings else {} GPG("--verify", *args, **kwargs) @_autoinit def list(trusted, signing): """List known keys. Args: trusted (bool): if True list public keys signing (bool): if True list private keys """ if trusted: GPG("--list-public-keys") if signing: GPG("--list-secret-keys") def _verify_exe_or_raise(exe): msg = ( "Spack requires gpgconf version >= 2\n" " To install a suitable version using Spack, run\n" " spack install gnupg@2:\n" " and load it by running\n" " spack load gnupg@2:" ) if not exe: raise SpackGPGError(msg) output = exe("--version", output=str) match = re.search(r"^gpg(conf)? \(GnuPG(?:/MacGPG2)?\) (.*)$", output, re.M) if not match: raise SpackGPGError('Could not determine "{0}" version'.format(exe.name)) if spack.version.Version(match.group(2)) < spack.version.Version("2"): raise SpackGPGError(msg) def _gpgconf(): exe = spack.util.executable.which("gpgconf", "gpg2conf", "gpgconf2") _verify_exe_or_raise(exe) # ensure that the gpgconf we found can run "gpgconf --create-socketdir" try: exe("--dry-run", "--create-socketdir", output=os.devnull, error=os.devnull) except spack.util.executable.ProcessError: # no dice exe = None return exe def _gpg(): exe = spack.util.executable.which("gpg2", "gpg") _verify_exe_or_raise(exe) return exe def _socket_dir(gpgconf): # Try to ensure that (/var)/run/user/$(id -u) exists so that # `gpgconf --create-socketdir` can be run later. # # NOTE(opadron): This action helps prevent a large class of # "file-name-too-long" errors in gpg. # If there is no suitable gpgconf, don't even bother trying to # pre-create a user run dir. if not gpgconf: return None result = None for var_run in ("/run", "/var/run"): if not os.path.exists(var_run): continue var_run_user = os.path.join(var_run, "user") try: if not os.path.exists(var_run_user): os.mkdir(var_run_user) os.chmod(var_run_user, 0o777) user_dir = os.path.join(var_run_user, str(spack.llnl.util.filesystem.getuid())) if not os.path.exists(user_dir): os.mkdir(user_dir) os.chmod(user_dir, 0o700) # If the above operation fails due to lack of permissions, then # just carry on without running gpgconf and hope for the best. # # NOTE(opadron): Without a dir in which to create a socket for IPC, # gnupg may fail if GNUPGHOME is set to a path that # is too long, where "too long" in this context is # actually quite short; somewhere in the # neighborhood of more than 100 characters. # # TODO(opadron): Maybe a warning should be printed in this case? except OSError as exc: if exc.errno not in (errno.EPERM, errno.EACCES): raise user_dir = None # return the last iteration that provides a usable user run dir if user_dir is not None: result = user_dir return result ================================================ FILE: lib/spack/spack/util/hash.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import base64 import hashlib import spack.util.crypto def b32_hash(content): """Return the b32 encoded sha1 hash of the input string as a string.""" sha = hashlib.sha1(content.encode("utf-8")) b32_hash = base64.b32encode(sha.digest()).lower() b32_hash = b32_hash.decode("utf-8") return b32_hash def base32_prefix_bits(hash_string, bits): """Return the first bits of a base32 string as an integer.""" if bits > len(hash_string) * 5: raise ValueError("Too many bits! Requested %d bit prefix of '%s'." % (bits, hash_string)) hash_bytes = base64.b32decode(hash_string, casefold=True) return spack.util.crypto.prefix_bits(hash_bytes, bits) ================================================ FILE: lib/spack/spack/util/ld_so_conf.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import glob import os import re import sys import spack.util.elf as elf_utils from spack.llnl.util.lang import dedupe def parse_ld_so_conf(conf_file="/etc/ld.so.conf"): """Parse glibc style ld.so.conf file, which specifies default search paths for the dynamic linker. This can in principle also be used for musl libc. Arguments: conf_file (str or bytes): Path to config file Returns: list: List of absolute search paths """ # Parse in binary mode since it's faster is_bytes = isinstance(conf_file, bytes) if not is_bytes: conf_file = conf_file.encode("utf-8") # For globbing in Python2 we need to chdir. cwd = os.getcwd() try: paths = _process_ld_so_conf_queue([conf_file]) finally: os.chdir(cwd) return list(paths) if is_bytes else [p.decode("utf-8") for p in paths] def _process_ld_so_conf_queue(queue): include_regex = re.compile(b"include\\s") paths = [] while queue: p = queue.pop(0) try: with open(p, "rb") as f: lines = f.readlines() except OSError: continue for line in lines: # Strip comments comment = line.find(b"#") if comment != -1: line = line[:comment] # Skip empty lines line = line.strip() if not line: continue is_include = include_regex.match(line) is not None # If not an include, it's a literal path (no globbing here). if not is_include: # We only allow absolute search paths. if os.path.isabs(line): paths.append(line) continue # Finally handle includes. include_path = line[8:].strip() if not include_path: continue cwd = os.path.dirname(p) os.chdir(cwd) queue.extend(os.path.join(cwd, p) for p in glob.glob(include_path)) return dedupe(paths) def get_conf_file_from_dynamic_linker(dynamic_linker_name): # We basically assume everything is glibc, except musl. if "ld-musl-" not in dynamic_linker_name: return "ld.so.conf" # Musl has a dynamic loader of the form ld-musl-.so.1 # and a corresponding config file ld-musl-.path idx = dynamic_linker_name.find(".") if idx != -1: return dynamic_linker_name[:idx] + ".path" def host_dynamic_linker_search_paths(): """Retrieve the current host runtime search paths for shared libraries; for GNU and musl Linux we try to retrieve the dynamic linker from the current Python interpreter and then find the corresponding config file (e.g. ld.so.conf or ld-musl-.path). Similar can be done for BSD and others, but this is not implemented yet. The default paths are always returned. We don't check if the listed directories exist.""" default_paths = ["/usr/lib", "/usr/lib64", "/lib", "/lib64"] # Currently only for Linux (gnu/musl) if not sys.platform.startswith("linux"): return default_paths # If everything fails, try this standard glibc path. conf_file = "/etc/ld.so.conf" # Try to improve on the default conf path by retrieving the location of the # dynamic linker from our current Python interpreter, and figure out the # config file location from there. try: with open(sys.executable, "rb") as f: elf = elf_utils.parse_elf(f, dynamic_section=False, interpreter=True) # If we have a dynamic linker, try to retrieve the config file relative # to its prefix. if elf.has_pt_interp: dynamic_linker = elf.pt_interp_str.decode("utf-8") dynamic_linker_name = os.path.basename(dynamic_linker) conf_name = get_conf_file_from_dynamic_linker(dynamic_linker_name) # Typically it is /lib/ld.so, but on Gentoo Prefix it is something # like /lib/ld.so. And on Debian /lib64 is actually # a symlink to /usr/lib64. So, best effort attempt is to just strip # two path components and join with etc/ld.so.conf. possible_prefix = os.path.dirname(os.path.dirname(dynamic_linker)) possible_conf = os.path.join(possible_prefix, "etc", conf_name) if os.path.exists(possible_conf): conf_file = possible_conf except (OSError, elf_utils.ElfParsingError): pass # Note: ld_so_conf doesn't error if the file does not exist. return list(dedupe(parse_ld_so_conf(conf_file) + default_paths)) ================================================ FILE: lib/spack/spack/util/libc.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import re import shlex import sys from subprocess import PIPE, run from typing import Dict, List, Optional import spack.spec import spack.util.elf from spack.llnl.util.lang import memoized #: Pattern to distinguish glibc from other libc implementations GLIBC_PATTERN = r"\b(?:Free Software Foundation|Roland McGrath|Ulrich Depper)\b" def _env() -> Dict[str, str]: """Currently only set LC_ALL=C without clearing further environment variables""" return {**os.environ, "LC_ALL": "C"} def _libc_from_ldd(ldd: str) -> Optional["spack.spec.Spec"]: try: result = run([ldd, "--version"], stdout=PIPE, stderr=PIPE, check=False, env=_env()) stdout = result.stdout.decode("utf-8") except Exception: return None # The string "Free Software Foundation" is sometimes translated and not detected, but the names # of the authors are typically present. if not re.search(GLIBC_PATTERN, stdout): return None version_str = re.match(r".+\(.+\) (.+)", stdout) if not version_str: return None try: return spack.spec.Spec(f"glibc@={version_str.group(1)}") except Exception: return None def default_search_paths_from_dynamic_linker(dynamic_linker: str) -> List[str]: """If the dynamic linker is glibc at a certain version, we can query the hard-coded library search paths""" try: result = run([dynamic_linker, "--help"], stdout=PIPE, stderr=PIPE, check=False, env=_env()) assert result.returncode == 0 out = result.stdout.decode("utf-8") except Exception: return [] return [ match.group(1).strip() for match in re.finditer(r"^ (/.+) \(system search path\)$", out, re.MULTILINE) ] def libc_from_dynamic_linker(dynamic_linker: str) -> Optional["spack.spec.Spec"]: """Get the libc spec from the dynamic linker path.""" maybe_spec = _libc_from_dynamic_linker(dynamic_linker) if maybe_spec: return maybe_spec.copy() return None @memoized def _libc_from_dynamic_linker(dynamic_linker: str) -> Optional["spack.spec.Spec"]: if not os.path.exists(dynamic_linker): return None # The dynamic linker is usually installed in the same /lib(64)?/ld-*.so path across all # distros. The rest of libc is elsewhere, e.g. /usr. Typically the dynamic linker is then # a symlink into /usr/lib, which we use to for determining the actual install prefix of # libc. realpath = os.path.realpath(dynamic_linker) prefix = os.path.dirname(realpath) # Remove the multiarch suffix if it exists if os.path.basename(prefix) not in ("lib", "lib64"): prefix = os.path.dirname(prefix) # Non-standard install layout -- just bail. if os.path.basename(prefix) not in ("lib", "lib64"): return None prefix = os.path.dirname(prefix) # Now try to figure out if glibc or musl, which is the only ones we support. # In recent glibc we can simply execute the dynamic loader. In musl that's always the case. try: result = run( [dynamic_linker, "--version"], stdout=PIPE, stderr=PIPE, check=False, env=_env() ) stdout = result.stdout.decode("utf-8") stderr = result.stderr.decode("utf-8") except Exception: return None # musl prints to stderr if stderr.startswith("musl libc"): version_str = re.search(r"^Version (.+)$", stderr, re.MULTILINE) if not version_str: return None try: spec = spack.spec.Spec(f"musl@={version_str.group(1)}") spec.external_path = prefix return spec except Exception: return None elif re.search(GLIBC_PATTERN, stdout): # output is like "ld.so (...) stable release version 2.33." match = re.search(r"version (\d+\.\d+(?:\.\d+)?)", stdout) if not match: return None try: version = match.group(1) spec = spack.spec.Spec(f"glibc@={version}") spec.external_path = prefix return spec except Exception: return None else: # Could not get the version by running the dynamic linker directly. Instead locate `ldd` # relative to the dynamic linker. ldd = os.path.join(prefix, "bin", "ldd") if not os.path.exists(ldd): # If `/lib64/ld.so` was not a symlink to `/usr/lib/ld.so` we can try to use /usr as # prefix. This is the case on ubuntu 18.04 where /lib != /usr/lib. if prefix != "/": return None prefix = "/usr" ldd = os.path.join(prefix, "bin", "ldd") if not os.path.exists(ldd): return None maybe_spec = _libc_from_ldd(ldd) if not maybe_spec: return None maybe_spec.external_path = prefix return maybe_spec def libc_from_current_python_process() -> Optional["spack.spec.Spec"]: if not sys.executable: return None dynamic_linker = spack.util.elf.pt_interp(sys.executable) if not dynamic_linker: return None return libc_from_dynamic_linker(dynamic_linker) def parse_dynamic_linker(output: str): """Parse ``-dynamic-linker /path/to/ld.so`` from compiler output""" for line in reversed(output.splitlines()): if "-dynamic-linker" not in line: continue args = shlex.split(line) for idx in reversed(range(1, len(args))): arg = args[idx] if arg == "-dynamic-linker" or args == "--dynamic-linker": return args[idx + 1] elif arg.startswith("--dynamic-linker=") or arg.startswith("-dynamic-linker="): return arg.split("=", 1)[1] ================================================ FILE: lib/spack/spack/util/lock.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Wrapper for ``spack.llnl.util.lock`` allows locking to be enabled/disabled.""" import os import stat import sys from typing import Optional, Tuple import spack.error from spack.llnl.util.lock import Lock as Llnl_lock from spack.llnl.util.lock import ( LockError, LockTimeoutError, LockUpgradeError, ReadTransaction, WriteTransaction, ) class Lock(Llnl_lock): """Lock that can be disabled. This overrides the ``_lock()`` and ``_unlock()`` methods from ``spack.llnl.util.lock`` so that all the lock API calls will succeed, but the actual locking mechanism can be disabled via ``_enable_locks``. """ def __init__( self, path: str, *, start: int = 0, length: int = 0, default_timeout: Optional[float] = None, debug: bool = False, desc: str = "", enable: bool = True, ) -> None: self._enable = sys.platform != "win32" and enable super().__init__( path, start=start, length=length, default_timeout=default_timeout, debug=debug, desc=desc, ) def _reaffirm_lock(self) -> None: if self._enable: super()._reaffirm_lock() def _lock(self, op: int, timeout: Optional[float] = 0.0) -> Tuple[float, int]: if self._enable: return super()._lock(op, timeout) return 0.0, 0 def _poll_lock(self, op: int) -> bool: if self._enable: return super()._poll_lock(op) return True def _unlock(self) -> None: """Unlock call that always succeeds.""" if self._enable: super()._unlock() def cleanup(self, *args) -> None: if self._enable: super().cleanup(*args) def check_lock_safety(path: str) -> None: """Do some extra checks to ensure disabling locks is safe. This will raise an error if ``path`` can is group- or world-writable AND the current user can write to the directory (i.e., if this user AND others could write to the path). This is intended to run on the Spack prefix, but can be run on any path for testing. """ if os.access(path, os.W_OK): stat_result = os.stat(path) uid, gid = stat_result.st_uid, stat_result.st_gid mode = stat_result[stat.ST_MODE] writable = None if (mode & stat.S_IWGRP) and (uid != gid): # spack is group-writeable and the group is not the owner writable = "group" elif mode & stat.S_IWOTH: # spack is world-writeable writable = "world" if writable: msg = f"Refusing to disable locks: spack is {writable}-writable." long_msg = ( f"Running a shared spack without locks is unsafe. You must " f"restrict permissions on {path} or enable locks." ) raise spack.error.SpackError(msg, long_msg) __all__ = [ "LockError", "LockTimeoutError", "LockUpgradeError", "ReadTransaction", "WriteTransaction", "Lock", "check_lock_safety", ] ================================================ FILE: lib/spack/spack/util/log_parse.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io from typing import List, TextIO, Union from spack.llnl.util.tty.color import cescape, colorize from spack.util.ctest_log_parser import CTestLogParser, LogEvent __all__ = ["parse_log_events", "make_log_context"] def parse_log_events(stream: Union[str, TextIO], context: int = 6, profile: bool = False): """Extract interesting events from a log file as a list of LogEvent. Args: stream: build log name or file object context: lines of context to extract around each log event profile: print out profile information for parsing Returns: two lists containing :class:`~spack.util.ctest_log_parser.BuildError` and :class:`~spack.util.ctest_log_parser.BuildWarning` objects. This is a wrapper around :class:`~spack.util.ctest_log_parser.CTestLogParser` that lazily constructs a single ``CTestLogParser`` object. This ensures that all the regex compilation is only done once. """ parser = getattr(parse_log_events, "ctest_parser", None) if parser is None: parser = CTestLogParser(profile=profile) setattr(parse_log_events, "ctest_parser", parser) result = parser.parse(stream, context) if profile: parser.print_timings() return result #: lazily constructed CTest log parser parse_log_events.ctest_parser = None # type: ignore[attr-defined] def make_log_context(log_events: List[LogEvent]) -> str: """Get error context from a log file. Args: log_events: list of events created by ``ctest_log_parser.parse()`` Returns: str: context from the build log with errors highlighted Parses the log file for lines containing errors, and prints them out with context. Errors are highlighted in red and warnings in yellow. Events are sorted by line number. """ event_colors = {e.line_no: e.color for e in log_events} log_events = sorted(log_events, key=lambda e: e.line_no) out = io.StringIO() next_line = 1 block_start = -1 block_lines: List[str] = [] def flush_block(): block_end = block_start + len(block_lines) - 1 out.write(colorize("@c{-- lines %d to %d --}\n" % (block_start, block_end))) out.writelines(block_lines) block_lines.clear() for event in log_events: start = event.start if start < next_line: start = next_line elif block_lines: flush_block() if not block_lines: block_start = start for i in range(start, event.end): if i in event_colors: color = event_colors[i] block_lines.append(colorize("@%s{> %s}\n" % (color, cescape(event[i])))) else: block_lines.append(" %s\n" % event[i]) next_line = event.end if block_lines: flush_block() return out.getvalue() ================================================ FILE: lib/spack/spack/util/module_cmd.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This module contains routines related to the module command for accessing and parsing environment modules. """ import os import re import subprocess from typing import MutableMapping, Optional import spack.llnl.util.tty as tty from spack.error import SpackError # This list is not exhaustive. Currently we only use load and unload # If we need another option that changes the environment, add it here. module_change_commands = ["load", "swap", "unload", "purge", "use", "unuse"] # This awk script is a posix alternative to `env -0` awk_cmd = r"""awk 'BEGIN{for(name in ENVIRON)""" r"""printf("%s=%s%c", name, ENVIRON[name], 0)}'""" def module( *args: str, module_template: Optional[str] = None, module_src_cmd: Optional[str] = None, environb: Optional[MutableMapping[bytes, bytes]] = None, ): """Run the ``module`` shell function in a ``/bin/bash`` subprocess, and either collect its changes to environment variables and apply them in the current process (for ``module load``, ``module swap``, etc.), or return its output as a string (for ``module show``, etc.). This requires ``/bin/bash`` to be available on the system and ``awk`` to be in ``PATH``. Args: args: Command line arguments for the module command. environb: (Binary) environment variables dictionary. If not provided, the current process's environment is modified. """ module_cmd = module_template or ("module " + " ".join(args)) environb = environb or os.environb if b"MODULESHOME" in environb: module_cmd = module_src_cmd or "source $MODULESHOME/init/bash; " + module_cmd if args[0] in module_change_commands: # Suppress module output module_cmd += r" >/dev/null 2>&1; " + awk_cmd module_p = subprocess.Popen( module_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, executable="/bin/bash", env=environb, ) new_environb = {} output = module_p.communicate()[0] # Loop over each environment variable key=value byte string for entry in output.strip(b"\0").split(b"\0"): # Split variable name and value parts = entry.split(b"=", 1) if len(parts) != 2: continue new_environb[parts[0]] = parts[1] # Update os.environ with new dict environb.clear() environb.update(new_environb) # novermin else: # Simply execute commands that don't change state and return output module_p = subprocess.Popen( module_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, executable="/bin/bash", ) # Decode and str to return a string object in both python 2 and 3 return str(module_p.communicate()[0].decode()) def load_module(mod): """Takes a module name and removes modules until it is possible to load that module. It then loads the provided module. Depends on the modulecmd implementation of modules used in cray and lmod. Raises: ModuleLoadError: if the module could not be loaded """ tty.debug("module_cmd.load_module: {0}".format(mod)) # Read the module and remove any conflicting modules # We do this without checking that they are already installed # for ease of programming because unloading a module that is not # loaded does nothing. text = module("show", mod).split() for i, word in enumerate(text): if word == "conflict": module("unload", text[i + 1]) # Store the LOADEDMODULES before trying to load the new module loaded_modules_before = os.environ.get("LOADEDMODULES", "") # Load the module now that there are no conflicts # Some module systems use stdout and some use stderr module("load", mod) # Check if the module was actually loaded by comparing LOADEDMODULES loaded_modules_after = os.environ.get("LOADEDMODULES", "") # If LOADEDMODULES didn't change, the module wasn't loaded if loaded_modules_before == loaded_modules_after: raise ModuleLoadError(mod) def get_path_args_from_module_line(line): if "(" in line and ")" in line: # Determine which lua quote symbol is being used for the argument comma_index = line.index(",") cline = line[comma_index:] try: quote_index = min(cline.find(q) for q in ['"', "'"] if q in cline) lua_quote = cline[quote_index] except ValueError: # Change error text to describe what is going on. raise ValueError("No lua quote symbol found in lmod module line.") words_and_symbols = line.split(lua_quote) path_arg = words_and_symbols[-2] else: # The path arg is the 3rd "word" of the line in a Tcl module # OPERATION VAR_NAME PATH_ARG words = line.split() if len(words) > 2: path_arg = words[2] else: return [] paths = path_arg.split(":") return paths def path_from_modules(modules): """Inspect a list of Tcl modules for entries that indicate the absolute path at which the library supported by said module can be found. Args: modules (list): module files to be loaded to get an external package Returns: Guess of the prefix path where the package """ assert isinstance(modules, list), 'the "modules" argument must be a list' best_choice = None for module_name in modules: # Read the current module and return a candidate path text = module("show", module_name).split("\n") candidate_path = get_path_from_module_contents(text, module_name) if candidate_path and not os.path.exists(candidate_path): msg = "Extracted path from module does not exist [module={0}, path={1}]" tty.warn(msg.format(module_name, candidate_path)) # If anything is found, then it's the best choice. This means # that we give preference to the last module to be loaded # for packages requiring to load multiple modules in sequence best_choice = candidate_path or best_choice return best_choice def get_path_from_module_contents(text, module_name): tty.debug("Module name: " + module_name) pkg_var_prefix = module_name.replace("-", "_").upper() components = pkg_var_prefix.split("/") # For modules with multiple components like foo/1.0.1, retrieve the package # name "foo" from the module name if len(components) > 1: pkg_var_prefix = components[-2] tty.debug("Package directory variable prefix: " + pkg_var_prefix) path_occurrences = {} def strip_path(path, endings): for ending in endings: if path.endswith(ending): return path[: -len(ending)] if path.endswith(ending + "/"): return path[: -(len(ending) + 1)] return path def match_pattern_and_strip(line, pattern, strip=[]): if re.search(pattern, line): paths = get_path_args_from_module_line(line) for path in paths: path = strip_path(path, strip) path_occurrences[path] = path_occurrences.get(path, 0) + 1 def match_flag_and_strip(line, flag, strip=[]): flag_idx = line.find(flag) if flag_idx >= 0: # Search for the first occurrence of any separator marking the end of # the path. separators = (" ", '"', "'") occurrences = [line.find(s, flag_idx) for s in separators] indices = [idx for idx in occurrences if idx >= 0] if indices: path = line[flag_idx + len(flag) : min(indices)] else: path = line[flag_idx + len(flag) :] path = strip_path(path, strip) path_occurrences[path] = path_occurrences.get(path, 0) + 1 lib_endings = ["/lib64", "/lib"] bin_endings = ["/bin"] man_endings = ["/share/man", "/man"] for line in text: # Check entries of LD_LIBRARY_PATH and CRAY_LD_LIBRARY_PATH pattern = r"\W(CRAY_)?LD_LIBRARY_PATH" match_pattern_and_strip(line, pattern, lib_endings) # Check {name}_DIR entries pattern = r"\W{0}_DIR".format(pkg_var_prefix) match_pattern_and_strip(line, pattern) # Check {name}_ROOT entries pattern = r"\W{0}_ROOT".format(pkg_var_prefix) match_pattern_and_strip(line, pattern) # Check entries that update the PATH variable pattern = r"\WPATH" match_pattern_and_strip(line, pattern, bin_endings) # Check entries that update the MANPATH variable pattern = r"MANPATH" match_pattern_and_strip(line, pattern, man_endings) # Check entries that add a `-rpath` flag to a variable match_flag_and_strip(line, "-rpath", lib_endings) # Check entries that add a `-L` flag to a variable match_flag_and_strip(line, "-L", lib_endings) # Whichever path appeared most in the module, we assume is the correct path if len(path_occurrences) > 0: return max(path_occurrences.items(), key=lambda x: x[1])[0] # Unable to find path in module return None class ModuleLoadError(SpackError): """Raised when a module cannot be loaded.""" def __init__(self, module): super().__init__(f"Module '{module}' could not be loaded.") ================================================ FILE: lib/spack/spack/util/naming.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import io import itertools import re import string from typing import List, Tuple __all__ = [ "pkg_name_to_class_name", "valid_module_name", "possible_spack_module_names", "simplify_name", "NamespaceTrie", ] #: see keyword.kwlist: https://github.com/python/cpython/blob/main/Lib/keyword.py RESERVED_NAMES_ONLY_LOWERCASE = frozenset( ( "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", ) ) RESERVED_NAMES_LIST_MIXED_CASE = ("False", "None", "True") # Valid module names can contain '-' but can't start with it. _VALID_MODULE_RE_V1 = re.compile(r"^\w[\w-]*$") _VALID_MODULE_RE_V2 = re.compile(r"^[a-z_][a-z0-9_]*$") def pkg_name_to_class_name(pkg_name: str): """Convert a Spack package name to a class name, based on `PEP-8 `_: * Module and package names use lowercase_with_underscores. * Class names use the CapWords convention. Not all package names are valid Python identifiers: * They can contain ``-``, but cannot start with ``-``. * They can start with numbers, e.g. ``3proxy``. This function converts from the package name to the class convention by removing ``_`` and ``-``, and converting surrounding lowercase text to CapWords. If package name starts with a number, the class name returned will be prepended with ``_`` to make a valid Python identifier. """ class_name = re.sub(r"[-_]+", "-", pkg_name) class_name = string.capwords(class_name, "-") class_name = class_name.replace("-", "") # Ensure that the class name is a valid Python identifier if re.match(r"^[0-9]", class_name) or class_name in RESERVED_NAMES_LIST_MIXED_CASE: class_name = f"_{class_name}" return class_name def pkg_dir_to_pkg_name(dirname: str, package_api: Tuple[int, int]) -> str: """Translate a package dir (pkg_dir/package.py) to its corresponding package name""" if package_api < (2, 0): return dirname return dirname.lstrip("_").replace("_", "-") def pkg_name_to_pkg_dir(name: str, package_api: Tuple[int, int]) -> str: """Translate a package name to its corresponding package dir (pkg_dir/package.py)""" if package_api < (2, 0): return name name = name.replace("-", "_") if re.match(r"^[0-9]", name) or name in RESERVED_NAMES_ONLY_LOWERCASE: name = f"_{name}" return name def possible_spack_module_names(python_mod_name: str) -> List[str]: """Given a Python module name, return a list of all possible spack module names that could correspond to it.""" mod_name = re.sub(r"^num(\d)", r"\1", python_mod_name) parts = re.split(r"(_)", mod_name) options = [["_", "-"]] * mod_name.count("_") results: List[str] = [] for subs in itertools.product(*options): s = list(parts) s[1::2] = subs results.append("".join(s)) return results def simplify_name(name: str) -> str: """Simplify package name to only lowercase, digits, and dashes. Simplifies a name which may include uppercase letters, periods, underscores, and pluses. In general, we want our package names to only contain lowercase letters, digits, and dashes. Args: name (str): The original name of the package Returns: str: The new name of the package """ # Convert CamelCase to Dashed-Names # e.g. ImageMagick -> Image-Magick # e.g. SuiteSparse -> Suite-Sparse # name = re.sub('([a-z])([A-Z])', r'\1-\2', name) # Rename Intel downloads # e.g. l_daal, l_ipp, l_mkl -> daal, ipp, mkl if name.startswith("l_"): name = name[2:] # Convert UPPERCASE to lowercase # e.g. SAMRAI -> samrai name = name.lower() # Replace '_' and '.' with '-' # e.g. backports.ssl_match_hostname -> backports-ssl-match-hostname name = name.replace("_", "-") name = name.replace(".", "-") # Replace "++" with "pp" and "+" with "-plus" # e.g. gtk+ -> gtk-plus # e.g. voro++ -> voropp name = name.replace("++", "pp") name = name.replace("+", "-plus") # Simplify Lua package names # We don't want "lua" to occur multiple times in the name name = re.sub("^(lua)([^-])", r"\1-\2", name) # Simplify Bio++ package names name = re.sub("^(bpp)([^-])", r"\1-\2", name) return name def valid_module_name(mod_name: str, package_api: Tuple[int, int]) -> bool: """Return whether mod_name is valid for use in Spack.""" if package_api < (2, 0): return bool(_VALID_MODULE_RE_V1.match(mod_name)) elif not _VALID_MODULE_RE_V2.match(mod_name) or "__" in mod_name: return False elif mod_name.startswith("_"): # it can only start with an underscore if followed by digit or reserved name return mod_name[1:] in RESERVED_NAMES_ONLY_LOWERCASE or mod_name[1].isdigit() else: return mod_name not in RESERVED_NAMES_ONLY_LOWERCASE class NamespaceTrie: class Element: def __init__(self, value): self.value = value def __init__(self, separator="."): self._subspaces = {} self._value = None self._sep = separator def __setitem__(self, namespace, value): first, sep, rest = namespace.partition(self._sep) if not first: self._value = NamespaceTrie.Element(value) return if first not in self._subspaces: self._subspaces[first] = NamespaceTrie() self._subspaces[first][rest] = value def _get_helper(self, namespace, full_name): first, sep, rest = namespace.partition(self._sep) if not first: if not self._value: raise KeyError("Can't find namespace '%s' in trie" % full_name) return self._value.value elif first not in self._subspaces: raise KeyError("Can't find namespace '%s' in trie" % full_name) else: return self._subspaces[first]._get_helper(rest, full_name) def __getitem__(self, namespace): return self._get_helper(namespace, namespace) def is_prefix(self, namespace): """True if the namespace has a value, or if it's the prefix of one that does.""" first, sep, rest = namespace.partition(self._sep) if not first: return True elif first not in self._subspaces: return False else: return self._subspaces[first].is_prefix(rest) def is_leaf(self, namespace): """True if this namespace has no children in the trie.""" first, sep, rest = namespace.partition(self._sep) if not first: return bool(self._subspaces) elif first not in self._subspaces: return False else: return self._subspaces[first].is_leaf(rest) def has_value(self, namespace): """True if there is a value set for the given namespace.""" first, sep, rest = namespace.partition(self._sep) if not first: return self._value is not None elif first not in self._subspaces: return False else: return self._subspaces[first].has_value(rest) def __contains__(self, namespace): """Returns whether a value has been set for the namespace.""" return self.has_value(namespace) def _str_helper(self, stream, level=0): indent = level * " " for name in sorted(self._subspaces): stream.write(indent + name + "\n") if self._value: stream.write(indent + " " + repr(self._value.value)) stream.write(self._subspaces[name]._str_helper(stream, level + 1)) def __str__(self): stream = io.StringIO() self._str_helper(stream) return stream.getvalue() ================================================ FILE: lib/spack/spack/util/package_hash.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import ast import sys from typing import Any, Dict, List, Optional, Tuple import spack.directives_meta import spack.error import spack.fetch_strategy import spack.repo import spack.spec import spack.util.hash from spack.util.unparse import unparse if sys.version_info >= (3, 8): def unused_string(node: ast.AST) -> bool: """Criteria for unassigned body strings.""" return ( isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str) ) else: def unused_string(node: ast.AST) -> bool: """Criteria for unassigned body strings.""" return isinstance(node, ast.Expr) and isinstance(node.value, ast.Str) class RemoveDocstrings(ast.NodeTransformer): """Transformer that removes docstrings from a Python AST. This removes *all* strings that aren't on the RHS of an assignment statement from the body of functions, classes, and modules -- even if they're not directly after the declaration. """ def remove_docstring(self, node): if node.body: node.body = [child for child in node.body if not unused_string(child)] if not node.body: node.body = [ast.Pass()] self.generic_visit(node) return node def visit_FunctionDef(self, node): return self.remove_docstring(node) def visit_ClassDef(self, node): return self.remove_docstring(node) def visit_Module(self, node): return self.remove_docstring(node) class RemoveDirectives(ast.NodeTransformer): """Remove Spack directives from a package AST. This removes Spack directives (e.g., ``depends_on``, ``conflicts``, etc.) and metadata attributes (e.g., ``tags``, ``homepage``, ``url``) in a top-level class definition within a ``package.py``, but it does not modify nested classes or functions. If removing directives causes a ``for``, ``with``, or ``while`` statement to have an empty body, we remove the entire statement. Similarly, If removing directives causes an ``if`` statement to have an empty body or ``else`` block, we'll remove the block (or replace the body with ``pass`` if there is an ``else`` block but no body). """ def __init__(self, spec): #: List of attributes to be excluded from a package's hash. self.metadata_attrs = [s.url_attr for s in spack.fetch_strategy.all_strategies] + [ "homepage", "url", "urls", "list_url", "extendable", "parallel", "make_jobs", "maintainers", "tags", ] self.spec = spec self.in_classdef = False # used to avoid nested classdefs def visit_Expr(self, node): # Directives are represented in the AST as named function call expressions (as # opposed to function calls through a variable callback). We remove them. # # Note that changes to directives (e.g., a preferred version change or a hash # change on an archive) are already represented in the spec *outside* the # package hash. return ( None if ( node.value and isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id in spack.directives_meta.directive_names ) else node ) def visit_Assign(self, node): # Remove assignments to metadata attributes, b/c they don't affect the build. return ( None if ( node.targets and isinstance(node.targets[0], ast.Name) and node.targets[0].id in self.metadata_attrs ) else node ) def visit_With(self, node): self.generic_visit(node) # visit children return node if node.body else None # remove with statement if it has no body def visit_For(self, node): self.generic_visit(node) # visit children return node if node.body else None # remove loop if it has no body def visit_While(self, node): self.generic_visit(node) # visit children return node if node.body else None # remove loop if it has no body def visit_If(self, node): self.generic_visit(node) # an empty orelse is ignored by unparsing, but an empty body with a full orelse # ends up unparsing as a syntax error, so we replace the empty body into `pass`. if not node.body: if node.orelse: node.body = [ast.Pass()] else: return None # if the node has a body, it's valid python code with or without an orelse return node def visit_FunctionDef(self, node): # do not descend into function definitions return node def visit_ClassDef(self, node): # packages are always top-level, and we do not descend # into nested class defs and their attributes if self.in_classdef: return node # guard against recursive class definitions self.in_classdef = True self.generic_visit(node) self.in_classdef = False # replace class definition with `pass` if it's empty (e.g., packages that only # have directives b/c they subclass a build system class) if not node.body: node.body = [ast.Pass()] return node def _is_when_decorator(node: ast.Call) -> bool: """Check if the node is a @when decorator.""" return isinstance(node.func, ast.Name) and node.func.id == "when" and len(node.args) == 1 class TagMultiMethods(ast.NodeVisitor): """Tag @when-decorated methods in a package AST.""" def __init__(self, spec: spack.spec.Spec) -> None: self.spec = spec # map from function name to (implementation, condition_list) tuples self.methods: Dict[str, List[Tuple[ast.FunctionDef, List[Optional[bool]]]]] = {} if sys.version_info >= (3, 8): def _get_when_condition(self, node: ast.expr) -> Optional[Any]: """Extract the first argument of a @when decorator.""" return node.value if isinstance(node, ast.Constant) else None else: def _get_when_condition(self, node: ast.expr) -> Optional[Any]: """Extract the first argument of a @when decorator.""" if isinstance(node, ast.Str): return node.s elif isinstance(node, ast.NameConstant): return node.value return None def _evaluate_decorator(self, dec: ast.AST) -> Optional[bool]: """Evaluates a single decorator node. Returns True/False if it's a statically evaluatable @when decorator, otherwise returns None.""" if not isinstance(dec, ast.Call) or not _is_when_decorator(dec): return None # Extract from the @when() decorator. cond = self._get_when_condition(dec.args[0]) # Statically evaluate the condition if possible. If not, return None. if isinstance(cond, str): try: return self.spec.satisfies(cond) except Exception: return None elif isinstance(cond, bool): return cond else: return None def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: conditions = [self._evaluate_decorator(dec) for dec in node.decorator_list] # anything defined without conditions will overwrite prior definitions if not conditions: self.methods[node.name] = [] # add all discovered conditions on this node to the node list self.methods.setdefault(node.name, []).append((node, conditions)) # don't modify the AST -- return the untouched function node return node class ResolveMultiMethods(ast.NodeTransformer): """Remove multi-methods when we know statically that they won't be used. Say we have multi-methods like this:: class SomePackage: def foo(self): print("implementation 1") @when("@1.0") def foo(self): print("implementation 2") @when("@2.0") @when(sys.platform == "darwin") def foo(self): print("implementation 3") @when("@3.0") def foo(self): print("implementation 4") The multimethod that will be chosen at runtime depends on the package spec and on whether we're on the darwin platform *at build time* (the darwin condition for implementation 3 is dynamic). We know the package spec statically; we don't know statically what the runtime environment will be. We need to include things that can possibly affect package behavior in the package hash, and we want to exclude things when we know that they will not affect package behavior. If we're at version 4.0, we know that implementation 1 will win, because some @when for 2, 3, and 4 will be ``False``. We should only include implementation 1. If we're at version 1.0, we know that implementation 2 will win, because it overrides implementation 1. We should only include implementation 2. If we're at version 3.0, we know that implementation 4 will win, because it overrides implementation 1 (the default), and some @when on all others will be False. If we're at version 2.0, it's a bit more complicated. We know we can remove implementations 2 and 4, because their @when's will never be satisfied. But, the choice between implementations 1 and 3 will happen at runtime (this is a bad example because the spec itself has platform information, and we should prefer to use that, but we allow arbitrary boolean expressions in @when's, so this example suffices). For this case, we end up needing to include *both* implementation 1 and 3 in the package hash, because either could be chosen. """ def __init__(self, methods): self.methods = methods def resolve(self, impl_conditions): """Given list of nodes and conditions, figure out which node will be chosen.""" result = [] default = None for impl, conditions in impl_conditions: # if there's a default implementation with no conditions, remember that. if not conditions: default = impl result.append(default) continue # any known-false @when means the method won't be used if any(c is False for c in conditions): continue # anything with all known-true conditions will be picked if it's first if all(c is True for c in conditions): if result and result[0] is default: return [impl] # we know the first MM will always win # if anything dynamic comes before it we don't know if it'll win, # so just let this result get appended # anything else has to be determined dynamically, so add it to a list result.append(impl) # if nothing was picked, the last definition wins. return result def visit_FunctionDef(self, node: ast.FunctionDef) -> Optional[ast.FunctionDef]: # if the function def wasn't visited on the first traversal there is a problem assert node.name in self.methods, "Inconsistent package traversal!" # if the function is a multimethod, need to resolve it statically impl_conditions = self.methods[node.name] resolutions = self.resolve(impl_conditions) if not any(r is node for r in resolutions): # multimethod did not resolve to this function; remove it return None # if we get here, this function is a possible resolution for a multi-method. # it might be the only one, or there might be several that have to be evaluated # dynamcially. Either way, we include the function. # strip the when decorators (preserve the rest) node.decorator_list = [ dec for dec in node.decorator_list if not (isinstance(dec, ast.Call) and _is_when_decorator(dec)) ] return node def canonical_source( spec: spack.spec.Spec, filter_multimethods: bool = True, source: Optional[bytes] = None ) -> str: """Get canonical source for a spec's package.py by unparsing its AST. Arguments: filter_multimethods: By default, filter multimethods out of the AST if they are known statically to be unused. Supply False to disable. source: Optionally provide a string to read python code from. """ return unparse(package_ast(spec, filter_multimethods, source=source), py_ver_consistent=True) def package_hash(spec: spack.spec.Spec, source: Optional[bytes] = None) -> str: """Get a hash of a package's canonical source code. This function is used to determine whether a spec needs a rebuild when a package's source code changes. Arguments: source: Optionally provide a string to read python code from. """ source = canonical_source(spec, filter_multimethods=True, source=source) return spack.util.hash.b32_hash(source) def package_ast( spec: spack.spec.Spec, filter_multimethods: bool = True, source: Optional[bytes] = None ) -> ast.AST: """Get the AST for the ``package.py`` file corresponding to ``spec``. Arguments: filter_multimethods: By default, filter multimethods out of the AST if they are known statically to be unused. Supply False to disable. source: Optionally provide a string to read python code from. """ if source is None: filename = spack.repo.PATH.filename_for_package_name(spec.name) with open(filename, "rb") as f: source = f.read() # create an AST root = ast.parse(source) # remove docstrings, comments, and directives from the package AST root = RemoveDocstrings().visit(root) root = RemoveDirectives(spec).visit(root) if filter_multimethods: # visit nodes and build up a dictionary of methods (no need to assign) tagger = TagMultiMethods(spec) tagger.visit(root) # transform AST using tagged methods root = ResolveMultiMethods(tagger.methods).visit(root) return root class PackageHashError(spack.error.SpackError): """Raised for all errors encountered during package hashing.""" ================================================ FILE: lib/spack/spack/util/parallel.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import concurrent.futures import multiprocessing import os import sys import traceback from typing import Optional import spack.config #: Used in tests to disable parallelism, as tests themselves are parallelized ENABLE_PARALLELISM = sys.platform != "win32" class ErrorFromWorker: """Wrapper class to report an error from a worker process""" def __init__(self, exc_cls, exc, tb): """Create an error object from an exception raised from the worker process. The attributes of the process error objects are all strings as they are easy to send over a pipe. Args: exc: exception raised from the worker process """ self.pid = os.getpid() self.error_message = str(exc) self.stacktrace_message = "".join(traceback.format_exception(exc_cls, exc, tb)) @property def stacktrace(self): msg = "[PID={0.pid}] {0.stacktrace_message}" return msg.format(self) def __str__(self): return self.error_message class Task: """Wrapped task that trap every Exception and return it as an ErrorFromWorker object. We are using a wrapper class instead of a decorator since the class is pickleable, while a decorator with an inner closure is not. """ def __init__(self, func): self.func = func def __call__(self, *args, **kwargs): try: value = self.func(*args, **kwargs) except Exception: value = ErrorFromWorker(*sys.exc_info()) return value def imap_unordered( f, list_of_args, *, processes: int, maxtaskperchild: Optional[int] = None, debug=False ): """Wrapper around multiprocessing.Pool.imap_unordered. Args: f: function to apply list_of_args: list of tuples of args for the task processes: maximum number of processes allowed debug: if False, raise an exception containing just the error messages from workers, if True an exception with complete stacktraces maxtaskperchild: number of tasks to be executed by a child before being killed and substituted Raises: RuntimeError: if any error occurred in the worker processes """ if not ENABLE_PARALLELISM or len(list_of_args) <= 1: yield from map(f, list_of_args) return from spack.subprocess_context import GlobalStateMarshaler marshaler = GlobalStateMarshaler() with multiprocessing.Pool( processes, initializer=marshaler.restore, maxtasksperchild=maxtaskperchild ) as p: for result in p.imap_unordered(Task(f), list_of_args): if isinstance(result, ErrorFromWorker): raise RuntimeError(result.stacktrace if debug else str(result)) yield result class SequentialExecutor(concurrent.futures.Executor): """Executor that runs tasks sequentially in the current thread.""" def submit(self, fn, *args, **kwargs): """Submit a function to be executed.""" future = concurrent.futures.Future() try: future.set_result(fn(*args, **kwargs)) except Exception as e: future.set_exception(e) return future def make_concurrent_executor( jobs: Optional[int] = None, *, require_fork: bool = True ) -> concurrent.futures.Executor: """Create a concurrent executor. If require_fork is True, then the executor is sequential if the platform does not enable forking as the default start method. Effectively require_fork=True makes the executor sequential in the current process on Windows, macOS, and Linux from Python 3.14+ (which changes defaults)""" if ( not ENABLE_PARALLELISM or (require_fork and multiprocessing.get_start_method() != "fork") or sys.version_info[:2] == (3, 6) ): return SequentialExecutor() from spack.subprocess_context import GlobalStateMarshaler jobs = jobs or spack.config.determine_number_of_jobs(parallel=True) marshaler = GlobalStateMarshaler() return concurrent.futures.ProcessPoolExecutor(jobs, initializer=marshaler.restore) # novermin ================================================ FILE: lib/spack/spack/util/path.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Utilities for managing paths in Spack. TODO: this is really part of spack.config. Consolidate it. """ import contextlib import getpass import os import pathlib import re import subprocess import sys import tempfile from datetime import date from typing import Optional, Union import spack.llnl.util.tty as tty import spack.util.spack_yaml as syaml from spack.llnl.util.lang import memoized __all__ = ["substitute_config_variables", "substitute_path_variables", "canonicalize_path"] def architecture(): # break circular import import spack.platforms import spack.spec host_platform = spack.platforms.host() host_os = host_platform.default_operating_system() host_target = host_platform.default_target() return spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target))) def get_user(): # User pwd where available because it accounts for effective uids when using ksu and similar try: # user pwd for unix systems import pwd return pwd.getpwuid(os.geteuid()).pw_name except ImportError: # fallback on getpass return getpass.getuser() # return value for replacements with no match NOMATCH = object() # Substitutions to perform def replacements(): # break circular imports import spack import spack.environment as ev import spack.paths arch = architecture() return { "spack": lambda: spack.paths.prefix, "user": lambda: get_user(), "tempdir": lambda: tempfile.gettempdir(), "user_cache_path": lambda: spack.paths.user_cache_path, "spack_instance_id": lambda: spack.paths.spack_instance_id, "architecture": lambda: arch, "arch": lambda: arch, "platform": lambda: arch.platform, "operating_system": lambda: arch.os, "os": lambda: arch.os, "target": lambda: arch.target, "target_family": lambda: arch.target.family, "date": lambda: date.today().strftime("%Y-%m-%d"), "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH, "spack_short_version": lambda: spack.get_short_version(), } # This is intended to be longer than the part of the install path # spack generates from the root path we give it. Included in the # estimate: # # os-arch -> 30 # compiler -> 30 # package name -> 50 (longest is currently 47 characters) # version -> 20 # hash -> 32 # buffer -> 138 # --------------------- # total -> 300 SPACK_MAX_INSTALL_PATH_LENGTH = 300 #: Padded paths comprise directories with this name (or some prefix of it). : #: It starts with two underscores to make it unlikely that prefix matches would #: include some other component of the installation path. SPACK_PATH_PADDING_CHARS = "__spack_path_placeholder__" #: Bytes equivalent of SPACK_PATH_PADDING_CHARS. SPACK_PATH_PADDING_BYTES = SPACK_PATH_PADDING_CHARS.encode("ascii") #: Special padding char if the padded string would otherwise end with a path #: separator (since the path separator would otherwise get collapsed out, #: causing inconsistent padding). SPACK_PATH_PADDING_EXTRA_CHAR = "_" def win_exe_ext(): return r"(?:\.bat|\.exe)" def sanitize_filename(filename: str) -> str: """ Replaces unsupported characters (for the host) in a filename with underscores. Criteria for legal files based on https://en.wikipedia.org/wiki/Filename#Comparison_of_filename_limitations Args: filename: string containing filename to be created on the host filesystem Return: filename that can be created on the host filesystem """ if sys.platform != "win32": # Only disallow null bytes and directory separators. return re.sub("[\0/]", "_", filename) # On Windows, things are more involved. # NOTE: this is incomplete, missing reserved names return re.sub(r'[\x00-\x1F\x7F"*/:<>?\\|]', "_", filename) @memoized def get_system_path_max(): # Choose a conservative default sys_max_path_length = 256 if sys.platform == "win32": sys_max_path_length = 260 else: try: path_max_proc = subprocess.Popen( ["getconf", "PATH_MAX", "/"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) proc_output = str(path_max_proc.communicate()[0].decode()) sys_max_path_length = int(proc_output) except (ValueError, subprocess.CalledProcessError, OSError): tty.msg( "Unable to find system max path length, using: {0}".format(sys_max_path_length) ) return sys_max_path_length def substitute_config_variables(path): """Substitute placeholders into paths. Spack allows paths in configs to have some placeholders, as follows: - $env The active Spack environment. - $spack The Spack instance's prefix - $tempdir Default temporary directory returned by tempfile.gettempdir() - $user The current user's username - $user_cache_path The user cache directory (~/.spack, unless overridden) - $spack_instance_id Hash that distinguishes Spack instances on the filesystem - $architecture The spack architecture triple for the current system - $arch The spack architecture triple for the current system - $platform The spack platform for the current system - $os The OS of the current system - $operating_system The OS of the current system - $target The ISA target detected for the system - $target_family The family of the target detected for the system - $date The current date (YYYY-MM-DD) - $spack_short_version The spack short version These are substituted case-insensitively into the path, and users can use either ``$var`` or ``${var}`` syntax for the variables. $env is only replaced if there is an active environment, and should only be used in environment yaml files. """ _replacements = replacements() # Look up replacements def repl(match): m = match.group(0) key = m.strip("${}").lower() repl = _replacements.get(key, lambda: m)() return m if repl is NOMATCH else str(repl) # Replace $var or ${var}. return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) def substitute_path_variables(path): """Substitute config vars, expand environment vars, expand user home.""" path = substitute_config_variables(path) path = os.path.expandvars(path) path = os.path.expanduser(path) return path def _get_padding_string(length): spack_path_padding_size = len(SPACK_PATH_PADDING_CHARS) num_reps = int(length / (spack_path_padding_size + 1)) extra_chars = length % (spack_path_padding_size + 1) reps_list = [SPACK_PATH_PADDING_CHARS for i in range(num_reps)] reps_list.append(SPACK_PATH_PADDING_CHARS[:extra_chars]) padding = os.path.sep.join(reps_list) if padding.endswith(os.path.sep): padding = padding[: len(padding) - 1] + SPACK_PATH_PADDING_EXTRA_CHAR return padding def add_padding(path, length): """Add padding subdirectories to path until total is length characters Returns the padded path. If path is length - 1 or more characters long, returns path. If path is length - 1 characters, warns that it is not padding to length Assumes path does not have a trailing path separator""" padding_length = length - len(path) if padding_length == 1: # The only 1 character addition we can make to a path is `/` # Spack internally runs normpath, so `foo/` will be reduced to `foo` # Even if we removed this behavior from Spack, the user could normalize # the path, removing the additional `/`. # Because we can't expect one character of padding to show up in the # resulting binaries, we warn the user and do not pad by a single char tty.warn("Cannot pad path by exactly one character.") if padding_length <= 0: return path # we subtract 1 from the padding_length to account for the path separator # coming from os.path.join below padding = _get_padding_string(padding_length - 1) return os.path.join(path, padding) def canonicalize_path(path: str, default_wd: Optional[str] = None) -> str: """Same as substitute_path_variables, but also take absolute path. If the string is a yaml object with file annotations, make absolute paths relative to that file's directory. Otherwise, use ``default_wd`` if specified, otherwise ``os.getcwd()`` Arguments: path: path being converted as needed default_wd: optional working directory/root for non-yaml string paths Returns: An absolute path or non-file URL with path variable substitution """ import urllib.parse import urllib.request # Get file in which path was written in case we need to make it absolute # relative to that path. filename = None if isinstance(path, syaml.syaml_str): filename = os.path.dirname(path._start_mark.name) # type: ignore[attr-defined] assert path._start_mark.name == path._end_mark.name # type: ignore[attr-defined] path = substitute_path_variables(path) # Ensure properly process a Windows path win_path = pathlib.PureWindowsPath(path) if win_path.drive: # Assume only absolute paths are supported with a Windows drive # (though DOS does allow drive-relative paths). return os.path.normpath(str(win_path)) # Now process linux-like paths and remote URLs url = urllib.parse.urlparse(path) url_path = urllib.request.url2pathname(url.path) if url.scheme: if url.scheme != "file": # Have a remote URL so simply return it with substitutions return path # Drop the URL scheme from the local path path = url_path if os.path.isabs(path): return os.path.normpath(path) # Have a relative path so prepend the appropriate dir to make it absolute if filename: # Prepend the directory of the syaml path return os.path.normpath(os.path.join(filename, path)) # Prepend the default, if provided, or current working directory. base = default_wd or os.getcwd() tty.debug(f"Using working directory {base} as base for abspath") return os.path.normpath(os.path.join(base, path)) def longest_prefix_re(string, capture=True): """Return a regular expression that matches a the longest possible prefix of string. i.e., if the input string is ``the_quick_brown_fox``, then:: m = re.compile(longest_prefix('the_quick_brown_fox')) m.match('the_').group(1) == 'the_' m.match('the_quick').group(1) == 'the_quick' m.match('the_quick_brown_fox').group(1) == 'the_quick_brown_fox' m.match('the_xquick_brown_fox').group(1) == 'the_' m.match('the_quickx_brown_fox').group(1) == 'the_quick' """ if len(string) < 2: return string return "(%s%s%s?)" % ( "" if capture else "?:", string[0], longest_prefix_re(string[1:], capture=False), ) def _build_padding_re(as_bytes: bool = False): """Build and return a compiled regex for filtering path padding placeholders.""" pad = re.escape(SPACK_PATH_PADDING_CHARS) extra = SPACK_PATH_PADDING_EXTRA_CHAR longest_prefix = longest_prefix_re(SPACK_PATH_PADDING_CHARS, capture=False) regex = ( r"((?:/[^/\s]*)*?)" # zero or more leading non-whitespace path components r"(?:/{pad})+" # the padding string repeated one or more times # trailing prefix of padding as path component r"(?:/{longest_prefix}|/{longest_prefix}{extra})?(?=/)" ) regex = regex.replace("/", re.escape(os.sep)) regex = regex.format(pad=pad, extra=extra, longest_prefix=longest_prefix) if as_bytes: return re.compile(regex.encode("ascii")) else: return re.compile(regex) class _PaddingFilter: """Callable that filters path-padding placeholders from a string or bytes buffer. This turns paths like this: /foo/bar/__spack_path_placeholder__/__spack_path_placeholder__/... Into paths like this: /foo/bar/[padded-to-512-chars]/... Where ``padded-to-512-chars`` indicates that the prefix was padded with placeholders until it hit 512 characters. The actual value of this number depends on what the ``install_tree``'s ``padded_length`` is configured to. For a path to match and be filtered, the placeholder must appear in its entirety at least one time. e.g., "/spack/" would not be filtered, but "/__spack_path_placeholder__/spack/" would be. Note that only the first padded path in the string is filtered. """ __slots__ = ("_re", "_needle", "_fmt") def __init__(self, as_bytes: bool = False) -> None: self._re = _build_padding_re(as_bytes=as_bytes) if as_bytes: self._needle: Union[str, bytes] = SPACK_PATH_PADDING_BYTES self._fmt: Union[str, bytes] = b"%b" + os.sep.encode("ascii") + b"[padded-to-%d-chars]" else: self._needle = SPACK_PATH_PADDING_CHARS self._fmt = "%s" + os.sep + "[padded-to-%d-chars]" def _replace(self, match): return self._fmt % (match.group(1), len(match.group(0))) def __call__(self, data): if self._needle not in data: return data return self._re.sub(self._replace, data) #: Callable that filters path-padding placeholders from strings padding_filter = _PaddingFilter(as_bytes=False) #: Callable that filters path-padding placeholders from bytes buffers padding_filter_bytes = _PaddingFilter(as_bytes=True) @contextlib.contextmanager def filter_padding(): """Context manager to safely disable path padding in all Spack output. This is needed because Spack's debug output gets extremely long when we use a long padded installation path. """ # circular import import spack.config padding = spack.config.get("config:install_tree:padded_length", None) if padding: # filter out all padding from the install command output with tty.output_filter(padding_filter): yield else: yield # no-op: don't filter unless padding is actually enabled def debug_padded_filter(string, level=1): """ Return string, path padding filtered if debug level and not windows Args: string (str): string containing path level (int): maximum debug level value for filtering (e.g., 1 means filter path padding if the current debug level is 0 or 1 but return the original string if it is 2 or more) Returns (str): filtered string if current debug level does not exceed level and not windows; otherwise, unfiltered string """ if sys.platform == "win32": return string return padding_filter(string) if tty.debug_level() <= level else string ================================================ FILE: lib/spack/spack/util/pattern.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) class Bunch: """Carries a bunch of named attributes (from Alex Martelli bunch)""" def __init__(self, **kwargs): self.__dict__.update(kwargs) class Args(Bunch): """Subclass of Bunch to write argparse args more naturally.""" def __init__(self, *flags, **kwargs): super().__init__(flags=tuple(flags), kwargs=kwargs) ================================================ FILE: lib/spack/spack/util/prefix.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ This file contains utilities for managing the installation prefix of a package. """ import os from typing import Dict class Prefix(str): """This class represents an installation prefix, but provides useful attributes for referring to directories inside the prefix. Attributes of this object are created on the fly when you request them, so any of the following are valid: >>> prefix = Prefix("/usr") >>> prefix.bin /usr/bin >>> prefix.lib64 /usr/lib64 >>> prefix.share.man /usr/share/man >>> prefix.foo.bar.baz /usr/foo/bar/baz >>> prefix.join("dashed-directory").bin64 /usr/dashed-directory/bin64 Prefix objects behave identically to strings. In fact, they subclass :class:`str`, so operators like ``+`` are legal:: print("foobar " + prefix) This prints ``foobar /usr``. All of this is meant to make custom installs easy. """ def __getattr__(self, name: str) -> "Prefix": """Concatenate a string to a prefix. Useful for strings that are valid variable names. Args: name: the string to append to the prefix Returns: the newly created installation prefix """ return Prefix(os.path.join(self, name)) def join(self, string: str) -> "Prefix": # type: ignore[override] """Concatenate a string to a prefix. Useful for strings that are not valid variable names. This includes strings containing characters like ``-`` and ``.``. Args: string: the string to append to the prefix Returns: the newly created installation prefix """ return Prefix(os.path.join(self, string)) def __getstate__(self) -> Dict[str, str]: """Control how object is pickled. Returns: current state of the object """ return self.__dict__ def __setstate__(self, state: Dict[str, str]) -> None: """Control how object is unpickled. Args: new state of the object """ self.__dict__.update(state) ================================================ FILE: lib/spack/spack/util/remote_file_cache.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import hashlib import os.path import pathlib import shutil import tempfile import urllib.parse import urllib.request from typing import Optional import spack.llnl.util.tty as tty import spack.util.crypto from spack.llnl.util.filesystem import copy, join_path, mkdirp from spack.util.path import canonicalize_path from spack.util.url import validate_scheme def raw_github_gitlab_url(url: str) -> str: """Transform a github URL to the raw form to avoid undesirable html. Args: url: url to be converted to raw form Returns: Raw github/gitlab url or the original url """ # Note we rely on GitHub to redirect the 'raw' URL returned here to the # actual URL under https://raw.githubusercontent.com/ with '/blob' # removed and or, '/blame' if needed. if "github" in url or "gitlab" in url: return url.replace("/blob/", "/raw/") return url def fetch_remote_text_file(url: str, dest_dir: str) -> str: """Retrieve the text file from the url into the destination directory. Arguments: url: URL for the remote text file dest_dir: destination directory in which to stage the file locally Returns: Path to the fetched file Raises: ValueError: if there are missing required arguments """ from spack.util.web import fetch_url_text # circular import if not url: raise ValueError("Cannot retrieve the remote file without the URL") raw_url = raw_github_gitlab_url(url) tty.debug(f"Fetching file from {raw_url} into {dest_dir}") return fetch_url_text(raw_url, dest_dir=dest_dir) def local_path(raw_path: str, sha256: str, dest: Optional[str] = None) -> str: """Determine the actual path and, if remote, stage its contents locally. Args: raw_path: raw path with possible variables needing substitution sha256: the expected sha256 if the file is remote dest: destination path Returns: resolved, normalized local path Raises: ValueError: missing or mismatched arguments, unsupported URL scheme """ if not raw_path: raise ValueError("path argument is required to cache remote files") file_schemes = ["", "file"] # Allow paths (and URLs) to contain spack config/environment variables, # etc. path = canonicalize_path(raw_path, dest) # Save off the Windows drive of the canonicalized path (since now absolute) # to ensure recognized by URL parsing as a valid file "scheme". win_path = pathlib.PureWindowsPath(path) if win_path.drive: file_schemes.append(win_path.drive.lower().strip(":")) url = urllib.parse.urlparse(path) # Path isn't remote so return normalized, absolute path with substitutions. if url.scheme in file_schemes: return os.path.normpath(path) # If scheme is not valid, path is not a supported url. if validate_scheme(url.scheme): # Fetch files from supported URL schemes. if url.scheme in ("http", "https", "ftp"): if not dest: raise ValueError("Requires the destination argument to cache remote files") assert os.path.isabs(dest), ( f"Remote file destination '{dest}' must be an absolute path" ) # Stage the remote configuration file tmpdir = tempfile.mkdtemp() try: staged_path = fetch_remote_text_file(path, tmpdir) # Ensure the sha256 is expected. checksum = spack.util.crypto.checksum(hashlib.sha256, staged_path) if sha256 and checksum != sha256: raise ValueError( f"Actual sha256 ('{checksum}') does not match expected ('{sha256}')" ) # Help the user by reporting the required checksum. if not sha256: raise ValueError(f"Requires sha256 ('{checksum}') to cache remote files.") # Copy the file to the destination directory dest_dir = join_path(dest, checksum) if not os.path.exists(dest_dir): mkdirp(dest_dir) cache_path = join_path(dest_dir, os.path.basename(staged_path)) copy(staged_path, cache_path) tty.debug(f"Cached {raw_path} in {cache_path}") # Stash the associated URL to aid with debugging with open(join_path(dest_dir, "source_url.txt"), "w", encoding="utf-8") as f: f.write(f"{raw_path}\n") return cache_path except ValueError as err: tty.warn(f"Unable to cache {raw_path}: {str(err)}") raise finally: shutil.rmtree(tmpdir) raise ValueError(f"Unsupported URL scheme ({url.scheme}) in {raw_path}") else: raise ValueError(f"Invalid URL scheme ({url.scheme}) in {raw_path}") ================================================ FILE: lib/spack/spack/util/s3.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os import urllib.error import urllib.parse import urllib.request import urllib.response from io import BufferedReader, BytesIO, IOBase from typing import Any, Dict, Tuple import spack.config #: Map (mirror name, method) tuples to s3 client instances. s3_client_cache: Dict[Tuple[str, str], Any] = dict() def get_s3_session(url, method="fetch"): # import boto and friends as late as possible. We don't want to require boto as a # dependency unless the user actually wants to access S3 mirrors. from boto3 import Session from botocore import UNSIGNED from botocore.client import Config from botocore.exceptions import ClientError # Circular dependency from spack.mirrors.mirror import MirrorCollection global s3_client_cache # Parse the URL if not already done. if not isinstance(url, urllib.parse.ParseResult): url = urllib.parse.urlparse(url) url_str = url.geturl() def get_mirror_url(mirror): return mirror.fetch_url if method == "fetch" else mirror.push_url # Get all configured mirrors that could match. all_mirrors = MirrorCollection() mirrors = [ (name, mirror) for name, mirror in all_mirrors.items() if url_str.startswith(get_mirror_url(mirror)) ] if not mirrors: name, mirror = None, {} else: # In case we have more than one mirror, we pick the longest matching url. # The heuristic being that it's more specific, and you can have different # credentials for a sub-bucket (if that is a thing). name, mirror = max( mirrors, key=lambda name_and_mirror: len(get_mirror_url(name_and_mirror[1])) ) key = (name, method) # Did we already create a client for this? Then return it. if key in s3_client_cache: return s3_client_cache[key] # Otherwise, create it. s3_connection, s3_client_args = get_mirror_s3_connection_info(mirror, method) session = Session(**s3_connection) # if no access credentials provided above, then access anonymously if not session.get_credentials(): s3_client_args["config"] = Config(signature_version=UNSIGNED) client = session.client("s3", **s3_client_args) client.ClientError = ClientError # Cache the client. s3_client_cache[key] = client return client def _parse_s3_endpoint_url(endpoint_url): if not urllib.parse.urlparse(endpoint_url, scheme="").scheme: endpoint_url = "://".join(("https", endpoint_url)) return endpoint_url def get_mirror_s3_connection_info(mirror, method): """Create s3 config for session/client from a Mirror instance (or just set defaults when no mirror is given.)""" from spack.mirrors.mirror import Mirror s3_connection = {} s3_client_args = {"use_ssl": spack.config.get("config:verify_ssl")} # access token if isinstance(mirror, Mirror): credentials = mirror.get_credentials(method) if credentials: if "access_token" in credentials: s3_connection["aws_session_token"] = credentials["access_token"] if "access_pair" in credentials: s3_connection["aws_access_key_id"] = credentials["access_pair"][0] s3_connection["aws_secret_access_key"] = credentials["access_pair"][1] if "profile" in credentials: s3_connection["profile_name"] = credentials["profile"] # endpoint url endpoint_url = mirror.get_endpoint_url(method) or os.environ.get("S3_ENDPOINT_URL") else: endpoint_url = os.environ.get("S3_ENDPOINT_URL") if endpoint_url: s3_client_args["endpoint_url"] = _parse_s3_endpoint_url(endpoint_url) return s3_connection, s3_client_args # NOTE(opadron): Workaround issue in boto where its StreamingBody # implementation is missing several APIs expected from IOBase. These missing # APIs prevent the streams returned by boto from being passed as-are along to # urllib. # # https://github.com/boto/botocore/issues/879 # https://github.com/python/cpython/pull/3249 class WrapStream(BufferedReader): def __init__(self, raw): # In botocore >=1.23.47, StreamingBody inherits from IOBase, so we # only add missing attributes in older versions. # https://github.com/boto/botocore/commit/a624815eabac50442ed7404f3c4f2664cd0aa784 if not isinstance(raw, IOBase): raw.readable = lambda: True raw.writable = lambda: False raw.seekable = lambda: False raw.closed = False raw.flush = lambda: None super().__init__(raw) def detach(self): self.raw = None def read(self, *args, **kwargs): return self.raw.read(*args, **kwargs) def __getattr__(self, key): return getattr(self.raw, key) def _s3_open(url, method="GET"): parsed = urllib.parse.urlparse(url) s3 = get_s3_session(url, method="fetch") bucket = parsed.netloc key = parsed.path if key.startswith("/"): key = key[1:] if method not in ("GET", "HEAD"): raise urllib.error.URLError( "Only GET and HEAD verbs are currently supported for the s3:// scheme" ) try: if method == "GET": obj = s3.get_object(Bucket=bucket, Key=key) # NOTE(opadron): Apply workaround here (see above) stream = WrapStream(obj["Body"]) elif method == "HEAD": obj = s3.head_object(Bucket=bucket, Key=key) stream = BytesIO() except s3.ClientError as e: raise urllib.error.URLError(e) from e headers = obj["ResponseMetadata"]["HTTPHeaders"] return url, headers, stream class UrllibS3Handler(urllib.request.BaseHandler): def s3_open(self, req): orig_url = req.get_full_url() url, headers, stream = _s3_open(orig_url, method=req.get_method()) return urllib.response.addinfourl(stream, headers, url) ================================================ FILE: lib/spack/spack/util/socket.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Optimized Spack implementations of methods from socket module.""" import socket import spack.llnl.util.lang @spack.llnl.util.lang.memoized def _gethostname(): """Memoized version of `getfqdn()`. If we call `getfqdn()` too many times, DNS can be very slow. We only need to call it one time per process, so we cache it here. """ return socket.gethostname() ================================================ FILE: lib/spack/spack/util/spack_json.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Simple wrapper around JSON to guarantee consistent use of load/dump.""" import json from typing import Any, Dict, Optional import spack.error __all__ = ["load", "dump", "SpackJSONError"] _json_dump_args = {"indent": None, "separators": (",", ":")} _pretty_dump_args = {"indent": " ", "separators": (", ", ": ")} def load(stream: Any) -> Dict: """Spack JSON needs to be ordered to support specs.""" if isinstance(stream, str): return json.loads(stream) return json.load(stream) def dump(data: Dict, stream: Optional[Any] = None, pretty: bool = False) -> Optional[str]: """Dump JSON with a reasonable amount of indentation and separation.""" dump_args = _pretty_dump_args if pretty else _json_dump_args if stream is None: return json.dumps(data, **dump_args) # type: ignore[arg-type] json.dump(data, stream, **dump_args) # type: ignore[arg-type] return None class SpackJSONError(spack.error.SpackError): """Raised when there are issues with JSON parsing.""" def __init__(self, msg: str, json_error: BaseException): super().__init__(msg, str(json_error)) ================================================ FILE: lib/spack/spack/util/spack_yaml.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Enhanced YAML parsing for Spack. - ``load()`` preserves YAML Marks on returned objects -- this allows us to access file and line information later. - ``Our load methods use ``OrderedDict`` class instead of YAML's default unordered dict. """ import ctypes import enum import functools import io import re from typing import IO, Any, Callable, Dict, List, Optional, Union from spack.vendor.ruamel.yaml import YAML, comments, constructor, emitter, error, representer import spack.error from spack.llnl.util.tty.color import cextra, clen, colorize # Only export load and dump __all__ = ["load", "dump", "SpackYAMLError"] # Make new classes so we can add custom attributes. class syaml_dict(dict): pass class syaml_list(list): pass class syaml_str(str): pass class syaml_int(int): pass #: mapping from syaml type -> primitive type syaml_types = {syaml_str: str, syaml_int: int, syaml_dict: dict, syaml_list: list} markable_types = set(syaml_types) | {comments.CommentedSeq, comments.CommentedMap} def syaml_type(obj): """Get the corresponding syaml wrapper type for a primitive type. Return: (object): syaml-typed copy of object, or the obj if no wrapper """ for syaml_t, t in syaml_types.items(): if type(obj) is not bool and isinstance(obj, t): return syaml_t(obj) if type(obj) is not syaml_t else obj return obj class DictWithLineInfo(dict): """A dictionary that preserves YAML line information.""" __slots__ = ("line_info",) def __init__(self, *args, line_info: str = "", **kwargs): super().__init__(*args, **kwargs) self.line_info = line_info def _represent_dict_with_line_info(dumper, data): return dumper.represent_dict(data) def deepcopy_as_builtin(obj: Any, *, line_info: bool = False) -> Any: """Deep copies a YAML object as built-in types (dict, list, str, int, ...). Args: obj: object to be copied line_info: if ``True``, add line information to the copied object """ if isinstance(obj, str): return str(obj) elif isinstance(obj, dict): result = DictWithLineInfo() result.update( { deepcopy_as_builtin(k): deepcopy_as_builtin(v, line_info=line_info) for k, v in obj.items() } ) if line_info: result.line_info = _line_info(obj) return result elif isinstance(obj, list): return [deepcopy_as_builtin(x, line_info=line_info) for x in obj] elif isinstance(obj, bool): return bool(obj) elif isinstance(obj, int): return int(obj) elif isinstance(obj, float): return float(obj) elif obj is None: return obj raise ValueError(f"cannot convert {type(obj)} to built-in type") def markable(obj): """Whether an object can be marked.""" return type(obj) in markable_types def mark(obj, node): """Add start and end markers to an object.""" if hasattr(node, "start_mark"): obj._start_mark = node.start_mark elif hasattr(node, "_start_mark"): obj._start_mark = node._start_mark if hasattr(node, "end_mark"): obj._end_mark = node.end_mark elif hasattr(node, "_end_mark"): obj._end_mark = node._end_mark def marked(obj): """Whether an object has been marked by spack_yaml.""" return ( hasattr(obj, "_start_mark") and obj._start_mark or hasattr(obj, "_end_mark") and obj._end_mark ) class OrderedLineConstructor(constructor.RoundTripConstructor): """YAML loader specifically intended for reading Spack configuration files. It preserves order and line numbers. It also has special-purpose logic for handling dictionary keys that indicate a Spack config override: namely any key that contains an "extra" ':' character. Mappings read in by this loader behave like an ordered dict. Sequences, mappings, and strings also have new attributes, ``_start_mark`` and ``_end_mark``, that preserve YAML line information in the output data. """ # # Override construct_yaml_* so that we can apply _start_mark/_end_mark to # them. The superclass returns CommentedMap/CommentedSeq objects that we # can add attributes to (and we depend on their behavior to preserve # comments). # # The inherited sequence/dictionary constructors return empty instances # and fill in with mappings later. We preserve this behavior. # def construct_yaml_str(self, node): value = super().construct_yaml_str(node) # There is no specific marker to indicate that we are parsing a key, # so this assumes we are talking about a Spack config override key if # it ends with a ':' and does not contain a '@' (which can appear # in config values that refer to Spack specs) if value and value.endswith(":") and "@" not in value: value = syaml_str(value[:-1]) value.override = True else: value = syaml_str(value) mark(value, node) return value def construct_yaml_seq(self, node): gen = super().construct_yaml_seq(node) data = next(gen) if markable(data): mark(data, node) yield data for x in gen: pass def construct_yaml_map(self, node): gen = super().construct_yaml_map(node) data = next(gen) if markable(data): mark(data, node) yield data for x in gen: pass # register above new constructors OrderedLineConstructor.add_constructor( "tag:yaml.org,2002:map", OrderedLineConstructor.construct_yaml_map ) OrderedLineConstructor.add_constructor( "tag:yaml.org,2002:seq", OrderedLineConstructor.construct_yaml_seq ) OrderedLineConstructor.add_constructor( "tag:yaml.org,2002:str", OrderedLineConstructor.construct_yaml_str ) class OrderedLineRepresenter(representer.RoundTripRepresenter): """Representer that preserves ordering and formats ``syaml_*`` objects. This representer preserves insertion ordering ``syaml_dict`` objects when they're written out. It also has some custom formatters for ``syaml_*`` objects so that they are formatted like their regular Python equivalents, instead of ugly YAML pyobjects. """ def ignore_aliases(self, _data): """Make the dumper NEVER print YAML aliases.""" return True def represent_data(self, data): result = super().represent_data(data) if data is None: result.value = syaml_str("null") return result def represent_str(self, data): if hasattr(data, "override") and data.override: data = data + ":" return super().represent_str(data) class SafeRepresenter(representer.RoundTripRepresenter): def ignore_aliases(self, _data): """Make the dumper NEVER print YAML aliases.""" return True # Make our special objects look like normal YAML ones. representer.RoundTripRepresenter.add_representer( syaml_dict, representer.RoundTripRepresenter.represent_dict ) representer.RoundTripRepresenter.add_representer( syaml_list, representer.RoundTripRepresenter.represent_list ) representer.RoundTripRepresenter.add_representer( syaml_int, representer.RoundTripRepresenter.represent_int ) representer.RoundTripRepresenter.add_representer( syaml_str, representer.RoundTripRepresenter.represent_str ) OrderedLineRepresenter.add_representer(syaml_str, OrderedLineRepresenter.represent_str) #: Max integer helps avoid passing too large a value to cyaml. maxint = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1 def return_string_when_no_stream(func): @functools.wraps(func) def wrapper(data, stream=None, **kwargs): if stream: return func(data, stream=stream, **kwargs) stream = io.StringIO() func(data, stream=stream, **kwargs) return stream.getvalue() return wrapper @return_string_when_no_stream def dump(data, stream=None, default_flow_style=False): handler = ConfigYAML(yaml_type=YAMLType.GENERIC_YAML) handler.yaml.default_flow_style = default_flow_style handler.yaml.width = maxint return handler.dump(data, stream=stream) def _line_info(obj): """Format a mark as : information.""" m = get_mark_from_yaml_data(obj) if m is None: return "" if m.line: return f"{m.name}:{m.line:d}" return m.name #: Global for interactions between LineAnnotationDumper and dump_annotated(). #: This is nasty but YAML doesn't give us many ways to pass arguments -- #: yaml.dump() takes a class (not an instance) and instantiates the dumper #: itself, so we can't just pass an instance _ANNOTATIONS: List[str] = [] class LineAnnotationRepresenter(OrderedLineRepresenter): """Representer that generates per-line annotations. Annotations are stored in the ``_annotations`` global. After one dump pass, the strings in ``_annotations`` will correspond one-to-one with the lines output by the dumper. LineAnnotationDumper records blame information after each line is generated. As each line is parsed, it saves file/line info for each object printed. At the end of each line, it creates an annotation based on the saved mark and stores it in ``_annotations``. For an example of how to use this, see ``dump_annotated()``, which writes to a ``StringIO`` then joins the lines from that with annotations. """ def represent_data(self, data): """Force syaml_str to be passed through with marks.""" result = super().represent_data(data) if data is None: result.value = syaml_str("null") elif isinstance(result.value, str): result.value = syaml_str(data) if markable(result.value): mark(result.value, data) return result class LineAnnotationEmitter(emitter.Emitter): saved = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) del _ANNOTATIONS[:] self.colors = "KgrbmcyGRBMCY" self.filename_colors = {} def process_scalar(self): super().process_scalar() if marked(self.event.value): self.saved = self.event.value def write_line_break(self, data=None): super().write_line_break(data) if self.saved is None: _ANNOTATIONS.append(colorize("@K{---}")) return # append annotations at the end of each line if self.saved: mark = self.saved._start_mark color = self.filename_colors.get(mark.name) if not color: ncolors = len(self.colors) color = self.colors[len(self.filename_colors) % ncolors] self.filename_colors[mark.name] = color fmt = "@%s{%%s}" % color ann = fmt % mark.name if mark.line is not None: ann += ":@c{%s}" % (mark.line + 1) _ANNOTATIONS.append(colorize(ann)) else: _ANNOTATIONS.append("") def write_comment(self, comment, pre=False): pass class YAMLType(enum.Enum): """YAML configurations handled by Spack""" #: Generic YAML configuration GENERIC_YAML = enum.auto() #: A Spack config file with overrides SPACK_CONFIG_FILE = enum.auto() #: A Spack config file with line annotations ANNOTATED_SPACK_CONFIG_FILE = enum.auto() class ConfigYAML: """Handles the loading and dumping of Spack's YAML files.""" def __init__(self, yaml_type: YAMLType) -> None: self.yaml = YAML(typ="rt", pure=True) if yaml_type == YAMLType.GENERIC_YAML: self.yaml.Representer = SafeRepresenter elif yaml_type == YAMLType.ANNOTATED_SPACK_CONFIG_FILE: self.yaml.Representer = LineAnnotationRepresenter self.yaml.Emitter = LineAnnotationEmitter self.yaml.Constructor = OrderedLineConstructor else: self.yaml.Representer = OrderedLineRepresenter self.yaml.Constructor = OrderedLineConstructor self.yaml.Representer.add_representer(DictWithLineInfo, _represent_dict_with_line_info) def load(self, stream: IO): """Loads the YAML data from a stream and returns it. Args: stream: stream to load from. Raises: SpackYAMLError: if anything goes wrong while loading """ try: return self.yaml.load(stream) except error.MarkedYAMLError as e: msg = "error parsing YAML" error_mark = e.context_mark if e.context_mark else e.problem_mark if error_mark: line, column = error_mark.line, error_mark.column filename = error_mark.name msg += f": near {filename}, {str(line)}, {str(column)}" else: filename = stream.name msg += f": {stream.name}" msg += f": {e.problem}" raise SpackYAMLError(msg, e, filename) from e except Exception as e: msg = "cannot load Spack YAML configuration" filename = stream.name raise SpackYAMLError(msg, e, filename) from e def dump(self, data, stream: Optional[IO] = None, *, transform=None) -> None: """Dumps the YAML data to a stream. Args: data: data to be dumped stream: stream to dump the data into. Raises: SpackYAMLError: if anything goes wrong while dumping """ try: return self.yaml.dump(data, stream=stream, transform=transform) except Exception as e: msg = "cannot dump Spack YAML configuration" filename = stream.name if stream else None raise SpackYAMLError(msg, str(e), filename) from e def as_string(self, data) -> str: """Returns a string representing the YAML data passed as input.""" result = io.StringIO() self.dump(data, stream=result) return result.getvalue() def load_config(str_or_file): """Load but modify the loader instance so that it will add __line__ attributes to the returned object.""" handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE) return handler.load(str_or_file) def load(*args, **kwargs): handler = ConfigYAML(yaml_type=YAMLType.GENERIC_YAML) return handler.load(*args, **kwargs) @return_string_when_no_stream def dump_config(data, stream, *, default_flow_style=False, blame=False): if blame: handler = ConfigYAML(yaml_type=YAMLType.ANNOTATED_SPACK_CONFIG_FILE) handler.yaml.default_flow_style = default_flow_style handler.yaml.width = maxint return _dump_annotated(handler, data, stream) handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE) handler.yaml.default_flow_style = default_flow_style handler.yaml.width = maxint return handler.dump(data, stream) def _dump_annotated(handler, data, stream=None): sio = io.StringIO() handler.dump(data, sio) # write_line_break() is not called by YAML for empty lines, so we # skip empty lines here with \n+. lines = re.split(r"\n+", sio.getvalue().rstrip()) getvalue = None if stream is None: stream = io.StringIO() getvalue = stream.getvalue # write out annotations and lines, accounting for color width = max(clen(a) for a in _ANNOTATIONS) formats = ["%%-%ds %%s\n" % (width + cextra(a)) for a in _ANNOTATIONS] for fmt, annotation, line in zip(formats, _ANNOTATIONS, lines): stream.write(fmt % (annotation, line)) if getvalue: return getvalue() def sorted_dict(data): """Descend into data and sort all dictionary keys.""" if isinstance(data, dict): return type(data)((k, sorted_dict(v)) for k, v in sorted(data.items())) elif isinstance(data, (list, tuple)): return type(data)(sorted_dict(v) for v in data) return data def extract_comments(data): """Extract and returns comments from some YAML data""" return getattr(data, comments.Comment.attrib, None) def set_comments(data, *, data_comments): """Set comments on some YAML data""" return setattr(data, comments.Comment.attrib, data_comments) def name_mark(name): """Returns a mark with just a name""" return error.StringMark(name, None, None, None, None, None) def anchorify(data: Union[dict, list], identifier: Callable[[Any], str] = repr) -> None: """Replace identical dict/list branches in tree with references to earlier instances. The YAML serializer generate anchors for them, resulting in small yaml files.""" anchors: Dict[str, Union[dict, list]] = {} stack: List[Union[dict, list]] = [data] while stack: item = stack.pop() for key, value in item.items() if isinstance(item, dict) else enumerate(item): if not isinstance(value, (dict, list)): continue id = identifier(value) anchor = anchors.get(id) if anchor is None: anchors[id] = value stack.append(value) else: item[key] = anchor # replace with reference class SpackYAMLError(spack.error.SpackError): """Raised when there are issues with YAML parsing.""" def __init__(self, msg, yaml_error, filename=None): self.filename = filename super().__init__(msg, str(yaml_error)) def get_mark_from_yaml_data(obj): """Try to get ``spack.util.spack_yaml`` mark from YAML data. We try the object, and if that fails we try its first member (if it's a container). Returns: mark if one is found, otherwise None. """ # mark of object itelf mark = getattr(obj, "_start_mark", None) if mark: return mark # mark of first member if it is a container if isinstance(obj, (list, dict)): first_member = next(iter(obj), None) if first_member: mark = getattr(first_member, "_start_mark", None) return mark ================================================ FILE: lib/spack/spack/util/timer.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Debug signal handler: prints a stack trace and enters interpreter. ``register_interrupt_handler()`` enables a ctrl-C handler that prints a stack trace and drops the user into an interpreter. """ import collections import sys import time from contextlib import contextmanager from typing import Callable, Dict, List import spack.util.spack_json as sjson from spack.llnl.util.lang import pretty_seconds_formatter TimerEvent = collections.namedtuple("TimerEvent", ("time", "running", "label")) TimeTracker = collections.namedtuple("TimeTracker", ("total", "start", "count", "path")) #: name for the global timer (used in start(), stop(), duration() without arguments) global_timer_name = "_global" class BaseTimer: def start(self, name=None): pass def stop(self, name=None): pass def duration(self, name=None): return 0.0 @contextmanager def measure(self, name): yield self @property def phases(self): return [] def write_json(self, out=sys.stdout): pass def write_tty(self, out=sys.stdout): pass class NullTimer(BaseTimer): """Timer interface that does nothing, useful in for "tell don't ask" style code when timers are optional.""" pass class Timer(BaseTimer): """Simple interval timer""" def __init__(self, now: Callable[[], float] = time.time): """ Arguments: now: function that gives the seconds since e.g. epoch """ self._now = now self._timers: Dict[str, TimeTracker] = {} self._timer_stack: List[str] = [] self._events: List[TimerEvent] = [] # Push start event self._events.append(TimerEvent(self._now(), True, global_timer_name)) def start(self, name=global_timer_name): """ Start or restart a named timer, or the global timer when no name is given. Arguments: name (str): Optional name of the timer. When no name is passed, the global timer is started. """ self._events.append(TimerEvent(self._now(), True, name)) def stop(self, name=global_timer_name): """ Stop a named timer, or all timers when no name is given. Stopping a timer that has not started has no effect. Arguments: name (str): Optional name of the timer. When no name is passed, all timers are stopped. """ self._events.append(TimerEvent(self._now(), False, name)) def duration(self, name=global_timer_name): """ Get the time in seconds of a named timer, or the total time if no name is passed. The duration is always 0 for timers that have not been started, no error is raised. Arguments: name (str): (Optional) name of the timer Returns: float: duration of timer. """ self._flatten() if name in self._timers: if name in self._timer_stack: return self._timers[name].total + (self._now() - self._timers[name].start) return self._timers[name].total else: return 0.0 @contextmanager def measure(self, name): """ Context manager that allows you to time a block of code. Arguments: name (str): Name of the timer """ self.start(name) yield self self.stop(name) @property def phases(self): """Get all named timers (excluding the global/total timer)""" self._flatten() return [k for k in self._timers.keys() if not k == global_timer_name] def _flatten(self): for event in self._events: if event.running: if event.label not in self._timer_stack: self._timer_stack.append(event.label) # Only start the timer if it is on top of the stack # restart doesn't work after a subtimer is started if event.label == self._timer_stack[-1]: timer_path = "/".join(self._timer_stack[1:]) tracker = self._timers.get( event.label, TimeTracker(0.0, event.time, 0, timer_path) ) assert tracker.path == timer_path self._timers[event.label] = TimeTracker( tracker.total, event.time, tracker.count, tracker.path ) else: # if not event.running: if event.label in self._timer_stack: index = self._timer_stack.index(event.label) for label in self._timer_stack[index:]: tracker = self._timers[label] self._timers[label] = TimeTracker( tracker.total + (event.time - tracker.start), None, tracker.count + 1, tracker.path, ) self._timer_stack = self._timer_stack[: max(0, index)] # clear events self._events = [] def write_json(self, out=sys.stdout, extra_attributes={}): """Write a json object with times to file""" self._flatten() data = { "total": self._timers[global_timer_name].total, "phases": [ { "name": phase, "path": self._timers[phase].path, "seconds": self._timers[phase].total, "count": self._timers[phase].count, } for phase in self.phases ], } if extra_attributes: data.update(extra_attributes) if out: out.write(sjson.dump(data)) else: return data def write_tty(self, out=sys.stdout): """Write a human-readable summary of timings (depth is 1)""" self._flatten() times = [self.duration(p) for p in self.phases] # Get a consistent unit for the time pretty_seconds = pretty_seconds_formatter(max(times)) # Tuples of (phase, time) including total. formatted = list(zip(self.phases, times)) formatted.append(("total", self.duration())) # Write to out for name, duration in formatted: out.write(f" {name:10s} {pretty_seconds(duration):>10s}\n") #: instance of a do-nothing timer NULL_TIMER = NullTimer() ================================================ FILE: lib/spack/spack/util/typing.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details.: object # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Extra support for type checking in Spack. Protocols here that have runtime overhead should be set to ``object`` when ``TYPE_CHECKING`` is not enabled, as they can incur unreasonable runtime overheads. In particular, Protocols intended for use on objects that have many ``isinstance()`` calls can be very expensive. """ from typing import TYPE_CHECKING, Any from spack.vendor.typing_extensions import Protocol if TYPE_CHECKING: class SupportsRichComparison(Protocol): """Objects that support =, !=, <, <=, >, and >=.""" def __eq__(self, other: Any) -> bool: raise NotImplementedError def __ne__(self, other: Any) -> bool: raise NotImplementedError def __lt__(self, other: Any) -> bool: raise NotImplementedError def __le__(self, other: Any) -> bool: raise NotImplementedError def __gt__(self, other: Any) -> bool: raise NotImplementedError def __ge__(self, other: Any) -> bool: raise NotImplementedError else: SupportsRichComparison = object ================================================ FILE: lib/spack/spack/util/unparse/LICENSE ================================================ LICENSE ======= Copyright (c) 2014, Simon Percivall All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of AST Unparser nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 -------------------------------------------- 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python. 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. ================================================ FILE: lib/spack/spack/util/unparse/__init__.py ================================================ # Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers. # # SPDX-License-Identifier: Python-2.0 from .unparser import Unparser __version__ = "1.6.3" def unparse(tree, py_ver_consistent=False): unparser = Unparser(py_ver_consistent=py_ver_consistent) return unparser.visit(tree) + "\n" ================================================ FILE: lib/spack/spack/util/unparse/unparser.py ================================================ # Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers. # # SPDX-License-Identifier: Python-2.0 "Usage: unparse.py " import ast import sys from ast import AST, FormattedValue, If, JoinedStr, Name, Tuple from contextlib import contextmanager from enum import IntEnum, auto from typing import Optional # TODO: if we require Python 3.7, use its `nullcontext()` @contextmanager def nullcontext(): yield def is_non_empty_non_star_tuple(slice_value): """True for `(1, 2)`, False for `()` and `(1, *b)`""" return ( isinstance(slice_value, Tuple) and slice_value.elts and not any(isinstance(elt, ast.Starred) for elt in slice_value.elts) ) def iter_fields(node): """ Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` that is present on *node*. """ for field in node._fields: try: yield field, getattr(node, field) except AttributeError: pass class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a visitor function for every node found. This function may return a value which is forwarded by the `visit` method. This class is meant to be subclassed, with the subclass adding visitor methods. Per default the visitor functions for the nodes are ``'visit_'`` + class name of the node. So a `TryFinally` node visit function would be `visit_TryFinally`. This behavior can be changed by overriding the `visit` method. If no visitor function exists for a node (return value `None`) the `generic_visit` visitor is used instead. Don't use the `NodeVisitor` if you want to apply changes to nodes during traversing. For this a special visitor exists (`NodeTransformer`) that allows modifications. """ def visit(self, node): """Visit a node.""" method = "visit_" + node.__class__.__name__ visitor = getattr(self, method, self.generic_visit) return visitor(node) def generic_visit(self, node): """Called if no explicit visitor function exists for a node.""" for field, value in iter_fields(node): if isinstance(value, list): for item in value: if isinstance(item, AST): self.visit(item) elif isinstance(value, AST): self.visit(value) # Large float and imaginary literals get turned into infinities in the AST. # We unparse those infinities to INFSTR. _INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) class _Precedence(IntEnum): """Precedence table that originated from python grammar.""" NAMED_EXPR = auto() # := TUPLE = auto() # , YIELD = auto() # 'yield', 'yield from' TEST = auto() # 'if'-'else', 'lambda' OR = auto() # 'or' AND = auto() # 'and' NOT = auto() # 'not' CMP = auto() # '<', '>', '==', '>=', '<=', '!=', # 'in', 'not in', 'is', 'is not' EXPR = auto() BOR = EXPR # '|' BXOR = auto() # '^' BAND = auto() # '&' SHIFT = auto() # '<<', '>>' ARITH = auto() # '+', '-' TERM = auto() # '*', '@', '/', '%', '//' FACTOR = auto() # unary '+', '-', '~' POWER = auto() # '**' AWAIT = auto() # 'await' ATOM = auto() def next(self): try: return self.__class__(self + 1) except ValueError: return self _SINGLE_QUOTES = ("'", '"') _MULTI_QUOTES = ('"""', "'''") _ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) class Unparser(NodeVisitor): """Methods in this class recursively traverse an AST and output source code for the abstract syntax; original formatting is disregarded.""" def __init__(self, py_ver_consistent=False, _avoid_backslashes=False): self._source = [] self._precedences = {} self._type_ignores = {} self._indent = 0 self._in_try_star = False self._py_ver_consistent = py_ver_consistent self._avoid_backslashes = _avoid_backslashes def interleave(self, inter, f, seq): """Call f on each item in seq, calling inter() in between.""" seq = iter(seq) try: f(next(seq)) except StopIteration: pass else: for x in seq: inter() f(x) def items_view(self, traverser, items): """Traverse and separate the given *items* with a comma and append it to the buffer. If *items* is a single item sequence, a trailing comma will be added.""" if len(items) == 1: traverser(items[0]) self.write(",") else: self.interleave(lambda: self.write(", "), traverser, items) def maybe_newline(self): """Adds a newline if it isn't the start of generated source""" if self._source: self.write("\n") def fill(self, text=""): """Indent a piece of text and append it, according to the current indentation level""" self.maybe_newline() self.write(" " * self._indent + text) def write(self, *text): """Add new source parts""" self._source.extend(text) @contextmanager def buffered(self, buffer=None): if buffer is None: buffer = [] original_source = self._source self._source = buffer yield buffer self._source = original_source @contextmanager def block(self, *, extra=None): """A context manager for preparing the source for blocks. It adds the character':', increases the indentation on enter and decreases the indentation on exit. If *extra* is given, it will be directly appended after the colon character. """ self.write(":") if extra: self.write(extra) self._indent += 1 yield self._indent -= 1 @contextmanager def delimit(self, start, end): """A context manager for preparing the source for expressions. It adds *start* to the buffer and enters, after exit it adds *end*.""" self.write(start) yield self.write(end) def delimit_if(self, start, end, condition): if condition: return self.delimit(start, end) else: return nullcontext() def require_parens(self, precedence, node): """Shortcut to adding precedence related parens""" return self.delimit_if("(", ")", self.get_precedence(node) > precedence) def get_precedence(self, node): return self._precedences.get(node, _Precedence.TEST) def set_precedence(self, precedence, *nodes): for node in nodes: self._precedences[node] = precedence def get_raw_docstring(self, node): """If a docstring node is found in the body of the *node* parameter, return that docstring node, None otherwise. Logic mirrored from ``_PyAST_GetDocString``.""" if ( not isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef, ast.ClassDef, ast.Module)) or len(node.body) < 1 ): return None node = node.body[0] if not isinstance(node, ast.Expr): return None node = node.value if _is_str_literal(node): return node def get_type_comment(self, node): # Python 3.8 introduced type_comment # (enabled on compile(... ast.PyCF_TYPE_COMMENTS)) comment = self._type_ignores.get(node.lineno) or getattr(node, "type_comment", None) if comment is not None: return f" # type: {comment}" def traverse(self, node): if isinstance(node, list): for item in node: self.traverse(item) else: super().visit(node) # Note: as visit() resets the output text, do NOT rely on # NodeVisitor.generic_visit to handle any nodes (as it calls back in to # the subclass visit() method, which resets self._source to an empty list) def visit(self, node): """Outputs a source code string that, if converted back to an ast (using ast.parse) will generate an AST equivalent to *node*""" self._source = [] self.traverse(node) return "".join(self._source) def _write_docstring_and_traverse_body(self, node): docstring = self.get_raw_docstring(node) if docstring: self._write_docstring(docstring) self.traverse(node.body[1:]) else: self.traverse(node.body) def visit_Module(self, node): # Python 3.8 introduced types self._type_ignores = { ignore.lineno: f"ignore{ignore.tag}" for ignore in getattr(node, "type_ignores", ()) } self._write_docstring_and_traverse_body(node) self._type_ignores.clear() def visit_FunctionType(self, node): with self.delimit("(", ")"): self.interleave(lambda: self.write(", "), self.traverse, node.argtypes) self.write(" -> ") self.traverse(node.returns) def visit_Expr(self, node): self.fill() self.set_precedence(_Precedence.YIELD, node.value) self.traverse(node.value) def visit_NamedExpr(self, node): with self.require_parens(_Precedence.NAMED_EXPR, node): self.set_precedence(_Precedence.ATOM, node.target, node.value) self.traverse(node.target) self.write(" := ") self.traverse(node.value) def visit_Import(self, node): self.fill("import ") self.interleave(lambda: self.write(", "), self.traverse, node.names) def visit_ImportFrom(self, node): self.fill("from ") self.write("." * (node.level or 0)) if node.module: self.write(node.module) self.write(" import ") self.interleave(lambda: self.write(", "), self.traverse, node.names) def visit_Assign(self, node): self.fill() for target in node.targets: self.set_precedence(_Precedence.TUPLE, target) self.traverse(target) self.write(" = ") self.traverse(node.value) type_comment = self.get_type_comment(node) if type_comment: self.write(type_comment) def visit_AugAssign(self, node): self.fill() self.traverse(node.target) self.write(" " + self.binop[node.op.__class__.__name__] + "= ") self.traverse(node.value) def visit_AnnAssign(self, node): self.fill() with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)): self.traverse(node.target) self.write(": ") self.traverse(node.annotation) if node.value: self.write(" = ") self.traverse(node.value) def visit_Return(self, node): self.fill("return") if node.value: self.write(" ") self.traverse(node.value) def visit_Pass(self, node): self.fill("pass") def visit_Break(self, node): self.fill("break") def visit_Continue(self, node): self.fill("continue") def visit_Delete(self, node): self.fill("del ") self.interleave(lambda: self.write(", "), self.traverse, node.targets) def visit_Assert(self, node): self.fill("assert ") self.traverse(node.test) if node.msg: self.write(", ") self.traverse(node.msg) def visit_Global(self, node): self.fill("global ") self.interleave(lambda: self.write(", "), self.write, node.names) def visit_Nonlocal(self, node): self.fill("nonlocal ") self.interleave(lambda: self.write(", "), self.write, node.names) def visit_Await(self, node): with self.require_parens(_Precedence.AWAIT, node): self.write("await") if node.value: self.write(" ") self.set_precedence(_Precedence.ATOM, node.value) self.traverse(node.value) def visit_Yield(self, node): with self.require_parens(_Precedence.YIELD, node): self.write("yield") if node.value: self.write(" ") self.set_precedence(_Precedence.ATOM, node.value) self.traverse(node.value) def visit_YieldFrom(self, node): with self.require_parens(_Precedence.YIELD, node): self.write("yield from ") if not node.value: raise ValueError("Node can't be used without a value attribute.") self.set_precedence(_Precedence.ATOM, node.value) self.traverse(node.value) def visit_Raise(self, node): self.fill("raise") if not node.exc: if node.cause: raise ValueError("Node can't use cause without an exception.") return self.write(" ") self.traverse(node.exc) if node.cause: self.write(" from ") self.traverse(node.cause) def do_visit_try(self, node): self.fill("try") with self.block(): self.traverse(node.body) for ex in node.handlers: self.traverse(ex) if node.orelse: self.fill("else") with self.block(): self.traverse(node.orelse) if node.finalbody: self.fill("finally") with self.block(): self.traverse(node.finalbody) def visit_Try(self, node): prev_in_try_star = self._in_try_star try: self._in_try_star = False self.do_visit_try(node) finally: self._in_try_star = prev_in_try_star def visit_TryStar(self, node): prev_in_try_star = self._in_try_star try: self._in_try_star = True self.do_visit_try(node) finally: self._in_try_star = prev_in_try_star def visit_ExceptHandler(self, node): self.fill("except*" if self._in_try_star else "except") if node.type: self.write(" ") self.traverse(node.type) if node.name: self.write(" as ") self.write(node.name) with self.block(): self.traverse(node.body) def visit_ClassDef(self, node): self.maybe_newline() for deco in node.decorator_list: self.fill("@") self.traverse(deco) self.fill("class " + node.name) if hasattr(node, "type_params"): self._type_params_helper(node.type_params) with self.delimit_if("(", ")", condition=node.bases or node.keywords): comma = False for e in node.bases: if comma: self.write(", ") else: comma = True self.traverse(e) for e in node.keywords: if comma: self.write(", ") else: comma = True self.traverse(e) with self.block(): self._write_docstring_and_traverse_body(node) def visit_FunctionDef(self, node): self._function_helper(node, "def") def visit_AsyncFunctionDef(self, node): self._function_helper(node, "async def") def _function_helper(self, node, fill_suffix): self.maybe_newline() for deco in node.decorator_list: self.fill("@") self.traverse(deco) def_str = fill_suffix + " " + node.name self.fill(def_str) if hasattr(node, "type_params"): self._type_params_helper(node.type_params) with self.delimit("(", ")"): self.traverse(node.args) if node.returns: self.write(" -> ") self.traverse(node.returns) with self.block(extra=self.get_type_comment(node)): self._write_docstring_and_traverse_body(node) def _type_params_helper(self, type_params): if type_params is not None and len(type_params) > 0: with self.delimit("[", "]"): self.interleave(lambda: self.write(", "), self.traverse, type_params) def visit_TypeVar(self, node): self.write(node.name) if node.bound: self.write(": ") self.traverse(node.bound) # Python 3.13 introduced default_value if getattr(node, "default_value", False): self.write(" = ") self.traverse(node.default_value) def visit_TypeVarTuple(self, node): self.write("*" + node.name) # Python 3.13 introduced default_value if getattr(node, "default_value", False): self.write(" = ") self.traverse(node.default_value) def visit_ParamSpec(self, node): self.write("**" + node.name) # Python 3.13 introduced default_value if getattr(node, "default_value", False): self.write(" = ") self.traverse(node.default_value) def visit_TypeAlias(self, node): self.fill("type ") self.traverse(node.name) self._type_params_helper(node.type_params) self.write(" = ") self.traverse(node.value) def visit_For(self, node): self._for_helper("for ", node) def visit_AsyncFor(self, node): self._for_helper("async for ", node) def _for_helper(self, fill, node): self.fill(fill) self.set_precedence(_Precedence.TUPLE, node.target) self.traverse(node.target) self.write(" in ") self.traverse(node.iter) with self.block(extra=self.get_type_comment(node)): self.traverse(node.body) if node.orelse: self.fill("else") with self.block(): self.traverse(node.orelse) def visit_If(self, node): self.fill("if ") self.traverse(node.test) with self.block(): self.traverse(node.body) # collapse nested ifs into equivalent elifs. while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If): node = node.orelse[0] self.fill("elif ") self.traverse(node.test) with self.block(): self.traverse(node.body) # final else if node.orelse: self.fill("else") with self.block(): self.traverse(node.orelse) def visit_While(self, node): self.fill("while ") self.traverse(node.test) with self.block(): self.traverse(node.body) if node.orelse: self.fill("else") with self.block(): self.traverse(node.orelse) def visit_With(self, node): self.fill("with ") self.interleave(lambda: self.write(", "), self.traverse, node.items) with self.block(extra=self.get_type_comment(node)): self.traverse(node.body) def visit_AsyncWith(self, node): self.fill("async with ") self.interleave(lambda: self.write(", "), self.traverse, node.items) with self.block(extra=self.get_type_comment(node)): self.traverse(node.body) def _str_literal_helper( self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False ): """Helper for writing string literals, minimizing escapes. Returns the tuple (string literal to write, possible quote types). """ def escape_char(c): # \n and \t are non-printable, but we only escape them if # escape_special_whitespace is True if not escape_special_whitespace and c in "\n\t": return c # Always escape backslashes and other non-printable characters if c == "\\" or not c.isprintable(): return c.encode("unicode_escape").decode("ascii") return c escaped_string = "".join(map(escape_char, string)) possible_quotes = quote_types if "\n" in escaped_string: possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] possible_quotes = [q for q in possible_quotes if q not in escaped_string] if not possible_quotes: # If there aren't any possible_quotes, fallback to using repr # on the original string. Try to use a quote from quote_types, # e.g., so that we use triple quotes for docstrings. string = repr(string) quote = next((q for q in quote_types if string[0] in q), string[0]) return string[1:-1], [quote] if escaped_string: # Sort so that we prefer '''"''' over """\"""" possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) # If we're using triple quotes and we'd need to escape a final # quote, escape it if possible_quotes[0][0] == escaped_string[-1]: assert len(possible_quotes[0]) == 3 escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] return escaped_string, possible_quotes def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): """Write string literal value with a best effort attempt to avoid backslashes.""" string, quote_types = self._str_literal_helper(string, quote_types=quote_types) quote_type = quote_types[0] self.write(f"{quote_type}{string}{quote_type}") # Python < 3.8. Num, Str, Bytes, NameConstant, Ellipsis replaced with Constant # https://github.com/python/cpython/commit/3f22811fef73aec848d961593d95fa877f77ecbf if sys.version_info < (3, 8): def visit_Num(self, node): repr_n = repr(node.n) self.write(repr_n.replace("inf", _INFSTR)) def visit_Str(self, node): self._write_constant(node.s) def visit_Bytes(self, node): self.write(repr(node.s)) def visit_NameConstant(self, node): self.write(repr(node.value)) def visit_Ellipsis(self, node): self.write("...") def _ftstring_helper(self, parts): new_parts = [] quote_types = list(_ALL_QUOTES) fallback_to_repr = False for value, is_constant in parts: # Python 3.12 allows `f'{''}'`. # But we unparse to `f'{""}'` for < 3.12 compat. if True: value, new_quote_types = self._str_literal_helper( value, quote_types=quote_types, escape_special_whitespace=is_constant ) if set(new_quote_types).isdisjoint(quote_types): fallback_to_repr = True break quote_types = new_quote_types elif "\n" in value: quote_types = [q for q in quote_types if q in _MULTI_QUOTES] assert quote_types new_parts.append(value) if fallback_to_repr: # If we weren't able to find a quote type that works for all parts # of the JoinedStr, fallback to using repr and triple single quotes. quote_types = ["'''"] new_parts.clear() for value, is_constant in parts: # Python 3.12 allows `f'{''}'`. # We need to unparse to `f'{""}'` for < 3.12 compat. if True: value = repr('"' + value) # force repr to use single quotes expected_prefix = "'\"" assert value.startswith(expected_prefix), repr(value) value = value[len(expected_prefix) : -1] new_parts.append(value) value = "".join(new_parts) quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") def _write_ftstring(self, node, prefix): self.write(prefix) # Python 3.12 added support for backslashes inside format parts. # We need to keep adding backslashes for python < 3.11 compat. if self._avoid_backslashes: with self.buffered() as buffer: self._write_ftstring_inner(node) return self._write_str_avoiding_backslashes("".join(buffer)) fstring_parts = [] for value in node.values: with self.buffered() as buffer: self._write_ftstring_inner(value) fstring_parts.append(("".join(buffer), _is_str_literal(value))) self._ftstring_helper(fstring_parts) def visit_JoinedStr(self, node): self._write_ftstring(node, "f") def visit_TemplateStr(self, node): self._write_ftstring(node, "t") def _write_ftstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: self._write_ftstring_inner(value, is_format_spec=is_format_spec) elif isinstance(node, FormattedValue): self.visit_FormattedValue(node) elif _is_interpolation(node): self.visit_Interpolation(node) else: # str literal maybe_string = _get_str_literal_value(node) if maybe_string is None: raise ValueError(f"Unexpected node inside JoinedStr, {node!r}") value = maybe_string.replace("{", "{{").replace("}", "}}") if is_format_spec: value = value.replace("\\", "\\\\") value = value.replace("'", "\\'") value = value.replace('"', '\\"') value = value.replace("\n", "\\n") self.write(value) def _unparse_interpolation_value(self, inner): # Python <= 3.11 does not support backslashes inside format parts unparser = type(self)(_avoid_backslashes=True) unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) def _write_interpolation(self, node, use_str_attr=False): with self.delimit("{", "}"): if use_str_attr: expr = node.str else: expr = self._unparse_interpolation_value(node.value) # Python <= 3.11 does not support backslash in formats part if "\\" in expr: raise ValueError( "Unable to avoid backslash in f-string expression part (python 3.11)" ) if expr.startswith("{"): # Separate pair of opening brackets as "{ {" self.write(" ") self.write(expr) if node.conversion != -1: self.write(f"!{chr(node.conversion)}") if node.format_spec: self.write(":") self._write_ftstring_inner(node.format_spec, is_format_spec=True) def visit_FormattedValue(self, node): self._write_interpolation(node) def visit_Interpolation(self, node): # If `str` is set to `None`, use the `value` to generate the source code. self._write_interpolation(node, use_str_attr=node.str is not None) def visit_Name(self, node): self.write(node.id) def _write_docstring(self, node): self.fill() # Don't emit `u""` because it's not avail in python AST <= 3.7 # Ubuntu 18's Python 3.6 doesn't have "kind" if not self._py_ver_consistent and getattr(node, "kind", None) == "u": self.write("u") # Python 3.8 replaced Str with Constant value = _get_str_literal_value(node) if value is None: raise ValueError(f"Node {node!r} is not a string literal.") self._write_str_avoiding_backslashes(value, quote_types=_MULTI_QUOTES) def _write_constant(self, value): if isinstance(value, (float, complex)): # Substitute overflowing decimal literal for AST infinities, # and inf - inf for NaNs. self.write( repr(value).replace("inf", _INFSTR).replace("nan", f"({_INFSTR}-{_INFSTR})") ) # Python <= 3.11 does not support backslashes inside format parts elif self._avoid_backslashes and isinstance(value, str): self._write_str_avoiding_backslashes(value) else: self.write(repr(value)) def visit_Constant(self, node): value = node.value if isinstance(value, tuple): with self.delimit("(", ")"): self.items_view(self._write_constant, value) elif value is ...: self.write("...") else: # Don't emit `u""` because it's not avail in python AST <= 3.7 # Ubuntu 18's Python 3.6 doesn't have "kind" if not self._py_ver_consistent and getattr(node, "kind", None) == "u": self.write("u") self._write_constant(node.value) def visit_List(self, node): with self.delimit("[", "]"): self.interleave(lambda: self.write(", "), self.traverse, node.elts) def visit_ListComp(self, node): with self.delimit("[", "]"): self.traverse(node.elt) for gen in node.generators: self.traverse(gen) def visit_GeneratorExp(self, node): with self.delimit("(", ")"): self.traverse(node.elt) for gen in node.generators: self.traverse(gen) def visit_SetComp(self, node): with self.delimit("{", "}"): self.traverse(node.elt) for gen in node.generators: self.traverse(gen) def visit_DictComp(self, node): with self.delimit("{", "}"): self.traverse(node.key) self.write(": ") self.traverse(node.value) for gen in node.generators: self.traverse(gen) def visit_comprehension(self, node): if node.is_async: self.write(" async for ") else: self.write(" for ") self.set_precedence(_Precedence.TUPLE, node.target) self.traverse(node.target) self.write(" in ") self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs) self.traverse(node.iter) for if_clause in node.ifs: self.write(" if ") self.traverse(if_clause) def visit_IfExp(self, node): with self.require_parens(_Precedence.TEST, node): self.set_precedence(_Precedence.TEST.next(), node.body, node.test) self.traverse(node.body) self.write(" if ") self.traverse(node.test) self.write(" else ") self.set_precedence(_Precedence.TEST, node.orelse) self.traverse(node.orelse) def visit_Set(self, node): if node.elts: with self.delimit("{", "}"): self.interleave(lambda: self.write(", "), self.traverse, node.elts) else: # `{}` would be interpreted as a dictionary literal, and # `set` might be shadowed. Thus: self.write("{*()}") def visit_Dict(self, node): def write_key_value_pair(k, v): self.traverse(k) self.write(": ") self.traverse(v) def write_item(item): k, v = item if k is None: # for dictionary unpacking operator in dicts {**{'y': 2}} # see PEP 448 for details self.write("**") self.set_precedence(_Precedence.EXPR, v) self.traverse(v) else: write_key_value_pair(k, v) with self.delimit("{", "}"): self.interleave(lambda: self.write(", "), write_item, zip(node.keys, node.values)) def visit_Tuple(self, node): with self.delimit_if( "(", ")", # Don't drop redundant parenthesis to mimic python <= 3.10 self._py_ver_consistent or len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE, ): self.items_view(self.traverse, node.elts) unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} unop_precedence = { "not": _Precedence.NOT, "~": _Precedence.FACTOR, "+": _Precedence.FACTOR, "-": _Precedence.FACTOR, } def visit_UnaryOp(self, node): operator = self.unop[node.op.__class__.__name__] operator_precedence = self.unop_precedence[operator] with self.require_parens(operator_precedence, node): self.write(operator) # factor prefixes (+, -, ~) shouldn't be separated # from the value they belong, (e.g: +1 instead of + 1) if operator_precedence is not _Precedence.FACTOR: self.write(" ") self.set_precedence(operator_precedence, node.operand) self.traverse(node.operand) binop = { "Add": "+", "Sub": "-", "Mult": "*", "MatMult": "@", "Div": "/", "Mod": "%", "LShift": "<<", "RShift": ">>", "BitOr": "|", "BitXor": "^", "BitAnd": "&", "FloorDiv": "//", "Pow": "**", } binop_precedence = { "+": _Precedence.ARITH, "-": _Precedence.ARITH, "*": _Precedence.TERM, "@": _Precedence.TERM, "/": _Precedence.TERM, "%": _Precedence.TERM, "<<": _Precedence.SHIFT, ">>": _Precedence.SHIFT, "|": _Precedence.BOR, "^": _Precedence.BXOR, "&": _Precedence.BAND, "//": _Precedence.TERM, "**": _Precedence.POWER, } binop_rassoc = frozenset(("**",)) def visit_BinOp(self, node): operator = self.binop[node.op.__class__.__name__] operator_precedence = self.binop_precedence[operator] with self.require_parens(operator_precedence, node): if operator in self.binop_rassoc: left_precedence = operator_precedence.next() right_precedence = operator_precedence else: left_precedence = operator_precedence right_precedence = operator_precedence.next() self.set_precedence(left_precedence, node.left) self.traverse(node.left) self.write(f" {operator} ") self.set_precedence(right_precedence, node.right) self.traverse(node.right) cmpops = { "Eq": "==", "NotEq": "!=", "Lt": "<", "LtE": "<=", "Gt": ">", "GtE": ">=", "Is": "is", "IsNot": "is not", "In": "in", "NotIn": "not in", } def visit_Compare(self, node): with self.require_parens(_Precedence.CMP, node): self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators) self.traverse(node.left) for o, e in zip(node.ops, node.comparators): self.write(" " + self.cmpops[o.__class__.__name__] + " ") self.traverse(e) boolops = {"And": "and", "Or": "or"} boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR} def visit_BoolOp(self, node): operator = self.boolops[node.op.__class__.__name__] operator_precedence = self.boolop_precedence[operator] def increasing_level_traverse(node): nonlocal operator_precedence operator_precedence = operator_precedence.next() self.set_precedence(operator_precedence, node) self.traverse(node) with self.require_parens(operator_precedence, node): s = f" {operator} " self.interleave(lambda: self.write(s), increasing_level_traverse, node.values) def visit_Attribute(self, node: ast.Attribute): self.set_precedence(_Precedence.ATOM, node.value) self.traverse(node.value) # Special case: 3.__abs__() is a syntax error, so if node.value # is an integer literal then we need to either parenthesize # it or add an extra space to get 3 .__abs__(). if _is_int_literal(node.value): self.write(" ") self.write(".") self.write(node.attr) def visit_Call(self, node): self.set_precedence(_Precedence.ATOM, node.func) self.traverse(node.func) with self.delimit("(", ")"): comma = False for e in node.args: if comma: self.write(", ") else: comma = True self.traverse(e) for e in node.keywords: if comma: self.write(", ") else: comma = True self.traverse(e) def visit_Subscript(self, node): def is_non_empty_tuple(slice_value): return isinstance(slice_value, Tuple) and slice_value.elts self.set_precedence(_Precedence.ATOM, node.value) self.traverse(node.value) with self.delimit("[", "]"): # Python >= 3.11 supports `a[42, *b]` (same AST as a[(42, *b)]), # but this is syntax error in 3.10. # So, always emit parenthesis `a[(42, *b)]` if is_non_empty_non_star_tuple(node.slice): self.items_view(self.traverse, node.slice.elts) else: self.traverse(node.slice) def visit_Starred(self, node): self.write("*") self.set_precedence(_Precedence.EXPR, node.value) self.traverse(node.value) # Python 3.9 simplified Subscript(Index(value)) to Subscript(value) # https://github.com/python/cpython/commit/13d52c268699f199a8e917a0f1dc4c51e5346c42 def visit_Index(self, node): if is_non_empty_non_star_tuple(node.value): self.items_view(self.traverse, node.value.elts) else: self.traverse(node.value) def visit_Slice(self, node): if node.lower: self.traverse(node.lower) self.write(":") if node.upper: self.traverse(node.upper) if node.step: self.write(":") self.traverse(node.step) def visit_Match(self, node): self.fill("match ") self.traverse(node.subject) with self.block(): for case in node.cases: self.traverse(case) # Python 3.9 replaced ExtSlice(slices) with Tuple(slices, Load()) # https://github.com/python/cpython/commit/13d52c268699f199a8e917a0f1dc4c51e5346c42 def visit_ExtSlice(self, node): self.interleave(lambda: self.write(", "), self.traverse, node.dims) def visit_arg(self, node): self.write(node.arg) if node.annotation: self.write(": ") self.traverse(node.annotation) def visit_arguments(self, node): first = True # normal arguments # Python 3.8 introduced position-only arguments (PEP 570) all_args = getattr(node, "posonlyargs", []) + node.args defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults for index, elements in enumerate(zip(all_args, defaults), 1): a, d = elements if first: first = False else: self.write(", ") self.traverse(a) if d: self.write("=") self.traverse(d) # Python 3.8 introduced position-only arguments (PEP 570) if index == len(getattr(node, "posonlyargs", ())): self.write(", /") # varargs, or bare '*' if no varargs but keyword-only arguments present if node.vararg or node.kwonlyargs: if first: first = False else: self.write(", ") self.write("*") if node.vararg: self.write(node.vararg.arg) if node.vararg.annotation: self.write(": ") self.traverse(node.vararg.annotation) # keyword-only arguments if node.kwonlyargs: for a, d in zip(node.kwonlyargs, node.kw_defaults): self.write(", ") self.traverse(a) if d: self.write("=") self.traverse(d) # kwargs if node.kwarg: if first: first = False else: self.write(", ") self.write("**" + node.kwarg.arg) if node.kwarg.annotation: self.write(": ") self.traverse(node.kwarg.annotation) def visit_keyword(self, node): if node.arg is None: self.write("**") else: self.write(node.arg) self.write("=") self.traverse(node.value) def visit_Lambda(self, node): with self.require_parens(_Precedence.TEST, node): self.write("lambda") with self.buffered() as buffer: self.traverse(node.args) # Don't omit extra space to keep old package hash # (extra space was removed in python 3.11) if buffer or self._py_ver_consistent: self.write(" ", *buffer) self.write(": ") self.set_precedence(_Precedence.TEST, node.body) self.traverse(node.body) def visit_alias(self, node): self.write(node.name) if node.asname: self.write(" as " + node.asname) def visit_withitem(self, node): self.traverse(node.context_expr) if node.optional_vars: self.write(" as ") self.traverse(node.optional_vars) def visit_match_case(self, node): self.fill("case ") self.traverse(node.pattern) if node.guard: self.write(" if ") self.traverse(node.guard) with self.block(): self.traverse(node.body) def visit_MatchValue(self, node): self.traverse(node.value) def visit_MatchSingleton(self, node): self._write_constant(node.value) def visit_MatchSequence(self, node): with self.delimit("[", "]"): self.interleave(lambda: self.write(", "), self.traverse, node.patterns) def visit_MatchStar(self, node): name = node.name if name is None: name = "_" self.write(f"*{name}") def visit_MatchMapping(self, node): def write_key_pattern_pair(pair): k, p = pair self.traverse(k) self.write(": ") self.traverse(p) with self.delimit("{", "}"): keys = node.keys self.interleave( lambda: self.write(", "), write_key_pattern_pair, # (zip strict is >= Python 3.10) zip(keys, node.patterns), ) rest = node.rest if rest is not None: if keys: self.write(", ") self.write(f"**{rest}") def visit_MatchClass(self, node): self.set_precedence(_Precedence.ATOM, node.cls) self.traverse(node.cls) with self.delimit("(", ")"): patterns = node.patterns self.interleave(lambda: self.write(", "), self.traverse, patterns) attrs = node.kwd_attrs if attrs: def write_attr_pattern(pair): attr, pattern = pair self.write(f"{attr}=") self.traverse(pattern) if patterns: self.write(", ") self.interleave( lambda: self.write(", "), write_attr_pattern, # (zip strict is >= Python 3.10) zip(attrs, node.kwd_patterns), ) def visit_MatchAs(self, node): name = node.name pattern = node.pattern if name is None: self.write("_") elif pattern is None: self.write(node.name) else: with self.require_parens(_Precedence.TEST, node): self.set_precedence(_Precedence.BOR, node.pattern) self.traverse(node.pattern) self.write(f" as {node.name}") def visit_MatchOr(self, node): with self.require_parens(_Precedence.BOR, node): self.set_precedence(_Precedence.BOR.next(), *node.patterns) self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) if sys.version_info >= (3, 8): def _is_int_literal(node: ast.AST) -> bool: """Check if a node represents a literal int.""" return isinstance(node, ast.Constant) and isinstance(node.value, int) def _is_str_literal(node: ast.AST) -> bool: """Check if a node represents a literal str.""" return isinstance(node, ast.Constant) and isinstance(node.value, str) def _get_str_literal_value(node: ast.AST) -> Optional[str]: """Get the string value of a literal str node.""" if isinstance(node, ast.Constant) and isinstance(node.value, str): return node.value return None else: def _is_int_literal(node: ast.AST) -> bool: """Check if a node represents a literal int.""" return isinstance(node, ast.Num) and isinstance(node.n, int) def _is_str_literal(node: ast.AST) -> bool: """Check if a node represents a literal str.""" return isinstance(node, ast.Str) def _get_str_literal_value(node: ast.AST) -> Optional[str]: """Get the string value of a literal str node.""" return node.s if isinstance(node, ast.Str) else None if sys.version_info >= (3, 14): def _is_interpolation(node: ast.AST) -> bool: """Check if a node represents a template string literal.""" return isinstance(node, ast.Interpolation) else: def _is_interpolation(node: ast.AST) -> bool: """Check if a node represents a template string literal.""" return False ================================================ FILE: lib/spack/spack/util/url.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ Utility functions for parsing, formatting, and manipulating URLs. """ import posixpath import re import urllib.parse import urllib.request from pathlib import Path from typing import Optional from spack.util.path import sanitize_filename def validate_scheme(scheme): """Returns true if the URL scheme is generally known to Spack. This function helps mostly in validation of paths vs urls, as Windows paths such as C:/x/y/z (with backward not forward slash) may parse as a URL with scheme C and path /x/y/z.""" return scheme in ("file", "http", "https", "ftp", "s3", "gs", "ssh", "git", "oci") def local_file_path(url): """Get a local file path from a url. If url is a ``file://`` URL, return the absolute path to the local file or directory referenced by it. Otherwise, return None. """ if isinstance(url, str): url = urllib.parse.urlparse(url) if url.scheme == "file": return urllib.request.url2pathname(url.path) return None def path_to_file_url(path): return Path(path).absolute().as_uri() def file_url_string_to_path(url): return urllib.request.url2pathname(urllib.parse.urlparse(url).path) def is_path_instead_of_url(path_or_url): """Historically some config files and spack commands used paths where urls should be used. This utility can be used to validate and promote paths to urls.""" return not validate_scheme(urllib.parse.urlparse(path_or_url).scheme) def format(parsed_url): """Format a URL string Returns a canonicalized format of the given URL as a string. """ if isinstance(parsed_url, str): parsed_url = urllib.parse.urlparse(parsed_url) return parsed_url.geturl() def join(base: str, *components: str, resolve_href: bool = False, **kwargs) -> str: """Convenience wrapper around :func:`urllib.parse.urljoin`, with a few differences: 1. By default ``resolve_href=False``, which makes the function like :func:`os.path.join`. For example ``https://example.com/a/b + c/d = https://example.com/a/b/c/d``. If ``resolve_href=True``, the behavior is how a browser would resolve the URL: ``https://example.com/a/c/d``. 2. ``s3://``, ``gs://``, ``oci://`` URLs are joined like ``http://`` URLs. 3. It accepts multiple components for convenience. Note that ``components[1:]`` are treated as literal path components and appended to ``components[0]`` separated by slashes.""" # Ensure a trailing slash in the path component of the base URL to get os.path.join-like # behavior instead of web browser behavior. if not resolve_href: parsed = urllib.parse.urlparse(base) if not parsed.path.endswith("/"): base = parsed._replace(path=f"{parsed.path}/").geturl() old_netloc = urllib.parse.uses_netloc old_relative = urllib.parse.uses_relative try: # NOTE: we temporarily modify urllib internals so s3 and gs schemes are treated like http. # This is non-portable, and may be forward incompatible with future cpython versions. urllib.parse.uses_netloc = [*old_netloc, "s3", "gs", "oci", "oci+http"] # type: ignore urllib.parse.uses_relative = [*old_relative, "s3", "gs", "oci", "oci+http"] # type: ignore return urllib.parse.urljoin(base, "/".join(components), **kwargs) finally: urllib.parse.uses_netloc = old_netloc # type: ignore urllib.parse.uses_relative = old_relative # type: ignore def default_download_filename(url: str) -> str: """This method computes a default file name for a given URL. Note that it makes no request, so this is not the same as the option curl -O, which uses the remote file name from the response header.""" parsed_url = urllib.parse.urlparse(url) # Only use the last path component + params + query + fragment name = urllib.parse.urlunparse( parsed_url._replace(scheme="", netloc="", path=posixpath.basename(parsed_url.path)) ) valid_name = sanitize_filename(name) # Don't download to hidden files please if valid_name[0] == ".": valid_name = "_" + valid_name[1:] return valid_name def parse_link_rel_next(link_value: str) -> Optional[str]: """Return the next link from a Link header value, if any.""" # Relaxed version of RFC5988 uri = re.compile(r"\s*<([^>]+)>\s*") param_key = r"[^;=\s]+" quoted_string = r"\"([^\"]+)\"" unquoted_param_value = r"([^;,\s]+)" param = re.compile(rf";\s*({param_key})\s*=\s*(?:{quoted_string}|{unquoted_param_value})\s*") data = link_value # Parse a list of ; key=value; key=value, ; key=value; key=value, ... links. while True: uri_match = re.match(uri, data) if not uri_match: break uri_reference = uri_match.group(1) data = data[uri_match.end() :] # Parse parameter list while True: param_match = re.match(param, data) if not param_match: break key, quoted_value, unquoted_value = param_match.groups() value = quoted_value or unquoted_value data = data[param_match.end() :] if key == "rel" and value == "next": return uri_reference if not data.startswith(","): break data = data[1:] return None ================================================ FILE: lib/spack/spack/util/web.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import email.message import errno import functools import io import json import os import re import shutil import socket import ssl import stat import sys import time import traceback import urllib.parse from html.parser import HTMLParser from http.client import IncompleteRead from pathlib import Path, PurePosixPath from typing import IO, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, Union from urllib.error import HTTPError, URLError from urllib.request import HTTPDefaultErrorHandler, HTTPSHandler, Request, build_opener from spack.vendor.typing_extensions import ParamSpec import spack import spack.config import spack.error import spack.llnl.url import spack.util.executable import spack.util.parallel import spack.util.path import spack.util.url as url_util from spack.llnl.util import lang, tty from spack.llnl.util.filesystem import mkdirp, rename, working_dir from .executable import CommandNotFoundError, Executable from .gcs import GCSBlob, GCSBucket, GCSHandler from .s3 import UrllibS3Handler, get_s3_session def is_transient_error(e: Exception) -> bool: """Return True for HTTP/network errors that are worth retrying.""" if isinstance(e, HTTPError) and (500 <= e.code < 600 or e.code == 429): return True if isinstance(e, URLError) and isinstance(e.reason, socket.timeout): return True if isinstance(e, (socket.timeout, IncompleteRead)): return True # exceptions not inherited from the above used in urllib3 and botocore. if type(e).__name__ in ( "ConnectionClosedError", "IncompleteReadError", "ProtocolError", "ReadTimeoutError", "ResponseStreamingError", ): return True return False _P = ParamSpec("_P") _R = TypeVar("_R") def retry_on_transient_error( f: Callable[_P, _R], retries: int = 5, sleep: Optional[Callable[[float], None]] = None ) -> Callable[_P, _R]: """Retry a function on transient HTTP/network errors with exponential backoff.""" sleep = sleep or time.sleep @functools.wraps(f) def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: for i in range(retries): try: return f(*args, **kwargs) except Exception as e: if i + 1 != retries and is_transient_error(e): sleep(2**i) # type: ignore[misc] # mypy still thinks it's possibly None. continue raise raise AssertionError("unreachable") return wrapper class DetailedHTTPError(HTTPError): def __init__( self, req: Request, code: int, msg: str, hdrs: email.message.Message, fp: Optional[IO] ) -> None: self.req = req super().__init__(req.get_full_url(), code, msg, hdrs, fp) def __str__(self): # Note: HTTPError, is actually a kind of non-seekable response object, so # best not to read the response body here (even if it may include a human-readable # error message). # Note: use self.filename, not self.url, because the latter requires fp to be an # IO object, which is not the case after unpickling. return f"{self.req.get_method()} {self.filename} returned {self.code}: {self.msg}" def __reduce__(self): # fp is an IO object and not picklable, the rest should be. return DetailedHTTPError, (self.req, self.code, self.msg, self.hdrs, None) class DetailedURLError(URLError): def __init__(self, req: Request, reason): super().__init__(reason) self.req = req def __str__(self): return f"{self.req.get_method()} {self.req.get_full_url()} errored with: {self.reason}" def __reduce__(self): return DetailedURLError, (self.req, self.reason) class SpackHTTPDefaultErrorHandler(HTTPDefaultErrorHandler): def http_error_default(self, req, fp, code, msg, hdrs): raise DetailedHTTPError(req, code, msg, hdrs, fp) class SpackHTTPSHandler(HTTPSHandler): """A custom HTTPS handler that shows more detailed error messages on connection failure.""" def https_open(self, req): try: return super().https_open(req) except HTTPError: raise except URLError as e: raise DetailedURLError(req, e.reason) from e def custom_ssl_certs() -> Optional[Tuple[bool, str]]: """Returns a tuple (is_file, path) if custom SSL certifates are configured and valid.""" ssl_certs = spack.config.get("config:ssl_certs") if not ssl_certs: return None path = spack.util.path.substitute_path_variables(ssl_certs) if not os.path.isabs(path): tty.debug(f"certs: relative path not allowed: {path}") return None try: st = os.stat(path) except OSError as e: tty.debug(f"certs: error checking path {path}: {e}") return None file_type = stat.S_IFMT(st.st_mode) if file_type != stat.S_IFREG and file_type != stat.S_IFDIR: tty.debug(f"certs: not a file or directory: {path}") return None return (file_type == stat.S_IFREG, path) def ssl_create_default_context(): """Create the default SSL context for urllib with custom certificates if configured.""" certs = custom_ssl_certs() if certs is None: return ssl.create_default_context() is_file, path = certs if is_file: tty.debug(f"urllib: certs: using cafile {path}") return ssl.create_default_context(cafile=path) else: tty.debug(f"urllib: certs: using capath {path}") return ssl.create_default_context(capath=path) def set_curl_env_for_ssl_certs(curl: Executable) -> None: """configure curl to use custom certs in a file at runtime. See: https://curl.se/docs/sslcerts.html item 4""" certs = custom_ssl_certs() if certs is None: return is_file, path = certs if not is_file: tty.debug(f"curl: {path} is not a file: default certs will be used.") return tty.debug(f"curl: using CURL_CA_BUNDLE={path}") curl.add_default_env("CURL_CA_BUNDLE", path) def _urlopen(): s3 = UrllibS3Handler() gcs = GCSHandler() error_handler = SpackHTTPDefaultErrorHandler() # One opener with HTTPS ssl enabled with_ssl = build_opener( s3, gcs, SpackHTTPSHandler(context=ssl_create_default_context()), error_handler ) # One opener with HTTPS ssl disabled without_ssl = build_opener( s3, gcs, SpackHTTPSHandler(context=ssl._create_unverified_context()), error_handler ) # And dynamically dispatch based on the config:verify_ssl. def dispatch_open(fullurl, data=None, timeout=None): opener = with_ssl if spack.config.get("config:verify_ssl", True) else without_ssl timeout = timeout or spack.config.get("config:connect_timeout", 10) return opener.open(fullurl, data, timeout) return dispatch_open #: Dispatches to the correct OpenerDirector.open, based on Spack configuration. urlopen = lang.Singleton(_urlopen) #: User-Agent used in Request objects SPACK_USER_AGENT = "Spackbot/{0}".format(spack.spack_version) # Also, HTMLParseError is deprecated and never raised. class HTMLParseError(Exception): pass class LinkParser(HTMLParser): """This parser just takes an HTML page and strips out the hrefs on the links, as well as some javascript tags used on GitLab servers. Good enough for a really simple spider.""" def __init__(self): super().__init__() self.links = [] def handle_starttag(self, tag, attrs): if tag == "a": self.links.extend(val for key, val in attrs if key == "href") # GitLab uses a javascript function to place dropdown links: #
# noqa: E501 if tag == "div" and ("class", "js-source-code-dropdown") in attrs: try: links_str = next(val for key, val in attrs if key == "data-download-links") links = json.loads(links_str) self.links.extend(x["path"] for x in links) except Exception: pass class ExtractMetadataParser(HTMLParser): """This parser takes an HTML page and selects the include-fragments, used on GitHub, https://github.github.io/include-fragment-element, as well as a possible base url.""" def __init__(self): super().__init__() self.fragments = [] self.base_url = None def handle_starttag(self, tag, attrs): # if tag == "include-fragment": for attr, val in attrs: if attr == "src": self.fragments.append(val) # elif tag == "base": for attr, val in attrs: if attr == "href": self.base_url = val def read_from_url(url, accept_content_type=None): if isinstance(url, str): url = urllib.parse.urlparse(url) # Timeout in seconds for web requests request = Request(url.geturl(), headers={"User-Agent": SPACK_USER_AGENT}) try: response = urlopen(request) except OSError as e: raise SpackWebError(f"Download of {url.geturl()} failed: {e.__class__.__name__}: {e}") if accept_content_type: try: content_type = get_header(response.headers, "Content-type") reject_content_type = not content_type.startswith(accept_content_type) except KeyError: content_type = None reject_content_type = True if reject_content_type: msg = "ignoring page {}".format(url.geturl()) if content_type: msg += " with content type {}".format(content_type) tty.debug(msg) return None, None, None return response.url, response.headers, response def _read_text(url: str) -> str: request = Request(url, headers={"User-Agent": SPACK_USER_AGENT}) with urlopen(request) as response: return io.TextIOWrapper(response, encoding="utf-8").read() def _read_json(url: str): request = Request(url, headers={"User-Agent": SPACK_USER_AGENT}) with urlopen(request) as response: return json.load(response) _read_text_with_retry = retry_on_transient_error(_read_text) _read_json_with_retry = retry_on_transient_error(_read_json) def read_text(url: str) -> str: """Fetch url and return the response body decoded as UTF-8 text.""" try: return _read_text_with_retry(url) except Exception as e: raise SpackWebError(f"Download of {url} failed: {e.__class__.__name__}: {e}") def read_json(url: str): """Fetch url and return the response body parsed as JSON.""" try: return _read_json_with_retry(url) except Exception as e: raise SpackWebError(f"Download of {url} failed: {e.__class__.__name__}: {e}") def push_to_url(local_file_path, remote_path, keep_original=True, extra_args=None): remote_url = urllib.parse.urlparse(remote_path) if remote_url.scheme == "file": remote_file_path = url_util.local_file_path(remote_url) mkdirp(os.path.dirname(remote_file_path)) if keep_original: shutil.copy(local_file_path, remote_file_path) else: try: rename(local_file_path, remote_file_path) except OSError as e: if e.errno == errno.EXDEV: # NOTE(opadron): The above move failed because it crosses # filesystem boundaries. Copy the file (plus original # metadata), and then delete the original. This operation # needs to be done in separate steps. shutil.copy2(local_file_path, remote_file_path) os.remove(local_file_path) else: raise elif remote_url.scheme == "s3": if extra_args is None: extra_args = {} remote_path = remote_url.path while remote_path.startswith("/"): remote_path = remote_path[1:] s3 = get_s3_session(remote_url, method="push") s3.upload_file(local_file_path, remote_url.netloc, remote_path, ExtraArgs=extra_args) if not keep_original: os.remove(local_file_path) elif remote_url.scheme == "gs": gcs = GCSBlob(remote_url) gcs.upload_to_blob(local_file_path) if not keep_original: os.remove(local_file_path) else: raise NotImplementedError(f"Unrecognized URL scheme: {remote_url.scheme}") def base_curl_fetch_args(url, timeout=0): """Return the basic fetch arguments typically used in calls to curl. The arguments include those for ensuring behaviors such as failing on errors for codes over 400, printing HTML headers, resolving 3xx redirects, status or failure handling, and connection timeouts. It also uses the following configuration option to set an additional argument as needed: * config:connect_timeout (int): connection timeout * config:verify_ssl (str): Perform SSL verification Arguments: url (str): URL whose contents will be fetched timeout (int): Connection timeout, which is only used if higher than config:connect_timeout Returns (list): list of argument strings """ curl_args = [ "-f", # fail on >400 errors "-D", "-", # "-D -" prints out HTML headers "-L", # resolve 3xx redirects url, ] if not spack.config.get("config:verify_ssl"): curl_args.append("-k") if sys.stdout.isatty() and tty.msg_enabled(): curl_args.append("-#") # status bar when using a tty else: curl_args.append("-sS") # show errors if fail connect_timeout = spack.config.get("config:connect_timeout", 10) if timeout: connect_timeout = max(int(connect_timeout), int(timeout)) if connect_timeout > 0: curl_args.extend(["--connect-timeout", str(connect_timeout)]) return curl_args def check_curl_code(returncode: int) -> None: """Check standard return code failures for provided arguments. Arguments: returncode: curl return code Raises FetchError if the curl returncode indicates failure """ if returncode == 0: return elif returncode == 22: # This is a 404. Curl will print the error. raise spack.error.FetchError("URL was not found!") elif returncode == 60: # This is a certificate error. Suggest spack -k raise spack.error.FetchError( "Curl was unable to fetch due to invalid certificate. " "This is either an attack, or your cluster's SSL " "configuration is bad. If you believe your SSL " "configuration is bad, you can try running spack -k, " "which will not check SSL certificates." "Use this at your own risk." ) raise spack.error.FetchError(f"Curl failed with error {returncode}") def require_curl() -> Executable: try: path = spack.util.executable.which_string("curl", required=True) except CommandNotFoundError as e: raise spack.error.FetchError(f"curl is required but not found: {e}") from e curl = spack.util.executable.Executable(path) set_curl_env_for_ssl_certs(curl) return curl def fetch_url_text(url, curl: Optional[Executable] = None, dest_dir="."): """Retrieves text-only URL content using the configured fetch method. It determines the fetch method from: * config:url_fetch_method (str): fetch method to use (e.g., 'curl') If the method is ``curl``, it also uses the following configuration options: * config:connect_timeout (int): connection time out * config:verify_ssl (str): Perform SSL verification Arguments: url (str): URL whose contents are to be fetched curl (spack.util.executable.Executable or None): (optional) curl executable if curl is the configured fetch method dest_dir (str): (optional) destination directory for fetched text file Returns (str or None): path to the fetched file Raises FetchError if the curl returncode indicates failure """ if not url: raise spack.error.FetchError("A URL is required to fetch its text") tty.debug("Fetching text at {0}".format(url)) filename = os.path.basename(url) path = os.path.join(dest_dir, filename) fetch_method = spack.config.get("config:url_fetch_method") tty.debug("Using '{0}' to fetch {1} into {2}".format(fetch_method, url, path)) if fetch_method and fetch_method.startswith("curl"): curl_exe = curl or require_curl() curl_args = fetch_method.split()[1:] + ["-O"] curl_args.extend(base_curl_fetch_args(url)) # Curl automatically downloads file contents as filename with working_dir(dest_dir, create=True): _ = curl_exe(*curl_args, fail_on_error=False, output=os.devnull) check_curl_code(curl_exe.returncode) return path else: try: output = read_text(url) if output: with working_dir(dest_dir, create=True): with open(filename, "w", encoding="utf-8") as f: f.write(output) return path except (SpackWebError, OSError, ValueError) as err: raise spack.error.FetchError(f"Urllib fetch failed: {err}") return None def _url_exists_urllib_impl(url): with urlopen( Request(url, method="HEAD", headers={"User-Agent": SPACK_USER_AGENT}), timeout=spack.config.get("config:connect_timeout", 10), ) as _: pass _url_exists_urllib = retry_on_transient_error(_url_exists_urllib_impl) def url_exists(url, curl=None): """Determines whether url exists. A scheme-specific process is used for Google Storage (``gs``) and Amazon Simple Storage Service (``s3``) URLs; otherwise, the configured fetch method defined by ``config:url_fetch_method`` is used. Arguments: url (str): URL whose existence is being checked curl (spack.util.executable.Executable or None): (optional) curl executable if curl is the configured fetch method Returns (bool): True if it exists; False otherwise. """ tty.debug("Checking existence of {0}".format(url)) url_result = urllib.parse.urlparse(url) # Use curl if configured to do so fetch_method = spack.config.get("config:url_fetch_method", "urllib") use_curl = fetch_method.startswith("curl") and url_result.scheme not in ("gs", "s3") if use_curl: curl_exe = curl or require_curl() # Telling curl to fetch the first byte (-r 0-0) is supposed to be # portable. curl_args = fetch_method.split()[1:] + ["--stderr", "-", "-s", "-f", "-r", "0-0", url] if not spack.config.get("config:verify_ssl"): curl_args.append("-k") _ = curl_exe(*curl_args, fail_on_error=False, output=os.devnull) return curl_exe.returncode == 0 # Otherwise use urllib. try: _url_exists_urllib(url) return True except Exception as e: tty.debug(f"Failure reading {url}: {e}") return False def _debug_print_delete_results(result): if "Deleted" in result: for d in result["Deleted"]: tty.debug("Deleted {0}".format(d["Key"])) if "Errors" in result: for e in result["Errors"]: tty.debug("Failed to delete {0} ({1})".format(e["Key"], e["Message"])) def remove_url(url, recursive=False): url = urllib.parse.urlparse(url) local_path = url_util.local_file_path(url) if local_path: if recursive: shutil.rmtree(local_path) else: os.remove(local_path) return if url.scheme == "s3": # Try to find a mirror for potential connection information s3 = get_s3_session(url, method="push") bucket = url.netloc if recursive: # Because list_objects_v2 can only return up to 1000 items # at a time, we have to paginate to make sure we get it all prefix = url.path.strip("/") paginator = s3.get_paginator("list_objects_v2") pages = paginator.paginate(Bucket=bucket, Prefix=prefix) delete_request = {"Objects": []} for item in pages.search("Contents"): if not item: continue delete_request["Objects"].append({"Key": item["Key"]}) # Make sure we do not try to hit S3 with a list of more # than 1000 items if len(delete_request["Objects"]) >= 1000: r = s3.delete_objects(Bucket=bucket, Delete=delete_request) _debug_print_delete_results(r) delete_request = {"Objects": []} # Delete any items that remain if len(delete_request["Objects"]): r = s3.delete_objects(Bucket=bucket, Delete=delete_request) _debug_print_delete_results(r) else: s3.delete_object(Bucket=bucket, Key=url.path.lstrip("/")) return elif url.scheme == "gs": if recursive: bucket = GCSBucket(url) bucket.destroy(recursive=recursive) else: blob = GCSBlob(url) blob.delete_blob() return # Don't even try for other URL schemes. def _iter_s3_contents(contents, prefix): for entry in contents: key = entry["Key"] if not key.startswith("/"): key = "/" + key key = os.path.relpath(key, prefix) if key == ".": continue yield key def _list_s3_objects(client, bucket, prefix, num_entries, start_after=None): list_args = dict(Bucket=bucket, Prefix=prefix[1:], MaxKeys=num_entries) if start_after is not None: list_args["StartAfter"] = start_after result = client.list_objects_v2(**list_args) last_key = None if result["IsTruncated"]: last_key = result["Contents"][-1]["Key"] iter = _iter_s3_contents(result["Contents"], prefix) return iter, last_key def _iter_s3_prefix(client, url, num_entries=1024): key = None bucket = url.netloc prefix = re.sub(r"^/*", "/", url.path) while True: contents, key = _list_s3_objects(client, bucket, prefix, num_entries, start_after=key) for x in contents: yield x if not key: break def _iter_local_prefix(path): for root, _, files in os.walk(path): for f in files: yield os.path.relpath(os.path.join(root, f), path) def list_url(url, recursive=False): url = urllib.parse.urlparse(url) local_path = url_util.local_file_path(url) if local_path: if recursive: # convert backslash to forward slash as required for URLs return [str(PurePosixPath(Path(p))) for p in _iter_local_prefix(local_path)] return [ subpath for subpath in os.listdir(local_path) if os.path.isfile(os.path.join(local_path, subpath)) ] if url.scheme == "s3": s3 = get_s3_session(url, method="fetch") if recursive: return list(_iter_s3_prefix(s3, url)) return list(set(key.split("/", 1)[0] for key in _iter_s3_prefix(s3, url))) elif url.scheme == "gs": gcs = GCSBucket(url) return gcs.get_all_blobs(recursive=recursive) def stat_url(url: str) -> Optional[Tuple[int, float]]: """Get stat result for a URL. Args: url: URL to get stat result for Returns: A tuple of (size, mtime) if the URL exists, None otherwise. """ parsed_url = urllib.parse.urlparse(url) if parsed_url.scheme == "file": local_file_path = url_util.local_file_path(parsed_url) assert isinstance(local_file_path, str) try: url_stat = Path(local_file_path).stat() except FileNotFoundError: return None return url_stat.st_size, url_stat.st_mtime elif parsed_url.scheme == "s3": s3_bucket = parsed_url.netloc s3_key = parsed_url.path.lstrip("/") s3 = get_s3_session(url, method="fetch") try: head_request = s3.head_object(Bucket=s3_bucket, Key=s3_key) except s3.ClientError as e: if e.response["Error"]["Code"] == "404": return None raise e mtime = head_request["LastModified"].timestamp() size = head_request["ContentLength"] return size, mtime else: raise NotImplementedError(f"Unrecognized URL scheme: {parsed_url.scheme}") def spider( root_urls: Union[str, Iterable[str]], depth: int = 0, concurrency: Optional[int] = None ): """Get web pages from root URLs. If depth is specified (e.g., depth=2), then this will also follow up to levels of links from each root. Args: root_urls: root urls used as a starting point for spidering depth: level of recursion into links concurrency: number of simultaneous requests that can be sent Returns: A dict of pages visited (URL) mapped to their full text and the set of visited links. """ if isinstance(root_urls, str): root_urls = [root_urls] current_depth = 0 pages, links, spider_args = {}, set(), [] _visited: Set[str] = set() go_deeper = current_depth < depth for root_str in root_urls: root = urllib.parse.urlparse(root_str) spider_args.append((root, go_deeper, _visited)) with spack.util.parallel.make_concurrent_executor(concurrency, require_fork=False) as tp: while current_depth <= depth: tty.debug( f"SPIDER: [depth={current_depth}, max_depth={depth}, urls={len(spider_args)}]" ) results = [tp.submit(_spider, *one_search_args) for one_search_args in spider_args] spider_args = [] go_deeper = current_depth < depth for future in results: sub_pages, sub_links, sub_spider_args, sub_visited = future.result() _visited.update(sub_visited) sub_spider_args = [(x, go_deeper, _visited) for x in sub_spider_args] pages.update(sub_pages) links.update(sub_links) spider_args.extend(sub_spider_args) current_depth += 1 return pages, links def _spider(url: urllib.parse.ParseResult, collect_nested: bool, _visited: Set[str]): """Fetches URL and any pages it links to. Prints out a warning only if the root can't be fetched; it ignores errors with pages that the root links to. Args: url: url being fetched and searched for links collect_nested: whether we want to collect arguments for nested spidering on the links found in this url _visited: links already visited Returns: A tuple of: - pages: dict of pages visited (URL) mapped to their full text. - links: set of links encountered while visiting the pages. - spider_args: argument for subsequent call to spider - visited: updated set of visited urls """ pages: Dict[str, str] = {} # dict from page URL -> text content. links: Set[str] = set() # set of all links seen on visited pages. subcalls: List[str] = [] try: response_url, _, response = read_from_url(url, "text/html") if not response_url or not response: return pages, links, subcalls, _visited with response: page = io.TextIOWrapper(response, encoding="utf-8").read() pages[response_url] = page # Parse out the include-fragments in the page # https://github.github.io/include-fragment-element metadata_parser = ExtractMetadataParser() metadata_parser.feed(page) # Change of base URL due to tag response_url = metadata_parser.base_url or response_url fragments = set() while metadata_parser.fragments: raw_link = metadata_parser.fragments.pop() abs_link = url_util.join(response_url, raw_link.strip(), resolve_href=True) fragment_response_url = None try: # This seems to be text/html, though text/fragment+html is also used fragment_response_url, _, fragment_response = read_from_url(abs_link, "text/html") except Exception as e: msg = f"Error reading fragment: {(type(e), str(e))}:{traceback.format_exc()}" tty.debug(msg) if not fragment_response_url or not fragment_response: continue with fragment_response: fragment = io.TextIOWrapper(fragment_response, encoding="utf-8").read() fragments.add(fragment) pages[fragment_response_url] = fragment # Parse out the links in the page and all fragments link_parser = LinkParser() link_parser.feed(page) for fragment in fragments: link_parser.feed(fragment) while link_parser.links: raw_link = link_parser.links.pop() abs_link = url_util.join(response_url, raw_link.strip(), resolve_href=True) links.add(abs_link) # Skip stuff that looks like an archive if any(raw_link.endswith(s) for s in spack.llnl.url.ALLOWED_ARCHIVE_TYPES): continue # Skip already-visited links if abs_link in _visited: continue # If we're not at max depth, follow links. if collect_nested: subcalls.append(abs_link) _visited.add(abs_link) except OSError as e: tty.debug(f"[SPIDER] Unable to read: {url}") tty.debug(str(e), level=2) if isinstance(e, URLError) and isinstance(e.reason, ssl.SSLError): tty.warn( "Spack was unable to fetch url list due to a " "certificate verification problem. You can try " "running spack -k, which will not check SSL " "certificates. Use this at your own risk." ) except HTMLParseError as e: # This error indicates that Python's HTML parser sucks. msg = "Got an error parsing HTML." tty.warn(msg, url, "HTMLParseError: " + str(e)) except Exception as e: # Other types of errors are completely ignored, # except in debug mode tty.debug(f"Error in _spider: {type(e)}:{str(e)}", traceback.format_exc()) finally: tty.debug(f"SPIDER: [url={url}]") return pages, links, subcalls, _visited def get_header(headers, header_name): """Looks up a dict of headers for the given header value. Looks up a dict of headers, [headers], for a header value given by [header_name]. Returns headers[header_name] if header_name is in headers. Otherwise, the first fuzzy match is returned, if any. This fuzzy matching is performed by discarding word separators and capitalization, so that for example, "Content-length", "content_length", "conTENtLength", etc., all match. In the case of multiple fuzzy-matches, the returned value is the "first" such match given the underlying mapping's ordering, or unspecified if no such ordering is defined. If header_name is not in headers, and no such fuzzy match exists, then a KeyError is raised. """ def unfuzz(header): return re.sub(r"[ _-]", "", header).lower() try: return headers[header_name] except KeyError: unfuzzed_header_name = unfuzz(header_name) for header, value in headers.items(): if unfuzz(header) == unfuzzed_header_name: return value raise def parse_etag(header_value): """Parse a strong etag from an ETag: header value. We don't allow for weakness indicators because it's unclear what that means for cache invalidation.""" if header_value is None: return None # First follow rfc7232 section 2.3 mostly: # ETag = entity-tag # entity-tag = [ weak ] opaque-tag # weak = %x57.2F ; "W/", case-sensitive # opaque-tag = DQUOTE *etagc DQUOTE # etagc = %x21 / %x23-7E / obs-text # ; VCHAR except double quotes, plus obs-text # obs-text = %x80-FF # That means quotes are required. valid = re.match(r'"([\x21\x23-\x7e\x80-\xFF]+)"$', header_value) if valid: return valid.group(1) # However, not everybody adheres to the RFC (some servers send # wrong etags, but also s3:// is simply a different standard). # In that case, it's common that quotes are omitted, everything # else stays the same. valid = re.match(r"([\x21\x23-\x7e\x80-\xFF]+)$", header_value) return valid.group(1) if valid else None class SpackWebError(spack.error.SpackError): """Superclass for Spack web spidering errors.""" class NoNetworkConnectionError(SpackWebError): """Raised when an operation can't get an internet connection.""" def __init__(self, message, url): super().__init__("No network connection: " + str(message), "URL was: " + str(url)) self.url = url ================================================ FILE: lib/spack/spack/util/windows_registry.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ Utility module for dealing with Windows Registry. """ import os import re import sys from contextlib import contextmanager from spack.llnl.util import tty if sys.platform == "win32": import winreg class RegistryValue: """ Class defining a Windows registry entry """ def __init__(self, name, value, parent_key): self.path = name self.value = value self.key = parent_key class RegistryKey: """ Class wrapping a Windows registry key """ def __init__(self, name, handle): self.path = name self.name = os.path.split(name)[-1] self._handle = handle self._keys = [] self._values = {} @property def values(self): """Returns all subvalues of this key as RegistryValue objects in dictionary of value name : RegistryValue object """ self._gather_value_info() return self._values @property def subkeys(self): """Returns list of all subkeys of this key as RegistryKey objects""" self._gather_subkey_info() return self._keys @property def hkey(self): return self._handle @contextmanager def winreg_error_handler(self, name, *args, **kwargs): try: yield except OSError as err: # Expected errors that occur on occasion, these are easily # debug-able and have sufficiently verbose reporting and obvious cause # [WinError 2]: the system cannot find the file specified - lookup item does # not exist # [WinError 5]: Access is denied - user not in key's ACL if hasattr(err, "winerror") and err.winerror in (5, 2): raise err # Other OS errors are more difficult to diagnose, so we wrap them in some extra # reporting raise InvalidRegistryOperation(name, err, *args, **kwargs) from err def OpenKeyEx(self, subname, **kwargs): """Convenience wrapper around winreg.OpenKeyEx""" tty.debug( f"[WINREG ACCESS] Accessing Reg Key {self.path}/{subname} with" f" {kwargs.get('access', 'default')} access" ) with self.winreg_error_handler("OpenKeyEx", subname, **kwargs): return winreg.OpenKeyEx(self.hkey, subname, **kwargs) def QueryInfoKey(self): """Convenience wrapper around winreg.QueryInfoKey""" tty.debug(f"[WINREG ACCESS] Obtaining key,value information from key {self.path}") with self.winreg_error_handler("QueryInfoKey"): return winreg.QueryInfoKey(self.hkey) def EnumKey(self, index): """Convenience wrapper around winreg.EnumKey""" tty.debug( "[WINREG ACCESS] Obtaining name of subkey at index " f"{index} from registry key {self.path}" ) with self.winreg_error_handler("EnumKey", index): return winreg.EnumKey(self.hkey, index) def EnumValue(self, index): """Convenience wrapper around winreg.EnumValue""" tty.debug( f"[WINREG ACCESS] Obtaining value at index {index} from registry key {self.path}" ) with self.winreg_error_handler("EnumValue", index): return winreg.EnumValue(self.hkey, index) def QueryValueEx(self, name, **kwargs): """Convenience wrapper around winreg.QueryValueEx""" tty.debug(f"[WINREG ACCESS] Obtaining value {name} from registry key {self.path}") with self.winreg_error_handler("QueryValueEx", name, **kwargs): return winreg.QueryValueEx(self.hkey, name, **kwargs) def __str__(self): return self.name def _gather_subkey_info(self): """Composes all subkeys into a list for access""" if self._keys: return sub_keys, _, _ = self.QueryInfoKey() for i in range(sub_keys): sub_name = self.EnumKey(i) try: sub_handle = self.OpenKeyEx(sub_name, access=winreg.KEY_READ) self._keys.append(RegistryKey(os.path.join(self.path, sub_name), sub_handle)) except OSError as e: if hasattr(e, "winerror") and e.winerror == 5: # This is a permission error, we can't read this key # move on pass else: raise def _gather_value_info(self): """Compose all values for this key into a dict of form value name: RegistryValue Object""" if self._values: return _, values, _ = self.QueryInfoKey() for i in range(values): value_name, value_data, _ = self.EnumValue(i) self._values[value_name] = RegistryValue(value_name, value_data, self) def get_subkey(self, sub_key): """Returns subkey of name sub_key in a RegistryKey objects""" return RegistryKey( os.path.join(self.path, sub_key), self.OpenKeyEx(sub_key, access=winreg.KEY_READ) ) def get_value(self, val_name): """Returns value associated with this key in RegistryValue object""" return RegistryValue(val_name, self.QueryValueEx(val_name)[0], self) class _HKEY_CONSTANT(RegistryKey): """Subclass of RegistryKey to represent the prebaked, always open registry HKEY constants""" def __init__(self, hkey_constant): hkey_name = hkey_constant # This class is instantiated at module import time # on non Windows platforms, winreg would not have been # imported. For this reason we can't reference winreg yet, # so handle is none for now to avoid invalid references to a module. # _handle provides a workaround to prevent null references to self.handle # when coupled with the handle property. super(_HKEY_CONSTANT, self).__init__(hkey_name, None) def _get_hkey(self, key): return getattr(winreg, key) @property def hkey(self): if not self._handle: self._handle = self._get_hkey(self.path) return self._handle class HKEY: """ Predefined, open registry HKEYs From the Microsoft docs: An application must open a key before it can read data from the registry. To open a key, an application must supply a handle to another key in the registry that is already open. The system defines predefined keys that are always open. Predefined keys help an application navigate in the registry.""" HKEY_CLASSES_ROOT = _HKEY_CONSTANT("HKEY_CLASSES_ROOT") HKEY_CURRENT_USER = _HKEY_CONSTANT("HKEY_CURRENT_USER") HKEY_USERS = _HKEY_CONSTANT("HKEY_USERS") HKEY_LOCAL_MACHINE = _HKEY_CONSTANT("HKEY_LOCAL_MACHINE") HKEY_CURRENT_CONFIG = _HKEY_CONSTANT("HKEY_CURRENT_CONFIG") HKEY_PERFORMANCE_DATA = _HKEY_CONSTANT("HKEY_PERFORMANCE_DATA") class WindowsRegistryView: """ Interface to provide access, querying, and searching to Windows registry entries. This class represents a single key entrypoint into the Windows registry and provides an interface to this key's values, its subkeys, and those subkey's values. This class cannot be used to move freely about the registry, only subkeys/values of the root key used to instantiate this class. """ def __init__(self, key, root_key=HKEY.HKEY_CURRENT_USER): """Constructs a Windows Registry entrypoint to key provided root_key should be an already open root key or an hkey constant if provided Args: key (str): registry key to provide root for registry key for this clas root_key: Already open registry key or HKEY constant to provide access into the Windows registry. Registry access requires an already open key to get an entrypoint, the HKEY constants are always open, or an already open key can be used instead. """ if sys.platform != "win32": raise RuntimeError( "Cannot instantiate Windows Registry class on non Windows platforms" ) self.key = key self.root = root_key self._reg = None class KeyMatchConditions: @staticmethod def regex_matcher(subkey_name): return lambda x: re.match(subkey_name, x.name) @staticmethod def name_matcher(subkey_name): return lambda x: subkey_name == x.name @contextmanager def invalid_reg_ref_error_handler(self): try: yield except FileNotFoundError as e: if sys.platform == "win32" and e.winerror == 2: tty.debug("Key %s at position %s does not exist" % (self.key, str(self.root))) else: raise e def __bool__(self): return self.reg != -1 def _load_key(self): try: self._reg = self.root.get_subkey(self.key) except FileNotFoundError as e: if sys.platform == "win32" and e.winerror == 2: self._reg = -1 tty.debug("Key %s at position %s does not exist" % (self.key, str(self.root))) else: raise e def _valid_reg_check(self): if self.reg == -1: tty.debug(f"[WINREG ACCESS] Cannot perform operation for nonexistent key {self.key}") return False return True def _regex_match_subkeys(self, subkey): r_subkey = re.compile(subkey) return [key for key in self.get_subkeys() if r_subkey.match(key.name)] @property def reg(self): if not self._reg: self._load_key() return self._reg def get_value(self, value_name): """Return registry value corresponding to provided argument (if it exists)""" if not self._valid_reg_check(): raise RegistryError(f"Cannot query value from invalid key {self.key}") with self.invalid_reg_ref_error_handler(): return self.reg.get_value(value_name) def get_subkey(self, subkey_name): if not self._valid_reg_check(): raise RegistryError(f"Cannot query subkey from invalid key {self.key}") with self.invalid_reg_ref_error_handler(): return self.reg.get_subkey(subkey_name) def get_subkeys(self): if not self._valid_reg_check(): raise RegistryError(f"Cannot query subkeys from invalid key {self.key}") with self.invalid_reg_ref_error_handler(): return self.reg.subkeys def get_matching_subkeys(self, subkey_name): """Returns all subkeys regex matching subkey name Note: this method obtains only direct subkeys of the given key and does not descend to transitive subkeys. For this behavior, see ``find_matching_subkeys``""" self._regex_match_subkeys(subkey_name) def get_values(self): if not self._valid_reg_check(): raise RegistryError(f"Cannot query values from invalid key {self.key}") with self.invalid_reg_ref_error_handler(): return self.reg.values def _traverse_subkeys(self, stop_condition, collect_all_matching=False, recursive=True): """Perform simple BFS of subkeys, returning the key that successfully triggers the stop condition. Args: stop_condition: lambda or function pointer that takes a single argument a key and returns a boolean value based on that key collect_all_matching: boolean value, if True, the traversal collects and returns all keys meeting stop condition. If false, once stop condition is met, the key that triggered the condition ' is returned. recursive: boolean value, if True perform a recursive search of subkeys Return: the key if stop_condition is triggered, or None if not """ collection = [] if not self._valid_reg_check(): raise InvalidKeyError(self.key) with self.invalid_reg_ref_error_handler(): queue = self.reg.subkeys for key in queue: if stop_condition(key): if collect_all_matching: collection.append(key) else: return key if recursive: queue.extend(key.subkeys) return collection if collection else None def find_subkey(self, subkey_name: str, recursive: bool = True): """Perform a BFS of subkeys until desired key is found Returns None or RegistryKey object corresponding to requested key name Args: subkey_name: subkey to be searched for recursive: perform a recursive search Return: the desired subkey as a RegistryKey object, or none """ return self._traverse_subkeys( WindowsRegistryView.KeyMatchConditions.name_matcher(subkey_name), recursive=recursive ) def find_matching_subkey(self, subkey_name: str, recursive: bool = True): """Perform a BFS of subkeys until a key matching subkey name regex is found Returns None or the first RegistryKey object corresponding to requested key name Args: subkey_name: subkey to be searched for recursive: perform a recursive search Return: the desired subkey as a RegistryKey object, or none """ return self._traverse_subkeys( WindowsRegistryView.KeyMatchConditions.regex_matcher(subkey_name), recursive=recursive ) def find_subkeys(self, subkey_name: str, recursive: bool = True): """Exactly the same as find_subkey, except this function tries to match a regex to multiple keys Args: subkey_name: subkey to be searched for Return: the desired subkeys as a list of RegistryKey object, or none """ kwargs = {"collect_all_matching": True, "recursive": recursive} return self._traverse_subkeys( WindowsRegistryView.KeyMatchConditions.regex_matcher(subkey_name), **kwargs ) def find_value(self, val_name: str, recursive: bool = True): """ If non recursive, return RegistryValue object corresponding to name Args: val_name: name of value desired from registry recursive: optional argument, if True, the registry is searched recursively for the value of name val_name, else only the current key is searched Return: The desired registry value as a RegistryValue object if it exists, otherwise, None """ if not recursive: return self.get_value(val_name) else: key = self._traverse_subkeys(lambda x: val_name in x.values) if not key: return None else: return key.values[val_name] class RegistryError(Exception): """RunTime Error concerning the Windows Registry""" class InvalidKeyError(RegistryError): """Runtime Error describing issue with invalid key access to Windows registry""" def __init__(self, key): message = f"Cannot query invalid key: {key}" super().__init__(message) class InvalidRegistryOperation(RegistryError): """A Runtime Error encountered when a registry operation is invalid for an indeterminate reason""" def __init__(self, name, e, *args, **kwargs): message = ( f"Windows registry operations: {name} encountered error: {str(e)}" "\nMethod invoked with parameters:\n" ) message += "\n\t".join([f"{k}:{v}" for k, v in kwargs.items()]) message += "\n" message += "\n\t".join(args) super().__init__(self, message) ================================================ FILE: lib/spack/spack/variant.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """The variant module contains data structures that are needed to manage variants both in packages and in specs. """ import collections.abc import enum import functools import inspect import itertools from typing import ( TYPE_CHECKING, Any, Callable, Collection, Iterable, List, Optional, Set, Tuple, Type, Union, ) import spack.error import spack.llnl.util.lang as lang import spack.llnl.util.tty.color import spack.spec_parser if TYPE_CHECKING: import spack.package_base import spack.spec #: These are variant names used by Spack internally; packages can't use them RESERVED_NAMES = { "arch", "architecture", "branch", "commit", "dev_path", "namespace", "operating_system", "os", "patches", "platform", "ref", "tag", "target", } class VariantType(enum.IntEnum): """Enum representing the three concrete variant types.""" BOOL = 1 SINGLE = 2 MULTI = 3 INDICATOR = 4 # special type for placeholder variant values @property def string(self) -> str: """Convert the variant type to a string.""" if self == VariantType.BOOL: return "bool" elif self == VariantType.SINGLE: return "single" elif self == VariantType.MULTI: return "multi" else: return "indicator" class Variant: """Represents a variant definition, created by the ``variant()`` directive. There can be multiple definitions of the same variant, and they are given precedence by order of appearance in the package. Later definitions have higher precedence. Similarly, definitions in derived classes have higher precedence than those in their superclasses. """ name: str default: Union[bool, str] description: str values: Optional[Collection] #: if None, valid values are defined only by validators multi: bool single_value_validator: Callable group_validator: Optional[Callable] sticky: bool precedence: int def __init__( self, name: str, *, default: Union[bool, str], description: str, values: Union[Collection, Callable] = (True, False), multi: bool = False, validator: Optional[Callable] = None, sticky: bool = False, precedence: int = 0, ): """Initialize a package variant. Args: name: name of the variant default: default value for the variant, used when nothing is explicitly specified description: purpose of the variant values: sequence of allowed values or a callable accepting a single value as argument and returning True if the value is good, False otherwise multi: whether multiple values are allowed validator: optional callable that can be used to perform additional validation sticky: if true the variant is set to the default value at concretization time precedence: int indicating precedence of this variant definition in the solve (definition with highest precedence is used when multiple definitions are possible) """ self.name = name self.default = default self.description = str(description) self.values = None if values == "*": # wildcard is a special case to make it easy to say any value is ok self.single_value_validator = lambda v: True elif isinstance(values, type): # supplying a type means any value *of that type* def isa_type(v): try: values(v) return True except ValueError: return False self.single_value_validator = isa_type elif callable(values): # If 'values' is a callable, assume it is a single value # validator and reset the values to be explicit during debug self.single_value_validator = values else: # Otherwise, assume values is the set of allowed explicit values values = _flatten(values) self.values = values self.single_value_validator = lambda v: v in values self.multi = multi self.group_validator = validator self.sticky = sticky self.precedence = precedence def values_defined_by_validator(self) -> bool: return self.values is None def validate_or_raise(self, vspec: "VariantValue", pkg_name: str): """Validate a variant spec against this package variant. Raises an exception if any error is found. Args: vspec: variant spec to be validated pkg_name: the name of the package class that required this validation (for errors) Raises: InconsistentValidationError: if ``vspec.name != self.name`` MultipleValuesInExclusiveVariantError: if ``vspec`` has multiple values but ``self.multi == False`` InvalidVariantValueError: if ``vspec.value`` contains invalid values """ # Check the name of the variant if self.name != vspec.name: raise InconsistentValidationError(vspec, self) # If the value is exclusive there must be at most one value = vspec.values if not self.multi and len(value) != 1: raise MultipleValuesInExclusiveVariantError(vspec, pkg_name) # Check and record the values that are not allowed invalid_vals = ", ".join( f"'{v}'" for v in value if v != "*" and self.single_value_validator(v) is False ) if invalid_vals: raise InvalidVariantValueError( f"invalid values for variant '{self.name}' in package {pkg_name}: {invalid_vals}\n" ) # Validate the group of values if needed if self.group_validator is not None and value != ("*",): self.group_validator(pkg_name, self.name, value) @property def allowed_values(self): """Returns a string representation of the allowed values for printing purposes Returns: str: representation of the allowed values """ # Join an explicit set of allowed values if self.values is not None: v = tuple(str(x) for x in self.values) return ", ".join(v) # In case we were given a single-value validator # print the docstring docstring = inspect.getdoc(self.single_value_validator) v = docstring if docstring else "" return v def make_default(self) -> "VariantValue": """Factory that creates a variant holding the default value(s).""" variant = VariantValue.from_string_or_bool(self.name, self.default) variant.type = self.variant_type return variant def make_variant(self, *value: Union[str, bool]) -> "VariantValue": """Factory that creates a variant holding the value(s) passed.""" return VariantValue(self.variant_type, self.name, value) @property def variant_type(self) -> VariantType: """String representation of the type of this variant (single/multi/bool)""" if self.multi: return VariantType.MULTI elif self.values == (True, False): return VariantType.BOOL else: return VariantType.SINGLE def __str__(self) -> str: return ( f"Variant('{self.name}', " f"default='{self.default}', " f"description='{self.description}', " f"values={self.values}, " f"multi={self.multi}, " f"single_value_validator={self.single_value_validator}, " f"group_validator={self.group_validator}, " f"sticky={self.sticky}, " f"precedence={self.precedence})" ) def _flatten(values) -> Collection: """Flatten instances of _ConditionalVariantValues for internal representation""" if isinstance(values, DisjointSetsOfValues): return values flattened: List = [] for item in values: if isinstance(item, ConditionalVariantValues): flattened.extend(item) else: flattened.append(item) # There are parts of the variant checking mechanism that expect to find tuples # here, so it is important to convert the type once we flattened the values. return tuple(flattened) #: Type for value of a variant ValueType = Tuple[Union[bool, str], ...] #: Type of variant value when output for JSON, YAML, etc. SerializedValueType = Union[str, bool, List[Union[str, bool]]] @lang.lazy_lexicographic_ordering class VariantValue: """A VariantValue is a key-value pair that represents a variant. It can have zero or more values. Values have set semantics, so they are unordered and unique. The variant type can be narrowed from multi to single to boolean, this limits the number of values that can be stored in the variant. Multi-valued variants can either be concrete or abstract: abstract means that the variant takes at least the values specified, but may take more when concretized. Concrete means that the variant takes exactly the values specified. Lastly, a variant can be marked as propagating, which means that it should be propagated to dependencies.""" name: str propagate: bool concrete: bool type: VariantType _values: ValueType slots = ("name", "propagate", "concrete", "type", "_values") def __init__( self, type: VariantType, name: str, value: ValueType, *, propagate: bool = False, concrete: bool = False, ) -> None: self.name = name self.type = type self.propagate = propagate # only multi-valued variants can be abstract self.concrete = concrete or type in (VariantType.BOOL, VariantType.SINGLE) # Invokes property setter self.set(*value) @staticmethod def from_node_dict( name: str, value: Union[str, List[str]], *, propagate: bool = False, abstract: bool = False ) -> "VariantValue": """Reconstruct a variant from a node dict.""" if isinstance(value, list): return VariantValue( VariantType.MULTI, name, tuple(value), propagate=propagate, concrete=not abstract ) # todo: is this necessary? not literal true / false in json/yaml? elif str(value).upper() == "TRUE" or str(value).upper() == "FALSE": return VariantValue( VariantType.BOOL, name, (str(value).upper() == "TRUE",), propagate=propagate ) return VariantValue(VariantType.SINGLE, name, (value,), propagate=propagate) @staticmethod def from_string_or_bool( name: str, value: Union[str, bool], *, propagate: bool = False, concrete: bool = False ) -> "VariantValue": if value is True or value is False: return VariantValue(VariantType.BOOL, name, (value,), propagate=propagate) elif value.upper() in ("TRUE", "FALSE"): return VariantValue( VariantType.BOOL, name, (value.upper() == "TRUE",), propagate=propagate ) elif value == "*": return VariantValue(VariantType.MULTI, name, (), propagate=propagate) return VariantValue( VariantType.MULTI, name, tuple(value.split(",")), propagate=propagate, concrete=concrete, ) @staticmethod def from_concretizer(name: str, value: str, type: str) -> "VariantValue": """Reconstruct a variant from concretizer output.""" if type == "bool": return VariantValue(VariantType.BOOL, name, (value == "True",)) elif type == "multi": return VariantValue(VariantType.MULTI, name, (value,), concrete=True) else: return VariantValue(VariantType.SINGLE, name, (value,)) def yaml_entry(self) -> Tuple[str, SerializedValueType]: """Returns a (key, value) tuple suitable to be an entry in a yaml dict. Returns: tuple: (name, value_representation) """ if self.type == VariantType.MULTI: return self.name, list(self.values) return self.name, self.values[0] @property def values(self) -> ValueType: return self._values @property def value(self) -> Union[ValueType, bool, str]: return self._values[0] if self.type != VariantType.MULTI else self._values def set(self, *value: Union[bool, str]) -> None: """Set the value(s) of the variant.""" if len(value) > 1: value = tuple(sorted(set(value))) if self.type != VariantType.MULTI: if len(value) != 1: raise MultipleValuesInExclusiveVariantError(self) unwrapped = value[0] if self.type == VariantType.BOOL and unwrapped not in (True, False): raise ValueError( f"cannot set a boolean variant to a value that is not a boolean: {unwrapped}" ) if "*" in value: raise InvalidVariantValueError("cannot use reserved value '*'") self._values = value def _cmp_iter(self) -> Iterable: yield self.name yield self.propagate yield self.concrete yield from (str(v) for v in self.values) def copy(self) -> "VariantValue": return VariantValue( self.type, self.name, self.values, propagate=self.propagate, concrete=self.concrete ) def satisfies(self, other: "VariantValue") -> bool: """The lhs satisfies the rhs if all possible concretizations of lhs are also possible concretizations of rhs.""" if self.name != other.name: return False if not other.concrete: # rhs abstract means the lhs must at least contain its values. # special-case patches with rhs abstract: their values may be prefixes of the lhs # values. if self.name == "patches": return all( isinstance(v, str) and any(isinstance(w, str) and w.startswith(v) for w in self.values) for v in other.values ) return all(v in self for v in other.values) if self.concrete: # both concrete: they must be equal return self.values == other.values return False def intersects(self, other: "VariantValue") -> bool: """True iff there exists a concretization that satisfies both lhs and rhs.""" if self.name != other.name: return False if self.concrete: if other.concrete: return self.values == other.values return all(v in self for v in other.values) if other.concrete: return all(v in other for v in self.values) # both abstract: the union is a valid concretization of both return True def constrain(self, other: "VariantValue") -> bool: """Constrain self with other if they intersect. Returns true iff self was changed.""" if not self.intersects(other): raise UnsatisfiableVariantSpecError(self, other) old_values = self.values self.set(*self.values, *other.values) changed = old_values != self.values if self.propagate and not other.propagate: self.propagate = False changed = True if not self.concrete and other.concrete: self.concrete = True changed = True if self.type > other.type: self.type = other.type changed = True return changed def append(self, value: Union[str, bool]) -> None: self.set(*self.values, value) def __contains__(self, item: Union[str, bool]) -> bool: return item in self.values def __str__(self) -> str: # boolean variants are printed +foo or ~foo if self.type == VariantType.BOOL: sigil = "+" if self.value else "~" if self.propagate: sigil *= 2 return f"{sigil}{self.name}" # concrete multi-valued foo:=bar,baz concrete = ":" if self.type == VariantType.MULTI and self.concrete else "" delim = "==" if self.propagate else "=" if not self.values: value_str = "*" elif self.name == "patches" and self.concrete: value_str = ",".join(str(x)[:7] for x in self.values) else: value_str = ",".join(str(x) for x in self.values) return f"{self.name}{concrete}{delim}{spack.spec_parser.quote_if_needed(value_str)}" def __repr__(self): return ( f"VariantValue({self.type!r}, {self.name!r}, {self.values!r}, " f"propagate={self.propagate!r}, concrete={self.concrete!r})" ) def MultiValuedVariant(name: str, value: ValueType, propagate: bool = False) -> VariantValue: return VariantValue(VariantType.MULTI, name, value, propagate=propagate, concrete=True) def SingleValuedVariant( name: str, value: Union[bool, str], propagate: bool = False ) -> VariantValue: return VariantValue(VariantType.SINGLE, name, (value,), propagate=propagate) def BoolValuedVariant(name: str, value: bool, propagate: bool = False) -> VariantValue: return VariantValue(VariantType.BOOL, name, (value,), propagate=propagate) class VariantValueRemoval(VariantValue): """Indicator class for Spec.mutate to remove a variant""" def __init__(self, name): super().__init__(VariantType.INDICATOR, name, (None,)) # The class below inherit from Sequence to disguise as a tuple and comply # with the semantic expected by the 'values' argument of the variant directive class DisjointSetsOfValues(collections.abc.Sequence): """Allows combinations from one of many mutually exclusive sets. The value ``('none',)`` is reserved to denote the empty set and therefore no other set can contain the item ``'none'``. Args: *sets (list): mutually exclusive sets of values """ _empty_set = ("none",) def __init__(self, *sets: Tuple[str, ...]) -> None: self.sets = [tuple(_flatten(x)) for x in sets] # 'none' is a special value and can appear only in a set of a single element if any("none" in s and s != self._empty_set for s in self.sets): raise spack.error.SpecError( "The value 'none' represents the empty set, and must appear alone in a set. " "Use the method 'allow_empty_set' to add it." ) # Sets should not intersect with each other cumulated: Set[str] = set() for current_set in self.sets: if not cumulated.isdisjoint(current_set): duplicates = ", ".join(sorted(cumulated.intersection(current_set))) raise spack.error.SpecError( f"sets in input must be disjoint, but {duplicates} appeared more than once" ) cumulated.update(current_set) #: Attribute used to track values which correspond to #: features which can be enabled or disabled as understood by the #: package's build system. self.feature_values = tuple(itertools.chain.from_iterable(self.sets)) self.default = None self.multi = True self.error_fmt = ( "this variant accepts combinations of values from " "exactly one of the following sets '{values}' " "@*r{{[{package}, variant '{variant}']}}" ) def with_default(self, default): """Sets the default value and returns self.""" self.default = default return self def with_error(self, error_fmt): """Sets the error message format and returns self.""" self.error_fmt = error_fmt return self def with_non_feature_values(self, *values): """Marks a few values as not being tied to a feature.""" self.feature_values = tuple(x for x in self.feature_values if x not in values) return self def allow_empty_set(self): """Adds the empty set to the current list of disjoint sets.""" if self._empty_set in self.sets: return self # Create a new object to be returned object_with_empty_set = type(self)(("none",), *self.sets) object_with_empty_set.error_fmt = self.error_fmt object_with_empty_set.feature_values = self.feature_values + ("none",) return object_with_empty_set def prohibit_empty_set(self): """Removes the empty set from the current list of disjoint sets.""" if self._empty_set not in self.sets: return self # Create a new object to be returned sets = [s for s in self.sets if s != self._empty_set] object_without_empty_set = type(self)(*sets) object_without_empty_set.error_fmt = self.error_fmt object_without_empty_set.feature_values = tuple( x for x in self.feature_values if x != "none" ) return object_without_empty_set def __getitem__(self, idx): return tuple(itertools.chain.from_iterable(self.sets))[idx] def __len__(self): return sum(len(x) for x in self.sets) @property def validator(self): def _disjoint_set_validator(pkg_name, variant_name, values): # If for any of the sets, all the values are in it return True if any(all(x in s for x in values) for s in self.sets): return format_args = {"variant": variant_name, "package": pkg_name, "values": values} msg = self.error_fmt + " @*r{{[{package}, variant '{variant}']}}" msg = spack.llnl.util.tty.color.colorize(msg.format(**format_args)) raise spack.error.SpecError(msg) return _disjoint_set_validator def _a_single_value_or_a_combination(single_value: str, *values: str) -> DisjointSetsOfValues: error = f"the value '{single_value}' is mutually exclusive with any of the other values" return ( DisjointSetsOfValues((single_value,), values) .with_default(single_value) .with_error(error) .with_non_feature_values(single_value) ) # TODO: The factories below are used by package writers to set values of # TODO: multi-valued variants. It could be worthwhile to gather them in # TODO: a common namespace (like 'multi') in the future. def any_combination_of(*values: str) -> DisjointSetsOfValues: """Multi-valued variant that allows either any combination of the specified values, or none at all (using ``variant=none``). The literal value ``none`` is used as sentinel for the empty set, since in the spec DSL we have to always specify a value for a variant. It is up to the package implementation to handle the value ``none`` specially, if at all. See also :func:`auto_or_any_combination_of` and :func:`disjoint_sets`. Args: *values: allowed variant values Example:: variant("cuda_arch", values=any_combination_of("10", "11")) Returns: a properly initialized instance of :class:`~spack.variant.DisjointSetsOfValues` """ return _a_single_value_or_a_combination("none", *values) def auto_or_any_combination_of(*values: str) -> DisjointSetsOfValues: """Multi-valued variant that allows any combination of a set of values (but not the empty set) or ``auto``. See also :func:`any_combination_of` and :func:`disjoint_sets`. Args: *values: allowed variant values Example:: variant( "file_systems", values=auto_or_any_combination_of("lustre", "gpfs", "nfs", "ufs"), ) Returns: a properly initialized instance of :class:`~spack.variant.DisjointSetsOfValues` """ return _a_single_value_or_a_combination("auto", *values) def disjoint_sets(*sets: Tuple[str, ...]) -> DisjointSetsOfValues: """Multi-valued variant that allows any combination picking from one of multiple disjoint sets of values, and also allows the user to specify ``none`` to choose none of them. It is up to the package implementation to handle the value ``none`` specially, if at all. See also :func:`any_combination_of` and :func:`auto_or_any_combination_of`. Args: *sets: sets of allowed values, each set is a tuple of strings Returns: a properly initialized instance of :class:`~spack.variant.DisjointSetsOfValues` """ return DisjointSetsOfValues(*sets).allow_empty_set().with_default("none") @functools.total_ordering class ConditionalValue: """Conditional value for a variant.""" value: Any # optional because statically disabled values (when=False) are set to None # when=True results in spack.spec.Spec() when: Optional["spack.spec.Spec"] def __init__(self, value: Any, when: Optional["spack.spec.Spec"]): self.value = value self.when = when def __repr__(self): return f"ConditionalValue({self.value}, when={self.when})" def __str__(self): return str(self.value) def __hash__(self): # Needed to allow testing the presence of a variant in a set by its value return hash(self.value) def __eq__(self, other): if isinstance(other, (str, bool)): return self.value == other return self.value == other.value def __lt__(self, other): if isinstance(other, str): return self.value < other return self.value < other.value def prevalidate_variant_value( pkg_cls: "Type[spack.package_base.PackageBase]", variant: VariantValue, spec: Optional["spack.spec.Spec"] = None, strict: bool = False, ) -> List[Variant]: """Do as much validation of a variant value as is possible before concretization. This checks that the variant value is valid for *some* definition of the variant, and it raises if we know *before* concretization that the value cannot occur. On success it returns the variant definitions for which the variant is valid. Arguments: pkg_cls: package in which variant is (potentially multiply) defined variant: variant spec with value to validate spec: optionally restrict validation only to variants defined for this spec strict: if True, raise an exception if no variant definition is valid for any constraint on the spec. Return: list of variant definitions that will accept the given value. List will be empty only if the variant is a reserved variant. """ # do not validate non-user variants or optional variants if variant.name in RESERVED_NAMES or variant.propagate: return [] # raise if there is no definition at all if not pkg_cls.has_variant(variant.name): raise UnknownVariantError( f"No such variant '{variant.name}' in package {pkg_cls.name}", [variant.name] ) # do as much prevalidation as we can -- check only those # variants whose when constraint intersects this spec errors = [] possible_definitions = [] valid_definitions = [] for when, pkg_variant_def in pkg_cls.variant_definitions(variant.name): if spec and not spec.intersects(when): continue possible_definitions.append(pkg_variant_def) try: pkg_variant_def.validate_or_raise(variant, pkg_cls.name) valid_definitions.append(pkg_variant_def) except spack.error.SpecError as e: errors.append(e) # value is valid for at least one definition -- return them all if valid_definitions: return valid_definitions # no when spec intersected, so no possible definition for the variant in this configuration if strict and not possible_definitions: when_clause = f" when {spec}" if spec else "" raise InvalidVariantValueError( f"variant '{variant.name}' does not exist for '{pkg_cls.name}'{when_clause}" ) # There are only no errors if we're not strict and there are no possible_definitions. # We are strict for audits but not for specs on the CLI or elsewhere. Being strict # in these cases would violate our rule of being able to *talk* about any configuration, # regardless of what the package.py currently says. if not errors: return [] # if there is just one error, raise the specific error if len(errors) == 1: raise errors[0] # otherwise combine all the errors and raise them together raise InvalidVariantValueError( "multiple variant issues:", "\n".join(e.message for e in errors) ) class ConditionalVariantValues(lang.TypedMutableSequence): """A list, just with a different type""" class DuplicateVariantError(spack.error.SpecError): """Raised when the same variant occurs in a spec twice.""" class UnknownVariantError(spack.error.SpecError): """Raised when an unknown variant occurs in a spec.""" def __init__(self, msg: str, unknown_variants: List[str]): super().__init__(msg) self.unknown_variants = unknown_variants class InconsistentValidationError(spack.error.SpecError): """Raised if the wrong validator is used to validate a variant.""" def __init__(self, vspec, variant): msg = 'trying to validate variant "{0.name}" with the validator of "{1.name}"' super().__init__(msg.format(vspec, variant)) class MultipleValuesInExclusiveVariantError(spack.error.SpecError, ValueError): """Raised when multiple values are present in a variant that wants only one. """ def __init__(self, variant: VariantValue, pkg_name: Optional[str] = None): pkg_info = "" if pkg_name is None else f" in package '{pkg_name}'" msg = f"multiple values are not allowed for variant '{variant.name}'{pkg_info}" super().__init__(msg.format(variant, pkg_info)) class InvalidVariantValueError(spack.error.SpecError): """Raised when variants have invalid values.""" class UnsatisfiableVariantSpecError(spack.error.UnsatisfiableSpecError): """Raised when a spec variant conflicts with package constraints.""" def __init__(self, provided, required): super().__init__(provided, required, "variant") ================================================ FILE: lib/spack/spack/vendor/__init__.py ================================================ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/vendor/_pyrsistent_version.py ================================================ __version__ = '0.18.0' ================================================ FILE: lib/spack/spack/vendor/altgraph/Dot.py ================================================ """ spack.vendor.altgraph.Dot - Interface to the dot language ============================================ The :py:mod:`~spack.vendor.altgraph.Dot` module provides a simple interface to the file format used in the `graphviz `_ program. The module is intended to offload the most tedious part of the process (the **dot** file generation) while transparently exposing most of its features. To display the graphs or to generate image files the `graphviz `_ package needs to be installed on the system, moreover the :command:`dot` and :command:`dotty` programs must be accesible in the program path so that they can be ran from processes spawned within the module. Example usage ------------- Here is a typical usage:: from spack.vendor.altgraph import Graph, Dot # create a graph edges = [ (1,2), (1,3), (3,4), (3,5), (4,5), (5,4) ] graph = Graph.Graph(edges) # create a dot representation of the graph dot = Dot.Dot(graph) # display the graph dot.display() # save the dot representation into the mydot.dot file dot.save_dot(file_name='mydot.dot') # save dot file as gif image into the graph.gif file dot.save_img(file_name='graph', file_type='gif') Directed graph and non-directed graph ------------------------------------- Dot class can use for both directed graph and non-directed graph by passing ``graphtype`` parameter. Example:: # create directed graph(default) dot = Dot.Dot(graph, graphtype="digraph") # create non-directed graph dot = Dot.Dot(graph, graphtype="graph") Customizing the output ---------------------- The graph drawing process may be customized by passing valid :command:`dot` parameters for the nodes and edges. For a list of all parameters see the `graphviz `_ documentation. Example:: # customizing the way the overall graph is drawn dot.style(size='10,10', rankdir='RL', page='5, 5' , ranksep=0.75) # customizing node drawing dot.node_style(1, label='BASE_NODE',shape='box', color='blue' ) dot.node_style(2, style='filled', fillcolor='red') # customizing edge drawing dot.edge_style(1, 2, style='dotted') dot.edge_style(3, 5, arrowhead='dot', label='binds', labelangle='90') dot.edge_style(4, 5, arrowsize=2, style='bold') .. note:: dotty (invoked via :py:func:`~spack.vendor.altgraph.Dot.display`) may not be able to display all graphics styles. To verify the output save it to an image file and look at it that way. Valid attributes ---------------- - dot styles, passed via the :py:meth:`Dot.style` method:: rankdir = 'LR' (draws the graph horizontally, left to right) ranksep = number (rank separation in inches) - node attributes, passed via the :py:meth:`Dot.node_style` method:: style = 'filled' | 'invisible' | 'diagonals' | 'rounded' shape = 'box' | 'ellipse' | 'circle' | 'point' | 'triangle' - edge attributes, passed via the :py:meth:`Dot.edge_style` method:: style = 'dashed' | 'dotted' | 'solid' | 'invis' | 'bold' arrowhead = 'box' | 'crow' | 'diamond' | 'dot' | 'inv' | 'none' | 'tee' | 'vee' weight = number (the larger the number the closer the nodes will be) - valid `graphviz colors `_ - for more details on how to control the graph drawing process see the `graphviz reference `_. """ import os import warnings from spack.vendor.altgraph import GraphError class Dot(object): """ A class providing a **graphviz** (dot language) representation allowing a fine grained control over how the graph is being displayed. If the :command:`dot` and :command:`dotty` programs are not in the current system path their location needs to be specified in the contructor. """ def __init__( self, graph=None, nodes=None, edgefn=None, nodevisitor=None, edgevisitor=None, name="G", dot="dot", dotty="dotty", neato="neato", graphtype="digraph", ): """ Initialization. """ self.name, self.attr = name, {} assert graphtype in ["graph", "digraph"] self.type = graphtype self.temp_dot = "tmp_dot.dot" self.temp_neo = "tmp_neo.dot" self.dot, self.dotty, self.neato = dot, dotty, neato # self.nodes: node styles # self.edges: edge styles self.nodes, self.edges = {}, {} if graph is not None and nodes is None: nodes = graph if graph is not None and edgefn is None: def edgefn(node, graph=graph): return graph.out_nbrs(node) if nodes is None: nodes = () seen = set() for node in nodes: if nodevisitor is None: style = {} else: style = nodevisitor(node) if style is not None: self.nodes[node] = {} self.node_style(node, **style) seen.add(node) if edgefn is not None: for head in seen: for tail in (n for n in edgefn(head) if n in seen): if edgevisitor is None: edgestyle = {} else: edgestyle = edgevisitor(head, tail) if edgestyle is not None: if head not in self.edges: self.edges[head] = {} self.edges[head][tail] = {} self.edge_style(head, tail, **edgestyle) def style(self, **attr): """ Changes the overall style """ self.attr = attr def display(self, mode="dot"): """ Displays the current graph via dotty """ if mode == "neato": self.save_dot(self.temp_neo) neato_cmd = "%s -o %s %s" % (self.neato, self.temp_dot, self.temp_neo) os.system(neato_cmd) else: self.save_dot(self.temp_dot) plot_cmd = "%s %s" % (self.dotty, self.temp_dot) os.system(plot_cmd) def node_style(self, node, **kwargs): """ Modifies a node style to the dot representation. """ if node not in self.edges: self.edges[node] = {} self.nodes[node] = kwargs def all_node_style(self, **kwargs): """ Modifies all node styles """ for node in self.nodes: self.node_style(node, **kwargs) def edge_style(self, head, tail, **kwargs): """ Modifies an edge style to the dot representation. """ if tail not in self.nodes: raise GraphError("invalid node %s" % (tail,)) try: if tail not in self.edges[head]: self.edges[head][tail] = {} self.edges[head][tail] = kwargs except KeyError: raise GraphError("invalid edge %s -> %s " % (head, tail)) def iterdot(self): # write graph title if self.type == "digraph": yield "digraph %s {\n" % (self.name,) elif self.type == "graph": yield "graph %s {\n" % (self.name,) else: raise GraphError("unsupported graphtype %s" % (self.type,)) # write overall graph attributes for attr_name, attr_value in sorted(self.attr.items()): yield '%s="%s";' % (attr_name, attr_value) yield "\n" # some reusable patterns cpatt = '%s="%s",' # to separate attributes epatt = "];\n" # to end attributes # write node attributes for node_name, node_attr in sorted(self.nodes.items()): yield '\t"%s" [' % (node_name,) for attr_name, attr_value in sorted(node_attr.items()): yield cpatt % (attr_name, attr_value) yield epatt # write edge attributes for head in sorted(self.edges): for tail in sorted(self.edges[head]): if self.type == "digraph": yield '\t"%s" -> "%s" [' % (head, tail) else: yield '\t"%s" -- "%s" [' % (head, tail) for attr_name, attr_value in sorted(self.edges[head][tail].items()): yield cpatt % (attr_name, attr_value) yield epatt # finish file yield "}\n" def __iter__(self): return self.iterdot() def save_dot(self, file_name=None): """ Saves the current graph representation into a file """ if not file_name: warnings.warn(DeprecationWarning, "always pass a file_name") file_name = self.temp_dot with open(file_name, "w") as fp: for chunk in self.iterdot(): fp.write(chunk) def save_img(self, file_name=None, file_type="gif", mode="dot"): """ Saves the dot file as an image file """ if not file_name: warnings.warn(DeprecationWarning, "always pass a file_name") file_name = "out" if mode == "neato": self.save_dot(self.temp_neo) neato_cmd = "%s -o %s %s" % (self.neato, self.temp_dot, self.temp_neo) os.system(neato_cmd) plot_cmd = self.dot else: self.save_dot(self.temp_dot) plot_cmd = self.dot file_name = "%s.%s" % (file_name, file_type) create_cmd = "%s -T%s %s -o %s" % ( plot_cmd, file_type, self.temp_dot, file_name, ) os.system(create_cmd) ================================================ FILE: lib/spack/spack/vendor/altgraph/Graph.py ================================================ """ spack.vendor.altgraph.Graph - Base Graph class ================================= .. #--Version 2.1 #--Bob Ippolito October, 2004 #--Version 2.0 #--Istvan Albert June, 2004 #--Version 1.0 #--Nathan Denny, May 27, 1999 """ from collections import deque from spack.vendor.altgraph import GraphError class Graph(object): """ The Graph class represents a directed graph with *N* nodes and *E* edges. Naming conventions: - the prefixes such as *out*, *inc* and *all* will refer to methods that operate on the outgoing, incoming or all edges of that node. For example: :py:meth:`inc_degree` will refer to the degree of the node computed over the incoming edges (the number of neighbours linking to the node). - the prefixes such as *forw* and *back* will refer to the orientation of the edges used in the method with respect to the node. For example: :py:meth:`forw_bfs` will start at the node then use the outgoing edges to traverse the graph (goes forward). """ def __init__(self, edges=None): """ Initialization """ self.next_edge = 0 self.nodes, self.edges = {}, {} self.hidden_edges, self.hidden_nodes = {}, {} if edges is not None: for item in edges: if len(item) == 2: head, tail = item self.add_edge(head, tail) elif len(item) == 3: head, tail, data = item self.add_edge(head, tail, data) else: raise GraphError("Cannot create edge from %s" % (item,)) def __repr__(self): return "" % ( self.number_of_nodes(), self.number_of_edges(), ) def add_node(self, node, node_data=None): """ Adds a new node to the graph. Arbitrary data can be attached to the node via the node_data parameter. Adding the same node twice will be silently ignored. The node must be a hashable value. """ # # the nodes will contain tuples that will store incoming edges, # outgoing edges and data # # index 0 -> incoming edges # index 1 -> outgoing edges if node in self.hidden_nodes: # Node is present, but hidden return if node not in self.nodes: self.nodes[node] = ([], [], node_data) def add_edge(self, head_id, tail_id, edge_data=1, create_nodes=True): """ Adds a directed edge going from head_id to tail_id. Arbitrary data can be attached to the edge via edge_data. It may create the nodes if adding edges between nonexisting ones. :param head_id: head node :param tail_id: tail node :param edge_data: (optional) data attached to the edge :param create_nodes: (optional) creates the head_id or tail_id node in case they did not exist """ # shorcut edge = self.next_edge # add nodes if on automatic node creation if create_nodes: self.add_node(head_id) self.add_node(tail_id) # update the corresponding incoming and outgoing lists in the nodes # index 0 -> incoming edges # index 1 -> outgoing edges try: self.nodes[tail_id][0].append(edge) self.nodes[head_id][1].append(edge) except KeyError: raise GraphError("Invalid nodes %s -> %s" % (head_id, tail_id)) # store edge information self.edges[edge] = (head_id, tail_id, edge_data) self.next_edge += 1 def hide_edge(self, edge): """ Hides an edge from the graph. The edge may be unhidden at some later time. """ try: head_id, tail_id, edge_data = self.hidden_edges[edge] = self.edges[edge] self.nodes[tail_id][0].remove(edge) self.nodes[head_id][1].remove(edge) del self.edges[edge] except KeyError: raise GraphError("Invalid edge %s" % edge) def hide_node(self, node): """ Hides a node from the graph. The incoming and outgoing edges of the node will also be hidden. The node may be unhidden at some later time. """ try: all_edges = self.all_edges(node) self.hidden_nodes[node] = (self.nodes[node], all_edges) for edge in all_edges: self.hide_edge(edge) del self.nodes[node] except KeyError: raise GraphError("Invalid node %s" % node) def restore_node(self, node): """ Restores a previously hidden node back into the graph and restores all of its incoming and outgoing edges. """ try: self.nodes[node], all_edges = self.hidden_nodes[node] for edge in all_edges: self.restore_edge(edge) del self.hidden_nodes[node] except KeyError: raise GraphError("Invalid node %s" % node) def restore_edge(self, edge): """ Restores a previously hidden edge back into the graph. """ try: head_id, tail_id, data = self.hidden_edges[edge] self.nodes[tail_id][0].append(edge) self.nodes[head_id][1].append(edge) self.edges[edge] = head_id, tail_id, data del self.hidden_edges[edge] except KeyError: raise GraphError("Invalid edge %s" % edge) def restore_all_edges(self): """ Restores all hidden edges. """ for edge in list(self.hidden_edges.keys()): try: self.restore_edge(edge) except GraphError: pass def restore_all_nodes(self): """ Restores all hidden nodes. """ for node in list(self.hidden_nodes.keys()): self.restore_node(node) def __contains__(self, node): """ Test whether a node is in the graph """ return node in self.nodes def edge_by_id(self, edge): """ Returns the edge that connects the head_id and tail_id nodes """ try: head, tail, data = self.edges[edge] except KeyError: head, tail = None, None raise GraphError("Invalid edge %s" % edge) return (head, tail) def edge_by_node(self, head, tail): """ Returns the edge that connects the head_id and tail_id nodes """ for edge in self.out_edges(head): if self.tail(edge) == tail: return edge return None def number_of_nodes(self): """ Returns the number of nodes """ return len(self.nodes) def number_of_edges(self): """ Returns the number of edges """ return len(self.edges) def __iter__(self): """ Iterates over all nodes in the graph """ return iter(self.nodes) def node_list(self): """ Return a list of the node ids for all visible nodes in the graph. """ return list(self.nodes.keys()) def edge_list(self): """ Returns an iterator for all visible nodes in the graph. """ return list(self.edges.keys()) def number_of_hidden_edges(self): """ Returns the number of hidden edges """ return len(self.hidden_edges) def number_of_hidden_nodes(self): """ Returns the number of hidden nodes """ return len(self.hidden_nodes) def hidden_node_list(self): """ Returns the list with the hidden nodes """ return list(self.hidden_nodes.keys()) def hidden_edge_list(self): """ Returns a list with the hidden edges """ return list(self.hidden_edges.keys()) def describe_node(self, node): """ return node, node data, outgoing edges, incoming edges for node """ incoming, outgoing, data = self.nodes[node] return node, data, outgoing, incoming def describe_edge(self, edge): """ return edge, edge data, head, tail for edge """ head, tail, data = self.edges[edge] return edge, data, head, tail def node_data(self, node): """ Returns the data associated with a node """ return self.nodes[node][2] def edge_data(self, edge): """ Returns the data associated with an edge """ return self.edges[edge][2] def update_edge_data(self, edge, edge_data): """ Replace the edge data for a specific edge """ self.edges[edge] = self.edges[edge][0:2] + (edge_data,) def head(self, edge): """ Returns the node of the head of the edge. """ return self.edges[edge][0] def tail(self, edge): """ Returns node of the tail of the edge. """ return self.edges[edge][1] def out_nbrs(self, node): """ List of nodes connected by outgoing edges """ return [self.tail(n) for n in self.out_edges(node)] def inc_nbrs(self, node): """ List of nodes connected by incoming edges """ return [self.head(n) for n in self.inc_edges(node)] def all_nbrs(self, node): """ List of nodes connected by incoming and outgoing edges """ return list(dict.fromkeys(self.inc_nbrs(node) + self.out_nbrs(node))) def out_edges(self, node): """ Returns a list of the outgoing edges """ try: return list(self.nodes[node][1]) except KeyError: raise GraphError("Invalid node %s" % node) def inc_edges(self, node): """ Returns a list of the incoming edges """ try: return list(self.nodes[node][0]) except KeyError: raise GraphError("Invalid node %s" % node) def all_edges(self, node): """ Returns a list of incoming and outging edges. """ return set(self.inc_edges(node) + self.out_edges(node)) def out_degree(self, node): """ Returns the number of outgoing edges """ return len(self.out_edges(node)) def inc_degree(self, node): """ Returns the number of incoming edges """ return len(self.inc_edges(node)) def all_degree(self, node): """ The total degree of a node """ return self.inc_degree(node) + self.out_degree(node) def _topo_sort(self, forward=True): """ Topological sort. Returns a list of nodes where the successors (based on outgoing and incoming edges selected by the forward parameter) of any given node appear in the sequence after that node. """ topo_list = [] queue = deque() indeg = {} # select the operation that will be performed if forward: get_edges = self.out_edges get_degree = self.inc_degree get_next = self.tail else: get_edges = self.inc_edges get_degree = self.out_degree get_next = self.head for node in self.node_list(): degree = get_degree(node) if degree: indeg[node] = degree else: queue.append(node) while queue: curr_node = queue.popleft() topo_list.append(curr_node) for edge in get_edges(curr_node): tail_id = get_next(edge) if tail_id in indeg: indeg[tail_id] -= 1 if indeg[tail_id] == 0: queue.append(tail_id) if len(topo_list) == len(self.node_list()): valid = True else: # the graph has cycles, invalid topological sort valid = False return (valid, topo_list) def forw_topo_sort(self): """ Topological sort. Returns a list of nodes where the successors (based on outgoing edges) of any given node appear in the sequence after that node. """ return self._topo_sort(forward=True) def back_topo_sort(self): """ Reverse topological sort. Returns a list of nodes where the successors (based on incoming edges) of any given node appear in the sequence after that node. """ return self._topo_sort(forward=False) def _bfs_subgraph(self, start_id, forward=True): """ Private method creates a subgraph in a bfs order. The forward parameter specifies whether it is a forward or backward traversal. """ if forward: get_bfs = self.forw_bfs get_nbrs = self.out_nbrs else: get_bfs = self.back_bfs get_nbrs = self.inc_nbrs g = Graph() bfs_list = get_bfs(start_id) for node in bfs_list: g.add_node(node) for node in bfs_list: for nbr_id in get_nbrs(node): if forward: g.add_edge(node, nbr_id) else: g.add_edge(nbr_id, node) return g def forw_bfs_subgraph(self, start_id): """ Creates and returns a subgraph consisting of the breadth first reachable nodes based on their outgoing edges. """ return self._bfs_subgraph(start_id, forward=True) def back_bfs_subgraph(self, start_id): """ Creates and returns a subgraph consisting of the breadth first reachable nodes based on the incoming edges. """ return self._bfs_subgraph(start_id, forward=False) def iterdfs(self, start, end=None, forward=True): """ Collecting nodes in some depth first traversal. The forward parameter specifies whether it is a forward or backward traversal. """ visited, stack = {start}, deque([start]) if forward: get_edges = self.out_edges get_next = self.tail else: get_edges = self.inc_edges get_next = self.head while stack: curr_node = stack.pop() yield curr_node if curr_node == end: break for edge in sorted(get_edges(curr_node)): tail = get_next(edge) if tail not in visited: visited.add(tail) stack.append(tail) def iterdata(self, start, end=None, forward=True, condition=None): """ Perform a depth-first walk of the graph (as ``iterdfs``) and yield the item data of every node where condition matches. The condition callback is only called when node_data is not None. """ visited, stack = {start}, deque([start]) if forward: get_edges = self.out_edges get_next = self.tail else: get_edges = self.inc_edges get_next = self.head get_data = self.node_data while stack: curr_node = stack.pop() curr_data = get_data(curr_node) if curr_data is not None: if condition is not None and not condition(curr_data): continue yield curr_data if curr_node == end: break for edge in get_edges(curr_node): tail = get_next(edge) if tail not in visited: visited.add(tail) stack.append(tail) def _iterbfs(self, start, end=None, forward=True): """ The forward parameter specifies whether it is a forward or backward traversal. Returns a list of tuples where the first value is the hop value the second value is the node id. """ queue, visited = deque([(start, 0)]), {start} # the direction of the bfs depends on the edges that are sampled if forward: get_edges = self.out_edges get_next = self.tail else: get_edges = self.inc_edges get_next = self.head while queue: curr_node, curr_step = queue.popleft() yield (curr_node, curr_step) if curr_node == end: break for edge in get_edges(curr_node): tail = get_next(edge) if tail not in visited: visited.add(tail) queue.append((tail, curr_step + 1)) def forw_bfs(self, start, end=None): """ Returns a list of nodes in some forward BFS order. Starting from the start node the breadth first search proceeds along outgoing edges. """ return [node for node, step in self._iterbfs(start, end, forward=True)] def back_bfs(self, start, end=None): """ Returns a list of nodes in some backward BFS order. Starting from the start node the breadth first search proceeds along incoming edges. """ return [node for node, _ in self._iterbfs(start, end, forward=False)] def forw_dfs(self, start, end=None): """ Returns a list of nodes in some forward DFS order. Starting with the start node the depth first search proceeds along outgoing edges. """ return list(self.iterdfs(start, end, forward=True)) def back_dfs(self, start, end=None): """ Returns a list of nodes in some backward DFS order. Starting from the start node the depth first search proceeds along incoming edges. """ return list(self.iterdfs(start, end, forward=False)) def connected(self): """ Returns :py:data:`True` if the graph's every node can be reached from every other node. """ node_list = self.node_list() for node in node_list: bfs_list = self.forw_bfs(node) if len(bfs_list) != len(node_list): return False return True def clust_coef(self, node): """ Computes and returns the local clustering coefficient of node. The local cluster coefficient is proportion of the actual number of edges between neighbours of node and the maximum number of edges between those neighbours. See "Local Clustering Coefficient" on for a formal definition. """ num = 0 nbr_set = set(self.out_nbrs(node)) if node in nbr_set: nbr_set.remove(node) # loop defense for nbr in nbr_set: sec_set = set(self.out_nbrs(nbr)) if nbr in sec_set: sec_set.remove(nbr) # loop defense num += len(nbr_set & sec_set) nbr_num = len(nbr_set) if nbr_num: clust_coef = float(num) / (nbr_num * (nbr_num - 1)) else: clust_coef = 0.0 return clust_coef def get_hops(self, start, end=None, forward=True): """ Computes the hop distance to all nodes centered around a node. First order neighbours are at hop 1, their neigbours are at hop 2 etc. Uses :py:meth:`forw_bfs` or :py:meth:`back_bfs` depending on the value of the forward parameter. If the distance between all neighbouring nodes is 1 the hop number corresponds to the shortest distance between the nodes. :param start: the starting node :param end: ending node (optional). When not specified will search the whole graph. :param forward: directionality parameter (optional). If C{True} (default) it uses L{forw_bfs} otherwise L{back_bfs}. :return: returns a list of tuples where each tuple contains the node and the hop. Typical usage:: >>> print (graph.get_hops(1, 8)) >>> [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)] # node 1 is at 0 hops # node 2 is at 1 hop # ... # node 8 is at 5 hops """ if forward: return list(self._iterbfs(start=start, end=end, forward=True)) else: return list(self._iterbfs(start=start, end=end, forward=False)) ================================================ FILE: lib/spack/spack/vendor/altgraph/GraphAlgo.py ================================================ """ spack.vendor.altgraph.GraphAlgo - Graph algorithms ===================================== """ from spack.vendor.altgraph import GraphError def dijkstra(graph, start, end=None): """ Dijkstra's algorithm for shortest paths `David Eppstein, UC Irvine, 4 April 2002 `_ `Python Cookbook Recipe `_ Find shortest paths from the start node to all nodes nearer than or equal to the end node. Dijkstra's algorithm is only guaranteed to work correctly when all edge lengths are positive. This code does not verify this property for all edges (only the edges examined until the end vertex is reached), but will correctly compute shortest paths even for some graphs with negative edges, and will raise an exception if it discovers that a negative edge has caused it to make a mistake. Adapted to spack.vendor.altgraph by Istvan Albert, Pennsylvania State University - June, 9 2004 """ D = {} # dictionary of final distances P = {} # dictionary of predecessors Q = _priorityDictionary() # estimated distances of non-final vertices Q[start] = 0 for v in Q: D[v] = Q[v] if v == end: break for w in graph.out_nbrs(v): edge_id = graph.edge_by_node(v, w) vwLength = D[v] + graph.edge_data(edge_id) if w in D: if vwLength < D[w]: raise GraphError( "Dijkstra: found better path to already-final vertex" ) elif w not in Q or vwLength < Q[w]: Q[w] = vwLength P[w] = v return (D, P) def shortest_path(graph, start, end): """ Find a single shortest path from the *start* node to the *end* node. The input has the same conventions as dijkstra(). The output is a list of the nodes in order along the shortest path. **Note that the distances must be stored in the edge data as numeric data** """ D, P = dijkstra(graph, start, end) Path = [] while 1: Path.append(end) if end == start: break end = P[end] Path.reverse() return Path # # Utility classes and functions # class _priorityDictionary(dict): """ Priority dictionary using binary heaps (internal use only) David Eppstein, UC Irvine, 8 Mar 2002 Implements a data structure that acts almost like a dictionary, with two modifications: 1. D.smallest() returns the value x minimizing D[x]. For this to work correctly, all values D[x] stored in the dictionary must be comparable. 2. iterating "for x in D" finds and removes the items from D in sorted order. Each item is not removed until the next item is requested, so D[x] will still return a useful value until the next iteration of the for-loop. Each operation takes logarithmic amortized time. """ def __init__(self): """ Initialize priorityDictionary by creating binary heap of pairs (value,key). Note that changing or removing a dict entry will not remove the old pair from the heap until it is found by smallest() or until the heap is rebuilt. """ self.__heap = [] dict.__init__(self) def smallest(self): """ Find smallest item after removing deleted items from front of heap. """ if len(self) == 0: raise IndexError("smallest of empty priorityDictionary") heap = self.__heap while heap[0][1] not in self or self[heap[0][1]] != heap[0][0]: lastItem = heap.pop() insertionPoint = 0 while 1: smallChild = 2 * insertionPoint + 1 if ( smallChild + 1 < len(heap) and heap[smallChild] > heap[smallChild + 1] ): smallChild += 1 if smallChild >= len(heap) or lastItem <= heap[smallChild]: heap[insertionPoint] = lastItem break heap[insertionPoint] = heap[smallChild] insertionPoint = smallChild return heap[0][1] def __iter__(self): """ Create destructive sorted iterator of priorityDictionary. """ def iterfn(): while len(self) > 0: x = self.smallest() yield x del self[x] return iterfn() def __setitem__(self, key, val): """ Change value stored in dictionary and add corresponding pair to heap. Rebuilds the heap if the number of deleted items gets large, to avoid memory leakage. """ dict.__setitem__(self, key, val) heap = self.__heap if len(heap) > 2 * len(self): self.__heap = [(v, k) for k, v in self.items()] self.__heap.sort() else: newPair = (val, key) insertionPoint = len(heap) heap.append(None) while insertionPoint > 0 and newPair < heap[(insertionPoint - 1) // 2]: heap[insertionPoint] = heap[(insertionPoint - 1) // 2] insertionPoint = (insertionPoint - 1) // 2 heap[insertionPoint] = newPair def setdefault(self, key, val): """ Reimplement setdefault to pass through our customized __setitem__. """ if key not in self: self[key] = val return self[key] ================================================ FILE: lib/spack/spack/vendor/altgraph/GraphStat.py ================================================ """ spack.vendor.altgraph.GraphStat - Functions providing various graph statistics ================================================================= """ def degree_dist(graph, limits=(0, 0), bin_num=10, mode="out"): """ Computes the degree distribution for a graph. Returns a list of tuples where the first element of the tuple is the center of the bin representing a range of degrees and the second element of the tuple are the number of nodes with the degree falling in the range. Example:: .... """ deg = [] if mode == "inc": get_deg = graph.inc_degree else: get_deg = graph.out_degree for node in graph: deg.append(get_deg(node)) if not deg: return [] results = _binning(values=deg, limits=limits, bin_num=bin_num) return results _EPS = 1.0 / (2.0 ** 32) def _binning(values, limits=(0, 0), bin_num=10): """ Bins data that falls between certain limits, if the limits are (0, 0) the minimum and maximum values are used. Returns a list of tuples where the first element of the tuple is the center of the bin and the second element of the tuple are the counts. """ if limits == (0, 0): min_val, max_val = min(values) - _EPS, max(values) + _EPS else: min_val, max_val = limits # get bin size bin_size = (max_val - min_val) / float(bin_num) bins = [0] * (bin_num) # will ignore these outliers for now for value in values: try: if (value - min_val) >= 0: index = int((value - min_val) / float(bin_size)) bins[index] += 1 except IndexError: pass # make it ready for an x,y plot result = [] center = (bin_size / 2) + min_val for i, y in enumerate(bins): x = center + bin_size * i result.append((x, y)) return result ================================================ FILE: lib/spack/spack/vendor/altgraph/GraphUtil.py ================================================ """ spack.vendor.altgraph.GraphUtil - Utility classes and functions ================================================== """ import random from collections import deque from spack.vendor.altgraph import Graph, GraphError def generate_random_graph(node_num, edge_num, self_loops=False, multi_edges=False): """ Generates and returns a :py:class:`~spack.vendor.altgraph.Graph.Graph` instance with *node_num* nodes randomly connected by *edge_num* edges. """ g = Graph.Graph() if not multi_edges: if self_loops: max_edges = node_num * node_num else: max_edges = node_num * (node_num - 1) if edge_num > max_edges: raise GraphError("inconsistent arguments to 'generate_random_graph'") nodes = range(node_num) for node in nodes: g.add_node(node) while 1: head = random.choice(nodes) tail = random.choice(nodes) # loop defense if head == tail and not self_loops: continue # multiple edge defense if g.edge_by_node(head, tail) is not None and not multi_edges: continue # add the edge g.add_edge(head, tail) if g.number_of_edges() >= edge_num: break return g def generate_scale_free_graph(steps, growth_num, self_loops=False, multi_edges=False): """ Generates and returns a :py:class:`~spack.vendor.altgraph.Graph.Graph` instance that will have *steps* \\* *growth_num* nodes and a scale free (powerlaw) connectivity. Starting with a fully connected graph with *growth_num* nodes at every step *growth_num* nodes are added to the graph and are connected to existing nodes with a probability proportional to the degree of these existing nodes. """ # The code doesn't seem to do what the documentation claims. graph = Graph.Graph() # initialize the graph store = [] for i in range(growth_num): for j in range(i + 1, growth_num): store.append(i) store.append(j) graph.add_edge(i, j) # generate for node in range(growth_num, steps * growth_num): graph.add_node(node) while graph.out_degree(node) < growth_num: nbr = random.choice(store) # loop defense if node == nbr and not self_loops: continue # multi edge defense if graph.edge_by_node(node, nbr) and not multi_edges: continue graph.add_edge(node, nbr) for nbr in graph.out_nbrs(node): store.append(node) store.append(nbr) return graph def filter_stack(graph, head, filters): """ Perform a walk in a depth-first order starting at *head*. Returns (visited, removes, orphans). * visited: the set of visited nodes * removes: the list of nodes where the node data does not all *filters* * orphans: tuples of (last_good, node), where node is not in removes, is directly reachable from a node in *removes* and *last_good* is the closest upstream node that is not in *removes*. """ visited, removes, orphans = {head}, set(), set() stack = deque([(head, head)]) get_data = graph.node_data get_edges = graph.out_edges get_tail = graph.tail while stack: last_good, node = stack.pop() data = get_data(node) if data is not None: for filtfunc in filters: if not filtfunc(data): removes.add(node) break else: last_good = node for edge in get_edges(node): tail = get_tail(edge) if last_good is not node: orphans.add((last_good, tail)) if tail not in visited: visited.add(tail) stack.append((last_good, tail)) orphans = [(lg, tl) for (lg, tl) in orphans if tl not in removes] return visited, removes, orphans ================================================ FILE: lib/spack/spack/vendor/altgraph/LICENSE ================================================ Copyright (c) 2004 Istvan Albert unless otherwise noted. Copyright (c) 2006-2010 Bob Ippolito Copyright (2) 2010-2020 Ronald Oussoren, et. al. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: lib/spack/spack/vendor/altgraph/ObjectGraph.py ================================================ """ spack.vendor.altgraph.ObjectGraph - Graph of objects with an identifier ========================================================== A graph of objects that have a "graphident" attribute. graphident is the key for the object in the graph """ from spack.vendor.altgraph import GraphError from spack.vendor.altgraph.Graph import Graph from spack.vendor.altgraph.GraphUtil import filter_stack class ObjectGraph(object): """ A graph of objects that have a "graphident" attribute. graphident is the key for the object in the graph """ def __init__(self, graph=None, debug=0): if graph is None: graph = Graph() self.graphident = self self.graph = graph self.debug = debug self.indent = 0 graph.add_node(self, None) def __repr__(self): return "<%s>" % (type(self).__name__,) def flatten(self, condition=None, start=None): """ Iterate over the subgraph that is entirely reachable by condition starting from the given start node or the ObjectGraph root """ if start is None: start = self start = self.getRawIdent(start) return self.graph.iterdata(start=start, condition=condition) def nodes(self): for ident in self.graph: node = self.graph.node_data(ident) if node is not None: yield self.graph.node_data(ident) def get_edges(self, node): if node is None: node = self start = self.getRawIdent(node) _, _, outraw, incraw = self.graph.describe_node(start) def iter_edges(lst, n): seen = set() for tpl in (self.graph.describe_edge(e) for e in lst): ident = tpl[n] if ident not in seen: yield self.findNode(ident) seen.add(ident) return iter_edges(outraw, 3), iter_edges(incraw, 2) def edgeData(self, fromNode, toNode): if fromNode is None: fromNode = self start = self.getRawIdent(fromNode) stop = self.getRawIdent(toNode) edge = self.graph.edge_by_node(start, stop) return self.graph.edge_data(edge) def updateEdgeData(self, fromNode, toNode, edgeData): if fromNode is None: fromNode = self start = self.getRawIdent(fromNode) stop = self.getRawIdent(toNode) edge = self.graph.edge_by_node(start, stop) self.graph.update_edge_data(edge, edgeData) def filterStack(self, filters): """ Filter the ObjectGraph in-place by removing all edges to nodes that do not match every filter in the given filter list Returns a tuple containing the number of: (nodes_visited, nodes_removed, nodes_orphaned) """ visited, removes, orphans = filter_stack(self.graph, self, filters) for last_good, tail in orphans: self.graph.add_edge(last_good, tail, edge_data="orphan") for node in removes: self.graph.hide_node(node) return len(visited) - 1, len(removes), len(orphans) def removeNode(self, node): """ Remove the given node from the graph if it exists """ ident = self.getIdent(node) if ident is not None: self.graph.hide_node(ident) def removeReference(self, fromnode, tonode): """ Remove all edges from fromnode to tonode """ if fromnode is None: fromnode = self fromident = self.getIdent(fromnode) toident = self.getIdent(tonode) if fromident is not None and toident is not None: while True: edge = self.graph.edge_by_node(fromident, toident) if edge is None: break self.graph.hide_edge(edge) def getIdent(self, node): """ Get the graph identifier for a node """ ident = self.getRawIdent(node) if ident is not None: return ident node = self.findNode(node) if node is None: return None return node.graphident def getRawIdent(self, node): """ Get the identifier for a node object """ if node is self: return node ident = getattr(node, "graphident", None) return ident def __contains__(self, node): return self.findNode(node) is not None def findNode(self, node): """ Find the node on the graph """ ident = self.getRawIdent(node) if ident is None: ident = node try: return self.graph.node_data(ident) except KeyError: return None def addNode(self, node): """ Add a node to the graph referenced by the root """ self.msg(4, "addNode", node) try: self.graph.restore_node(node.graphident) except GraphError: self.graph.add_node(node.graphident, node) def createReference(self, fromnode, tonode, edge_data=None): """ Create a reference from fromnode to tonode """ if fromnode is None: fromnode = self fromident, toident = self.getIdent(fromnode), self.getIdent(tonode) if fromident is None or toident is None: return self.msg(4, "createReference", fromnode, tonode, edge_data) self.graph.add_edge(fromident, toident, edge_data=edge_data) def createNode(self, cls, name, *args, **kw): """ Add a node of type cls to the graph if it does not already exist by the given name """ m = self.findNode(name) if m is None: m = cls(name, *args, **kw) self.addNode(m) return m def msg(self, level, s, *args): """ Print a debug message with the given level """ if s and level <= self.debug: print("%s%s %s" % (" " * self.indent, s, " ".join(map(repr, args)))) def msgin(self, level, s, *args): """ Print a debug message and indent """ if level <= self.debug: self.msg(level, s, *args) self.indent = self.indent + 1 def msgout(self, level, s, *args): """ Dedent and print a debug message """ if level <= self.debug: self.indent = self.indent - 1 self.msg(level, s, *args) ================================================ FILE: lib/spack/spack/vendor/altgraph/__init__.py ================================================ """ spack.vendor.altgraph - a python graph library ================================= spack.vendor.altgraph is a fork of `graphlib `_ tailored to use newer Python 2.3+ features, including additional support used by the py2app suite (modulegraph and spack.vendor.macholib, specifically). spack.vendor.altgraph is a python based graph (network) representation and manipulation package. It has started out as an extension to the `graph_lib module `_ written by Nathan Denny it has been significantly optimized and expanded. The :class:`spack.vendor.altgraph.Graph.Graph` class is loosely modeled after the `LEDA `_ (Library of Efficient Datatypes) representation. The library includes methods for constructing graphs, BFS and DFS traversals, topological sort, finding connected components, shortest paths as well as a number graph statistics functions. The library can also visualize graphs via `graphviz `_. The package contains the following modules: - the :py:mod:`spack.vendor.altgraph.Graph` module contains the :class:`~spack.vendor.altgraph.Graph.Graph` class that stores the graph data - the :py:mod:`spack.vendor.altgraph.GraphAlgo` module implements graph algorithms operating on graphs (:py:class:`~spack.vendor.altgraph.Graph.Graph`} instances) - the :py:mod:`spack.vendor.altgraph.GraphStat` module contains functions for computing statistical measures on graphs - the :py:mod:`spack.vendor.altgraph.GraphUtil` module contains functions for generating, reading and saving graphs - the :py:mod:`spack.vendor.altgraph.Dot` module contains functions for displaying graphs via `graphviz `_ - the :py:mod:`spack.vendor.altgraph.ObjectGraph` module implements a graph of objects with a unique identifier Installation ------------ Download and unpack the archive then type:: python setup.py install This will install the library in the default location. For instructions on how to customize the install procedure read the output of:: python setup.py --help install To verify that the code works run the test suite:: python setup.py test Example usage ------------- Lets assume that we want to analyze the graph below (links to the full picture) GRAPH_IMG. Our script then might look the following way:: from spack.vendor.altgraph import Graph, GraphAlgo, Dot # these are the edges edges = [ (1,2), (2,4), (1,3), (2,4), (3,4), (4,5), (6,5), (6,14), (14,15), (6, 15), (5,7), (7, 8), (7,13), (12,8), (8,13), (11,12), (11,9), (13,11), (9,13), (13,10) ] # creates the graph graph = Graph.Graph() for head, tail in edges: graph.add_edge(head, tail) # do a forward bfs from 1 at most to 20 print(graph.forw_bfs(1)) This will print the nodes in some breadth first order:: [1, 2, 3, 4, 5, 7, 8, 13, 11, 10, 12, 9] If we wanted to get the hop-distance from node 1 to node 8 we coud write:: print(graph.get_hops(1, 8)) This will print the following:: [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)] Node 1 is at 0 hops since it is the starting node, nodes 2,3 are 1 hop away ... node 8 is 5 hops away. To find the shortest distance between two nodes you can use:: print(GraphAlgo.shortest_path(graph, 1, 12)) It will print the nodes on one (if there are more) the shortest paths:: [1, 2, 4, 5, 7, 13, 11, 12] To display the graph we can use the GraphViz backend:: dot = Dot.Dot(graph) # display the graph on the monitor dot.display() # save it in an image file dot.save_img(file_name='graph', file_type='gif') .. @author: U{Istvan Albert} @license: MIT License Copyright (c) 2004 Istvan Albert unless otherwise noted. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @requires: Python 2.3 or higher @newfield contributor: Contributors: @contributor: U{Reka Albert } """ __version__ = "0.17.3" class GraphError(ValueError): pass ================================================ FILE: lib/spack/spack/vendor/archspec/LICENSE ================================================ The MIT License (MIT) Copyright (c) 2014 Anders Høst Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: lib/spack/spack/vendor/archspec/LICENSE-APACHE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: lib/spack/spack/vendor/archspec/LICENSE-MIT ================================================ Copyright 2019-2020 Lawrence Livermore National Security, LLC and other Archspec Project Developers. See the top-level COPYRIGHT file for details. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: lib/spack/spack/vendor/archspec/__init__.py ================================================ """Init file to avoid namespace packages""" __version__ = "0.2.5" ================================================ FILE: lib/spack/spack/vendor/archspec/__main__.py ================================================ """ Run the `archspec` CLI as a module. """ import sys from .cli import main sys.exit(main()) ================================================ FILE: lib/spack/spack/vendor/archspec/cli.py ================================================ # Copyright 2019-2020 Lawrence Livermore National Security, LLC and other # Archspec Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """ archspec command line interface """ import argparse import typing from . import __version__ as archspec_version from .cpu import host def _make_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( "archspec", description="archspec command line interface", add_help=False, ) parser.add_argument( "--version", "-V", help="Show the version and exit.", action="version", version=f"archspec, version {archspec_version}", ) parser.add_argument("--help", "-h", help="Show the help and exit.", action="help") subcommands = parser.add_subparsers( title="command", metavar="COMMAND", dest="command", ) cpu_command = subcommands.add_parser( "cpu", help="archspec command line interface for CPU", description="archspec command line interface for CPU", ) cpu_command.set_defaults(run=cpu) return parser def cpu() -> int: """Run the `archspec cpu` subcommand.""" try: print(host()) except FileNotFoundError as exc: print(exc) return 1 return 0 def main(argv: typing.Optional[typing.List[str]] = None) -> int: """Run the `archspec` command line interface.""" parser = _make_parser() try: args = parser.parse_args(argv) except SystemExit as err: return err.code if args.command is None: parser.print_help() return 0 return args.run() ================================================ FILE: lib/spack/spack/vendor/archspec/cpu/__init__.py ================================================ # Copyright 2019-2020 Lawrence Livermore National Security, LLC and other # Archspec Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """The "cpu" package permits to query and compare different CPU microarchitectures. """ from .detect import brand_string, host from .microarchitecture import ( TARGETS, InvalidCompilerVersion, Microarchitecture, UnsupportedMicroarchitecture, generic_microarchitecture, version_components, ) __all__ = [ "brand_string", "host", "TARGETS", "InvalidCompilerVersion", "Microarchitecture", "UnsupportedMicroarchitecture", "generic_microarchitecture", "version_components", ] ================================================ FILE: lib/spack/spack/vendor/archspec/cpu/alias.py ================================================ # Copyright 2019-2020 Lawrence Livermore National Security, LLC and other # Archspec Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Aliases for microarchitecture features.""" from typing import Callable, Dict from .schema import TARGETS_JSON, LazyDictionary _FEATURE_ALIAS_PREDICATE: Dict[str, Callable] = {} class FeatureAliasTest: """A test that must be passed for a feature alias to succeed. Args: rules (dict): dictionary of rules to be met. Each key must be a valid alias predicate """ # pylint: disable=too-few-public-methods def __init__(self, rules): self.rules = rules self.predicates = [] for name, args in rules.items(): self.predicates.append(_FEATURE_ALIAS_PREDICATE[name](args)) def __call__(self, microarchitecture): return all(feature_test(microarchitecture) for feature_test in self.predicates) def _feature_aliases(): """Returns the dictionary of all defined feature aliases.""" json_data = TARGETS_JSON["feature_aliases"] aliases = {} for alias, rules in json_data.items(): aliases[alias] = FeatureAliasTest(rules) return aliases FEATURE_ALIASES = LazyDictionary(_feature_aliases) def alias_predicate(func): """Decorator to register a predicate that can be used to evaluate feature aliases. """ name = func.__name__ # Check we didn't register anything else with the same name if name in _FEATURE_ALIAS_PREDICATE: msg = f'the alias predicate "{name}" already exists' raise KeyError(msg) _FEATURE_ALIAS_PREDICATE[name] = func return func @alias_predicate def reason(_): """This predicate returns always True and it's there to allow writing a documentation string in the JSON file to explain why an alias is needed. """ return lambda x: True @alias_predicate def any_of(list_of_features): """Returns a predicate that is True if any of the feature in the list is in the microarchitecture being tested, False otherwise. """ def _impl(microarchitecture): return any(x in microarchitecture for x in list_of_features) return _impl @alias_predicate def families(list_of_families): """Returns a predicate that is True if the architecture family of the microarchitecture being tested is in the list, False otherwise. """ def _impl(microarchitecture): return str(microarchitecture.family) in list_of_families return _impl ================================================ FILE: lib/spack/spack/vendor/archspec/cpu/detect.py ================================================ # Copyright 2019-2020 Lawrence Livermore National Security, LLC and other # Archspec Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Detection of CPU microarchitectures""" import collections import os import platform import re import struct import subprocess import warnings from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union from ..vendor.cpuid.cpuid import CPUID from .microarchitecture import TARGETS, Microarchitecture, generic_microarchitecture from .schema import CPUID_JSON, TARGETS_JSON #: Mapping from operating systems to chain of commands #: to obtain a dictionary of raw info on the current cpu INFO_FACTORY = collections.defaultdict(list) #: Mapping from micro-architecture families (x86_64, ppc64le, etc.) to #: functions checking the compatibility of the host with a given target COMPATIBILITY_CHECKS: Dict[str, Callable[[Microarchitecture, Microarchitecture], bool]] = {} # Constants for commonly used architectures X86_64 = "x86_64" AARCH64 = "aarch64" PPC64LE = "ppc64le" PPC64 = "ppc64" RISCV64 = "riscv64" def detection(operating_system: str): """Decorator to mark functions that are meant to return partial information on the current cpu. Args: operating_system: operating system where this function can be used. """ def decorator(factory): INFO_FACTORY[operating_system].append(factory) return factory return decorator def partial_uarch( name: str = "", vendor: str = "", features: Optional[Set[str]] = None, generation: int = 0, cpu_part: str = "", ) -> Microarchitecture: """Construct a partial microarchitecture, from information gathered during system scan.""" return Microarchitecture( name=name, parents=[], vendor=vendor, features=features or set(), compilers={}, generation=generation, cpu_part=cpu_part, ) @detection(operating_system="Linux") def proc_cpuinfo() -> Microarchitecture: """Returns a partial Microarchitecture, obtained from scanning ``/proc/cpuinfo``""" data: Dict[str, Any] = {} with open("/proc/cpuinfo") as file: # pylint: disable=unspecified-encoding for line in file: key, separator, value = line.partition(":") # If there's no separator and info was already populated # according to what's written here: # # http://www.linfo.org/proc_cpuinfo.html # # we are on a blank line separating two cpus. Exit early as # we want to read just the first entry in /proc/cpuinfo if separator != ":" and data: break data[key.strip()] = value.strip() architecture = _machine() if architecture == X86_64: return partial_uarch( vendor=data.get("vendor_id", "generic"), features=_feature_set(data, key="flags") ) if architecture == AARCH64: return partial_uarch( vendor=_canonicalize_aarch64_vendor(data), features=_feature_set(data, key="Features"), cpu_part=data.get("CPU part", ""), ) if architecture in (PPC64LE, PPC64): generation_match = re.search(r"POWER(\d+)", data.get("cpu", "")) # There might be no match under emulated environments. For instance # emulating a ppc64le with QEMU and Docker still reports the host # /proc/cpuinfo and not a Power if generation_match is None: return partial_uarch(generation=0) try: generation = int(generation_match.group(1)) except ValueError: generation = 0 return partial_uarch(generation=generation) if architecture == RISCV64: if data.get("uarch") == "sifive,u74-mc": data["uarch"] = "u74mc" return partial_uarch(name=data.get("uarch", RISCV64)) return generic_microarchitecture(architecture) class CpuidInfoCollector: """Collects the information we need on the host CPU from cpuid""" # pylint: disable=too-few-public-methods def __init__(self): self.cpuid = CPUID() registers = self.cpuid.registers_for(**CPUID_JSON["vendor"]["input"]) self.highest_basic_support = registers.eax self.vendor = struct.pack("III", registers.ebx, registers.edx, registers.ecx).decode( "utf-8" ) registers = self.cpuid.registers_for(**CPUID_JSON["highest_extension_support"]["input"]) self.highest_extension_support = registers.eax self.features = self._features() def _features(self): result = set() def check_features(data): registers = self.cpuid.registers_for(**data["input"]) for feature_check in data["bits"]: current = getattr(registers, feature_check["register"]) if self._is_bit_set(current, feature_check["bit"]): result.add(feature_check["name"]) for call_data in CPUID_JSON["flags"]: if call_data["input"]["eax"] > self.highest_basic_support: continue check_features(call_data) for call_data in CPUID_JSON["extension-flags"]: if call_data["input"]["eax"] > self.highest_extension_support: continue check_features(call_data) return result def _is_bit_set(self, register: int, bit: int) -> bool: mask = 1 << bit return register & mask > 0 def brand_string(self) -> Optional[str]: """Returns the brand string, if available.""" if self.highest_extension_support < 0x80000004: return None r1 = self.cpuid.registers_for(eax=0x80000002, ecx=0) r2 = self.cpuid.registers_for(eax=0x80000003, ecx=0) r3 = self.cpuid.registers_for(eax=0x80000004, ecx=0) result = struct.pack( "IIIIIIIIIIII", r1.eax, r1.ebx, r1.ecx, r1.edx, r2.eax, r2.ebx, r2.ecx, r2.edx, r3.eax, r3.ebx, r3.ecx, r3.edx, ).decode("utf-8") return result.strip("\x00") @detection(operating_system="Windows") def cpuid_info(): """Returns a partial Microarchitecture, obtained from running the cpuid instruction""" architecture = _machine() if architecture == X86_64: data = CpuidInfoCollector() return partial_uarch(vendor=data.vendor, features=data.features) return generic_microarchitecture(architecture) def _check_output(args, env): with subprocess.Popen(args, stdout=subprocess.PIPE, env=env) as proc: output = proc.communicate()[0] return str(output.decode("utf-8")) WINDOWS_MAPPING = { "AMD64": X86_64, "ARM64": AARCH64, } def _machine() -> str: """Return the machine architecture we are on""" operating_system = platform.system() # If we are not on Darwin or Windows, trust what Python tells us if operating_system not in ("Darwin", "Windows"): return platform.machine() # Normalize windows specific names if operating_system == "Windows": platform_machine = platform.machine() return WINDOWS_MAPPING.get(platform_machine, platform_machine) # On Darwin it might happen that we are on M1, but using an interpreter # built for x86_64. In that case "platform.machine() == 'x86_64'", so we # need to fix that. # # See: https://bugs.python.org/issue42704 output = _check_output( ["sysctl", "-n", "machdep.cpu.brand_string"], env=_ensure_bin_usrbin_in_path() ).strip() if "Apple" in output: # Note that a native Python interpreter on Apple M1 would return # "arm64" instead of "aarch64". Here we normalize to the latter. return AARCH64 return X86_64 @detection(operating_system="Darwin") def sysctl_info() -> Microarchitecture: """Returns a raw info dictionary parsing the output of sysctl.""" child_environment = _ensure_bin_usrbin_in_path() def sysctl(*args: str) -> str: return _check_output(["sysctl", *args], env=child_environment).strip() if _machine() == X86_64: raw_features = sysctl( "-n", "machdep.cpu.features", "machdep.cpu.leaf7_features", "machdep.cpu.extfeatures", ) features = set(raw_features.lower().split()) # Flags detected on Darwin turned to their linux counterpart for darwin_flags, linux_flags in TARGETS_JSON["conversions"]["darwin_flags"].items(): if all(x in features for x in darwin_flags.split()): features.update(linux_flags.split()) return partial_uarch(vendor=sysctl("-n", "machdep.cpu.vendor"), features=features) model = "unknown" model_str = sysctl("-n", "machdep.cpu.brand_string").lower() if "m4" in model_str: model = "m4" elif "m3" in model_str: model = "m3" elif "m2" in model_str: model = "m2" elif "m1" in model_str: model = "m1" elif "apple" in model_str: model = "m1" return partial_uarch(name=model, vendor="Apple") def _ensure_bin_usrbin_in_path(): # Make sure that /sbin and /usr/sbin are in PATH as sysctl is usually found there child_environment = dict(os.environ.items()) search_paths = child_environment.get("PATH", "").split(os.pathsep) for additional_path in ("/sbin", "/usr/sbin"): if additional_path not in search_paths: search_paths.append(additional_path) child_environment["PATH"] = os.pathsep.join(search_paths) return child_environment def _canonicalize_aarch64_vendor(data: Dict[str, str]) -> str: """Adjust the vendor field to make it human-readable""" if "CPU implementer" not in data: return "generic" # Mapping numeric codes to vendor (ARM). This list is a merge from # different sources: # # https://github.com/karelzak/util-linux/blob/master/sys-utils/lscpu-arm.c # https://developer.arm.com/docs/ddi0487/latest/arm-architecture-reference-manual-armv8-for-armv8-a-architecture-profile # https://github.com/gcc-mirror/gcc/blob/master/gcc/config/aarch64/aarch64-cores.def # https://patchwork.kernel.org/patch/10524949/ arm_vendors = TARGETS_JSON["conversions"]["arm_vendors"] arm_code = data["CPU implementer"] return arm_vendors.get(arm_code, arm_code) def _feature_set(data: Dict[str, str], key: str) -> Set[str]: return set(data.get(key, "").split()) def detected_info() -> Microarchitecture: """Returns a partial Microarchitecture with information on the CPU of the current host. This function calls all the viable factories one after the other until there's one that is able to produce the requested information. Falls-back to a generic microarchitecture, if none of the calls succeed. """ # pylint: disable=broad-except for factory in INFO_FACTORY[platform.system()]: try: return factory() except Exception as exc: warnings.warn(str(exc)) return generic_microarchitecture(_machine()) def compatible_microarchitectures(info: Microarchitecture) -> List[Microarchitecture]: """Returns an unordered list of known micro-architectures that are compatible with the partial Microarchitecture passed as input. """ architecture_family = _machine() # If a tester is not registered, assume no known target is compatible with the host tester = COMPATIBILITY_CHECKS.get(architecture_family, lambda x, y: False) return [x for x in TARGETS.values() if tester(info, x)] or [ generic_microarchitecture(architecture_family) ] def host() -> Microarchitecture: """Detects the host micro-architecture and returns it.""" # Retrieve information on the host's cpu info = detected_info() # Get a list of possible candidates for this micro-architecture candidates = compatible_microarchitectures(info) # Sorting criteria for candidates def sorting_fn(item): return len(item.ancestors), len(item.features) # Get the best generic micro-architecture generic_candidates = [c for c in candidates if c.vendor == "generic"] best_generic = max(generic_candidates, key=sorting_fn) # Relevant for AArch64. Filter on "cpu_part" if we have any match if info.cpu_part != "" and any(c for c in candidates if info.cpu_part == c.cpu_part): candidates = [c for c in candidates if info.cpu_part == c.cpu_part] # Filter the candidates to be descendant of the best generic candidate. # This is to avoid that the lack of a niche feature that can be disabled # from e.g. BIOS prevents detection of a reasonably performant architecture candidates = [c for c in candidates if c > best_generic] # If we don't have candidates, return the best generic micro-architecture if not candidates: return best_generic # Reverse sort of the depth for the inheritance tree among only targets we # can use. This gets the newest target we satisfy. return max(candidates, key=sorting_fn) def compatibility_check(architecture_family: Union[str, Tuple[str, ...]]): """Decorator to register a function as a proper compatibility check. A compatibility check function takes a partial Microarchitecture object as a first argument, and an arbitrary target Microarchitecture as the second argument. It returns True if the target is compatible with the first argument, False otherwise. Args: architecture_family: architecture family for which this test can be used """ # Turn the argument into something iterable if isinstance(architecture_family, str): architecture_family = (architecture_family,) def decorator(func): COMPATIBILITY_CHECKS.update({family: func for family in architecture_family}) return func return decorator @compatibility_check(architecture_family=(PPC64LE, PPC64)) def compatibility_check_for_power(info, target): """Compatibility check for PPC64 and PPC64LE architectures.""" # We can use a target if it descends from our machine type, and our # generation (9 for POWER9, etc.) is at least its generation. arch_root = TARGETS[_machine()] return ( target == arch_root or arch_root in target.ancestors ) and target.generation <= info.generation @compatibility_check(architecture_family=X86_64) def compatibility_check_for_x86_64(info, target): """Compatibility check for x86_64 architectures.""" # We can use a target if it descends from our machine type, is from our # vendor, and we have all of its features arch_root = TARGETS[X86_64] return ( (target == arch_root or arch_root in target.ancestors) and target.vendor in (info.vendor, "generic") and target.features.issubset(info.features) ) @compatibility_check(architecture_family=AARCH64) def compatibility_check_for_aarch64(info, target): """Compatibility check for AARCH64 architectures.""" # At the moment, it's not clear how to detect compatibility with # a specific version of the architecture if target.vendor == "generic" and target.name != AARCH64: return False arch_root = TARGETS[AARCH64] arch_root_and_vendor = arch_root == target.family and target.vendor in ( info.vendor, "generic", ) # On macOS it seems impossible to get all the CPU features # with syctl info, but for ARM we can get the exact model if platform.system() == "Darwin": model = TARGETS[info.name] return arch_root_and_vendor and (target == model or target in model.ancestors) return arch_root_and_vendor and target.features.issubset(info.features) @compatibility_check(architecture_family=RISCV64) def compatibility_check_for_riscv64(info, target): """Compatibility check for riscv64 architectures.""" arch_root = TARGETS[RISCV64] return (target == arch_root or arch_root in target.ancestors) and ( target.name == info.name or target.vendor == "generic" ) def brand_string() -> Optional[str]: """Returns the brand string of the host, if detected, or None.""" if platform.system() == "Darwin": return _check_output( ["sysctl", "-n", "machdep.cpu.brand_string"], env=_ensure_bin_usrbin_in_path() ).strip() if host().family == X86_64: return CpuidInfoCollector().brand_string() return None ================================================ FILE: lib/spack/spack/vendor/archspec/cpu/microarchitecture.py ================================================ # Copyright 2019-2020 Lawrence Livermore National Security, LLC and other # Archspec Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Types and functions to manage information on CPU microarchitectures.""" import functools import platform import re import sys import warnings from typing import IO, Any, Dict, List, Optional, Set, Tuple, Union from . import schema from .alias import FEATURE_ALIASES from .schema import LazyDictionary def coerce_target_names(func): """Decorator that automatically converts a known target name to a proper Microarchitecture object. """ @functools.wraps(func) def _impl(self, other): if isinstance(other, str): if other not in TARGETS: msg = '"{0}" is not a valid target name' raise ValueError(msg.format(other)) other = TARGETS[other] return func(self, other) return _impl class Microarchitecture: """A specific CPU micro-architecture""" # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-instance-attributes #: Aliases for micro-architecture's features feature_aliases = FEATURE_ALIASES def __init__( self, name: str, parents: List["Microarchitecture"], vendor: str, features: Set[str], compilers: Dict[str, List[Dict[str, str]]], generation: int = 0, cpu_part: str = "", ): """ Args: name: name of the micro-architecture (e.g. ``icelake``) parents: list of parent micro-architectures, if any. Parenthood is considered by cpu features and not chronologically. As such, each micro-architecture is compatible with its ancestors. For example, ``skylake``, which has ``broadwell`` as a parent, supports running binaries optimized for ``broadwell``. vendor: vendor of the micro-architecture features: supported CPU flags. Note that the semantic of the flags in this field might vary among architectures, if at all present. For instance, x86_64 processors will list all the flags supported by a given CPU, while Arm processors will list instead only the flags that have been added on top of the base model for the current micro-architecture. compilers: compiler support to generate tuned code for this micro-architecture. This dictionary has as keys names of supported compilers, while values are a list of dictionaries with fields: * name: name of the micro-architecture according to the compiler. This is the name passed to the ``-march`` option or similar. Not needed if it is the same as ``self.name``. * versions: versions that support this micro-architecture. * flags: flags to be passed to the compiler to generate optimized code generation: generation of the micro-architecture, if relevant. cpu_part: cpu part of the architecture, if relevant. """ self.name = name self.parents = parents self.vendor = vendor self.features = features self.compilers = compilers # Only relevant for PowerPC self.generation = generation # Only relevant for AArch64 self.cpu_part = cpu_part # Cache the "ancestor" computation self._ancestors: Optional[List["Microarchitecture"]] = None # Cache the "generic" computation self._generic: Optional["Microarchitecture"] = None # Cache the "family" computation self._family: Optional["Microarchitecture"] = None # ssse3 implies sse3; on Linux sse3 is not mentioned in /proc/cpuinfo, so add it ad-hoc. if "ssse3" in self.features: self.features.add("sse3") @property def ancestors(self) -> List["Microarchitecture"]: """All the ancestors of this microarchitecture.""" if self._ancestors is None: value = self.parents[:] for parent in self.parents: value.extend(a for a in parent.ancestors if a not in value) self._ancestors = value return self._ancestors def _to_set(self) -> Set[str]: """Returns a set of the nodes in this microarchitecture DAG.""" # This function is used to implement subset semantics with # comparison operators return set([str(self)] + [str(x) for x in self.ancestors]) @coerce_target_names def __eq__(self, other: Union[str, "Microarchitecture"]) -> bool: if not isinstance(other, Microarchitecture): return NotImplemented return ( self.name == other.name and self.vendor == other.vendor and self.features == other.features and self.parents == other.parents # avoid ancestors here and self.compilers == other.compilers and self.generation == other.generation and self.cpu_part == other.cpu_part ) def __hash__(self) -> int: return hash(self.name) @coerce_target_names def __ne__(self, other: Union[str, "Microarchitecture"]) -> bool: return not self == other @coerce_target_names def __lt__(self, other: Union[str, "Microarchitecture"]) -> bool: if not isinstance(other, Microarchitecture): return NotImplemented return self._to_set() < other._to_set() @coerce_target_names def __le__(self, other: Union[str, "Microarchitecture"]) -> bool: return (self == other) or (self < other) @coerce_target_names def __gt__(self, other: Union[str, "Microarchitecture"]) -> bool: if not isinstance(other, Microarchitecture): return NotImplemented return self._to_set() > other._to_set() @coerce_target_names def __ge__(self, other: Union[str, "Microarchitecture"]) -> bool: return (self == other) or (self > other) def __repr__(self) -> str: return f"{self.__class__.__name__}({self.name!r})" def __str__(self) -> str: return self.name def tree(self, fp: IO[str] = sys.stdout, indent: int = 4) -> None: """Format the partial order of this microarchitecture's ancestors as a tree.""" seen: Set[str] = set() stack: List[Tuple[int, Microarchitecture]] = [(0, self)] while stack: level, current = stack.pop() print(f"{'':>{level}}{current.name}", file=fp) if current.name in seen: continue for parent in reversed(current.parents): stack.append((level + indent, parent)) def __contains__(self, feature: str) -> bool: # Feature must be of a string type, so be defensive about that if not isinstance(feature, str): msg = "only objects of string types are accepted [got {0}]" raise TypeError(msg.format(str(type(feature)))) # Here we look first in the raw features, and fall-back to # feature aliases if not match was found if feature in self.features: return True # Check if the alias is defined, if not it will return False match_alias = Microarchitecture.feature_aliases.get(feature, lambda x: False) return match_alias(self) @property def family(self) -> "Microarchitecture": """Returns the architecture family a given target belongs to""" if self._family is None: roots = [x for x in [self] + self.ancestors if not x.ancestors] msg = "a target is expected to belong to just one architecture family" msg += f"[found {', '.join(str(x) for x in roots)}]" assert len(roots) == 1, msg self._family = roots.pop() return self._family @property def generic(self) -> "Microarchitecture": """Returns the best generic architecture that is compatible with self""" if self._generic is None: generics = [x for x in [self] + self.ancestors if x.vendor == "generic"] self._generic = max(generics, key=lambda x: len(x.ancestors)) return self._generic def to_dict(self) -> Dict[str, Any]: """Returns a dictionary representation of this object.""" return { "name": str(self.name), "vendor": str(self.vendor), "features": sorted(str(x) for x in self.features), "generation": self.generation, "parents": [str(x) for x in self.parents], "compilers": self.compilers, "cpupart": self.cpu_part, } @staticmethod def from_dict(data) -> "Microarchitecture": """Construct a microarchitecture from a dictionary representation.""" return Microarchitecture( name=data["name"], parents=[TARGETS[x] for x in data["parents"]], vendor=data["vendor"], features=set(data["features"]), compilers=data.get("compilers", {}), generation=data.get("generation", 0), cpu_part=data.get("cpupart", ""), ) def optimization_flags(self, compiler: str, version: str) -> str: """Returns a string containing the optimization flags that needs to be used to produce code optimized for this micro-architecture. The version is expected to be a string of dot-separated digits. If there is no information on the compiler passed as an argument, the function returns an empty string. If it is known that the compiler version we want to use does not support this architecture, the function raises an exception. Args: compiler: name of the compiler to be used version: version of the compiler to be used Raises: UnsupportedMicroarchitecture: if the requested compiler does not support this micro-architecture. ValueError: if the version doesn't match the expected format """ # If we don't have information on compiler at all return an empty string if compiler not in self.family.compilers: return "" # If we have information, but it stops before this # microarchitecture, fall back to the best known target if compiler not in self.compilers: best_target = [x for x in self.ancestors if compiler in x.compilers][0] msg = ( "'{0}' compiler is known to optimize up to the '{1}'" " microarchitecture in the '{2}' architecture family" ) msg = msg.format(compiler, best_target, best_target.family) raise UnsupportedMicroarchitecture(msg) # Check that the version matches the expected format if not re.match(r"^(?:\d+\.)*\d+$", version): msg = ( "invalid format for the compiler version argument. " "Only dot separated digits are allowed." ) raise InvalidCompilerVersion(msg) # If we have information on this compiler we need to check the # version being used compiler_info = self.compilers[compiler] def satisfies_constraint(entry, version): min_version, max_version = entry["versions"].split(":") # Check version suffixes min_version, _ = version_components(min_version) max_version, _ = version_components(max_version) version, _ = version_components(version) # Assume compiler versions fit into semver def tuplify(ver): return tuple(int(y) for y in ver.split(".")) version = tuplify(version) if min_version: min_version = tuplify(min_version) if min_version > version: return False if max_version: max_version = tuplify(max_version) if max_version < version: return False return True for compiler_entry in compiler_info: if satisfies_constraint(compiler_entry, version): flags_fmt = compiler_entry["flags"] # If there's no field name, use the name of the # micro-architecture compiler_entry.setdefault("name", self.name) # Check if we need to emit a warning warning_message = compiler_entry.get("warnings", None) if warning_message: warnings.warn(warning_message) flags = flags_fmt.format(**compiler_entry) return flags msg = "cannot produce optimized binary for micro-architecture '{0}' with {1}@{2}" if compiler_info: versions = [x["versions"] for x in compiler_info] msg += f' [supported compiler versions are {", ".join(versions)}]' else: msg += " [no supported compiler versions]" msg = msg.format(self.name, compiler, version) raise UnsupportedMicroarchitecture(msg) def generic_microarchitecture(name: str) -> Microarchitecture: """Returns a generic micro-architecture with no vendor and no features. Args: name: name of the micro-architecture """ return Microarchitecture(name, parents=[], vendor="generic", features=set(), compilers={}) def version_components(version: str) -> Tuple[str, str]: """Decomposes the version passed as input in version number and suffix and returns them. If the version number or the suffix are not present, an empty string is returned. Args: version: version to be decomposed into its components """ match = re.match(r"([\d.]*)(-?)(.*)", str(version)) if not match: return "", "" version_number = match.group(1) suffix = match.group(3) return version_number, suffix def _known_microarchitectures(): """Returns a dictionary of the known micro-architectures. If the current host platform is unknown, add it too as a generic target. """ def fill_target_from_dict(name, data, targets): """Recursively fills targets by adding the micro-architecture passed as argument and all its ancestors. Args: name (str): micro-architecture to be added to targets. data (dict): raw data loaded from JSON. targets (dict): dictionary that maps micro-architecture names to ``Microarchitecture`` objects """ values = data[name] # Get direct parents of target parent_names = values["from"] for parent in parent_names: # Recursively fill parents so they exist before we add them if parent in targets: continue fill_target_from_dict(parent, data, targets) parents = [targets.get(parent) for parent in parent_names] vendor = values["vendor"] features = set(values["features"]) compilers = values.get("compilers", {}) generation = values.get("generation", 0) cpu_part = values.get("cpupart", "") targets[name] = Microarchitecture( name, parents, vendor, features, compilers, generation=generation, cpu_part=cpu_part ) known_targets = {} data = schema.TARGETS_JSON["microarchitectures"] for name in data: if name in known_targets: # name was already brought in as ancestor to a target continue fill_target_from_dict(name, data, known_targets) # Add the host platform if not present host_platform = platform.machine() known_targets.setdefault(host_platform, generic_microarchitecture(host_platform)) return known_targets #: Dictionary of known micro-architectures TARGETS = LazyDictionary(_known_microarchitectures) class ArchspecError(Exception): """Base class for errors within archspec""" class UnsupportedMicroarchitecture(ArchspecError, ValueError): """Raised if a compiler version does not support optimization for a given micro-architecture. """ class InvalidCompilerVersion(ArchspecError, ValueError): """Raised when an invalid format is used for compiler versions in archspec.""" ================================================ FILE: lib/spack/spack/vendor/archspec/cpu/schema.py ================================================ # Copyright 2019-2020 Lawrence Livermore National Security, LLC and other # Archspec Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Global objects with the content of the microarchitecture JSON file and its schema """ import collections.abc import json import os import pathlib from typing import Optional, Tuple class LazyDictionary(collections.abc.MutableMapping): """Lazy dictionary that gets constructed on first access to any object key Args: factory (callable): factory function to construct the dictionary """ def __init__(self, factory, *args, **kwargs): self.factory = factory self.args = args self.kwargs = kwargs self._data = None @property def data(self): """Returns the lazily constructed dictionary""" if self._data is None: self._data = self.factory(*self.args, **self.kwargs) return self._data def __getitem__(self, key): return self.data[key] def __setitem__(self, key, value): self.data[key] = value def __delitem__(self, key): del self.data[key] def __iter__(self): return iter(self.data) def __len__(self): return len(self.data) #: Environment variable that might point to a directory with a user defined JSON file DIR_FROM_ENVIRONMENT = "ARCHSPEC_CPU_DIR" #: Environment variable that might point to a directory with extensions to JSON files EXTENSION_DIR_FROM_ENVIRONMENT = "ARCHSPEC_EXTENSION_CPU_DIR" def _json_file( filename: str, allow_custom: bool = False ) -> Tuple[pathlib.Path, Optional[pathlib.Path]]: """Given a filename, returns the absolute path for the main JSON file, and an optional absolute path for an extension JSON file. Args: filename: filename for the JSON file allow_custom: if True, allows overriding the location where the file resides """ json_dir = pathlib.Path(__file__).parent / ".." / "json" / "cpu" if allow_custom and DIR_FROM_ENVIRONMENT in os.environ: json_dir = pathlib.Path(os.environ[DIR_FROM_ENVIRONMENT]) json_dir = json_dir.absolute() json_file = json_dir / filename extension_file = None if allow_custom and EXTENSION_DIR_FROM_ENVIRONMENT in os.environ: extension_dir = pathlib.Path(os.environ[EXTENSION_DIR_FROM_ENVIRONMENT]) extension_dir.absolute() extension_file = extension_dir / filename return json_file, extension_file def _load(json_file: pathlib.Path, extension_file: pathlib.Path): with open(json_file, "r", encoding="utf-8") as file: data = json.load(file) if not extension_file or not extension_file.exists(): return data with open(extension_file, "r", encoding="utf-8") as file: extension_data = json.load(file) top_level_sections = list(data.keys()) for key in top_level_sections: if key not in extension_data: continue data[key].update(extension_data[key]) return data #: In memory representation of the data in microarchitectures.json, loaded on first access TARGETS_JSON = LazyDictionary(_load, *_json_file("microarchitectures.json", allow_custom=True)) #: JSON schema for microarchitectures.json, loaded on first access TARGETS_JSON_SCHEMA = LazyDictionary(_load, *_json_file("microarchitectures_schema.json")) #: Information on how to call 'cpuid' to get information on the HOST CPU CPUID_JSON = LazyDictionary(_load, *_json_file("cpuid.json", allow_custom=True)) #: JSON schema for cpuid.json, loaded on first access CPUID_JSON_SCHEMA = LazyDictionary(_load, *_json_file("cpuid_schema.json")) ================================================ FILE: lib/spack/spack/vendor/archspec/json/COPYRIGHT ================================================ Intellectual Property Notice ------------------------------ Archspec is licensed under the Apache License, Version 2.0 (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or the MIT license, (LICENSE-MIT or http://opensource.org/licenses/MIT), at your option. Copyrights and patents in the Archspec project are retained by contributors. No copyright assignment is required to contribute to Archspec. SPDX usage ------------ Individual files contain SPDX tags instead of the full license text. This enables machine processing of license information based on the SPDX License Identifiers that are available here: https://spdx.org/licenses/ Files that are dual-licensed as Apache-2.0 OR MIT contain the following text in the license header: SPDX-License-Identifier: (Apache-2.0 OR MIT) ================================================ FILE: lib/spack/spack/vendor/archspec/json/LICENSE-APACHE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: lib/spack/spack/vendor/archspec/json/LICENSE-MIT ================================================ Copyright 2019-2020 Lawrence Livermore National Security, LLC and other Archspec Project Developers. See the top-level COPYRIGHT file for details. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: lib/spack/spack/vendor/archspec/json/NOTICE ================================================ This work was produced under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344. This work was prepared as an account of work sponsored by an agency of the United States Government. Neither the United States Government nor Lawrence Livermore National Security, LLC, nor any of their employees makes any warranty, expressed or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness of any information, apparatus, product, or process disclosed, or represents that its use would not infringe privately owned rights. Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or Lawrence Livermore National Security, LLC. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or Lawrence Livermore National Security, LLC, and shall not be used for advertising or product endorsement purposes. ================================================ FILE: lib/spack/spack/vendor/archspec/json/README.md ================================================ [![](https://github.com/archspec/archspec-json/workflows/JSON%20Validation/badge.svg)](https://github.com/archspec/archspec-json/actions) # Archspec-json The [archspec-json](https://github.com/archspec/archspec-json) repository is part of the [Archspec](https://github.com/archspec) project. It contains data on various architectural aspects of a platform stored in JSON format and is meant to be used as a base to develop language specific APIs. Currently the repository contains the following JSON files: ```console cpu/ ├── cpuid.json # Contains information on CPUID calls to retrieve vendor and features on x86_64 ├── cpuid_schema.json # Schema for the file above ├── microarchitectures.json # Contains information on CPU microarchitectures └── microarchitectures_schema.json # Schema for the file above ``` ## License Archspec is distributed under the terms of both the MIT license and the Apache License (Version 2.0). Users may choose either license, at their option. All new contributions must be made under both the MIT and Apache-2.0 licenses. See [LICENSE-MIT](https://github.com/archspec/archspec-json/blob/master/LICENSE-MIT), [LICENSE-APACHE](https://github.com/archspec/archspec-json/blob/master/LICENSE-APACHE), [COPYRIGHT](https://github.com/archspec/archspec-json/blob/master/COPYRIGHT), and [NOTICE](https://github.com/archspec/archspec-json/blob/master/NOTICE) for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) LLNL-CODE-811653 ================================================ FILE: lib/spack/spack/vendor/archspec/json/cpu/cpuid.json ================================================ { "vendor": { "description": "https://en.wikipedia.org/wiki/CPUID#EAX=0:_Highest_Function_Parameter_and_Manufacturer_ID", "input": { "eax": 0, "ecx": 0 } }, "highest_extension_support": { "description": "https://en.wikipedia.org/wiki/CPUID#EAX=80000000h:_Get_Highest_Extended_Function_Implemented", "input": { "eax": 2147483648, "ecx": 0 } }, "flags": [ { "description": "https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits", "input": { "eax": 1, "ecx": 0 }, "bits": [ { "name": "fpu", "register": "edx", "bit": 0 }, { "name": "vme", "register": "edx", "bit": 1 }, { "name": "de", "register": "edx", "bit": 2 }, { "name": "pse", "register": "edx", "bit": 3 }, { "name": "tsc", "register": "edx", "bit": 4 }, { "name": "msr", "register": "edx", "bit": 5 }, { "name": "pae", "register": "edx", "bit": 6 }, { "name": "mce", "register": "edx", "bit": 7 }, { "name": "cx8", "register": "edx", "bit": 8 }, { "name": "apic", "register": "edx", "bit": 9 }, { "name": "sep", "register": "edx", "bit": 11 }, { "name": "mtrr", "register": "edx", "bit": 12 }, { "name": "pge", "register": "edx", "bit": 13 }, { "name": "mca", "register": "edx", "bit": 14 }, { "name": "cmov", "register": "edx", "bit": 15 }, { "name": "pat", "register": "edx", "bit": 16 }, { "name": "pse36", "register": "edx", "bit": 17 }, { "name": "pn", "register": "edx", "bit": 18 }, { "name": "clflush", "register": "edx", "bit": 19 }, { "name": "dts", "register": "edx", "bit": 21 }, { "name": "acpi", "register": "edx", "bit": 22 }, { "name": "mmx", "register": "edx", "bit": 23 }, { "name": "fxsr", "register": "edx", "bit": 24 }, { "name": "sse", "register": "edx", "bit": 25 }, { "name": "sse2", "register": "edx", "bit": 26 }, { "name": "ss", "register": "edx", "bit": 27 }, { "name": "ht", "register": "edx", "bit": 28 }, { "name": "tm", "register": "edx", "bit": 29 }, { "name": "ia64", "register": "edx", "bit": 30 }, { "name": "pbe", "register": "edx", "bit": 31 }, { "name": "pni", "register": "ecx", "bit": 0 }, { "name": "pclmulqdq", "register": "ecx", "bit": 1 }, { "name": "dtes64", "register": "ecx", "bit": 2 }, { "name": "monitor", "register": "ecx", "bit": 3 }, { "name": "ds_cpl", "register": "ecx", "bit": 4 }, { "name": "vmx", "register": "ecx", "bit": 5 }, { "name": "smx", "register": "ecx", "bit": 6 }, { "name": "est", "register": "ecx", "bit": 7 }, { "name": "tm2", "register": "ecx", "bit": 8 }, { "name": "ssse3", "register": "ecx", "bit": 9 }, { "name": "cid", "register": "ecx", "bit": 10 }, { "name": "fma", "register": "ecx", "bit": 12 }, { "name": "cx16", "register": "ecx", "bit": 13 }, { "name": "xtpr", "register": "ecx", "bit": 14 }, { "name": "pdcm", "register": "ecx", "bit": 15 }, { "name": "pcid", "register": "ecx", "bit": 17 }, { "name": "dca", "register": "ecx", "bit": 18 }, { "name": "sse4_1", "register": "ecx", "bit": 19 }, { "name": "sse4_2", "register": "ecx", "bit": 20 }, { "name": "x2apic", "register": "ecx", "bit": 21 }, { "name": "movbe", "register": "ecx", "bit": 22 }, { "name": "popcnt", "register": "ecx", "bit": 23 }, { "name": "tscdeadline", "register": "ecx", "bit": 24 }, { "name": "aes", "register": "ecx", "bit": 25 }, { "name": "xsave", "register": "ecx", "bit": 26 }, { "name": "osxsave", "register": "ecx", "bit": 27 }, { "name": "avx", "register": "ecx", "bit": 28 }, { "name": "f16c", "register": "ecx", "bit": 29 }, { "name": "rdrand", "register": "ecx", "bit": 30 }, { "name": "hypervisor", "register": "ecx", "bit": 31 } ] }, { "description": "https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features", "input": { "eax": 7, "ecx": 0 }, "bits": [ { "name": "fsgsbase", "register": "ebx", "bit": 0 }, { "name": "sgx", "register": "ebx", "bit": 2 }, { "name": "bmi1", "register": "ebx", "bit": 3 }, { "name": "hle", "register": "ebx", "bit": 4 }, { "name": "avx2", "register": "ebx", "bit": 5 }, { "name": "fdp-excptn-only", "register": "ebx", "bit": 6 }, { "name": "smep", "register": "ebx", "bit": 7 }, { "name": "bmi2", "register": "ebx", "bit": 8 }, { "name": "erms", "register": "ebx", "bit": 9 }, { "name": "invpcid", "register": "ebx", "bit": 10 }, { "name": "rtm", "register": "ebx", "bit": 11 }, { "name": "pqm", "register": "ebx", "bit": 12 }, { "name": "mpx", "register": "ebx", "bit": 14 }, { "name": "pqe", "register": "ebx", "bit": 15 }, { "name": "avx512f", "register": "ebx", "bit": 16 }, { "name": "avx512dq", "register": "ebx", "bit": 17 }, { "name": "rdseed", "register": "ebx", "bit": 18 }, { "name": "adx", "register": "ebx", "bit": 19 }, { "name": "smap", "register": "ebx", "bit": 20 }, { "name": "avx512ifma", "register": "ebx", "bit": 21 }, { "name": "pcommit", "register": "ebx", "bit": 22 }, { "name": "clflushopt", "register": "ebx", "bit": 23 }, { "name": "clwb", "register": "ebx", "bit": 24 }, { "name": "intel_pt", "register": "ebx", "bit": 25 }, { "name": "avx512pf", "register": "ebx", "bit": 26 }, { "name": "avx512er", "register": "ebx", "bit": 27 }, { "name": "avx512cd", "register": "ebx", "bit": 28 }, { "name": "sha_ni", "register": "ebx", "bit": 29 }, { "name": "avx512bw", "register": "ebx", "bit": 30 }, { "name": "avx512vl", "register": "ebx", "bit": 31 }, { "name": "prefetchwt1", "register": "ecx", "bit": 0 }, { "name": "avx512vbmi", "register": "ecx", "bit": 1 }, { "name": "umip", "register": "ecx", "bit": 2 }, { "name": "pku", "register": "ecx", "bit": 3 }, { "name": "ospke", "register": "ecx", "bit": 4 }, { "name": "waitpkg", "register": "ecx", "bit": 5 }, { "name": "avx512_vbmi2", "register": "ecx", "bit": 6 }, { "name": "cet_ss", "register": "ecx", "bit": 7 }, { "name": "gfni", "register": "ecx", "bit": 8 }, { "name": "vaes", "register": "ecx", "bit": 9 }, { "name": "vpclmulqdq", "register": "ecx", "bit": 10 }, { "name": "avx512_vnni", "register": "ecx", "bit": 11 }, { "name": "avx512_bitalg", "register": "ecx", "bit": 12 }, { "name": "tme", "register": "ecx", "bit": 13 }, { "name": "avx512_vpopcntdq", "register": "ecx", "bit": 14 }, { "name": "rdpid", "register": "ecx", "bit": 22 }, { "name": "cldemote", "register": "ecx", "bit": 25 }, { "name": "movdiri", "register": "ecx", "bit": 27 }, { "name": "movdir64b", "register": "ecx", "bit": 28 }, { "name": "enqcmd", "register": "ecx", "bit": 29 }, { "name": "sgx_lc", "register": "ecx", "bit": 30 }, { "name": "pks", "register": "ecx", "bit": 31 }, { "name": "fsrm", "register": "edx", "bit": 4 }, { "name": "avx512_vp2intersect", "register": "edx", "bit": 8 }, { "name": "md_clear", "register": "edx", "bit": 10 }, { "name": "serialize", "register": "edx", "bit": 14 }, { "name": "tsxldtrk", "register": "edx", "bit": 16 }, { "name": "amx_bf16", "register": "edx", "bit": 22 }, { "name": "avx512_fp16", "register": "edx", "bit": 23 }, { "name": "amx_tile", "register": "edx", "bit": 24 }, { "name": "amx_int8", "register": "edx", "bit": 25 }, { "name": "ssbd", "register": "edx", "bit": 31 } ] }, { "description": "https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features", "input": { "eax": 7, "ecx": 1 }, "bits": [ { "name": "sha512", "register": "eax", "bit": 0 }, { "name": "sm3", "register": "eax", "bit": 1 }, { "name": "sm4", "register": "eax", "bit": 2 }, { "name": "rao_int", "register": "eax", "bit": 3 }, { "name": "avx_vnni", "register": "eax", "bit": 4 }, { "name": "avx512_bf16", "register": "eax", "bit": 5 }, { "name": "cmpccxadd", "register": "eax", "bit": 7 }, { "name": "arch_perfmon_ext", "register": "eax", "bit": 8 }, { "name": "fzrm", "register": "eax", "bit": 10 }, { "name": "fsrs", "register": "eax", "bit": 11 }, { "name": "fsrc", "register": "eax", "bit": 12 }, { "name": "lkgs", "register": "eax", "bit": 18 }, { "name": "amx_fp16", "register": "eax", "bit": 21 }, { "name": "avx_ifma", "register": "eax", "bit": 23 }, { "name": "lam", "register": "eax", "bit": 26 } ] }, { "description": "https://en.wikipedia.org/wiki/CPUID#EAX=0Dh:_XSAVE_features_and_state-components", "input": { "eax": 13, "ecx": 1 }, "bits": [ { "name": "xsaveopt", "register": "eax", "bit": 0 }, { "name": "xsavec", "register": "eax", "bit": 1 }, { "name": "xgetbv1", "register": "eax", "bit": 2 }, { "name": "xsaves", "register": "eax", "bit": 3 }, { "name": "xfd", "register": "eax", "bit": 4 } ] } ], "extension-flags": [ { "description": "https://en.wikipedia.org/wiki/CPUID#EAX=0Dh:_XSAVE_features_and_state-components", "input": { "eax": 2147483649, "ecx": 0 }, "bits": [ { "name": "fpu", "register": "edx", "bit": 0 }, { "name": "vme", "register": "edx", "bit": 1 }, { "name": "de", "register": "edx", "bit": 2 }, { "name": "pse", "register": "edx", "bit": 3 }, { "name": "tsc", "register": "edx", "bit": 4 }, { "name": "msr", "register": "edx", "bit": 5 }, { "name": "pae", "register": "edx", "bit": 6 }, { "name": "mce", "register": "edx", "bit": 7 }, { "name": "cx8", "register": "edx", "bit": 8 }, { "name": "apic", "register": "edx", "bit": 9 }, { "name": "syscall", "register": "edx", "bit": 10 }, { "name": "syscall", "register": "edx", "bit": 11 }, { "name": "mtrr", "register": "edx", "bit": 12 }, { "name": "pge", "register": "edx", "bit": 13 }, { "name": "mca", "register": "edx", "bit": 14 }, { "name": "cmov", "register": "edx", "bit": 15 }, { "name": "pat", "register": "edx", "bit": 16 }, { "name": "pse36", "register": "edx", "bit": 17 }, { "name": "mp", "register": "edx", "bit": 19 }, { "name": "nx", "register": "edx", "bit": 20 }, { "name": "mmxext", "register": "edx", "bit": 22 }, { "name": "mmx", "register": "edx", "bit": 23 }, { "name": "fxsr", "register": "edx", "bit": 24 }, { "name": "fxsr_opt", "register": "edx", "bit": 25 }, { "name": "pdpe1gp", "register": "edx", "bit": 26 }, { "name": "rdtscp", "register": "edx", "bit": 27 }, { "name": "lm", "register": "edx", "bit": 29 }, { "name": "3dnowext", "register": "edx", "bit": 30 }, { "name": "3dnow", "register": "edx", "bit": 31 }, { "name": "lahf_lm", "register": "ecx", "bit": 0 }, { "name": "cmp_legacy", "register": "ecx", "bit": 1 }, { "name": "svm", "register": "ecx", "bit": 2 }, { "name": "extapic", "register": "ecx", "bit": 3 }, { "name": "cr8_legacy", "register": "ecx", "bit": 4 }, { "name": "abm", "register": "ecx", "bit": 5 }, { "name": "sse4a", "register": "ecx", "bit": 6 }, { "name": "misalignsse", "register": "ecx", "bit": 7 }, { "name": "3dnowprefetch", "register": "ecx", "bit": 8 }, { "name": "osvw", "register": "ecx", "bit": 9 }, { "name": "ibs", "register": "ecx", "bit": 10 }, { "name": "xop", "register": "ecx", "bit": 11 }, { "name": "skinit", "register": "ecx", "bit": 12 }, { "name": "wdt", "register": "ecx", "bit": 13 }, { "name": "lwp", "register": "ecx", "bit": 15 }, { "name": "fma4", "register": "ecx", "bit": 16 }, { "name": "tce", "register": "ecx", "bit": 17 }, { "name": "nodeid_msr", "register": "ecx", "bit": 19 }, { "name": "tbm", "register": "ecx", "bit": 21 }, { "name": "topoext", "register": "ecx", "bit": 22 }, { "name": "perfctr_core", "register": "ecx", "bit": 23 }, { "name": "perfctr_nb", "register": "ecx", "bit": 24 }, { "name": "dbx", "register": "ecx", "bit": 26 }, { "name": "perftsc", "register": "ecx", "bit": 27 }, { "name": "pci_l2i", "register": "ecx", "bit": 28 }, { "name": "mwaitx", "register": "ecx", "bit": 29 } ] } ] } ================================================ FILE: lib/spack/spack/vendor/archspec/json/cpu/cpuid_schema.json ================================================ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Schema for microarchitecture definitions and feature aliases", "type": "object", "additionalProperties": false, "properties": { "vendor": { "type": "object", "additionalProperties": false, "properties": { "description": { "type": "string" }, "input": { "type": "object", "additionalProperties": false, "properties": { "eax": { "type": "integer" }, "ecx": { "type": "integer" } } } } }, "highest_extension_support": { "type": "object", "additionalProperties": false, "properties": { "description": { "type": "string" }, "input": { "type": "object", "additionalProperties": false, "properties": { "eax": { "type": "integer" }, "ecx": { "type": "integer" } } } } }, "flags": { "type": "array", "items": { "type": "object", "additionalProperties": false, "properties": { "description": { "type": "string" }, "input": { "type": "object", "additionalProperties": false, "properties": { "eax": { "type": "integer" }, "ecx": { "type": "integer" } } }, "bits": { "type": "array", "items": { "type": "object", "additionalProperties": false, "properties": { "name": { "type": "string" }, "register": { "type": "string" }, "bit": { "type": "integer" } } } } } } }, "extension-flags": { "type": "array", "items": { "type": "object", "additionalProperties": false, "properties": { "description": { "type": "string" }, "input": { "type": "object", "additionalProperties": false, "properties": { "eax": { "type": "integer" }, "ecx": { "type": "integer" } } }, "bits": { "type": "array", "items": { "type": "object", "additionalProperties": false, "properties": { "name": { "type": "string" }, "register": { "type": "string" }, "bit": { "type": "integer" } } } } } } } } } ================================================ FILE: lib/spack/spack/vendor/archspec/json/cpu/microarchitectures.json ================================================ { "microarchitectures": { "x86": { "from": [], "vendor": "generic", "features": [] }, "i686": { "from": [ "x86" ], "vendor": "GenuineIntel", "features": [] }, "pentium2": { "from": [ "i686" ], "vendor": "GenuineIntel", "features": [ "mmx" ] }, "pentium3": { "from": [ "pentium2" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse" ] }, "pentium4": { "from": [ "pentium3" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2" ] }, "prescott": { "from": [ "pentium4" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "sse3" ] }, "x86_64": { "from": [], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "4.2.0:", "name": "x86-64", "flags": "-march={name} -mtune=generic" }, { "versions": ":4.1.2", "name": "x86-64", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": ":", "name": "x86-64", "flags": "-march={name}" } ], "clang": [ { "versions": ":", "name": "x86-64", "flags": "-march={name} -mtune=generic" } ], "aocc": [ { "versions": "2.2:", "name": "x86-64", "flags": "-march={name} -mtune=generic" } ], "intel": [ { "versions": ":", "name": "pentium4", "flags": "-march={name} -mtune=generic" } ], "oneapi": [ { "versions": ":", "name": "x86-64", "flags": "-march={name} -mtune=generic" } ], "dpcpp": [ { "versions": ":", "name": "x86-64", "flags": "-march={name} -mtune=generic" } ], "nvhpc": [] } }, "x86_64_v2": { "from": [ "x86_64" ], "vendor": "generic", "features": [ "cx16", "lahf_lm", "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt" ], "compilers": { "gcc": [ { "versions": "11.1:", "name": "x86-64-v2", "flags": "-march={name} -mtune=generic" }, { "versions": "4.6:11.0", "name": "x86-64", "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3" } ], "clang": [ { "versions": "12.0:", "name": "x86-64-v2", "flags": "-march={name} -mtune=generic" }, { "versions": "3.9:11.1", "name": "x86-64", "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3" } ], "aocc": [ { "versions": "2.2:", "name": "x86-64-v2", "flags": "-march={name} -mtune=generic" } ], "intel": [ { "versions": "16.0:", "name": "corei7", "flags": "-march={name} -mtune=generic -mpopcnt" } ], "oneapi": [ { "versions": "2021.2.0:", "name": "x86-64-v2", "flags": "-march={name} -mtune=generic" } ], "dpcpp": [ { "versions": "2021.2.0:", "name": "x86-64-v2", "flags": "-march={name} -mtune=generic" } ], "nvhpc": [] } }, "x86_64_v3": { "from": [ "x86_64_v2" ], "vendor": "generic", "features": [ "cx16", "lahf_lm", "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "abm", "movbe", "xsave" ], "compilers": { "gcc": [ { "versions": "11.1:", "name": "x86-64-v3", "flags": "-march={name} -mtune=generic" }, { "versions": "4.8:11.0", "name": "x86-64", "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave" } ], "clang": [ { "versions": "12.0:", "name": "x86-64-v3", "flags": "-march={name} -mtune=generic" }, { "versions": "3.9:11.1", "name": "x86-64", "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave" } ], "aocc": [ { "versions": "2.2:", "name": "x86-64-v3", "flags": "-march={name} -mtune=generic" } ], "apple-clang": [ { "versions": "8.0:", "name": "x86-64", "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave" } ], "intel": [ { "versions": "16.0:", "name": "core-avx2", "flags": "-march={name} -mtune={name} -fma -mf16c" } ], "oneapi": [ { "versions": "2021.2.0:", "name": "x86-64-v3", "flags": "-march={name} -mtune=generic" } ], "dpcpp": [ { "versions": "2021.2.0:", "name": "x86-64-v3", "flags": "-march={name} -mtune=generic" } ], "nvhpc": [ { "versions": ":", "name": "px", "flags": "-tp {name} -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mxsave" } ] } }, "x86_64_v4": { "from": [ "x86_64_v3" ], "vendor": "generic", "features": [ "cx16", "lahf_lm", "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "abm", "movbe", "xsave", "avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl" ], "compilers": { "gcc": [ { "versions": "11.1:", "name": "x86-64-v4", "flags": "-march={name} -mtune=generic" }, { "versions": "6.0:11.0", "name": "x86-64", "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl" } ], "clang": [ { "versions": "12.0:", "name": "x86-64-v4", "flags": "-march={name} -mtune=generic" }, { "versions": "3.9:11.1", "name": "x86-64", "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl" } ], "aocc": [ { "versions": "4:", "name": "x86-64-v4", "flags": "-march={name} -mtune=generic" } ], "apple-clang": [ { "versions": "8.0:", "name": "x86-64", "flags": "-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl" } ], "intel": [ { "versions": "16.0:", "name": "skylake-avx512", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": "2021.2.0:", "name": "x86-64-v4", "flags": "-march={name} -mtune=generic" } ], "dpcpp": [ { "versions": "2021.2.0:", "name": "x86-64-v4", "flags": "-march={name} -mtune=generic" } ], "nvhpc": [ { "versions": ":", "name": "px", "flags": "-tp {name} -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl" } ] } }, "nocona": { "from": [ "x86_64" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "sse3" ], "compilers": { "gcc": [ { "versions": "4.0.4:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune=generic" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [] } }, "core2": { "from": [ "nocona" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3" ], "compilers": { "gcc": [ { "versions": "4.3.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune=generic" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [] } }, "nehalem": { "from": [ "core2", "x86_64_v2" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "lahf_lm", "cx16" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, { "versions": "4.6:4.8.5", "name": "corei7", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune=generic" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "name": "corei7", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "name": "corei7", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "name": "corei7", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [] } }, "westmere": { "from": [ "nehalem" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "lahf_lm", "cx16", "aes", "pclmulqdq" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune=generic" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "name": "corei7", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "name": "corei7", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "name": "corei7", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [] } }, "sandybridge": { "from": [ "westmere" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "lahf_lm", "cx16", "aes", "pclmulqdq", "avx" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, { "versions": "4.6:4.8.5", "name": "corei7-avx", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:17.9.0", "name": "corei7-avx", "flags": "-march={name} -mtune={name}" }, { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "flags": "-tp {name}" } ] } }, "ivybridge": { "from": [ "sandybridge" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "lahf_lm", "cx16", "aes", "pclmulqdq", "avx", "rdrand", "f16c" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, { "versions": "4.6:4.8.5", "name": "core-avx-i", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:17.9.0", "name": "core-avx-i", "flags": "-march={name} -mtune={name}" }, { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "flags": "-tp {name}" } ] } }, "haswell": { "from": [ "ivybridge", "x86_64_v3" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "abm", "lahf_lm", "xsave", "cx16", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" }, { "versions": "4.8:4.8.5", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:17.9.0", "name": "core-avx2", "flags": "-march={name} -mtune={name}" }, { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "flags": "-tp {name}" } ] } }, "broadwell": { "from": [ "haswell" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "abm", "popcnt", "xsave", "lahf_lm", "cx16", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx" ], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "haswell", "flags": "-tp {name}" } ] } }, "skylake": { "from": [ "broadwell" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "abm", "lahf_lm", "xsave", "cx16", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx", "clflushopt", "xsavec", "xsaveopt" ], "compilers": { "gcc": [ { "versions": "6.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "haswell", "flags": "-tp {name}" } ] } }, "mic_knl": { "from": [ "broadwell" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "abm", "lahf_lm", "cx16", "aes", "xsave", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "avx2", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx", "avx512f", "avx512pf", "avx512er", "avx512cd" ], "compilers": { "gcc": [ { "versions": "5.1:", "name": "knl", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "name": "knl", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "knl", "flags": "-march={name} -mtune=generic" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "18.0:2021.2", "name": "knl", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":2021.2", "name": "knl", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":2021.2", "name": "knl", "flags": "-march={name} -mtune={name}" } ] } }, "skylake_avx512": { "from": [ "skylake", "x86_64_v4" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "abm", "lahf_lm", "cx16", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx", "clflushopt", "xsave", "xsavec", "xsaveopt", "avx512f", "clwb", "avx512vl", "avx512bw", "avx512dq", "avx512cd" ], "compilers": { "gcc": [ { "name": "skylake-avx512", "versions": "6.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "name": "skylake-avx512", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "skylake-avx512", "flags": "-march={name} -mtune=generic" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "18.0:", "name": "skylake-avx512", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "name": "skylake-avx512", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "name": "skylake-avx512", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "skylake", "flags": "-tp {name}" } ] } }, "cannonlake": { "from": [ "skylake" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "abm", "lahf_lm", "cx16", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx", "clflushopt", "xsave", "xsavec", "xsaveopt", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512cd", "avx512vbmi", "avx512ifma", "sha_ni" ], "compilers": { "gcc": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "18.0:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "skylake", "flags": "-tp {name}" } ] } }, "cascadelake": { "from": [ "skylake_avx512" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "abm", "lahf_lm", "cx16", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx", "clflushopt", "xsave", "xsavec", "xsaveopt", "avx512f", "clwb", "avx512vl", "avx512bw", "avx512dq", "avx512cd", "avx512_vnni" ], "compilers": { "gcc": [ { "versions": "9.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": "11.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "19.0.1:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "skylake", "flags": "-tp {name}" } ] } }, "icelake": { "from": [ "cascadelake", "cannonlake" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "abm", "lahf_lm", "cx16", "aes", "sha_ni", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx", "clflushopt", "xsave", "xsavec", "xsaveopt", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512cd", "avx512vbmi", "avx512ifma", "sha_ni", "clwb", "rdpid", "gfni", "avx512_vbmi2", "avx512_vpopcntdq", "avx512_bitalg", "avx512_vnni", "vpclmulqdq", "vaes" ], "compilers": { "gcc": [ { "name": "icelake-client", "versions": "8.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "7.0:", "name": "icelake-client", "flags": "-march={name} -mtune={name}" }, { "versions": "6.0:6.9", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "icelake-client", "flags": "-march={name} -mtune={name}" } ], "apple-clang": [ { "versions": "10.0.1:", "name": "icelake-client", "flags": "-march={name} -mtune={name}" }, { "versions": "10.0.0:10.0.99", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "18.0:", "name": "icelake-client", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "name": "icelake-client", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "name": "icelake-client", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "skylake", "flags": "-tp {name}" } ] } }, "sapphirerapids": { "from": [ "icelake" ], "vendor": "GenuineIntel", "features": [ "mmx", "sse", "sse2", "ssse3", "sse4_1", "sse4_2", "popcnt", "abm", "lahf_lm", "cx16", "sha_ni", "aes", "pclmulqdq", "avx", "rdrand", "f16c", "movbe", "fma", "avx2", "bmi1", "bmi2", "rdseed", "adx", "clflushopt", "xsave", "xsavec", "xsaveopt", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512cd", "avx512vbmi", "avx512ifma", "sha_ni", "clwb", "rdpid", "gfni", "avx512_vbmi2", "avx512_vpopcntdq", "avx512_bitalg", "avx512_vnni", "vpclmulqdq", "vaes", "avx512_bf16", "cldemote", "movdir64b", "movdiri", "serialize", "waitpkg", "amx_bf16", "amx_tile", "amx_int8" ], "compilers": { "gcc": [ { "versions": "11.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "12.0:", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "2021.2:", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": "2021.2:", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": "2021.2:", "flags": "-march={name} -mtune={name}" } ] } }, "k10": { "from": [ "x86_64" ], "vendor": "AuthenticAMD", "features": [ "mmx", "sse", "sse2", "sse4a", "abm", "cx16", "3dnow", "3dnowext" ], "compilers": { "gcc": [ { "name": "amdfam10", "versions": "4.3:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "name": "amdfam10", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "amdfam10", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse2" } ], "oneapi": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse2" } ], "dpcpp": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse2" } ], "nvhpc": [] } }, "bulldozer": { "from": [ "x86_64_v2" ], "vendor": "AuthenticAMD", "features": [ "mmx", "sse", "sse2", "sse4a", "popcnt", "lahf_lm", "cx16", "xsave", "abm", "avx", "xop", "fma4", "aes", "pclmulqdq", "cx16", "ssse3", "sse4_1", "sse4_2" ], "compilers": { "gcc": [ { "name": "bdver1", "versions": "4.7:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "name": "bdver1", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "bdver1", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse3" } ], "oneapi": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse3" } ], "dpcpp": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse3" } ], "nvhpc": [ { "versions": ":", "flags": "-tp {name}" } ] } }, "piledriver": { "from": [ "bulldozer" ], "vendor": "AuthenticAMD", "features": [ "mmx", "sse", "sse2", "sse4a", "popcnt", "lahf_lm", "cx16", "xsave", "abm", "avx", "xop", "fma4", "aes", "pclmulqdq", "cx16", "ssse3", "sse4_1", "sse4_2", "bmi1", "f16c", "fma", "tbm" ], "compilers": { "gcc": [ { "name": "bdver2", "versions": "4.7:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "name": "bdver2", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "bdver2", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse3" } ], "oneapi": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse3" } ], "dpcpp": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse3" } ], "nvhpc": [ { "versions": ":", "flags": "-tp {name}" } ] } }, "steamroller": { "from": [ "piledriver" ], "vendor": "AuthenticAMD", "features": [ "mmx", "sse", "sse2", "sse4a", "popcnt", "lahf_lm", "cx16", "xsave", "abm", "avx", "xop", "fma4", "aes", "pclmulqdq", "cx16", "ssse3", "sse4_1", "sse4_2", "bmi1", "f16c", "fma", "fsgsbase", "tbm" ], "compilers": { "gcc": [ { "name": "bdver3", "versions": "4.8:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "name": "bdver3", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "bdver3", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse4.2" } ], "oneapi": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse4.2" } ], "dpcpp": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "flags": "-msse4.2" } ], "nvhpc": [ { "versions": ":", "name": "piledriver", "flags": "-tp {name}" } ] } }, "excavator": { "from": [ "steamroller", "x86_64_v3" ], "vendor": "AuthenticAMD", "features": [ "mmx", "sse", "sse2", "sse4a", "popcnt", "lahf_lm", "cx16", "xsave", "abm", "avx", "xop", "fma4", "aes", "pclmulqdq", "cx16", "ssse3", "sse4_1", "sse4_2", "bmi1", "f16c", "fma", "fsgsbase", "bmi2", "avx2", "movbe", "tbm" ], "compilers": { "gcc": [ { "name": "bdver4", "versions": "4.9:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "name": "bdver4", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "bdver4", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "piledriver", "flags": "-tp {name}" } ] } }, "zen": { "from": [ "x86_64_v3" ], "vendor": "AuthenticAMD", "features": [ "bmi1", "bmi2", "f16c", "fma", "fsgsbase", "avx", "avx2", "rdseed", "clzero", "aes", "pclmulqdq", "cx16", "movbe", "mmx", "sse", "sse2", "sse4a", "ssse3", "sse4_1", "sse4_2", "abm", "xsave", "xsavec", "xsaveopt", "clflushopt", "popcnt", "lahf_lm", "cx16" ], "compilers": { "gcc": [ { "name": "znver1", "versions": "6.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "4.0:", "name": "znver1", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "znver1", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "flags": "-tp {name}" } ] } }, "zen2": { "from": [ "zen" ], "vendor": "AuthenticAMD", "features": [ "bmi1", "bmi2", "f16c", "fma", "fsgsbase", "avx", "avx2", "rdseed", "clzero", "aes", "pclmulqdq", "cx16", "movbe", "mmx", "sse", "sse2", "sse4a", "ssse3", "sse4_1", "sse4_2", "abm", "xsave", "xsavec", "xsaveopt", "clflushopt", "popcnt", "clwb", "lahf_lm" ], "compilers": { "gcc": [ { "name": "znver2", "versions": "9.0:", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "9.0:", "name": "znver2", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "2.2:", "name": "znver2", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": "20.5:", "flags": "-tp {name}" } ] } }, "zen3": { "from": [ "zen2" ], "vendor": "AuthenticAMD", "features": [ "bmi1", "bmi2", "f16c", "fma", "fsgsbase", "avx", "avx2", "rdseed", "clzero", "aes", "pclmulqdq", "cx16", "movbe", "mmx", "sse", "sse2", "sse4a", "ssse3", "sse4_1", "sse4_2", "abm", "xsave", "xsavec", "xsaveopt", "clflushopt", "popcnt", "lahf_lm", "clwb", "vaes", "vpclmulqdq", "pku" ], "compilers": { "gcc": [ { "versions": "10.3:", "name": "znver3", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "12.0:", "name": "znver3", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "3.0:", "name": "znver3", "flags": "-march={name} -mtune={name}" } ], "intel": [ { "versions": "16.0:", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "oneapi": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "dpcpp": [ { "versions": ":", "warnings": "Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors", "name": "core-avx2", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": "21.11:", "flags": "-tp {name}" } ] } }, "zen4": { "from": [ "zen3", "x86_64_v4" ], "vendor": "AuthenticAMD", "features": [ "bmi1", "bmi2", "f16c", "fma", "fsgsbase", "avx", "avx2", "rdseed", "clzero", "aes", "pclmulqdq", "cx16", "movbe", "mmx", "sse", "sse2", "sse4a", "ssse3", "sse4_1", "sse4_2", "abm", "xsave", "xsavec", "xsaveopt", "clflushopt", "popcnt", "lahf_lm", "clwb", "vaes", "vpclmulqdq", "pku", "gfni", "flush_l1d", "avx512f", "avx512dq", "avx512ifma", "avx512cd", "avx512bw", "avx512vl", "avx512_bf16", "avx512vbmi", "avx512_vbmi2", "avx512_vnni", "avx512_bitalg", "avx512_vpopcntdq" ], "compilers": { "gcc": [ { "versions": "10.3:12.2", "name": "znver3", "flags": "-march={name} -mtune={name} -mavx512f -mavx512dq -mavx512ifma -mavx512cd -mavx512bw -mavx512vl -mavx512vbmi -mavx512vbmi2 -mavx512vnni -mavx512bitalg" }, { "versions": "12.3:", "name": "znver4", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "12.0:15.9", "name": "znver3", "flags": "-march={name} -mtune={name} -mavx512f -mavx512dq -mavx512ifma -mavx512cd -mavx512bw -mavx512vl -mavx512vbmi -mavx512vbmi2 -mavx512vnni -mavx512bitalg" }, { "versions": "16.0:", "name": "znver4", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "3.0:3.9", "name": "znver3", "flags": "-march={name} -mtune={name} -mavx512f -mavx512dq -mavx512ifma -mavx512cd -mavx512bw -mavx512vl -mavx512vbmi -mavx512vbmi2 -mavx512vnni -mavx512bitalg", "warnings": "Zen4 processors are not fully supported by AOCC versions < 4.0. For optimal performance please upgrade to a newer version of AOCC" }, { "versions": "4.0:", "name": "znver4", "flags": "-march={name} -mtune={name}" } ], "nvhpc": [ { "versions": "21.11:23.8", "name": "zen3", "flags": "-tp {name}", "warnings": "zen4 is not fully supported by nvhpc versions < 23.9, falling back to zen3" }, { "versions": "23.9:", "flags": "-tp {name}" } ] } }, "zen5": { "from": [ "zen4" ], "vendor": "AuthenticAMD", "features": [ "abm", "aes", "avx", "avx2", "avx512_bf16", "avx512_bitalg", "avx512bw", "avx512cd", "avx512dq", "avx512f", "avx512ifma", "avx512vbmi", "avx512_vbmi2", "avx512vl", "avx512_vnni", "avx512_vp2intersect", "avx512_vpopcntdq", "avx_vnni", "bmi1", "bmi2", "clflushopt", "clwb", "clzero", "cx16", "f16c", "flush_l1d", "fma", "fsgsbase", "gfni", "ibrs_enhanced", "mmx", "movbe", "movdir64b", "lahf_lm", "movdiri", "pclmulqdq", "popcnt", "pku", "rdseed", "sse", "sse2", "sse4_1", "sse4_2", "sse4a", "ssse3", "tsc_adjust", "vaes", "vpclmulqdq", "xsave", "xsavec", "xsaveopt" ], "compilers": { "gcc": [ { "versions": "14.1:", "name": "znver5", "flags": "-march={name} -mtune={name}" } ], "aocc": [ { "versions": "5.0:", "name": "znver5", "flags": "-march={name} -mtune={name}" } ], "clang": [ { "versions": "19.1:", "name": "znver5", "flags": "-march={name} -mtune={name}" } ] } }, "ppc64": { "from": [], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "name": "powerpc64", "versions": ":", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": ":", "flags": "-mcpu={name} -mtune={name}" } ] } }, "power7": { "from": [ "ppc64" ], "vendor": "IBM", "generation": 7, "features": [], "compilers": { "gcc": [ { "versions": "4.4:", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-mcpu={name} -mtune={name}" } ] } }, "power8": { "from": [ "power7" ], "vendor": "IBM", "generation": 8, "features": [], "compilers": { "gcc": [ { "versions": "4.9:", "flags": "-mcpu={name} -mtune={name}" }, { "versions": "4.8:4.8.5", "warnings": "Using GCC 4.8 to optimize for Power 8 might not work if you are not on Red Hat Enterprise Linux 7, where a custom backport of the feature has been done. Upstream support from GCC starts in version 4.9", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-mcpu={name} -mtune={name}" } ] } }, "power9": { "from": [ "power8" ], "vendor": "IBM", "generation": 9, "features": [], "compilers": { "gcc": [ { "versions": "6.0:", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "flags": "-mcpu={name} -mtune={name}" } ] } }, "power10": { "from": [ "power9" ], "vendor": "IBM", "generation": 10, "features": [], "compilers": { "gcc": [ { "versions": "11.1:", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": "11.0:", "flags": "-mcpu={name} -mtune={name}" } ] } }, "ppc64le": { "from": [], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "name": "powerpc64le", "versions": "4.8:", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": ":", "flags": "-mcpu={name} -mtune={name}" } ], "nvhpc": [] } }, "power8le": { "from": [ "ppc64le" ], "vendor": "IBM", "generation": 8, "features": [], "compilers": { "gcc": [ { "versions": "4.9:", "name": "power8", "flags": "-mcpu={name} -mtune={name}" }, { "versions": "4.8:4.8.5", "warnings": "Using GCC 4.8 to optimize for Power 8 might not work if you are not on Red Hat Enterprise Linux 7, where a custom backport of the feature has been done. Upstream support from GCC starts in version 4.9", "name": "power8", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "family": "ppc64le", "name": "power8", "flags": "-mcpu={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "pwr8", "flags": "-tp {name}" } ] } }, "power9le": { "from": [ "power8le" ], "vendor": "IBM", "generation": 9, "features": [], "compilers": { "gcc": [ { "name": "power9", "versions": "6.0:", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": "3.9:", "family": "ppc64le", "name": "power9", "flags": "-mcpu={name} -mtune={name}" } ], "nvhpc": [ { "versions": ":", "name": "pwr9", "flags": "-tp {name}" } ] } }, "power10le": { "from": [ "power9le" ], "vendor": "IBM", "generation": 10, "features": [], "compilers": { "gcc": [ { "name": "power10", "versions": "11.1:", "flags": "-mcpu={name} -mtune={name}" } ], "clang": [ { "versions": "11.0:", "family": "ppc64le", "name": "power10", "flags": "-mcpu={name} -mtune={name}" } ] } }, "aarch64": { "from": [], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "4.8.0:", "flags": "-march=armv8-a -mtune=generic" } ], "clang": [ { "versions": ":", "flags": "-march=armv8-a -mtune=generic" } ], "apple-clang": [ { "versions": ":", "flags": "-march=armv8-a -mtune=generic" } ], "arm": [ { "versions": ":", "flags": "-march=armv8-a -mtune=generic" } ], "nvhpc": [] } }, "armv8.1a": { "from": [ "aarch64" ], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "5:", "flags": "-march=armv8.1-a -mtune=generic" } ], "clang": [ { "versions": ":", "flags": "-march=armv8.1-a -mtune=generic" } ], "apple-clang": [ { "versions": ":", "flags": "-march=armv8.1-a -mtune=generic" } ], "arm": [ { "versions": ":", "flags": "-march=armv8.1-a -mtune=generic" } ] } }, "armv8.2a": { "from": [ "armv8.1a" ], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "6:", "flags": "-march=armv8.2-a -mtune=generic" } ], "clang": [ { "versions": ":", "flags": "-march=armv8.2-a -mtune=generic" } ], "apple-clang": [ { "versions": ":", "flags": "-march=armv8.2-a -mtune=generic" } ], "arm": [ { "versions": ":", "flags": "-march=armv8.2-a -mtune=generic" } ] } }, "armv8.3a": { "from": [ "armv8.2a" ], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "6:", "flags": "-march=armv8.3-a -mtune=generic" } ], "clang": [ { "versions": "6:", "flags": "-march=armv8.3-a -mtune=generic" } ], "apple-clang": [ { "versions": ":", "flags": "-march=armv8.3-a -mtune=generic" } ], "arm": [ { "versions": ":", "flags": "-march=armv8.3-a -mtune=generic" } ] } }, "armv8.4a": { "from": [ "armv8.3a" ], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "8:", "flags": "-march=armv8.4-a -mtune=generic" } ], "clang": [ { "versions": "8:", "flags": "-march=armv8.4-a -mtune=generic" } ], "apple-clang": [ { "versions": ":", "flags": "-march=armv8.4-a -mtune=generic" } ], "arm": [ { "versions": ":", "flags": "-march=armv8.4-a -mtune=generic" } ] } }, "armv8.5a": { "from": [ "armv8.4a" ], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "9:", "flags": "-march=armv8.5-a -mtune=generic" } ], "clang": [ { "versions": "11:", "flags": "-march=armv8.5-a -mtune=generic" } ], "apple-clang": [ { "versions": ":", "flags": "-march=armv8.5-a -mtune=generic" } ], "arm": [ { "versions": ":", "flags": "-march=armv8.5-a -mtune=generic" } ] } }, "armv8.6a": { "from": [ "armv8.5a" ], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "10.1:", "flags": "-march=armv8.6-a -mtune=generic" } ], "clang": [ { "versions": "11:", "flags": "-march=armv8.6-a -mtune=generic" } ] } }, "armv9.0a": { "from": [ "armv8.5a" ], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "12:", "flags": "-march=armv9-a -mtune=generic" } ], "clang": [ { "versions": "14:", "flags": "-march=armv9-a -mtune=generic" } ], "apple-clang": [ { "versions": ":", "flags": "-march=armv9-a -mtune=generic" } ], "arm": [ { "versions": ":", "flags": "-march=armv9-a -mtune=generic" } ] } }, "thunderx2": { "from": [ "armv8.1a" ], "vendor": "Cavium", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "cpuid", "asimdrdm" ], "compilers": { "gcc": [ { "versions": "4.8:4.8.9", "flags": "-march=armv8-a" }, { "versions": "4.9:5.9", "flags": "-march=armv8-a+crc+crypto" }, { "versions": "6:6.9", "flags": "-march=armv8.1-a+crc+crypto" }, { "versions": "7:", "flags": "-mcpu=thunderx2t99" } ], "clang": [ { "versions": "3.9:4.9", "flags": "-march=armv8.1-a+crc+crypto" }, { "versions": "5:", "flags": "-mcpu=thunderx2t99" } ] }, "cpupart": "0x0af" }, "a64fx": { "from": [ "armv8.2a" ], "vendor": "Fujitsu", "features": [ "fp", "asimd", "evtstrm", "sha1", "sha2", "crc32", "atomics", "cpuid", "asimdrdm", "fphp", "asimdhp", "fcma", "dcpop", "sve" ], "compilers": { "gcc": [ { "versions": "4.8:4.8.9", "flags": "-march=armv8-a" }, { "versions": "4.9:5.9", "flags": "-march=armv8-a+crc+crypto" }, { "versions": "6:6.9", "flags": "-march=armv8.1-a+crc+crypto" }, { "versions": "7:7.9", "flags": "-march=armv8.2-a+crc+crypto+fp16" }, { "versions": "8:10.2", "flags": "-march=armv8.2-a+crc+sha2+fp16+sve -msve-vector-bits=512" }, { "versions": "10.3:", "flags": "-mcpu=a64fx -msve-vector-bits=512" } ], "clang": [ { "versions": "3.9:4.9", "flags": "-march=armv8.2-a+crc+sha2+fp16" }, { "versions": "5:10", "flags": "-march=armv8.2-a+crc+sha2+fp16+sve" }, { "versions": "11:", "flags": "-mcpu=a64fx" } ], "arm": [ { "versions": "20:", "flags": "-march=armv8.2-a+crc+crypto+fp16+sve" } ] }, "cpupart": "0x001" }, "cortex_a72": { "from": [ "aarch64" ], "vendor": "ARM", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "cpuid" ], "compilers": { "gcc": [ { "versions": "4.8:4.8.9", "flags": "-march=armv8-a" }, { "versions": "4.9:5.9", "flags": "-march=armv8-a+crc+crypto" }, { "versions": "6:", "flags": "-mcpu=cortex-a72" } ], "clang": [ { "versions": "3.9:", "flags": "-mcpu=cortex-a72" } ] }, "cpupart": "0xd08" }, "neoverse_n1": { "from": [ "cortex_a72", "armv8.2a" ], "vendor": "ARM", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "fphp", "asimdhp", "cpuid", "asimdrdm", "lrcpc", "dcpop", "asimddp" ], "compilers": { "gcc": [ { "versions": "4.8:4.8.9", "flags": "-march=armv8-a" }, { "versions": "4.9:5.9", "flags": "-march=armv8-a+crc+crypto" }, { "versions": "6:6.9", "flags": "-march=armv8.1-a" }, { "versions": "7:7.9", "flags": "-march=armv8.2-a+fp16 -mtune=cortex-a72" }, { "versions": "8.0:8.0", "flags": "-march=armv8.2-a+fp16+dotprod+crypto -mtune=cortex-a72" }, { "versions": "8.1:8.9", "flags": "-march=armv8.2-a+fp16+rcpc+dotprod+crypto -mtune=cortex-a72" }, { "versions": "9.0:", "flags": "-mcpu=neoverse-n1" } ], "clang": [ { "versions": "3.9:4.9", "flags": "-march=armv8.2-a+fp16+crc+crypto" }, { "versions": "5:", "flags": "-march=armv8.2-a+fp16+rcpc+dotprod+crypto" }, { "versions": "10:", "flags": "-mcpu=neoverse-n1" } ], "arm": [ { "versions": "20:21.9", "flags": "-march=armv8.2-a+fp16+rcpc+dotprod+crypto" }, { "versions": "22:", "flags": "-mcpu=neoverse-n1" } ], "nvhpc": [ { "versions": "22.5:", "name": "neoverse-n1", "flags": "-tp {name}" } ] }, "cpupart": "0xd0c" }, "neoverse_v1": { "from": [ "neoverse_n1", "armv8.4a" ], "vendor": "ARM", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "fphp", "asimdhp", "cpuid", "asimdrdm", "jscvt", "fcma", "lrcpc", "dcpop", "sha3", "asimddp", "sha512", "sve", "asimdfhm", "dit", "uscat", "ilrcpc", "flagm", "dcpodp", "svei8mm", "svebf16", "i8mm", "bf16", "dgh", "rng" ], "compilers": { "gcc": [ { "versions": "4.8:4.8.9", "flags": "-march=armv8-a" }, { "versions": "4.9:5.9", "flags": "-march=armv8-a+crc+crypto" }, { "versions": "6:6.9", "flags": "-march=armv8.1-a" }, { "versions": "7:7.9", "flags": "-march=armv8.2-a+crypto+fp16 -mtune=cortex-a72" }, { "versions": "8.0:8.4", "flags": "-march=armv8.2-a+fp16+dotprod+crypto -mtune=cortex-a72" }, { "versions": "8.5:8.9", "flags": "-mcpu=neoverse-v1" }, { "versions": "9.0:9.3", "flags": "-march=armv8.2-a+fp16+dotprod+crypto -mtune=cortex-a72" }, { "versions": "9.4:9.9", "flags": "-mcpu=neoverse-v1" }, { "versions": "10.0:10.1", "flags": "-march=armv8.2-a+fp16+dotprod+crypto -mtune=cortex-a72" }, { "versions": "10.2:10.2.99", "flags": "-mcpu=zeus" }, { "versions": "10.3:", "flags": "-mcpu=neoverse-v1" } ], "clang": [ { "versions": "3.9:4.9", "flags": "-march=armv8.2-a+fp16+crc+crypto" }, { "versions": "5:10", "flags": "-march=armv8.2-a+fp16+rcpc+dotprod+crypto" }, { "versions": "11:", "flags": "-march=armv8.4-a+sve+fp16+bf16+crypto+i8mm+rng" }, { "versions": "12:", "flags": "-mcpu=neoverse-v1" } ], "arm": [ { "versions": "20:21.9", "flags": "-march=armv8.2-a+sve+fp16+rcpc+dotprod+crypto" }, { "versions": "22:", "flags": "-mcpu=neoverse-v1" } ], "nvhpc": [ { "versions": "22.5:", "name": "neoverse-n1", "flags": "-tp {name}" } ] }, "cpupart": "0xd40" }, "neoverse_v2": { "from": [ "neoverse_n1", "armv9.0a" ], "vendor": "ARM", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "fphp", "asimdhp", "cpuid", "asimdrdm", "jscvt", "fcma", "lrcpc", "dcpop", "sha3", "asimddp", "sha512", "sve", "asimdfhm", "uscat", "ilrcpc", "flagm", "sb", "dcpodp", "sve2", "flagm2", "frint", "svei8mm", "svebf16", "i8mm", "bf16" ], "compilers": { "gcc": [ { "versions": "4.8:5.99", "flags": "-march=armv8-a" }, { "versions": "6:6.99", "flags": "-march=armv8.1-a" }, { "versions": "7.0:7.99", "flags": "-march=armv8.2-a -mtune=cortex-a72" }, { "versions": "8.0:8.99", "flags": "-march=armv8.4-a+sve -mtune=cortex-a72" }, { "versions": "9.0:9.99", "flags": "-march=armv8.5-a+sve -mtune=cortex-a76" }, { "versions": "10.0:11.3.99", "flags": "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77" }, { "versions": "11.4:11.99", "flags": "-mcpu=neoverse-v2" }, { "versions": "12.0:12.2.99", "flags": "-march=armv9-a+i8mm+bf16 -mtune=cortex-a710" }, { "versions": "12.3:", "flags": "-mcpu=neoverse-v2" } ], "clang": [ { "versions": "9.0:10.99", "flags": "-march=armv8.5-a+sve" }, { "versions": "11.0:13.99", "flags": "-march=armv8.5-a+sve+sve2+i8mm+bf16" }, { "versions": "14.0:15.99", "flags": "-march=armv9-a+i8mm+bf16" }, { "versions": "16.0:", "flags": "-mcpu=neoverse-v2" } ], "arm": [ { "versions": "23.04.0:", "flags": "-mcpu=neoverse-v2" } ], "nvhpc": [ { "versions": "23.3:", "name": "neoverse-v2", "flags": "-tp {name}" } ] }, "cpupart": "0xd4f" }, "neoverse_n2": { "from": [ "neoverse_n1", "armv9.0a" ], "vendor": "ARM", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "fphp", "asimdhp", "cpuid", "asimdrdm", "jscvt", "fcma", "lrcpc", "dcpop", "sha3", "asimddp", "sha512", "sve", "asimdfhm", "uscat", "ilrcpc", "flagm", "sb", "dcpodp", "sve2", "flagm2", "frint", "svei8mm", "svebf16", "i8mm", "bf16" ], "compilers": { "gcc": [ { "versions": "4.8:5.99", "flags": "-march=armv8-a" }, { "versions": "6:6.99", "flags": "-march=armv8.1-a" }, { "versions": "7.0:7.99", "flags": "-march=armv8.2-a -mtune=cortex-a72" }, { "versions": "8.0:8.99", "flags": "-march=armv8.4-a+sve -mtune=cortex-a72" }, { "versions": "9.0:9.99", "flags": "-march=armv8.5-a+sve -mtune=cortex-a76" }, { "versions": "10.0:10.99", "flags": "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77" }, { "versions": "11.0:", "flags": "-mcpu=neoverse-n2" } ], "clang": [ { "versions": "9.0:10.99", "flags": "-march=armv8.5-a+sve" }, { "versions": "11.0:13.99", "flags": "-march=armv8.5-a+sve+sve2+i8mm+bf16" }, { "versions": "14.0:15.99", "flags": "-march=armv9-a+i8mm+bf16" }, { "versions": "16.0:", "flags": "-mcpu=neoverse-n2" } ], "arm": [ { "versions": "23.04.0:", "flags": "-mcpu=neoverse-n2" } ], "nvhpc": [ { "versions": "23.3:", "name": "neoverse-n1", "flags": "-tp {name}" } ] }, "cpupart": "0xd49" }, "m1": { "from": [ "armv8.4a" ], "vendor": "Apple", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "fphp", "asimdhp", "cpuid", "asimdrdm", "jscvt", "fcma", "lrcpc", "dcpop", "sha3", "asimddp", "sha512", "asimdfhm", "dit", "uscat", "ilrcpc", "flagm", "ssbs", "sb", "paca", "pacg", "dcpodp", "flagm2", "frint" ], "compilers": { "gcc": [ { "versions": "8.0:", "flags": "-march=armv8.4-a -mtune=generic" } ], "clang": [ { "versions": "9.0:12.0", "flags": "-march=armv8.4-a" }, { "versions": "13.0:", "flags": "-mcpu=apple-m1" } ], "apple-clang": [ { "versions": "11.0:12.5", "flags": "-march=armv8.4-a" }, { "versions": "13.0:", "flags": "-mcpu=apple-m1" } ] }, "cpupart": "0x022" }, "m2": { "from": [ "m1", "armv8.5a" ], "vendor": "Apple", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "fphp", "asimdhp", "cpuid", "asimdrdm", "jscvt", "fcma", "lrcpc", "dcpop", "sha3", "asimddp", "sha512", "asimdfhm", "dit", "uscat", "ilrcpc", "flagm", "ssbs", "sb", "paca", "pacg", "dcpodp", "flagm2", "frint", "ecv", "bf16", "i8mm", "bti" ], "compilers": { "gcc": [ { "versions": "8.0:", "flags": "-march=armv8.5-a -mtune=generic" } ], "clang": [ { "versions": "9.0:12.0", "flags": "-march=armv8.5-a" }, { "versions": "13.0:", "flags": "-mcpu=apple-m1" }, { "versions": "16.0:", "flags": "-mcpu=apple-m2" } ], "apple-clang": [ { "versions": "11.0:12.5", "flags": "-march=armv8.5-a" }, { "versions": "13.0:14.0.2", "flags": "-mcpu=apple-m1" }, { "versions": "14.0.2:", "flags": "-mcpu=apple-m2" } ] }, "cpupart": "0x032" }, "m3": { "from": [ "m2", "armv8.6a" ], "vendor": "Apple", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "fphp", "asimdhp", "cpuid", "asimdrdm", "jscvt", "fcma", "lrcpc", "dcpop", "sha3", "asimddp", "sha512", "asimdfhm", "dit", "uscat", "ilrcpc", "flagm", "ssbs", "sb", "paca", "pacg", "dcpodp", "flagm2", "frint", "ecv", "bf16", "i8mm", "bti" ], "compilers": { "gcc": [ { "versions": "8.0:", "flags": "-march=armv8.5-a -mtune=generic" } ], "clang": [ { "versions": "9.0:12.0", "flags": "-march=armv8.5-a" }, { "versions": "13.0:", "flags": "-mcpu=apple-m1" }, { "versions": "16.0:", "flags": "-mcpu=apple-m3" } ], "apple-clang": [ { "versions": "11.0:12.5", "flags": "-march=armv8.5-a" }, { "versions": "13.0:14.0.2", "flags": "-mcpu=apple-m1" }, { "versions": "14.0.2:15", "flags": "-mcpu=apple-m2" }, { "versions": "16:", "flags": "-mcpu=apple-m3" } ] }, "cpupart": "Unknown" }, "m4": { "from": [ "m3", "armv8.6a" ], "vendor": "Apple", "features": [ "fp", "asimd", "evtstrm", "aes", "pmull", "sha1", "sha2", "crc32", "atomics", "fphp", "asimdhp", "cpuid", "asimdrdm", "jscvt", "fcma", "lrcpc", "dcpop", "sha3", "asimddp", "sha512", "asimdfhm", "dit", "uscat", "ilrcpc", "flagm", "ssbs", "sb", "paca", "pacg", "dcpodp", "flagm2", "frint", "ecv", "bf16", "i8mm", "bti", "sme", "sme2" ], "compilers": { "clang": [ { "versions": "9.0:12.0", "flags": "-march=armv8.5-a" }, { "versions": "13.0:", "flags": "-mcpu=apple-m1" }, { "versions": "16.0:18", "flags": "-mcpu=apple-m3" }, { "versions": "19:", "flags": "-mcpu=apple-m4" } ], "apple-clang": [ { "versions": "11.0:12.5", "flags": "-march=armv8.5-a" }, { "versions": "13.0:14.0.2", "flags": "-mcpu=apple-m1" }, { "versions": "14.0.2:15", "flags": "-mcpu=apple-m2" }, { "versions": "16:", "flags": "-mcpu=apple-m3" }, { "versions": "17:", "flags": "-mcpu=apple-m4" } ] }, "cpupart": "Unknown" }, "arm": { "from": [], "vendor": "generic", "features": [], "compilers": { "clang": [ { "versions": ":", "family": "arm", "flags": "-march={family} -mcpu=generic" } ] } }, "ppc": { "from": [], "vendor": "generic", "features": [], "compilers": {} }, "ppcle": { "from": [], "vendor": "generic", "features": [], "compilers": {} }, "sparc": { "from": [], "vendor": "generic", "features": [], "compilers": {} }, "sparc64": { "from": [], "vendor": "generic", "features": [], "compilers": {} }, "riscv64": { "from": [], "vendor": "generic", "features": [], "compilers": { "gcc": [ { "versions": "7.1:", "flags": "-march=rv64gc" } ], "clang": [ { "versions": "9.0:", "flags": "-march=rv64gc" } ] } }, "u74mc": { "from": [ "riscv64" ], "vendor": "SiFive", "features": [], "compilers": { "gcc": [ { "versions": "10.2:", "flags": "-march=rv64gc -mtune=sifive-7-series" } ], "clang": [ { "versions": "12.0:", "flags": "-march=rv64gc -mtune=sifive-7-series" } ] } } }, "feature_aliases": { "sse3": { "reason": "ssse3 is a superset of sse3 and might be the only one listed", "any_of": [ "ssse3" ] }, "avx512": { "reason": "avx512 indicates generic support for any of the avx512 instruction sets", "any_of": [ "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512cd" ] }, "altivec": { "reason": "altivec is supported by Power PC architectures, but might not be listed in features", "families": [ "ppc64le", "ppc64" ] }, "vsx": { "reason": "VSX alitvec extensions are supported by PowerISA from v2.06 (Power7+), but might not be listed in features", "families": [ "ppc64le", "ppc64" ] }, "fma": { "reason": "FMA has been supported by PowerISA since Power1, but might not be listed in features", "families": [ "ppc64le", "ppc64" ] }, "sse4.1": { "reason": "permits to refer to sse4_1 also as sse4.1", "any_of": [ "sse4_1" ] }, "sse4.2": { "reason": "permits to refer to sse4_2 also as sse4.2", "any_of": [ "sse4_2" ] }, "neon": { "reason": "NEON is required in all standard ARMv8 implementations", "families": [ "aarch64" ] } }, "conversions": { "description": "Conversions that map some platform specific values to canonical values", "arm_vendors": { "0x41": "ARM", "0x42": "Broadcom", "0x43": "Cavium", "0x44": "DEC", "0x46": "Fujitsu", "0x48": "HiSilicon", "0x49": "Infineon Technologies AG", "0x4d": "Motorola", "0x4e": "Nvidia", "0x50": "APM", "0x51": "Qualcomm", "0x53": "Samsung", "0x56": "Marvell", "0x61": "Apple", "0x66": "Faraday", "0x68": "HXT", "0x69": "Intel" }, "darwin_flags": { "sse4.1": "sse4_1", "sse4.2": "sse4_2", "avx1.0": "avx", "lahf": "lahf_lm", "sha": "sha_ni", "popcnt lzcnt": "abm", "clfsopt": "clflushopt", "xsave": "xsavec xsaveopt" } } } ================================================ FILE: lib/spack/spack/vendor/archspec/json/cpu/microarchitectures_schema.json ================================================ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Schema for microarchitecture definitions and feature aliases", "type": "object", "additionalProperties": false, "properties": { "microarchitectures": { "type": "object", "patternProperties": { "([\\w]*)": { "type": "object", "properties": { "from": { "$comment": "More than one parent", "type": "array", "items": { "type": "string" } }, "vendor": { "type": "string" }, "features": { "type": "array", "items": { "type": "string" } }, "compilers": { "type": "object", "patternProperties": { "([\\w]*)": { "$comment": "Permit multiple entries since compilers change options across versions", "type": "array", "items": { "type": "object", "properties": { "versions": { "type": "string" }, "name": { "type": "string" }, "flags": { "type": "string" } }, "required": [ "versions", "flags" ] } } } }, "cpupart": { "type": "string" } }, "required": [ "from", "vendor", "features" ] } } }, "feature_aliases": { "type": "object", "patternProperties": { "([\\w]*)": { "type": "object", "properties": { "reason": { "$comment": "Comment containing the reason why an alias is there", "type": "string" }, "any_of": { "$comment": "The alias is true if any of the items is a feature of the target", "type": "array", "items": { "type": "string" } }, "families": { "$comment": "The alias is true if the family of the target is in this list", "type": "array", "items": { "type": "string" } } }, "additionalProperties": false } } }, "conversions": { "type": "object", "properties": { "description": { "type": "string" }, "arm_vendors": { "type": "object" }, "darwin_flags": { "type": "object" } }, "additionalProperties": false } } } ================================================ FILE: lib/spack/spack/vendor/archspec/vendor/cpuid/LICENSE ================================================ The MIT License (MIT) Copyright (c) 2014 Anders Høst Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: lib/spack/spack/vendor/archspec/vendor/cpuid/README.md ================================================ cpuid.py ======== Now, this is silly! Pure Python library for accessing information about x86 processors by querying the [CPUID](http://en.wikipedia.org/wiki/CPUID) instruction. Well, not exactly pure Python... It works by allocating a small piece of virtual memory, copying a raw x86 function to that memory, giving the memory execute permissions and then calling the memory as a function. The injected function executes the CPUID instruction and copies the result back to a ctypes.Structure where is can be read by Python. It should work fine on both 32 and 64 bit versions of Windows and Linux running x86 processors. Apple OS X and other BSD systems should also work, not tested though... Why? ---- For poops and giggles. Plus, having access to a low-level feature without having to compile a C wrapper is pretty neat. Examples -------- Getting info with eax=0: import cpuid q = cpuid.CPUID() eax, ebx, ecx, edx = q(0) Running the files: $ python example.py Vendor ID : GenuineIntel CPU name : Intel(R) Xeon(R) CPU W3550 @ 3.07GHz Vector instructions supported: SSE : Yes SSE2 : Yes SSE3 : Yes SSSE3 : Yes SSE4.1 : Yes SSE4.2 : Yes SSE4a : -- AVX : -- AVX2 : -- $ python cpuid.py CPUID A B C D 00000000 0000000b 756e6547 6c65746e 49656e69 00000001 000106a5 00100800 009ce3bd bfebfbff 00000002 55035a01 00f0b2e4 00000000 09ca212c 00000003 00000000 00000000 00000000 00000000 00000004 00000000 00000000 00000000 00000000 00000005 00000040 00000040 00000003 00001120 00000006 00000003 00000002 00000001 00000000 00000007 00000000 00000000 00000000 00000000 00000008 00000000 00000000 00000000 00000000 00000009 00000000 00000000 00000000 00000000 0000000a 07300403 00000044 00000000 00000603 0000000b 00000000 00000000 00000095 00000000 80000000 80000008 00000000 00000000 00000000 80000001 00000000 00000000 00000001 28100800 80000002 65746e49 2952286c 6f655820 2952286e 80000003 55504320 20202020 20202020 57202020 80000004 30353533 20402020 37302e33 007a4847 80000005 00000000 00000000 00000000 00000000 80000006 00000000 00000000 01006040 00000000 80000007 00000000 00000000 00000000 00000100 80000008 00003024 00000000 00000000 00000000 ================================================ FILE: lib/spack/spack/vendor/archspec/vendor/cpuid/cpuid.py ================================================ # -*- coding: utf-8 -*- # # Copyright (c) 2024 Anders Høst # from __future__ import print_function import platform import os import ctypes from ctypes import c_uint32, c_long, c_ulong, c_size_t, c_void_p, POINTER, CFUNCTYPE # Posix x86_64: # Three first call registers : RDI, RSI, RDX # Volatile registers : RAX, RCX, RDX, RSI, RDI, R8-11 # Windows x86_64: # Three first call registers : RCX, RDX, R8 # Volatile registers : RAX, RCX, RDX, R8-11 # cdecl 32 bit: # Three first call registers : Stack (%esp) # Volatile registers : EAX, ECX, EDX _POSIX_64_OPC = [ 0x53, # push %rbx 0x89, 0xf0, # mov %esi,%eax 0x89, 0xd1, # mov %edx,%ecx 0x0f, 0xa2, # cpuid 0x89, 0x07, # mov %eax,(%rdi) 0x89, 0x5f, 0x04, # mov %ebx,0x4(%rdi) 0x89, 0x4f, 0x08, # mov %ecx,0x8(%rdi) 0x89, 0x57, 0x0c, # mov %edx,0xc(%rdi) 0x5b, # pop %rbx 0xc3 # retq ] _WINDOWS_64_OPC = [ 0x53, # push %rbx 0x89, 0xd0, # mov %edx,%eax 0x49, 0x89, 0xc9, # mov %rcx,%r9 0x44, 0x89, 0xc1, # mov %r8d,%ecx 0x0f, 0xa2, # cpuid 0x41, 0x89, 0x01, # mov %eax,(%r9) 0x41, 0x89, 0x59, 0x04, # mov %ebx,0x4(%r9) 0x41, 0x89, 0x49, 0x08, # mov %ecx,0x8(%r9) 0x41, 0x89, 0x51, 0x0c, # mov %edx,0xc(%r9) 0x5b, # pop %rbx 0xc3 # retq ] _CDECL_32_OPC = [ 0x53, # push %ebx 0x57, # push %edi 0x8b, 0x7c, 0x24, 0x0c, # mov 0xc(%esp),%edi 0x8b, 0x44, 0x24, 0x10, # mov 0x10(%esp),%eax 0x8b, 0x4c, 0x24, 0x14, # mov 0x14(%esp),%ecx 0x0f, 0xa2, # cpuid 0x89, 0x07, # mov %eax,(%edi) 0x89, 0x5f, 0x04, # mov %ebx,0x4(%edi) 0x89, 0x4f, 0x08, # mov %ecx,0x8(%edi) 0x89, 0x57, 0x0c, # mov %edx,0xc(%edi) 0x5f, # pop %edi 0x5b, # pop %ebx 0xc3 # ret ] is_windows = os.name == "nt" is_64bit = ctypes.sizeof(ctypes.c_voidp) == 8 class CPUID_struct(ctypes.Structure): _register_names = ("eax", "ebx", "ecx", "edx") _fields_ = [(r, c_uint32) for r in _register_names] def __getitem__(self, item): if item not in self._register_names: raise KeyError(item) return getattr(self, item) def __repr__(self): return "eax=0x{:x}, ebx=0x{:x}, ecx=0x{:x}, edx=0x{:x}".format(self.eax, self.ebx, self.ecx, self.edx) class CPUID(object): def __init__(self): if platform.machine() not in ("AMD64", "x86_64", "x86", "i686"): raise SystemError("Only available for x86") if is_windows: if is_64bit: # VirtualAlloc seems to fail under some weird # circumstances when ctypes.windll.kernel32 is # used under 64 bit Python. CDLL fixes this. self.win = ctypes.CDLL("kernel32.dll") opc = _WINDOWS_64_OPC else: # Here ctypes.windll.kernel32 is needed to get the # right DLL. Otherwise it will fail when running # 32 bit Python on 64 bit Windows. self.win = ctypes.windll.kernel32 opc = _CDECL_32_OPC else: opc = _POSIX_64_OPC if is_64bit else _CDECL_32_OPC size = len(opc) code = (ctypes.c_ubyte * size)(*opc) if is_windows: self.win.VirtualAlloc.restype = c_void_p self.win.VirtualAlloc.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_ulong, ctypes.c_ulong] self.addr = self.win.VirtualAlloc(None, size, 0x1000, 0x40) if not self.addr: raise MemoryError("Could not allocate RWX memory") ctypes.memmove(self.addr, code, size) else: from mmap import ( mmap, MAP_PRIVATE, MAP_ANONYMOUS, PROT_WRITE, PROT_READ, PROT_EXEC, ) self.mm = mmap( -1, size, flags=MAP_PRIVATE | MAP_ANONYMOUS, prot=PROT_WRITE | PROT_READ | PROT_EXEC, ) self.mm.write(code) self.addr = ctypes.addressof(ctypes.c_int.from_buffer(self.mm)) func_type = CFUNCTYPE(None, POINTER(CPUID_struct), c_uint32, c_uint32) self.func_ptr = func_type(self.addr) def __call__(self, eax, ecx=0): struct = self.registers_for(eax=eax, ecx=ecx) return struct.eax, struct.ebx, struct.ecx, struct.edx def registers_for(self, eax, ecx=0): """Calls cpuid with eax and ecx set as the input arguments, and returns a structure containing eax, ebx, ecx, and edx. """ struct = CPUID_struct() self.func_ptr(struct, eax, ecx) return struct def __del__(self): if is_windows: self.win.VirtualFree.restype = c_long self.win.VirtualFree.argtypes = [c_void_p, c_size_t, c_ulong] self.win.VirtualFree(self.addr, 0, 0x8000) else: self.mm.close() if __name__ == "__main__": def valid_inputs(): cpuid = CPUID() for eax in (0x0, 0x80000000): highest, _, _, _ = cpuid(eax) while eax <= highest: regs = cpuid(eax) yield (eax, regs) eax += 1 print(" ".join(x.ljust(8) for x in ("CPUID", "A", "B", "C", "D")).strip()) for eax, regs in valid_inputs(): print("%08x" % eax, " ".join("%08x" % reg for reg in regs)) ================================================ FILE: lib/spack/spack/vendor/archspec/vendor/cpuid/example.py ================================================ # -*- coding: utf-8 -*- # # Copyright (c) 2024 Anders Høst # from __future__ import print_function import struct import cpuid def cpu_vendor(cpu): _, b, c, d = cpu(0) return struct.pack("III", b, d, c).decode("utf-8") def cpu_name(cpu): name = "".join((struct.pack("IIII", *cpu(0x80000000 + i)).decode("utf-8") for i in range(2, 5))) return name.split('\x00', 1)[0] def is_set(cpu, leaf, subleaf, reg_idx, bit): """ @param {leaf} %eax @param {sublead} %ecx, 0 in most cases @param {reg_idx} idx of [%eax, %ebx, %ecx, %edx], 0-based @param {bit} bit of reg selected by {reg_idx}, 0-based """ regs = cpu(leaf, subleaf) if (1 << bit) & regs[reg_idx]: return "Yes" else: return "--" if __name__ == "__main__": cpu = cpuid.CPUID() print("Vendor ID : %s" % cpu_vendor(cpu)) print("CPU name : %s" % cpu_name(cpu)) print() print("Vector instructions supported:") print("SSE : %s" % is_set(cpu, 1, 0, 3, 25)) print("SSE2 : %s" % is_set(cpu, 1, 0, 3, 26)) print("SSE3 : %s" % is_set(cpu, 1, 0, 2, 0)) print("SSSE3 : %s" % is_set(cpu, 1, 0, 2, 9)) print("SSE4.1 : %s" % is_set(cpu, 1, 0, 2, 19)) print("SSE4.2 : %s" % is_set(cpu, 1, 0, 2, 20)) print("SSE4a : %s" % is_set(cpu, 0x80000001, 0, 2, 6)) print("AVX : %s" % is_set(cpu, 1, 0, 2, 28)) print("AVX2 : %s" % is_set(cpu, 7, 0, 1, 5)) print("BMI1 : %s" % is_set(cpu, 7, 0, 1, 3)) print("BMI2 : %s" % is_set(cpu, 7, 0, 1, 8)) # Intel RDT CMT/MBM print("L3 Monitoring : %s" % is_set(cpu, 0xf, 0, 3, 1)) print("L3 Occupancy : %s" % is_set(cpu, 0xf, 1, 3, 0)) print("L3 Total BW : %s" % is_set(cpu, 0xf, 1, 3, 1)) print("L3 Local BW : %s" % is_set(cpu, 0xf, 1, 3, 2)) ================================================ FILE: lib/spack/spack/vendor/attr/__init__.py ================================================ # SPDX-License-Identifier: MIT import sys from functools import partial from . import converters, exceptions, filters, setters, validators from ._cmp import cmp_using from ._config import get_run_validators, set_run_validators from ._funcs import asdict, assoc, astuple, evolve, has, resolve_types from ._make import ( NOTHING, Attribute, Factory, attrib, attrs, fields, fields_dict, make_class, validate, ) from ._version_info import VersionInfo __version__ = "22.1.0" __version_info__ = VersionInfo._from_version_string(__version__) __title__ = "attrs" __description__ = "Classes Without Boilerplate" __url__ = "https://www.attrs.org/" __uri__ = __url__ __doc__ = __description__ + " <" + __uri__ + ">" __author__ = "Hynek Schlawack" __email__ = "hs@ox.cx" __license__ = "MIT" __copyright__ = "Copyright (c) 2015 Hynek Schlawack" s = attributes = attrs ib = attr = attrib dataclass = partial(attrs, auto_attribs=True) # happy Easter ;) __all__ = [ "Attribute", "Factory", "NOTHING", "asdict", "assoc", "astuple", "attr", "attrib", "attributes", "attrs", "cmp_using", "converters", "evolve", "exceptions", "fields", "fields_dict", "filters", "get_run_validators", "has", "ib", "make_class", "resolve_types", "s", "set_run_validators", "setters", "validate", "validators", ] if sys.version_info[:2] >= (3, 6): from ._next_gen import define, field, frozen, mutable # noqa: F401 __all__.extend(("define", "field", "frozen", "mutable")) ================================================ FILE: lib/spack/spack/vendor/attr/__init__.pyi ================================================ import sys from typing import ( Any, Callable, ClassVar, Dict, Generic, List, Mapping, Optional, Protocol, Sequence, Tuple, Type, TypeVar, Union, overload, ) # `import X as X` is required to make these public from . import converters as converters from . import exceptions as exceptions from . import filters as filters from . import setters as setters from . import validators as validators from ._cmp import cmp_using as cmp_using from ._version_info import VersionInfo __version__: str __version_info__: VersionInfo __title__: str __description__: str __url__: str __uri__: str __author__: str __email__: str __license__: str __copyright__: str _T = TypeVar("_T") _C = TypeVar("_C", bound=type) _EqOrderType = Union[bool, Callable[[Any], Any]] _ValidatorType = Callable[[Any, Attribute[_T], _T], Any] _ConverterType = Callable[[Any], Any] _FilterType = Callable[[Attribute[_T], _T], bool] _ReprType = Callable[[Any], str] _ReprArgType = Union[bool, _ReprType] _OnSetAttrType = Callable[[Any, Attribute[Any], Any], Any] _OnSetAttrArgType = Union[ _OnSetAttrType, List[_OnSetAttrType], setters._NoOpType ] _FieldTransformer = Callable[ [type, List[Attribute[Any]]], List[Attribute[Any]] ] # FIXME: in reality, if multiple validators are passed they must be in a list # or tuple, but those are invariant and so would prevent subtypes of # _ValidatorType from working when passed in a list or tuple. _ValidatorArgType = Union[_ValidatorType[_T], Sequence[_ValidatorType[_T]]] # A protocol to be able to statically accept an attrs class. class AttrsInstance(Protocol): __attrs_attrs__: ClassVar[Any] # _make -- NOTHING: object # NOTE: Factory lies about its return type to make this possible: # `x: List[int] # = Factory(list)` # Work around mypy issue #4554 in the common case by using an overload. if sys.version_info >= (3, 8): from typing import Literal @overload def Factory(factory: Callable[[], _T]) -> _T: ... @overload def Factory( factory: Callable[[Any], _T], takes_self: Literal[True], ) -> _T: ... @overload def Factory( factory: Callable[[], _T], takes_self: Literal[False], ) -> _T: ... else: @overload def Factory(factory: Callable[[], _T]) -> _T: ... @overload def Factory( factory: Union[Callable[[Any], _T], Callable[[], _T]], takes_self: bool = ..., ) -> _T: ... # Static type inference support via __dataclass_transform__ implemented as per: # https://github.com/microsoft/pyright/blob/1.1.135/specs/dataclass_transforms.md # This annotation must be applied to all overloads of "define" and "attrs" # # NOTE: This is a typing construct and does not exist at runtime. Extensions # wrapping attrs decorators should declare a separate __dataclass_transform__ # signature in the extension module using the specification linked above to # provide pyright support. def __dataclass_transform__( *, eq_default: bool = True, order_default: bool = False, kw_only_default: bool = False, field_descriptors: Tuple[Union[type, Callable[..., Any]], ...] = (()), ) -> Callable[[_T], _T]: ... class Attribute(Generic[_T]): name: str default: Optional[_T] validator: Optional[_ValidatorType[_T]] repr: _ReprArgType cmp: _EqOrderType eq: _EqOrderType order: _EqOrderType hash: Optional[bool] init: bool converter: Optional[_ConverterType] metadata: Dict[Any, Any] type: Optional[Type[_T]] kw_only: bool on_setattr: _OnSetAttrType def evolve(self, **changes: Any) -> "Attribute[Any]": ... # NOTE: We had several choices for the annotation to use for type arg: # 1) Type[_T] # - Pros: Handles simple cases correctly # - Cons: Might produce less informative errors in the case of conflicting # TypeVars e.g. `attr.ib(default='bad', type=int)` # 2) Callable[..., _T] # - Pros: Better error messages than #1 for conflicting TypeVars # - Cons: Terrible error messages for validator checks. # e.g. attr.ib(type=int, validator=validate_str) # -> error: Cannot infer function type argument # 3) type (and do all of the work in the mypy plugin) # - Pros: Simple here, and we could customize the plugin with our own errors. # - Cons: Would need to write mypy plugin code to handle all the cases. # We chose option #1. # `attr` lies about its return type to make the following possible: # attr() -> Any # attr(8) -> int # attr(validator=) -> Whatever the callable expects. # This makes this type of assignments possible: # x: int = attr(8) # # This form catches explicit None or no default but with no other arguments # returns Any. @overload def attrib( default: None = ..., validator: None = ..., repr: _ReprArgType = ..., cmp: Optional[_EqOrderType] = ..., hash: Optional[bool] = ..., init: bool = ..., metadata: Optional[Mapping[Any, Any]] = ..., type: None = ..., converter: None = ..., factory: None = ..., kw_only: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., ) -> Any: ... # This form catches an explicit None or no default and infers the type from the # other arguments. @overload def attrib( default: None = ..., validator: Optional[_ValidatorArgType[_T]] = ..., repr: _ReprArgType = ..., cmp: Optional[_EqOrderType] = ..., hash: Optional[bool] = ..., init: bool = ..., metadata: Optional[Mapping[Any, Any]] = ..., type: Optional[Type[_T]] = ..., converter: Optional[_ConverterType] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., ) -> _T: ... # This form catches an explicit default argument. @overload def attrib( default: _T, validator: Optional[_ValidatorArgType[_T]] = ..., repr: _ReprArgType = ..., cmp: Optional[_EqOrderType] = ..., hash: Optional[bool] = ..., init: bool = ..., metadata: Optional[Mapping[Any, Any]] = ..., type: Optional[Type[_T]] = ..., converter: Optional[_ConverterType] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., ) -> _T: ... # This form covers type=non-Type: e.g. forward references (str), Any @overload def attrib( default: Optional[_T] = ..., validator: Optional[_ValidatorArgType[_T]] = ..., repr: _ReprArgType = ..., cmp: Optional[_EqOrderType] = ..., hash: Optional[bool] = ..., init: bool = ..., metadata: Optional[Mapping[Any, Any]] = ..., type: object = ..., converter: Optional[_ConverterType] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., ) -> Any: ... @overload def field( *, default: None = ..., validator: None = ..., repr: _ReprArgType = ..., hash: Optional[bool] = ..., init: bool = ..., metadata: Optional[Mapping[Any, Any]] = ..., converter: None = ..., factory: None = ..., kw_only: bool = ..., eq: Optional[bool] = ..., order: Optional[bool] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., ) -> Any: ... # This form catches an explicit None or no default and infers the type from the # other arguments. @overload def field( *, default: None = ..., validator: Optional[_ValidatorArgType[_T]] = ..., repr: _ReprArgType = ..., hash: Optional[bool] = ..., init: bool = ..., metadata: Optional[Mapping[Any, Any]] = ..., converter: Optional[_ConverterType] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., ) -> _T: ... # This form catches an explicit default argument. @overload def field( *, default: _T, validator: Optional[_ValidatorArgType[_T]] = ..., repr: _ReprArgType = ..., hash: Optional[bool] = ..., init: bool = ..., metadata: Optional[Mapping[Any, Any]] = ..., converter: Optional[_ConverterType] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., ) -> _T: ... # This form covers type=non-Type: e.g. forward references (str), Any @overload def field( *, default: Optional[_T] = ..., validator: Optional[_ValidatorArgType[_T]] = ..., repr: _ReprArgType = ..., hash: Optional[bool] = ..., init: bool = ..., metadata: Optional[Mapping[Any, Any]] = ..., converter: Optional[_ConverterType] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., ) -> Any: ... @overload @__dataclass_transform__(order_default=True, field_descriptors=(attrib, field)) def attrs( maybe_cls: _C, these: Optional[Dict[str, Any]] = ..., repr_ns: Optional[str] = ..., repr: bool = ..., cmp: Optional[_EqOrderType] = ..., hash: Optional[bool] = ..., init: bool = ..., slots: bool = ..., frozen: bool = ..., weakref_slot: bool = ..., str: bool = ..., auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., auto_exc: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., auto_detect: bool = ..., collect_by_mro: bool = ..., getstate_setstate: Optional[bool] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., field_transformer: Optional[_FieldTransformer] = ..., match_args: bool = ..., ) -> _C: ... @overload @__dataclass_transform__(order_default=True, field_descriptors=(attrib, field)) def attrs( maybe_cls: None = ..., these: Optional[Dict[str, Any]] = ..., repr_ns: Optional[str] = ..., repr: bool = ..., cmp: Optional[_EqOrderType] = ..., hash: Optional[bool] = ..., init: bool = ..., slots: bool = ..., frozen: bool = ..., weakref_slot: bool = ..., str: bool = ..., auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., auto_exc: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., auto_detect: bool = ..., collect_by_mro: bool = ..., getstate_setstate: Optional[bool] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., field_transformer: Optional[_FieldTransformer] = ..., match_args: bool = ..., ) -> Callable[[_C], _C]: ... @overload @__dataclass_transform__(field_descriptors=(attrib, field)) def define( maybe_cls: _C, *, these: Optional[Dict[str, Any]] = ..., repr: bool = ..., hash: Optional[bool] = ..., init: bool = ..., slots: bool = ..., frozen: bool = ..., weakref_slot: bool = ..., str: bool = ..., auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., auto_exc: bool = ..., eq: Optional[bool] = ..., order: Optional[bool] = ..., auto_detect: bool = ..., getstate_setstate: Optional[bool] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., field_transformer: Optional[_FieldTransformer] = ..., match_args: bool = ..., ) -> _C: ... @overload @__dataclass_transform__(field_descriptors=(attrib, field)) def define( maybe_cls: None = ..., *, these: Optional[Dict[str, Any]] = ..., repr: bool = ..., hash: Optional[bool] = ..., init: bool = ..., slots: bool = ..., frozen: bool = ..., weakref_slot: bool = ..., str: bool = ..., auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., auto_exc: bool = ..., eq: Optional[bool] = ..., order: Optional[bool] = ..., auto_detect: bool = ..., getstate_setstate: Optional[bool] = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., field_transformer: Optional[_FieldTransformer] = ..., match_args: bool = ..., ) -> Callable[[_C], _C]: ... mutable = define frozen = define # they differ only in their defaults def fields(cls: Type[AttrsInstance]) -> Any: ... def fields_dict(cls: Type[AttrsInstance]) -> Dict[str, Attribute[Any]]: ... def validate(inst: AttrsInstance) -> None: ... def resolve_types( cls: _C, globalns: Optional[Dict[str, Any]] = ..., localns: Optional[Dict[str, Any]] = ..., attribs: Optional[List[Attribute[Any]]] = ..., ) -> _C: ... # TODO: add support for returning a proper attrs class from the mypy plugin # we use Any instead of _CountingAttr so that e.g. `make_class('Foo', # [attr.ib()])` is valid def make_class( name: str, attrs: Union[List[str], Tuple[str, ...], Dict[str, Any]], bases: Tuple[type, ...] = ..., repr_ns: Optional[str] = ..., repr: bool = ..., cmp: Optional[_EqOrderType] = ..., hash: Optional[bool] = ..., init: bool = ..., slots: bool = ..., frozen: bool = ..., weakref_slot: bool = ..., str: bool = ..., auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., auto_exc: bool = ..., eq: Optional[_EqOrderType] = ..., order: Optional[_EqOrderType] = ..., collect_by_mro: bool = ..., on_setattr: Optional[_OnSetAttrArgType] = ..., field_transformer: Optional[_FieldTransformer] = ..., ) -> type: ... # _funcs -- # TODO: add support for returning TypedDict from the mypy plugin # FIXME: asdict/astuple do not honor their factory args. Waiting on one of # these: # https://github.com/python/mypy/issues/4236 # https://github.com/python/typing/issues/253 # XXX: remember to fix attrs.asdict/astuple too! def asdict( inst: AttrsInstance, recurse: bool = ..., filter: Optional[_FilterType[Any]] = ..., dict_factory: Type[Mapping[Any, Any]] = ..., retain_collection_types: bool = ..., value_serializer: Optional[ Callable[[type, Attribute[Any], Any], Any] ] = ..., tuple_keys: Optional[bool] = ..., ) -> Dict[str, Any]: ... # TODO: add support for returning NamedTuple from the mypy plugin def astuple( inst: AttrsInstance, recurse: bool = ..., filter: Optional[_FilterType[Any]] = ..., tuple_factory: Type[Sequence[Any]] = ..., retain_collection_types: bool = ..., ) -> Tuple[Any, ...]: ... def has(cls: type) -> bool: ... def assoc(inst: _T, **changes: Any) -> _T: ... def evolve(inst: _T, **changes: Any) -> _T: ... # _config -- def set_run_validators(run: bool) -> None: ... def get_run_validators() -> bool: ... # aliases -- s = attributes = attrs ib = attr = attrib dataclass = attrs # Technically, partial(attrs, auto_attribs=True) ;) ================================================ FILE: lib/spack/spack/vendor/attr/_cmp.py ================================================ # SPDX-License-Identifier: MIT import functools import types from ._make import _make_ne _operation_names = {"eq": "==", "lt": "<", "le": "<=", "gt": ">", "ge": ">="} def cmp_using( eq=None, lt=None, le=None, gt=None, ge=None, require_same_type=True, class_name="Comparable", ): """ Create a class that can be passed into `attr.ib`'s ``eq``, ``order``, and ``cmp`` arguments to customize field comparison. The resulting class will have a full set of ordering methods if at least one of ``{lt, le, gt, ge}`` and ``eq`` are provided. :param Optional[callable] eq: `callable` used to evaluate equality of two objects. :param Optional[callable] lt: `callable` used to evaluate whether one object is less than another object. :param Optional[callable] le: `callable` used to evaluate whether one object is less than or equal to another object. :param Optional[callable] gt: `callable` used to evaluate whether one object is greater than another object. :param Optional[callable] ge: `callable` used to evaluate whether one object is greater than or equal to another object. :param bool require_same_type: When `True`, equality and ordering methods will return `NotImplemented` if objects are not of the same type. :param Optional[str] class_name: Name of class. Defaults to 'Comparable'. See `comparison` for more details. .. versionadded:: 21.1.0 """ body = { "__slots__": ["value"], "__init__": _make_init(), "_requirements": [], "_is_comparable_to": _is_comparable_to, } # Add operations. num_order_functions = 0 has_eq_function = False if eq is not None: has_eq_function = True body["__eq__"] = _make_operator("eq", eq) body["__ne__"] = _make_ne() if lt is not None: num_order_functions += 1 body["__lt__"] = _make_operator("lt", lt) if le is not None: num_order_functions += 1 body["__le__"] = _make_operator("le", le) if gt is not None: num_order_functions += 1 body["__gt__"] = _make_operator("gt", gt) if ge is not None: num_order_functions += 1 body["__ge__"] = _make_operator("ge", ge) type_ = types.new_class( class_name, (object,), {}, lambda ns: ns.update(body) ) # Add same type requirement. if require_same_type: type_._requirements.append(_check_same_type) # Add total ordering if at least one operation was defined. if 0 < num_order_functions < 4: if not has_eq_function: # functools.total_ordering requires __eq__ to be defined, # so raise early error here to keep a nice stack. raise ValueError( "eq must be define is order to complete ordering from " "lt, le, gt, ge." ) type_ = functools.total_ordering(type_) return type_ def _make_init(): """ Create __init__ method. """ def __init__(self, value): """ Initialize object with *value*. """ self.value = value return __init__ def _make_operator(name, func): """ Create operator method. """ def method(self, other): if not self._is_comparable_to(other): return NotImplemented result = func(self.value, other.value) if result is NotImplemented: return NotImplemented return result method.__name__ = "__%s__" % (name,) method.__doc__ = "Return a %s b. Computed by attrs." % ( _operation_names[name], ) return method def _is_comparable_to(self, other): """ Check whether `other` is comparable to `self`. """ for func in self._requirements: if not func(self, other): return False return True def _check_same_type(self, other): """ Return True if *self* and *other* are of the same type, False otherwise. """ return other.value.__class__ is self.value.__class__ ================================================ FILE: lib/spack/spack/vendor/attr/_cmp.pyi ================================================ from typing import Any, Callable, Optional, Type _CompareWithType = Callable[[Any, Any], bool] def cmp_using( eq: Optional[_CompareWithType], lt: Optional[_CompareWithType], le: Optional[_CompareWithType], gt: Optional[_CompareWithType], ge: Optional[_CompareWithType], require_same_type: bool, class_name: str, ) -> Type: ... ================================================ FILE: lib/spack/spack/vendor/attr/_compat.py ================================================ # SPDX-License-Identifier: MIT import inspect import platform import sys import threading import types import warnings from collections.abc import Mapping, Sequence # noqa PYPY = platform.python_implementation() == "PyPy" PY36 = sys.version_info[:2] >= (3, 6) HAS_F_STRINGS = PY36 PY310 = sys.version_info[:2] >= (3, 10) if PYPY or PY36: ordered_dict = dict else: from collections import OrderedDict ordered_dict = OrderedDict def just_warn(*args, **kw): warnings.warn( "Running interpreter doesn't sufficiently support code object " "introspection. Some features like bare super() or accessing " "__class__ will not work with slotted classes.", RuntimeWarning, stacklevel=2, ) class _AnnotationExtractor: """ Extract type annotations from a callable, returning None whenever there is none. """ __slots__ = ["sig"] def __init__(self, callable): try: self.sig = inspect.signature(callable) except (ValueError, TypeError): # inspect failed self.sig = None def get_first_param_type(self): """ Return the type annotation of the first argument if it's not empty. """ if not self.sig: return None params = list(self.sig.parameters.values()) if params and params[0].annotation is not inspect.Parameter.empty: return params[0].annotation return None def get_return_type(self): """ Return the return type if it's not empty. """ if ( self.sig and self.sig.return_annotation is not inspect.Signature.empty ): return self.sig.return_annotation return None def make_set_closure_cell(): """Return a function of two arguments (cell, value) which sets the value stored in the closure cell `cell` to `value`. """ # pypy makes this easy. (It also supports the logic below, but # why not do the easy/fast thing?) if PYPY: def set_closure_cell(cell, value): cell.__setstate__((value,)) return set_closure_cell # Otherwise gotta do it the hard way. # Create a function that will set its first cellvar to `value`. def set_first_cellvar_to(value): x = value return # This function will be eliminated as dead code, but # not before its reference to `x` forces `x` to be # represented as a closure cell rather than a local. def force_x_to_be_a_cell(): # pragma: no cover return x try: # Extract the code object and make sure our assumptions about # the closure behavior are correct. co = set_first_cellvar_to.__code__ if co.co_cellvars != ("x",) or co.co_freevars != (): raise AssertionError # pragma: no cover # Convert this code object to a code object that sets the # function's first _freevar_ (not cellvar) to the argument. if sys.version_info >= (3, 8): def set_closure_cell(cell, value): cell.cell_contents = value else: args = [co.co_argcount] args.append(co.co_kwonlyargcount) args.extend( [ co.co_nlocals, co.co_stacksize, co.co_flags, co.co_code, co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name, co.co_firstlineno, co.co_lnotab, # These two arguments are reversed: co.co_cellvars, co.co_freevars, ] ) set_first_freevar_code = types.CodeType(*args) def set_closure_cell(cell, value): # Create a function using the set_first_freevar_code, # whose first closure cell is `cell`. Calling it will # change the value of that cell. setter = types.FunctionType( set_first_freevar_code, {}, "setter", (), (cell,) ) # And call it to set the cell. setter(value) # Make sure it works on this interpreter: def make_func_with_cell(): x = None def func(): return x # pragma: no cover return func cell = make_func_with_cell().__closure__[0] set_closure_cell(cell, 100) if cell.cell_contents != 100: raise AssertionError # pragma: no cover except Exception: return just_warn else: return set_closure_cell set_closure_cell = make_set_closure_cell() # Thread-local global to track attrs instances which are already being repr'd. # This is needed because there is no other (thread-safe) way to pass info # about the instances that are already being repr'd through the call stack # in order to ensure we don't perform infinite recursion. # # For instance, if an instance contains a dict which contains that instance, # we need to know that we're already repr'ing the outside instance from within # the dict's repr() call. # # This lives here rather than in _make.py so that the functions in _make.py # don't have a direct reference to the thread-local in their globals dict. # If they have such a reference, it breaks cloudpickle. repr_context = threading.local() ================================================ FILE: lib/spack/spack/vendor/attr/_config.py ================================================ # SPDX-License-Identifier: MIT __all__ = ["set_run_validators", "get_run_validators"] _run_validators = True def set_run_validators(run): """ Set whether or not validators are run. By default, they are run. .. deprecated:: 21.3.0 It will not be removed, but it also will not be moved to new ``attrs`` namespace. Use `attrs.validators.set_disabled()` instead. """ if not isinstance(run, bool): raise TypeError("'run' must be bool.") global _run_validators _run_validators = run def get_run_validators(): """ Return whether or not validators are run. .. deprecated:: 21.3.0 It will not be removed, but it also will not be moved to new ``attrs`` namespace. Use `attrs.validators.get_disabled()` instead. """ return _run_validators ================================================ FILE: lib/spack/spack/vendor/attr/_funcs.py ================================================ # SPDX-License-Identifier: MIT import copy from ._make import NOTHING, _obj_setattr, fields from .exceptions import AttrsAttributeNotFoundError def asdict( inst, recurse=True, filter=None, dict_factory=dict, retain_collection_types=False, value_serializer=None, ): """ Return the ``attrs`` attribute values of *inst* as a dict. Optionally recurse into other ``attrs``-decorated classes. :param inst: Instance of an ``attrs``-decorated class. :param bool recurse: Recurse into classes that are also ``attrs``-decorated. :param callable filter: A callable whose return code determines whether an attribute or element is included (``True``) or dropped (``False``). Is called with the `attrs.Attribute` as the first argument and the value as the second argument. :param callable dict_factory: A callable to produce dictionaries from. For example, to produce ordered dictionaries instead of normal Python dictionaries, pass in ``collections.OrderedDict``. :param bool retain_collection_types: Do not convert to ``list`` when encountering an attribute whose type is ``tuple`` or ``set``. Only meaningful if ``recurse`` is ``True``. :param Optional[callable] value_serializer: A hook that is called for every attribute or dict key/value. It receives the current instance, field and value and must return the (updated) value. The hook is run *after* the optional *filter* has been applied. :rtype: return type of *dict_factory* :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` class. .. versionadded:: 16.0.0 *dict_factory* .. versionadded:: 16.1.0 *retain_collection_types* .. versionadded:: 20.3.0 *value_serializer* .. versionadded:: 21.3.0 If a dict has a collection for a key, it is serialized as a tuple. """ attrs = fields(inst.__class__) rv = dict_factory() for a in attrs: v = getattr(inst, a.name) if filter is not None and not filter(a, v): continue if value_serializer is not None: v = value_serializer(inst, a, v) if recurse is True: if has(v.__class__): rv[a.name] = asdict( v, recurse=True, filter=filter, dict_factory=dict_factory, retain_collection_types=retain_collection_types, value_serializer=value_serializer, ) elif isinstance(v, (tuple, list, set, frozenset)): cf = v.__class__ if retain_collection_types is True else list rv[a.name] = cf( [ _asdict_anything( i, is_key=False, filter=filter, dict_factory=dict_factory, retain_collection_types=retain_collection_types, value_serializer=value_serializer, ) for i in v ] ) elif isinstance(v, dict): df = dict_factory rv[a.name] = df( ( _asdict_anything( kk, is_key=True, filter=filter, dict_factory=df, retain_collection_types=retain_collection_types, value_serializer=value_serializer, ), _asdict_anything( vv, is_key=False, filter=filter, dict_factory=df, retain_collection_types=retain_collection_types, value_serializer=value_serializer, ), ) for kk, vv in v.items() ) else: rv[a.name] = v else: rv[a.name] = v return rv def _asdict_anything( val, is_key, filter, dict_factory, retain_collection_types, value_serializer, ): """ ``asdict`` only works on attrs instances, this works on anything. """ if getattr(val.__class__, "__attrs_attrs__", None) is not None: # Attrs class. rv = asdict( val, recurse=True, filter=filter, dict_factory=dict_factory, retain_collection_types=retain_collection_types, value_serializer=value_serializer, ) elif isinstance(val, (tuple, list, set, frozenset)): if retain_collection_types is True: cf = val.__class__ elif is_key: cf = tuple else: cf = list rv = cf( [ _asdict_anything( i, is_key=False, filter=filter, dict_factory=dict_factory, retain_collection_types=retain_collection_types, value_serializer=value_serializer, ) for i in val ] ) elif isinstance(val, dict): df = dict_factory rv = df( ( _asdict_anything( kk, is_key=True, filter=filter, dict_factory=df, retain_collection_types=retain_collection_types, value_serializer=value_serializer, ), _asdict_anything( vv, is_key=False, filter=filter, dict_factory=df, retain_collection_types=retain_collection_types, value_serializer=value_serializer, ), ) for kk, vv in val.items() ) else: rv = val if value_serializer is not None: rv = value_serializer(None, None, rv) return rv def astuple( inst, recurse=True, filter=None, tuple_factory=tuple, retain_collection_types=False, ): """ Return the ``attrs`` attribute values of *inst* as a tuple. Optionally recurse into other ``attrs``-decorated classes. :param inst: Instance of an ``attrs``-decorated class. :param bool recurse: Recurse into classes that are also ``attrs``-decorated. :param callable filter: A callable whose return code determines whether an attribute or element is included (``True``) or dropped (``False``). Is called with the `attrs.Attribute` as the first argument and the value as the second argument. :param callable tuple_factory: A callable to produce tuples from. For example, to produce lists instead of tuples. :param bool retain_collection_types: Do not convert to ``list`` or ``dict`` when encountering an attribute which type is ``tuple``, ``dict`` or ``set``. Only meaningful if ``recurse`` is ``True``. :rtype: return type of *tuple_factory* :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` class. .. versionadded:: 16.2.0 """ attrs = fields(inst.__class__) rv = [] retain = retain_collection_types # Very long. :/ for a in attrs: v = getattr(inst, a.name) if filter is not None and not filter(a, v): continue if recurse is True: if has(v.__class__): rv.append( astuple( v, recurse=True, filter=filter, tuple_factory=tuple_factory, retain_collection_types=retain, ) ) elif isinstance(v, (tuple, list, set, frozenset)): cf = v.__class__ if retain is True else list rv.append( cf( [ astuple( j, recurse=True, filter=filter, tuple_factory=tuple_factory, retain_collection_types=retain, ) if has(j.__class__) else j for j in v ] ) ) elif isinstance(v, dict): df = v.__class__ if retain is True else dict rv.append( df( ( astuple( kk, tuple_factory=tuple_factory, retain_collection_types=retain, ) if has(kk.__class__) else kk, astuple( vv, tuple_factory=tuple_factory, retain_collection_types=retain, ) if has(vv.__class__) else vv, ) for kk, vv in v.items() ) ) else: rv.append(v) else: rv.append(v) return rv if tuple_factory is list else tuple_factory(rv) def has(cls): """ Check whether *cls* is a class with ``attrs`` attributes. :param type cls: Class to introspect. :raise TypeError: If *cls* is not a class. :rtype: bool """ return getattr(cls, "__attrs_attrs__", None) is not None def assoc(inst, **changes): """ Copy *inst* and apply *changes*. :param inst: Instance of a class with ``attrs`` attributes. :param changes: Keyword changes in the new copy. :return: A copy of inst with *changes* incorporated. :raise attr.exceptions.AttrsAttributeNotFoundError: If *attr_name* couldn't be found on *cls*. :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` class. .. deprecated:: 17.1.0 Use `attrs.evolve` instead if you can. This function will not be removed du to the slightly different approach compared to `attrs.evolve`. """ import warnings warnings.warn( "assoc is deprecated and will be removed after 2018/01.", DeprecationWarning, stacklevel=2, ) new = copy.copy(inst) attrs = fields(inst.__class__) for k, v in changes.items(): a = getattr(attrs, k, NOTHING) if a is NOTHING: raise AttrsAttributeNotFoundError( "{k} is not an attrs attribute on {cl}.".format( k=k, cl=new.__class__ ) ) _obj_setattr(new, k, v) return new def evolve(inst, **changes): """ Create a new instance, based on *inst* with *changes* applied. :param inst: Instance of a class with ``attrs`` attributes. :param changes: Keyword changes in the new copy. :return: A copy of inst with *changes* incorporated. :raise TypeError: If *attr_name* couldn't be found in the class ``__init__``. :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` class. .. versionadded:: 17.1.0 """ cls = inst.__class__ attrs = fields(cls) for a in attrs: if not a.init: continue attr_name = a.name # To deal with private attributes. init_name = attr_name if attr_name[0] != "_" else attr_name[1:] if init_name not in changes: changes[init_name] = getattr(inst, attr_name) return cls(**changes) def resolve_types(cls, globalns=None, localns=None, attribs=None): """ Resolve any strings and forward annotations in type annotations. This is only required if you need concrete types in `Attribute`'s *type* field. In other words, you don't need to resolve your types if you only use them for static type checking. With no arguments, names will be looked up in the module in which the class was created. If this is not what you want, e.g. if the name only exists inside a method, you may pass *globalns* or *localns* to specify other dictionaries in which to look up these names. See the docs of `typing.get_type_hints` for more details. :param type cls: Class to resolve. :param Optional[dict] globalns: Dictionary containing global variables. :param Optional[dict] localns: Dictionary containing local variables. :param Optional[list] attribs: List of attribs for the given class. This is necessary when calling from inside a ``field_transformer`` since *cls* is not an ``attrs`` class yet. :raise TypeError: If *cls* is not a class. :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` class and you didn't pass any attribs. :raise NameError: If types cannot be resolved because of missing variables. :returns: *cls* so you can use this function also as a class decorator. Please note that you have to apply it **after** `attrs.define`. That means the decorator has to come in the line **before** `attrs.define`. .. versionadded:: 20.1.0 .. versionadded:: 21.1.0 *attribs* """ # Since calling get_type_hints is expensive we cache whether we've # done it already. if getattr(cls, "__attrs_types_resolved__", None) != cls: import typing hints = typing.get_type_hints(cls, globalns=globalns, localns=localns) for field in fields(cls) if attribs is None else attribs: if field.name in hints: # Since fields have been frozen we must work around it. _obj_setattr(field, "type", hints[field.name]) # We store the class we resolved so that subclasses know they haven't # been resolved. cls.__attrs_types_resolved__ = cls # Return the class so you can use it as a decorator too. return cls ================================================ FILE: lib/spack/spack/vendor/attr/_make.py ================================================ # SPDX-License-Identifier: MIT import copy import linecache import sys import types import typing from operator import itemgetter # We need to import _compat itself in addition to the _compat members to avoid # having the thread-local in the globals here. from . import _compat, _config, setters from ._compat import ( HAS_F_STRINGS, PY310, PYPY, _AnnotationExtractor, ordered_dict, set_closure_cell, ) from .exceptions import ( DefaultAlreadySetError, FrozenInstanceError, NotAnAttrsClassError, UnannotatedAttributeError, ) # This is used at least twice, so cache it here. _obj_setattr = object.__setattr__ _init_converter_pat = "__attr_converter_%s" _init_factory_pat = "__attr_factory_{}" _tuple_property_pat = ( " {attr_name} = _attrs_property(_attrs_itemgetter({index}))" ) _classvar_prefixes = ( "typing.ClassVar", "t.ClassVar", "ClassVar", "spack.vendor.typing_extensions.ClassVar", ) # we don't use a double-underscore prefix because that triggers # name mangling when trying to create a slot for the field # (when slots=True) _hash_cache_field = "_attrs_cached_hash" _empty_metadata_singleton = types.MappingProxyType({}) # Unique object for unequivocal getattr() defaults. _sentinel = object() _ng_default_on_setattr = setters.pipe(setters.convert, setters.validate) class _Nothing: """ Sentinel class to indicate the lack of a value when ``None`` is ambiguous. ``_Nothing`` is a singleton. There is only ever one of it. .. versionchanged:: 21.1.0 ``bool(NOTHING)`` is now False. """ _singleton = None def __new__(cls): if _Nothing._singleton is None: _Nothing._singleton = super().__new__(cls) return _Nothing._singleton def __repr__(self): return "NOTHING" def __bool__(self): return False NOTHING = _Nothing() """ Sentinel to indicate the lack of a value when ``None`` is ambiguous. """ class _CacheHashWrapper(int): """ An integer subclass that pickles / copies as None This is used for non-slots classes with ``cache_hash=True``, to avoid serializing a potentially (even likely) invalid hash value. Since ``None`` is the default value for uncalculated hashes, whenever this is copied, the copy's value for the hash should automatically reset. See GH #613 for more details. """ def __reduce__(self, _none_constructor=type(None), _args=()): return _none_constructor, _args def attrib( default=NOTHING, validator=None, repr=True, cmp=None, hash=None, init=True, metadata=None, type=None, converter=None, factory=None, kw_only=False, eq=None, order=None, on_setattr=None, ): """ Create a new attribute on a class. .. warning:: Does *not* do anything unless the class is also decorated with `attr.s`! :param default: A value that is used if an ``attrs``-generated ``__init__`` is used and no value is passed while instantiating or the attribute is excluded using ``init=False``. If the value is an instance of `attrs.Factory`, its callable will be used to construct a new value (useful for mutable data types like lists or dicts). If a default is not set (or set manually to `attrs.NOTHING`), a value *must* be supplied when instantiating; otherwise a `TypeError` will be raised. The default can also be set using decorator notation as shown below. :type default: Any value :param callable factory: Syntactic sugar for ``default=attr.Factory(factory)``. :param validator: `callable` that is called by ``attrs``-generated ``__init__`` methods after the instance has been initialized. They receive the initialized instance, the :func:`~attrs.Attribute`, and the passed value. The return value is *not* inspected so the validator has to throw an exception itself. If a `list` is passed, its items are treated as validators and must all pass. Validators can be globally disabled and re-enabled using `get_run_validators`. The validator can also be set using decorator notation as shown below. :type validator: `callable` or a `list` of `callable`\\ s. :param repr: Include this attribute in the generated ``__repr__`` method. If ``True``, include the attribute; if ``False``, omit it. By default, the built-in ``repr()`` function is used. To override how the attribute value is formatted, pass a ``callable`` that takes a single value and returns a string. Note that the resulting string is used as-is, i.e. it will be used directly *instead* of calling ``repr()`` (the default). :type repr: a `bool` or a `callable` to use a custom function. :param eq: If ``True`` (default), include this attribute in the generated ``__eq__`` and ``__ne__`` methods that check two instances for equality. To override how the attribute value is compared, pass a ``callable`` that takes a single value and returns the value to be compared. :type eq: a `bool` or a `callable`. :param order: If ``True`` (default), include this attributes in the generated ``__lt__``, ``__le__``, ``__gt__`` and ``__ge__`` methods. To override how the attribute value is ordered, pass a ``callable`` that takes a single value and returns the value to be ordered. :type order: a `bool` or a `callable`. :param cmp: Setting *cmp* is equivalent to setting *eq* and *order* to the same value. Must not be mixed with *eq* or *order*. :type cmp: a `bool` or a `callable`. :param Optional[bool] hash: Include this attribute in the generated ``__hash__`` method. If ``None`` (default), mirror *eq*'s value. This is the correct behavior according the Python spec. Setting this value to anything else than ``None`` is *discouraged*. :param bool init: Include this attribute in the generated ``__init__`` method. It is possible to set this to ``False`` and set a default value. In that case this attributed is unconditionally initialized with the specified default value or factory. :param callable converter: `callable` that is called by ``attrs``-generated ``__init__`` methods to convert attribute's value to the desired format. It is given the passed-in value, and the returned value will be used as the new value of the attribute. The value is converted before being passed to the validator, if any. :param metadata: An arbitrary mapping, to be used by third-party components. See `extending_metadata`. :param type: The type of the attribute. In Python 3.6 or greater, the preferred method to specify the type is using a variable annotation (see :pep:`526`). This argument is provided for backward compatibility. Regardless of the approach used, the type will be stored on ``Attribute.type``. Please note that ``attrs`` doesn't do anything with this metadata by itself. You can use it as part of your own code or for `static type checking `. :param kw_only: Make this attribute keyword-only (Python 3+) in the generated ``__init__`` (if ``init`` is ``False``, this parameter is ignored). :param on_setattr: Allows to overwrite the *on_setattr* setting from `attr.s`. If left `None`, the *on_setattr* value from `attr.s` is used. Set to `attrs.setters.NO_OP` to run **no** `setattr` hooks for this attribute -- regardless of the setting in `attr.s`. :type on_setattr: `callable`, or a list of callables, or `None`, or `attrs.setters.NO_OP` .. versionadded:: 15.2.0 *convert* .. versionadded:: 16.3.0 *metadata* .. versionchanged:: 17.1.0 *validator* can be a ``list`` now. .. versionchanged:: 17.1.0 *hash* is ``None`` and therefore mirrors *eq* by default. .. versionadded:: 17.3.0 *type* .. deprecated:: 17.4.0 *convert* .. versionadded:: 17.4.0 *converter* as a replacement for the deprecated *convert* to achieve consistency with other noun-based arguments. .. versionadded:: 18.1.0 ``factory=f`` is syntactic sugar for ``default=attr.Factory(f)``. .. versionadded:: 18.2.0 *kw_only* .. versionchanged:: 19.2.0 *convert* keyword argument removed. .. versionchanged:: 19.2.0 *repr* also accepts a custom callable. .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. .. versionadded:: 19.2.0 *eq* and *order* .. versionadded:: 20.1.0 *on_setattr* .. versionchanged:: 20.3.0 *kw_only* backported to Python 2 .. versionchanged:: 21.1.0 *eq*, *order*, and *cmp* also accept a custom callable .. versionchanged:: 21.1.0 *cmp* undeprecated """ eq, eq_key, order, order_key = _determine_attrib_eq_order( cmp, eq, order, True ) if hash is not None and hash is not True and hash is not False: raise TypeError( "Invalid value for hash. Must be True, False, or None." ) if factory is not None: if default is not NOTHING: raise ValueError( "The `default` and `factory` arguments are mutually " "exclusive." ) if not callable(factory): raise ValueError("The `factory` argument must be a callable.") default = Factory(factory) if metadata is None: metadata = {} # Apply syntactic sugar by auto-wrapping. if isinstance(on_setattr, (list, tuple)): on_setattr = setters.pipe(*on_setattr) if validator and isinstance(validator, (list, tuple)): validator = and_(*validator) if converter and isinstance(converter, (list, tuple)): converter = pipe(*converter) return _CountingAttr( default=default, validator=validator, repr=repr, cmp=None, hash=hash, init=init, converter=converter, metadata=metadata, type=type, kw_only=kw_only, eq=eq, eq_key=eq_key, order=order, order_key=order_key, on_setattr=on_setattr, ) def _compile_and_eval(script, globs, locs=None, filename=""): """ "Exec" the script with the given global (globs) and local (locs) variables. """ bytecode = compile(script, filename, "exec") eval(bytecode, globs, locs) def _make_method(name, script, filename, globs): """ Create the method with the script given and return the method object. """ locs = {} # In order of debuggers like PDB being able to step through the code, # we add a fake linecache entry. count = 1 base_filename = filename while True: linecache_tuple = ( len(script), None, script.splitlines(True), filename, ) old_val = linecache.cache.setdefault(filename, linecache_tuple) if old_val == linecache_tuple: break else: filename = "{}-{}>".format(base_filename[:-1], count) count += 1 _compile_and_eval(script, globs, locs, filename) return locs[name] def _make_attr_tuple_class(cls_name, attr_names): """ Create a tuple subclass to hold `Attribute`s for an `attrs` class. The subclass is a bare tuple with properties for names. class MyClassAttributes(tuple): __slots__ = () x = property(itemgetter(0)) """ attr_class_name = "{}Attributes".format(cls_name) attr_class_template = [ "class {}(tuple):".format(attr_class_name), " __slots__ = ()", ] if attr_names: for i, attr_name in enumerate(attr_names): attr_class_template.append( _tuple_property_pat.format(index=i, attr_name=attr_name) ) else: attr_class_template.append(" pass") globs = {"_attrs_itemgetter": itemgetter, "_attrs_property": property} _compile_and_eval("\n".join(attr_class_template), globs) return globs[attr_class_name] # Tuple class for extracted attributes from a class definition. # `base_attrs` is a subset of `attrs`. _Attributes = _make_attr_tuple_class( "_Attributes", [ # all attributes to build dunder methods for "attrs", # attributes that have been inherited "base_attrs", # map inherited attributes to their originating classes "base_attrs_map", ], ) def _is_class_var(annot): """ Check whether *annot* is a typing.ClassVar. The string comparison hack is used to avoid evaluating all string annotations which would put attrs-based classes at a performance disadvantage compared to plain old classes. """ annot = str(annot) # Annotation can be quoted. if annot.startswith(("'", '"')) and annot.endswith(("'", '"')): annot = annot[1:-1] return annot.startswith(_classvar_prefixes) def _has_own_attribute(cls, attrib_name): """ Check whether *cls* defines *attrib_name* (and doesn't just inherit it). Requires Python 3. """ attr = getattr(cls, attrib_name, _sentinel) if attr is _sentinel: return False for base_cls in cls.__mro__[1:]: a = getattr(base_cls, attrib_name, None) if attr is a: return False return True def _get_annotations(cls): """ Get annotations for *cls*. """ if _has_own_attribute(cls, "__annotations__"): return cls.__annotations__ return {} def _counter_getter(e): """ Key function for sorting to avoid re-creating a lambda for every class. """ return e[1].counter def _collect_base_attrs(cls, taken_attr_names): """ Collect attr.ibs from base classes of *cls*, except *taken_attr_names*. """ base_attrs = [] base_attr_map = {} # A dictionary of base attrs to their classes. # Traverse the MRO and collect attributes. for base_cls in reversed(cls.__mro__[1:-1]): for a in getattr(base_cls, "__attrs_attrs__", []): if a.inherited or a.name in taken_attr_names: continue a = a.evolve(inherited=True) base_attrs.append(a) base_attr_map[a.name] = base_cls # For each name, only keep the freshest definition i.e. the furthest at the # back. base_attr_map is fine because it gets overwritten with every new # instance. filtered = [] seen = set() for a in reversed(base_attrs): if a.name in seen: continue filtered.insert(0, a) seen.add(a.name) return filtered, base_attr_map def _collect_base_attrs_broken(cls, taken_attr_names): """ Collect attr.ibs from base classes of *cls*, except *taken_attr_names*. N.B. *taken_attr_names* will be mutated. Adhere to the old incorrect behavior. Notably it collects from the front and considers inherited attributes which leads to the buggy behavior reported in #428. """ base_attrs = [] base_attr_map = {} # A dictionary of base attrs to their classes. # Traverse the MRO and collect attributes. for base_cls in cls.__mro__[1:-1]: for a in getattr(base_cls, "__attrs_attrs__", []): if a.name in taken_attr_names: continue a = a.evolve(inherited=True) taken_attr_names.add(a.name) base_attrs.append(a) base_attr_map[a.name] = base_cls return base_attrs, base_attr_map def _transform_attrs( cls, these, auto_attribs, kw_only, collect_by_mro, field_transformer ): """ Transform all `_CountingAttr`s on a class into `Attribute`s. If *these* is passed, use that and don't look for them on the class. *collect_by_mro* is True, collect them in the correct MRO order, otherwise use the old -- incorrect -- order. See #428. Return an `_Attributes`. """ cd = cls.__dict__ anns = _get_annotations(cls) if these is not None: ca_list = [(name, ca) for name, ca in these.items()] if not isinstance(these, ordered_dict): ca_list.sort(key=_counter_getter) elif auto_attribs is True: ca_names = { name for name, attr in cd.items() if isinstance(attr, _CountingAttr) } ca_list = [] annot_names = set() for attr_name, type in anns.items(): if _is_class_var(type): continue annot_names.add(attr_name) a = cd.get(attr_name, NOTHING) if not isinstance(a, _CountingAttr): if a is NOTHING: a = attrib() else: a = attrib(default=a) ca_list.append((attr_name, a)) unannotated = ca_names - annot_names if len(unannotated) > 0: raise UnannotatedAttributeError( "The following `attr.ib`s lack a type annotation: " + ", ".join( sorted(unannotated, key=lambda n: cd.get(n).counter) ) + "." ) else: ca_list = sorted( ( (name, attr) for name, attr in cd.items() if isinstance(attr, _CountingAttr) ), key=lambda e: e[1].counter, ) own_attrs = [ Attribute.from_counting_attr( name=attr_name, ca=ca, type=anns.get(attr_name) ) for attr_name, ca in ca_list ] if collect_by_mro: base_attrs, base_attr_map = _collect_base_attrs( cls, {a.name for a in own_attrs} ) else: base_attrs, base_attr_map = _collect_base_attrs_broken( cls, {a.name for a in own_attrs} ) if kw_only: own_attrs = [a.evolve(kw_only=True) for a in own_attrs] base_attrs = [a.evolve(kw_only=True) for a in base_attrs] attrs = base_attrs + own_attrs # Mandatory vs non-mandatory attr order only matters when they are part of # the __init__ signature and when they aren't kw_only (which are moved to # the end and can be mandatory or non-mandatory in any order, as they will # be specified as keyword args anyway). Check the order of those attrs: had_default = False for a in (a for a in attrs if a.init is not False and a.kw_only is False): if had_default is True and a.default is NOTHING: raise ValueError( "No mandatory attributes allowed after an attribute with a " "default value or factory. Attribute in question: %r" % (a,) ) if had_default is False and a.default is not NOTHING: had_default = True if field_transformer is not None: attrs = field_transformer(cls, attrs) # Create AttrsClass *after* applying the field_transformer since it may # add or remove attributes! attr_names = [a.name for a in attrs] AttrsClass = _make_attr_tuple_class(cls.__name__, attr_names) return _Attributes((AttrsClass(attrs), base_attrs, base_attr_map)) if PYPY: def _frozen_setattrs(self, name, value): """ Attached to frozen classes as __setattr__. """ if isinstance(self, BaseException) and name in ( "__cause__", "__context__", ): BaseException.__setattr__(self, name, value) return raise FrozenInstanceError() else: def _frozen_setattrs(self, name, value): """ Attached to frozen classes as __setattr__. """ raise FrozenInstanceError() def _frozen_delattrs(self, name): """ Attached to frozen classes as __delattr__. """ raise FrozenInstanceError() class _ClassBuilder: """ Iteratively build *one* class. """ __slots__ = ( "_attr_names", "_attrs", "_base_attr_map", "_base_names", "_cache_hash", "_cls", "_cls_dict", "_delete_attribs", "_frozen", "_has_pre_init", "_has_post_init", "_is_exc", "_on_setattr", "_slots", "_weakref_slot", "_wrote_own_setattr", "_has_custom_setattr", ) def __init__( self, cls, these, slots, frozen, weakref_slot, getstate_setstate, auto_attribs, kw_only, cache_hash, is_exc, collect_by_mro, on_setattr, has_custom_setattr, field_transformer, ): attrs, base_attrs, base_map = _transform_attrs( cls, these, auto_attribs, kw_only, collect_by_mro, field_transformer, ) self._cls = cls self._cls_dict = dict(cls.__dict__) if slots else {} self._attrs = attrs self._base_names = {a.name for a in base_attrs} self._base_attr_map = base_map self._attr_names = tuple(a.name for a in attrs) self._slots = slots self._frozen = frozen self._weakref_slot = weakref_slot self._cache_hash = cache_hash self._has_pre_init = bool(getattr(cls, "__attrs_pre_init__", False)) self._has_post_init = bool(getattr(cls, "__attrs_post_init__", False)) self._delete_attribs = not bool(these) self._is_exc = is_exc self._on_setattr = on_setattr self._has_custom_setattr = has_custom_setattr self._wrote_own_setattr = False self._cls_dict["__attrs_attrs__"] = self._attrs if frozen: self._cls_dict["__setattr__"] = _frozen_setattrs self._cls_dict["__delattr__"] = _frozen_delattrs self._wrote_own_setattr = True elif on_setattr in ( _ng_default_on_setattr, setters.validate, setters.convert, ): has_validator = has_converter = False for a in attrs: if a.validator is not None: has_validator = True if a.converter is not None: has_converter = True if has_validator and has_converter: break if ( ( on_setattr == _ng_default_on_setattr and not (has_validator or has_converter) ) or (on_setattr == setters.validate and not has_validator) or (on_setattr == setters.convert and not has_converter) ): # If class-level on_setattr is set to convert + validate, but # there's no field to convert or validate, pretend like there's # no on_setattr. self._on_setattr = None if getstate_setstate: ( self._cls_dict["__getstate__"], self._cls_dict["__setstate__"], ) = self._make_getstate_setstate() def __repr__(self): return "<_ClassBuilder(cls={cls})>".format(cls=self._cls.__name__) def build_class(self): """ Finalize class based on the accumulated configuration. Builder cannot be used after calling this method. """ if self._slots is True: return self._create_slots_class() else: return self._patch_original_class() def _patch_original_class(self): """ Apply accumulated methods and return the class. """ cls = self._cls base_names = self._base_names # Clean class of attribute definitions (`attr.ib()`s). if self._delete_attribs: for name in self._attr_names: if ( name not in base_names and getattr(cls, name, _sentinel) is not _sentinel ): try: delattr(cls, name) except AttributeError: # This can happen if a base class defines a class # variable and we want to set an attribute with the # same name by using only a type annotation. pass # Attach our dunder methods. for name, value in self._cls_dict.items(): setattr(cls, name, value) # If we've inherited an attrs __setattr__ and don't write our own, # reset it to object's. if not self._wrote_own_setattr and getattr( cls, "__attrs_own_setattr__", False ): cls.__attrs_own_setattr__ = False if not self._has_custom_setattr: cls.__setattr__ = _obj_setattr return cls def _create_slots_class(self): """ Build and return a new class with a `__slots__` attribute. """ cd = { k: v for k, v in self._cls_dict.items() if k not in tuple(self._attr_names) + ("__dict__", "__weakref__") } # If our class doesn't have its own implementation of __setattr__ # (either from the user or by us), check the bases, if one of them has # an attrs-made __setattr__, that needs to be reset. We don't walk the # MRO because we only care about our immediate base classes. # XXX: This can be confused by subclassing a slotted attrs class with # XXX: a non-attrs class and subclass the resulting class with an attrs # XXX: class. See `test_slotted_confused` for details. For now that's # XXX: OK with us. if not self._wrote_own_setattr: cd["__attrs_own_setattr__"] = False if not self._has_custom_setattr: for base_cls in self._cls.__bases__: if base_cls.__dict__.get("__attrs_own_setattr__", False): cd["__setattr__"] = _obj_setattr break # Traverse the MRO to collect existing slots # and check for an existing __weakref__. existing_slots = dict() weakref_inherited = False for base_cls in self._cls.__mro__[1:-1]: if base_cls.__dict__.get("__weakref__", None) is not None: weakref_inherited = True existing_slots.update( { name: getattr(base_cls, name) for name in getattr(base_cls, "__slots__", []) } ) base_names = set(self._base_names) names = self._attr_names if ( self._weakref_slot and "__weakref__" not in getattr(self._cls, "__slots__", ()) and "__weakref__" not in names and not weakref_inherited ): names += ("__weakref__",) # We only add the names of attributes that aren't inherited. # Setting __slots__ to inherited attributes wastes memory. slot_names = [name for name in names if name not in base_names] # There are slots for attributes from current class # that are defined in parent classes. # As their descriptors may be overridden by a child class, # we collect them here and update the class dict reused_slots = { slot: slot_descriptor for slot, slot_descriptor in existing_slots.items() if slot in slot_names } slot_names = [name for name in slot_names if name not in reused_slots] cd.update(reused_slots) if self._cache_hash: slot_names.append(_hash_cache_field) cd["__slots__"] = tuple(slot_names) cd["__qualname__"] = self._cls.__qualname__ # Create new class based on old class and our methods. cls = type(self._cls)(self._cls.__name__, self._cls.__bases__, cd) # The following is a fix for # . On Python 3, # if a method mentions `__class__` or uses the no-arg super(), the # compiler will bake a reference to the class in the method itself # as `method.__closure__`. Since we replace the class with a # clone, we rewrite these references so it keeps working. for item in cls.__dict__.values(): if isinstance(item, (classmethod, staticmethod)): # Class- and staticmethods hide their functions inside. # These might need to be rewritten as well. closure_cells = getattr(item.__func__, "__closure__", None) elif isinstance(item, property): # Workaround for property `super()` shortcut (PY3-only). # There is no universal way for other descriptors. closure_cells = getattr(item.fget, "__closure__", None) else: closure_cells = getattr(item, "__closure__", None) if not closure_cells: # Catch None or the empty list. continue for cell in closure_cells: try: match = cell.cell_contents is self._cls except ValueError: # ValueError: Cell is empty pass else: if match: set_closure_cell(cell, cls) return cls def add_repr(self, ns): self._cls_dict["__repr__"] = self._add_method_dunders( _make_repr(self._attrs, ns, self._cls) ) return self def add_str(self): repr = self._cls_dict.get("__repr__") if repr is None: raise ValueError( "__str__ can only be generated if a __repr__ exists." ) def __str__(self): return self.__repr__() self._cls_dict["__str__"] = self._add_method_dunders(__str__) return self def _make_getstate_setstate(self): """ Create custom __setstate__ and __getstate__ methods. """ # __weakref__ is not writable. state_attr_names = tuple( an for an in self._attr_names if an != "__weakref__" ) def slots_getstate(self): """ Automatically created by attrs. """ return tuple(getattr(self, name) for name in state_attr_names) hash_caching_enabled = self._cache_hash def slots_setstate(self, state): """ Automatically created by attrs. """ __bound_setattr = _obj_setattr.__get__(self, Attribute) for name, value in zip(state_attr_names, state): __bound_setattr(name, value) # The hash code cache is not included when the object is # serialized, but it still needs to be initialized to None to # indicate that the first call to __hash__ should be a cache # miss. if hash_caching_enabled: __bound_setattr(_hash_cache_field, None) return slots_getstate, slots_setstate def make_unhashable(self): self._cls_dict["__hash__"] = None return self def add_hash(self): self._cls_dict["__hash__"] = self._add_method_dunders( _make_hash( self._cls, self._attrs, frozen=self._frozen, cache_hash=self._cache_hash, ) ) return self def add_init(self): self._cls_dict["__init__"] = self._add_method_dunders( _make_init( self._cls, self._attrs, self._has_pre_init, self._has_post_init, self._frozen, self._slots, self._cache_hash, self._base_attr_map, self._is_exc, self._on_setattr, attrs_init=False, ) ) return self def add_match_args(self): self._cls_dict["__match_args__"] = tuple( field.name for field in self._attrs if field.init and not field.kw_only ) def add_attrs_init(self): self._cls_dict["__attrs_init__"] = self._add_method_dunders( _make_init( self._cls, self._attrs, self._has_pre_init, self._has_post_init, self._frozen, self._slots, self._cache_hash, self._base_attr_map, self._is_exc, self._on_setattr, attrs_init=True, ) ) return self def add_eq(self): cd = self._cls_dict cd["__eq__"] = self._add_method_dunders( _make_eq(self._cls, self._attrs) ) cd["__ne__"] = self._add_method_dunders(_make_ne()) return self def add_order(self): cd = self._cls_dict cd["__lt__"], cd["__le__"], cd["__gt__"], cd["__ge__"] = ( self._add_method_dunders(meth) for meth in _make_order(self._cls, self._attrs) ) return self def add_setattr(self): if self._frozen: return self sa_attrs = {} for a in self._attrs: on_setattr = a.on_setattr or self._on_setattr if on_setattr and on_setattr is not setters.NO_OP: sa_attrs[a.name] = a, on_setattr if not sa_attrs: return self if self._has_custom_setattr: # We need to write a __setattr__ but there already is one! raise ValueError( "Can't combine custom __setattr__ with on_setattr hooks." ) # docstring comes from _add_method_dunders def __setattr__(self, name, val): try: a, hook = sa_attrs[name] except KeyError: nval = val else: nval = hook(self, a, val) _obj_setattr(self, name, nval) self._cls_dict["__attrs_own_setattr__"] = True self._cls_dict["__setattr__"] = self._add_method_dunders(__setattr__) self._wrote_own_setattr = True return self def _add_method_dunders(self, method): """ Add __module__ and __qualname__ to a *method* if possible. """ try: method.__module__ = self._cls.__module__ except AttributeError: pass try: method.__qualname__ = ".".join( (self._cls.__qualname__, method.__name__) ) except AttributeError: pass try: method.__doc__ = "Method generated by attrs for class %s." % ( self._cls.__qualname__, ) except AttributeError: pass return method def _determine_attrs_eq_order(cmp, eq, order, default_eq): """ Validate the combination of *cmp*, *eq*, and *order*. Derive the effective values of eq and order. If *eq* is None, set it to *default_eq*. """ if cmp is not None and any((eq is not None, order is not None)): raise ValueError("Don't mix `cmp` with `eq' and `order`.") # cmp takes precedence due to bw-compatibility. if cmp is not None: return cmp, cmp # If left None, equality is set to the specified default and ordering # mirrors equality. if eq is None: eq = default_eq if order is None: order = eq if eq is False and order is True: raise ValueError("`order` can only be True if `eq` is True too.") return eq, order def _determine_attrib_eq_order(cmp, eq, order, default_eq): """ Validate the combination of *cmp*, *eq*, and *order*. Derive the effective values of eq and order. If *eq* is None, set it to *default_eq*. """ if cmp is not None and any((eq is not None, order is not None)): raise ValueError("Don't mix `cmp` with `eq' and `order`.") def decide_callable_or_boolean(value): """ Decide whether a key function is used. """ if callable(value): value, key = True, value else: key = None return value, key # cmp takes precedence due to bw-compatibility. if cmp is not None: cmp, cmp_key = decide_callable_or_boolean(cmp) return cmp, cmp_key, cmp, cmp_key # If left None, equality is set to the specified default and ordering # mirrors equality. if eq is None: eq, eq_key = default_eq, None else: eq, eq_key = decide_callable_or_boolean(eq) if order is None: order, order_key = eq, eq_key else: order, order_key = decide_callable_or_boolean(order) if eq is False and order is True: raise ValueError("`order` can only be True if `eq` is True too.") return eq, eq_key, order, order_key def _determine_whether_to_implement( cls, flag, auto_detect, dunders, default=True ): """ Check whether we should implement a set of methods for *cls*. *flag* is the argument passed into @attr.s like 'init', *auto_detect* the same as passed into @attr.s and *dunders* is a tuple of attribute names whose presence signal that the user has implemented it themselves. Return *default* if no reason for either for or against is found. """ if flag is True or flag is False: return flag if flag is None and auto_detect is False: return default # Logically, flag is None and auto_detect is True here. for dunder in dunders: if _has_own_attribute(cls, dunder): return False return default def attrs( maybe_cls=None, these=None, repr_ns=None, repr=None, cmp=None, hash=None, init=None, slots=False, frozen=False, weakref_slot=True, str=False, auto_attribs=False, kw_only=False, cache_hash=False, auto_exc=False, eq=None, order=None, auto_detect=False, collect_by_mro=False, getstate_setstate=None, on_setattr=None, field_transformer=None, match_args=True, ): r""" A class decorator that adds `dunder `_\ -methods according to the specified attributes using `attr.ib` or the *these* argument. :param these: A dictionary of name to `attr.ib` mappings. This is useful to avoid the definition of your attributes within the class body because you can't (e.g. if you want to add ``__repr__`` methods to Django models) or don't want to. If *these* is not ``None``, ``attrs`` will *not* search the class body for attributes and will *not* remove any attributes from it. If *these* is an ordered dict (`dict` on Python 3.6+, `collections.OrderedDict` otherwise), the order is deduced from the order of the attributes inside *these*. Otherwise the order of the definition of the attributes is used. :type these: `dict` of `str` to `attr.ib` :param str repr_ns: When using nested classes, there's no way in Python 2 to automatically detect that. Therefore it's possible to set the namespace explicitly for a more meaningful ``repr`` output. :param bool auto_detect: Instead of setting the *init*, *repr*, *eq*, *order*, and *hash* arguments explicitly, assume they are set to ``True`` **unless any** of the involved methods for one of the arguments is implemented in the *current* class (i.e. it is *not* inherited from some base class). So for example by implementing ``__eq__`` on a class yourself, ``attrs`` will deduce ``eq=False`` and will create *neither* ``__eq__`` *nor* ``__ne__`` (but Python classes come with a sensible ``__ne__`` by default, so it *should* be enough to only implement ``__eq__`` in most cases). .. warning:: If you prevent ``attrs`` from creating the ordering methods for you (``order=False``, e.g. by implementing ``__le__``), it becomes *your* responsibility to make sure its ordering is sound. The best way is to use the `functools.total_ordering` decorator. Passing ``True`` or ``False`` to *init*, *repr*, *eq*, *order*, *cmp*, or *hash* overrides whatever *auto_detect* would determine. *auto_detect* requires Python 3. Setting it ``True`` on Python 2 raises an `attrs.exceptions.PythonTooOldError`. :param bool repr: Create a ``__repr__`` method with a human readable representation of ``attrs`` attributes.. :param bool str: Create a ``__str__`` method that is identical to ``__repr__``. This is usually not necessary except for `Exception`\ s. :param Optional[bool] eq: If ``True`` or ``None`` (default), add ``__eq__`` and ``__ne__`` methods that check two instances for equality. They compare the instances as if they were tuples of their ``attrs`` attributes if and only if the types of both classes are *identical*! :param Optional[bool] order: If ``True``, add ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` methods that behave like *eq* above and allow instances to be ordered. If ``None`` (default) mirror value of *eq*. :param Optional[bool] cmp: Setting *cmp* is equivalent to setting *eq* and *order* to the same value. Must not be mixed with *eq* or *order*. :param Optional[bool] hash: If ``None`` (default), the ``__hash__`` method is generated according how *eq* and *frozen* are set. 1. If *both* are True, ``attrs`` will generate a ``__hash__`` for you. 2. If *eq* is True and *frozen* is False, ``__hash__`` will be set to None, marking it unhashable (which it is). 3. If *eq* is False, ``__hash__`` will be left untouched meaning the ``__hash__`` method of the base class will be used (if base class is ``object``, this means it will fall back to id-based hashing.). Although not recommended, you can decide for yourself and force ``attrs`` to create one (e.g. if the class is immutable even though you didn't freeze it programmatically) by passing ``True`` or not. Both of these cases are rather special and should be used carefully. See our documentation on `hashing`, Python's documentation on `object.__hash__`, and the `GitHub issue that led to the default \ behavior `_ for more details. :param bool init: Create a ``__init__`` method that initializes the ``attrs`` attributes. Leading underscores are stripped for the argument name. If a ``__attrs_pre_init__`` method exists on the class, it will be called before the class is initialized. If a ``__attrs_post_init__`` method exists on the class, it will be called after the class is fully initialized. If ``init`` is ``False``, an ``__attrs_init__`` method will be injected instead. This allows you to define a custom ``__init__`` method that can do pre-init work such as ``super().__init__()``, and then call ``__attrs_init__()`` and ``__attrs_post_init__()``. :param bool slots: Create a `slotted class ` that's more memory-efficient. Slotted classes are generally superior to the default dict classes, but have some gotchas you should know about, so we encourage you to read the `glossary entry `. :param bool frozen: Make instances immutable after initialization. If someone attempts to modify a frozen instance, `attr.exceptions.FrozenInstanceError` is raised. .. note:: 1. This is achieved by installing a custom ``__setattr__`` method on your class, so you can't implement your own. 2. True immutability is impossible in Python. 3. This *does* have a minor a runtime performance `impact ` when initializing new instances. In other words: ``__init__`` is slightly slower with ``frozen=True``. 4. If a class is frozen, you cannot modify ``self`` in ``__attrs_post_init__`` or a self-written ``__init__``. You can circumvent that limitation by using ``object.__setattr__(self, "attribute_name", value)``. 5. Subclasses of a frozen class are frozen too. :param bool weakref_slot: Make instances weak-referenceable. This has no effect unless ``slots`` is also enabled. :param bool auto_attribs: If ``True``, collect :pep:`526`-annotated attributes (Python 3.6 and later only) from the class body. In this case, you **must** annotate every field. If ``attrs`` encounters a field that is set to an `attr.ib` but lacks a type annotation, an `attr.exceptions.UnannotatedAttributeError` is raised. Use ``field_name: typing.Any = attr.ib(...)`` if you don't want to set a type. If you assign a value to those attributes (e.g. ``x: int = 42``), that value becomes the default value like if it were passed using ``attr.ib(default=42)``. Passing an instance of `attrs.Factory` also works as expected in most cases (see warning below). Attributes annotated as `typing.ClassVar`, and attributes that are neither annotated nor set to an `attr.ib` are **ignored**. .. warning:: For features that use the attribute name to create decorators (e.g. `validators `), you still *must* assign `attr.ib` to them. Otherwise Python will either not find the name or try to use the default value to call e.g. ``validator`` on it. These errors can be quite confusing and probably the most common bug report on our bug tracker. :param bool kw_only: Make all attributes keyword-only (Python 3+) in the generated ``__init__`` (if ``init`` is ``False``, this parameter is ignored). :param bool cache_hash: Ensure that the object's hash code is computed only once and stored on the object. If this is set to ``True``, hashing must be either explicitly or implicitly enabled for this class. If the hash code is cached, avoid any reassignments of fields involved in hash code computation or mutations of the objects those fields point to after object creation. If such changes occur, the behavior of the object's hash code is undefined. :param bool auto_exc: If the class subclasses `BaseException` (which implicitly includes any subclass of any exception), the following happens to behave like a well-behaved Python exceptions class: - the values for *eq*, *order*, and *hash* are ignored and the instances compare and hash by the instance's ids (N.B. ``attrs`` will *not* remove existing implementations of ``__hash__`` or the equality methods. It just won't add own ones.), - all attributes that are either passed into ``__init__`` or have a default value are additionally available as a tuple in the ``args`` attribute, - the value of *str* is ignored leaving ``__str__`` to base classes. :param bool collect_by_mro: Setting this to `True` fixes the way ``attrs`` collects attributes from base classes. The default behavior is incorrect in certain cases of multiple inheritance. It should be on by default but is kept off for backward-compatibility. See issue `#428 `_ for more details. :param Optional[bool] getstate_setstate: .. note:: This is usually only interesting for slotted classes and you should probably just set *auto_detect* to `True`. If `True`, ``__getstate__`` and ``__setstate__`` are generated and attached to the class. This is necessary for slotted classes to be pickleable. If left `None`, it's `True` by default for slotted classes and ``False`` for dict classes. If *auto_detect* is `True`, and *getstate_setstate* is left `None`, and **either** ``__getstate__`` or ``__setstate__`` is detected directly on the class (i.e. not inherited), it is set to `False` (this is usually what you want). :param on_setattr: A callable that is run whenever the user attempts to set an attribute (either by assignment like ``i.x = 42`` or by using `setattr` like ``setattr(i, "x", 42)``). It receives the same arguments as validators: the instance, the attribute that is being modified, and the new value. If no exception is raised, the attribute is set to the return value of the callable. If a list of callables is passed, they're automatically wrapped in an `attrs.setters.pipe`. :type on_setattr: `callable`, or a list of callables, or `None`, or `attrs.setters.NO_OP` :param Optional[callable] field_transformer: A function that is called with the original class object and all fields right before ``attrs`` finalizes the class. You can use this, e.g., to automatically add converters or validators to fields based on their types. See `transform-fields` for more details. :param bool match_args: If `True` (default), set ``__match_args__`` on the class to support :pep:`634` (Structural Pattern Matching). It is a tuple of all non-keyword-only ``__init__`` parameter names on Python 3.10 and later. Ignored on older Python versions. .. versionadded:: 16.0.0 *slots* .. versionadded:: 16.1.0 *frozen* .. versionadded:: 16.3.0 *str* .. versionadded:: 16.3.0 Support for ``__attrs_post_init__``. .. versionchanged:: 17.1.0 *hash* supports ``None`` as value which is also the default now. .. versionadded:: 17.3.0 *auto_attribs* .. versionchanged:: 18.1.0 If *these* is passed, no attributes are deleted from the class body. .. versionchanged:: 18.1.0 If *these* is ordered, the order is retained. .. versionadded:: 18.2.0 *weakref_slot* .. deprecated:: 18.2.0 ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now raise a `DeprecationWarning` if the classes compared are subclasses of each other. ``__eq`` and ``__ne__`` never tried to compared subclasses to each other. .. versionchanged:: 19.2.0 ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now do not consider subclasses comparable anymore. .. versionadded:: 18.2.0 *kw_only* .. versionadded:: 18.2.0 *cache_hash* .. versionadded:: 19.1.0 *auto_exc* .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. .. versionadded:: 19.2.0 *eq* and *order* .. versionadded:: 20.1.0 *auto_detect* .. versionadded:: 20.1.0 *collect_by_mro* .. versionadded:: 20.1.0 *getstate_setstate* .. versionadded:: 20.1.0 *on_setattr* .. versionadded:: 20.3.0 *field_transformer* .. versionchanged:: 21.1.0 ``init=False`` injects ``__attrs_init__`` .. versionchanged:: 21.1.0 Support for ``__attrs_pre_init__`` .. versionchanged:: 21.1.0 *cmp* undeprecated .. versionadded:: 21.3.0 *match_args* """ eq_, order_ = _determine_attrs_eq_order(cmp, eq, order, None) hash_ = hash # work around the lack of nonlocal if isinstance(on_setattr, (list, tuple)): on_setattr = setters.pipe(*on_setattr) def wrap(cls): is_frozen = frozen or _has_frozen_base_class(cls) is_exc = auto_exc is True and issubclass(cls, BaseException) has_own_setattr = auto_detect and _has_own_attribute( cls, "__setattr__" ) if has_own_setattr and is_frozen: raise ValueError("Can't freeze a class with a custom __setattr__.") builder = _ClassBuilder( cls, these, slots, is_frozen, weakref_slot, _determine_whether_to_implement( cls, getstate_setstate, auto_detect, ("__getstate__", "__setstate__"), default=slots, ), auto_attribs, kw_only, cache_hash, is_exc, collect_by_mro, on_setattr, has_own_setattr, field_transformer, ) if _determine_whether_to_implement( cls, repr, auto_detect, ("__repr__",) ): builder.add_repr(repr_ns) if str is True: builder.add_str() eq = _determine_whether_to_implement( cls, eq_, auto_detect, ("__eq__", "__ne__") ) if not is_exc and eq is True: builder.add_eq() if not is_exc and _determine_whether_to_implement( cls, order_, auto_detect, ("__lt__", "__le__", "__gt__", "__ge__") ): builder.add_order() builder.add_setattr() if ( hash_ is None and auto_detect is True and _has_own_attribute(cls, "__hash__") ): hash = False else: hash = hash_ if hash is not True and hash is not False and hash is not None: # Can't use `hash in` because 1 == True for example. raise TypeError( "Invalid value for hash. Must be True, False, or None." ) elif hash is False or (hash is None and eq is False) or is_exc: # Don't do anything. Should fall back to __object__'s __hash__ # which is by id. if cache_hash: raise TypeError( "Invalid value for cache_hash. To use hash caching," " hashing must be either explicitly or implicitly " "enabled." ) elif hash is True or ( hash is None and eq is True and is_frozen is True ): # Build a __hash__ if told so, or if it's safe. builder.add_hash() else: # Raise TypeError on attempts to hash. if cache_hash: raise TypeError( "Invalid value for cache_hash. To use hash caching," " hashing must be either explicitly or implicitly " "enabled." ) builder.make_unhashable() if _determine_whether_to_implement( cls, init, auto_detect, ("__init__",) ): builder.add_init() else: builder.add_attrs_init() if cache_hash: raise TypeError( "Invalid value for cache_hash. To use hash caching," " init must be True." ) if ( PY310 and match_args and not _has_own_attribute(cls, "__match_args__") ): builder.add_match_args() return builder.build_class() # maybe_cls's type depends on the usage of the decorator. It's a class # if it's used as `@attrs` but ``None`` if used as `@attrs()`. if maybe_cls is None: return wrap else: return wrap(maybe_cls) _attrs = attrs """ Internal alias so we can use it in functions that take an argument called *attrs*. """ def _has_frozen_base_class(cls): """ Check whether *cls* has a frozen ancestor by looking at its __setattr__. """ return cls.__setattr__ is _frozen_setattrs def _generate_unique_filename(cls, func_name): """ Create a "filename" suitable for a function being generated. """ unique_filename = "".format( func_name, cls.__module__, getattr(cls, "__qualname__", cls.__name__), ) return unique_filename def _make_hash(cls, attrs, frozen, cache_hash): attrs = tuple( a for a in attrs if a.hash is True or (a.hash is None and a.eq is True) ) tab = " " unique_filename = _generate_unique_filename(cls, "hash") type_hash = hash(unique_filename) # If eq is custom generated, we need to include the functions in globs globs = {} hash_def = "def __hash__(self" hash_func = "hash((" closing_braces = "))" if not cache_hash: hash_def += "):" else: hash_def += ", *" hash_def += ( ", _cache_wrapper=" + "__import__('attr._make')._make._CacheHashWrapper):" ) hash_func = "_cache_wrapper(" + hash_func closing_braces += ")" method_lines = [hash_def] def append_hash_computation_lines(prefix, indent): """ Generate the code for actually computing the hash code. Below this will either be returned directly or used to compute a value which is then cached, depending on the value of cache_hash """ method_lines.extend( [ indent + prefix + hash_func, indent + " %d," % (type_hash,), ] ) for a in attrs: if a.eq_key: cmp_name = "_%s_key" % (a.name,) globs[cmp_name] = a.eq_key method_lines.append( indent + " %s(self.%s)," % (cmp_name, a.name) ) else: method_lines.append(indent + " self.%s," % a.name) method_lines.append(indent + " " + closing_braces) if cache_hash: method_lines.append(tab + "if self.%s is None:" % _hash_cache_field) if frozen: append_hash_computation_lines( "object.__setattr__(self, '%s', " % _hash_cache_field, tab * 2 ) method_lines.append(tab * 2 + ")") # close __setattr__ else: append_hash_computation_lines( "self.%s = " % _hash_cache_field, tab * 2 ) method_lines.append(tab + "return self.%s" % _hash_cache_field) else: append_hash_computation_lines("return ", tab) script = "\n".join(method_lines) return _make_method("__hash__", script, unique_filename, globs) def _add_hash(cls, attrs): """ Add a hash method to *cls*. """ cls.__hash__ = _make_hash(cls, attrs, frozen=False, cache_hash=False) return cls def _make_ne(): """ Create __ne__ method. """ def __ne__(self, other): """ Check equality and either forward a NotImplemented or return the result negated. """ result = self.__eq__(other) if result is NotImplemented: return NotImplemented return not result return __ne__ def _make_eq(cls, attrs): """ Create __eq__ method for *cls* with *attrs*. """ attrs = [a for a in attrs if a.eq] unique_filename = _generate_unique_filename(cls, "eq") lines = [ "def __eq__(self, other):", " if other.__class__ is not self.__class__:", " return NotImplemented", ] # We can't just do a big self.x = other.x and... clause due to # irregularities like nan == nan is false but (nan,) == (nan,) is true. globs = {} if attrs: lines.append(" return (") others = [" ) == ("] for a in attrs: if a.eq_key: cmp_name = "_%s_key" % (a.name,) # Add the key function to the global namespace # of the evaluated function. globs[cmp_name] = a.eq_key lines.append( " %s(self.%s)," % ( cmp_name, a.name, ) ) others.append( " %s(other.%s)," % ( cmp_name, a.name, ) ) else: lines.append(" self.%s," % (a.name,)) others.append(" other.%s," % (a.name,)) lines += others + [" )"] else: lines.append(" return True") script = "\n".join(lines) return _make_method("__eq__", script, unique_filename, globs) def _make_order(cls, attrs): """ Create ordering methods for *cls* with *attrs*. """ attrs = [a for a in attrs if a.order] def attrs_to_tuple(obj): """ Save us some typing. """ return tuple( key(value) if key else value for value, key in ( (getattr(obj, a.name), a.order_key) for a in attrs ) ) def __lt__(self, other): """ Automatically created by attrs. """ if other.__class__ is self.__class__: return attrs_to_tuple(self) < attrs_to_tuple(other) return NotImplemented def __le__(self, other): """ Automatically created by attrs. """ if other.__class__ is self.__class__: return attrs_to_tuple(self) <= attrs_to_tuple(other) return NotImplemented def __gt__(self, other): """ Automatically created by attrs. """ if other.__class__ is self.__class__: return attrs_to_tuple(self) > attrs_to_tuple(other) return NotImplemented def __ge__(self, other): """ Automatically created by attrs. """ if other.__class__ is self.__class__: return attrs_to_tuple(self) >= attrs_to_tuple(other) return NotImplemented return __lt__, __le__, __gt__, __ge__ def _add_eq(cls, attrs=None): """ Add equality methods to *cls* with *attrs*. """ if attrs is None: attrs = cls.__attrs_attrs__ cls.__eq__ = _make_eq(cls, attrs) cls.__ne__ = _make_ne() return cls if HAS_F_STRINGS: def _make_repr(attrs, ns, cls): unique_filename = _generate_unique_filename(cls, "repr") # Figure out which attributes to include, and which function to use to # format them. The a.repr value can be either bool or a custom # callable. attr_names_with_reprs = tuple( (a.name, (repr if a.repr is True else a.repr), a.init) for a in attrs if a.repr is not False ) globs = { name + "_repr": r for name, r, _ in attr_names_with_reprs if r != repr } globs["_compat"] = _compat globs["AttributeError"] = AttributeError globs["NOTHING"] = NOTHING attribute_fragments = [] for name, r, i in attr_names_with_reprs: accessor = ( "self." + name if i else 'getattr(self, "' + name + '", NOTHING)' ) fragment = ( "%s={%s!r}" % (name, accessor) if r == repr else "%s={%s_repr(%s)}" % (name, name, accessor) ) attribute_fragments.append(fragment) repr_fragment = ", ".join(attribute_fragments) if ns is None: cls_name_fragment = ( '{self.__class__.__qualname__.rsplit(">.", 1)[-1]}' ) else: cls_name_fragment = ns + ".{self.__class__.__name__}" lines = [ "def __repr__(self):", " try:", " already_repring = _compat.repr_context.already_repring", " except AttributeError:", " already_repring = {id(self),}", " _compat.repr_context.already_repring = already_repring", " else:", " if id(self) in already_repring:", " return '...'", " else:", " already_repring.add(id(self))", " try:", " return f'%s(%s)'" % (cls_name_fragment, repr_fragment), " finally:", " already_repring.remove(id(self))", ] return _make_method( "__repr__", "\n".join(lines), unique_filename, globs=globs ) else: def _make_repr(attrs, ns, _): """ Make a repr method that includes relevant *attrs*, adding *ns* to the full name. """ # Figure out which attributes to include, and which function to use to # format them. The a.repr value can be either bool or a custom # callable. attr_names_with_reprs = tuple( (a.name, repr if a.repr is True else a.repr) for a in attrs if a.repr is not False ) def __repr__(self): """ Automatically created by attrs. """ try: already_repring = _compat.repr_context.already_repring except AttributeError: already_repring = set() _compat.repr_context.already_repring = already_repring if id(self) in already_repring: return "..." real_cls = self.__class__ if ns is None: class_name = real_cls.__qualname__.rsplit(">.", 1)[-1] else: class_name = ns + "." + real_cls.__name__ # Since 'self' remains on the stack (i.e.: strongly referenced) # for the duration of this call, it's safe to depend on id(...) # stability, and not need to track the instance and therefore # worry about properties like weakref- or hash-ability. already_repring.add(id(self)) try: result = [class_name, "("] first = True for name, attr_repr in attr_names_with_reprs: if first: first = False else: result.append(", ") result.extend( (name, "=", attr_repr(getattr(self, name, NOTHING))) ) return "".join(result) + ")" finally: already_repring.remove(id(self)) return __repr__ def _add_repr(cls, ns=None, attrs=None): """ Add a repr method to *cls*. """ if attrs is None: attrs = cls.__attrs_attrs__ cls.__repr__ = _make_repr(attrs, ns, cls) return cls def fields(cls): """ Return the tuple of ``attrs`` attributes for a class. The tuple also allows accessing the fields by their names (see below for examples). :param type cls: Class to introspect. :raise TypeError: If *cls* is not a class. :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` class. :rtype: tuple (with name accessors) of `attrs.Attribute` .. versionchanged:: 16.2.0 Returned tuple allows accessing the fields by name. """ if not isinstance(cls, type): raise TypeError("Passed object must be a class.") attrs = getattr(cls, "__attrs_attrs__", None) if attrs is None: raise NotAnAttrsClassError( "{cls!r} is not an attrs-decorated class.".format(cls=cls) ) return attrs def fields_dict(cls): """ Return an ordered dictionary of ``attrs`` attributes for a class, whose keys are the attribute names. :param type cls: Class to introspect. :raise TypeError: If *cls* is not a class. :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` class. :rtype: an ordered dict where keys are attribute names and values are `attrs.Attribute`\\ s. This will be a `dict` if it's naturally ordered like on Python 3.6+ or an :class:`~collections.OrderedDict` otherwise. .. versionadded:: 18.1.0 """ if not isinstance(cls, type): raise TypeError("Passed object must be a class.") attrs = getattr(cls, "__attrs_attrs__", None) if attrs is None: raise NotAnAttrsClassError( "{cls!r} is not an attrs-decorated class.".format(cls=cls) ) return ordered_dict((a.name, a) for a in attrs) def validate(inst): """ Validate all attributes on *inst* that have a validator. Leaves all exceptions through. :param inst: Instance of a class with ``attrs`` attributes. """ if _config._run_validators is False: return for a in fields(inst.__class__): v = a.validator if v is not None: v(inst, a, getattr(inst, a.name)) def _is_slot_cls(cls): return "__slots__" in cls.__dict__ def _is_slot_attr(a_name, base_attr_map): """ Check if the attribute name comes from a slot class. """ return a_name in base_attr_map and _is_slot_cls(base_attr_map[a_name]) def _make_init( cls, attrs, pre_init, post_init, frozen, slots, cache_hash, base_attr_map, is_exc, cls_on_setattr, attrs_init, ): has_cls_on_setattr = ( cls_on_setattr is not None and cls_on_setattr is not setters.NO_OP ) if frozen and has_cls_on_setattr: raise ValueError("Frozen classes can't use on_setattr.") needs_cached_setattr = cache_hash or frozen filtered_attrs = [] attr_dict = {} for a in attrs: if not a.init and a.default is NOTHING: continue filtered_attrs.append(a) attr_dict[a.name] = a if a.on_setattr is not None: if frozen is True: raise ValueError("Frozen classes can't use on_setattr.") needs_cached_setattr = True elif has_cls_on_setattr and a.on_setattr is not setters.NO_OP: needs_cached_setattr = True unique_filename = _generate_unique_filename(cls, "init") script, globs, annotations = _attrs_to_init_script( filtered_attrs, frozen, slots, pre_init, post_init, cache_hash, base_attr_map, is_exc, has_cls_on_setattr, attrs_init, ) if cls.__module__ in sys.modules: # This makes typing.get_type_hints(CLS.__init__) resolve string types. globs.update(sys.modules[cls.__module__].__dict__) globs.update({"NOTHING": NOTHING, "attr_dict": attr_dict}) if needs_cached_setattr: # Save the lookup overhead in __init__ if we need to circumvent # setattr hooks. globs["_setattr"] = _obj_setattr init = _make_method( "__attrs_init__" if attrs_init else "__init__", script, unique_filename, globs, ) init.__annotations__ = annotations return init def _setattr(attr_name, value_var, has_on_setattr): """ Use the cached object.setattr to set *attr_name* to *value_var*. """ return "_setattr(self, '%s', %s)" % (attr_name, value_var) def _setattr_with_converter(attr_name, value_var, has_on_setattr): """ Use the cached object.setattr to set *attr_name* to *value_var*, but run its converter first. """ return "_setattr(self, '%s', %s(%s))" % ( attr_name, _init_converter_pat % (attr_name,), value_var, ) def _assign(attr_name, value, has_on_setattr): """ Unless *attr_name* has an on_setattr hook, use normal assignment. Otherwise relegate to _setattr. """ if has_on_setattr: return _setattr(attr_name, value, True) return "self.%s = %s" % (attr_name, value) def _assign_with_converter(attr_name, value_var, has_on_setattr): """ Unless *attr_name* has an on_setattr hook, use normal assignment after conversion. Otherwise relegate to _setattr_with_converter. """ if has_on_setattr: return _setattr_with_converter(attr_name, value_var, True) return "self.%s = %s(%s)" % ( attr_name, _init_converter_pat % (attr_name,), value_var, ) def _attrs_to_init_script( attrs, frozen, slots, pre_init, post_init, cache_hash, base_attr_map, is_exc, has_cls_on_setattr, attrs_init, ): """ Return a script of an initializer for *attrs* and a dict of globals. The globals are expected by the generated script. If *frozen* is True, we cannot set the attributes directly so we use a cached ``object.__setattr__``. """ lines = [] if pre_init: lines.append("self.__attrs_pre_init__()") if frozen is True: if slots is True: fmt_setter = _setattr fmt_setter_with_converter = _setattr_with_converter else: # Dict frozen classes assign directly to __dict__. # But only if the attribute doesn't come from an ancestor slot # class. # Note _inst_dict will be used again below if cache_hash is True lines.append("_inst_dict = self.__dict__") def fmt_setter(attr_name, value_var, has_on_setattr): if _is_slot_attr(attr_name, base_attr_map): return _setattr(attr_name, value_var, has_on_setattr) return "_inst_dict['%s'] = %s" % (attr_name, value_var) def fmt_setter_with_converter( attr_name, value_var, has_on_setattr ): if has_on_setattr or _is_slot_attr(attr_name, base_attr_map): return _setattr_with_converter( attr_name, value_var, has_on_setattr ) return "_inst_dict['%s'] = %s(%s)" % ( attr_name, _init_converter_pat % (attr_name,), value_var, ) else: # Not frozen. fmt_setter = _assign fmt_setter_with_converter = _assign_with_converter args = [] kw_only_args = [] attrs_to_validate = [] # This is a dictionary of names to validator and converter callables. # Injecting this into __init__ globals lets us avoid lookups. names_for_globals = {} annotations = {"return": None} for a in attrs: if a.validator: attrs_to_validate.append(a) attr_name = a.name has_on_setattr = a.on_setattr is not None or ( a.on_setattr is not setters.NO_OP and has_cls_on_setattr ) arg_name = a.name.lstrip("_") has_factory = isinstance(a.default, Factory) if has_factory and a.default.takes_self: maybe_self = "self" else: maybe_self = "" if a.init is False: if has_factory: init_factory_name = _init_factory_pat.format(a.name) if a.converter is not None: lines.append( fmt_setter_with_converter( attr_name, init_factory_name + "(%s)" % (maybe_self,), has_on_setattr, ) ) conv_name = _init_converter_pat % (a.name,) names_for_globals[conv_name] = a.converter else: lines.append( fmt_setter( attr_name, init_factory_name + "(%s)" % (maybe_self,), has_on_setattr, ) ) names_for_globals[init_factory_name] = a.default.factory else: if a.converter is not None: lines.append( fmt_setter_with_converter( attr_name, "attr_dict['%s'].default" % (attr_name,), has_on_setattr, ) ) conv_name = _init_converter_pat % (a.name,) names_for_globals[conv_name] = a.converter else: lines.append( fmt_setter( attr_name, "attr_dict['%s'].default" % (attr_name,), has_on_setattr, ) ) elif a.default is not NOTHING and not has_factory: arg = "%s=attr_dict['%s'].default" % (arg_name, attr_name) if a.kw_only: kw_only_args.append(arg) else: args.append(arg) if a.converter is not None: lines.append( fmt_setter_with_converter( attr_name, arg_name, has_on_setattr ) ) names_for_globals[ _init_converter_pat % (a.name,) ] = a.converter else: lines.append(fmt_setter(attr_name, arg_name, has_on_setattr)) elif has_factory: arg = "%s=NOTHING" % (arg_name,) if a.kw_only: kw_only_args.append(arg) else: args.append(arg) lines.append("if %s is not NOTHING:" % (arg_name,)) init_factory_name = _init_factory_pat.format(a.name) if a.converter is not None: lines.append( " " + fmt_setter_with_converter( attr_name, arg_name, has_on_setattr ) ) lines.append("else:") lines.append( " " + fmt_setter_with_converter( attr_name, init_factory_name + "(" + maybe_self + ")", has_on_setattr, ) ) names_for_globals[ _init_converter_pat % (a.name,) ] = a.converter else: lines.append( " " + fmt_setter(attr_name, arg_name, has_on_setattr) ) lines.append("else:") lines.append( " " + fmt_setter( attr_name, init_factory_name + "(" + maybe_self + ")", has_on_setattr, ) ) names_for_globals[init_factory_name] = a.default.factory else: if a.kw_only: kw_only_args.append(arg_name) else: args.append(arg_name) if a.converter is not None: lines.append( fmt_setter_with_converter( attr_name, arg_name, has_on_setattr ) ) names_for_globals[ _init_converter_pat % (a.name,) ] = a.converter else: lines.append(fmt_setter(attr_name, arg_name, has_on_setattr)) if a.init is True: if a.type is not None and a.converter is None: annotations[arg_name] = a.type elif a.converter is not None: # Try to get the type from the converter. t = _AnnotationExtractor(a.converter).get_first_param_type() if t: annotations[arg_name] = t if attrs_to_validate: # we can skip this if there are no validators. names_for_globals["_config"] = _config lines.append("if _config._run_validators is True:") for a in attrs_to_validate: val_name = "__attr_validator_" + a.name attr_name = "__attr_" + a.name lines.append( " %s(self, %s, self.%s)" % (val_name, attr_name, a.name) ) names_for_globals[val_name] = a.validator names_for_globals[attr_name] = a if post_init: lines.append("self.__attrs_post_init__()") # because this is set only after __attrs_post_init__ is called, a crash # will result if post-init tries to access the hash code. This seemed # preferable to setting this beforehand, in which case alteration to # field values during post-init combined with post-init accessing the # hash code would result in silent bugs. if cache_hash: if frozen: if slots: # if frozen and slots, then _setattr defined above init_hash_cache = "_setattr(self, '%s', %s)" else: # if frozen and not slots, then _inst_dict defined above init_hash_cache = "_inst_dict['%s'] = %s" else: init_hash_cache = "self.%s = %s" lines.append(init_hash_cache % (_hash_cache_field, "None")) # For exceptions we rely on BaseException.__init__ for proper # initialization. if is_exc: vals = ",".join("self." + a.name for a in attrs if a.init) lines.append("BaseException.__init__(self, %s)" % (vals,)) args = ", ".join(args) if kw_only_args: args += "%s*, %s" % ( ", " if args else "", # leading comma ", ".join(kw_only_args), # kw_only args ) return ( """\ def {init_name}(self, {args}): {lines} """.format( init_name=("__attrs_init__" if attrs_init else "__init__"), args=args, lines="\n ".join(lines) if lines else "pass", ), names_for_globals, annotations, ) class Attribute: """ *Read-only* representation of an attribute. The class has *all* arguments of `attr.ib` (except for ``factory`` which is only syntactic sugar for ``default=Factory(...)`` plus the following: - ``name`` (`str`): The name of the attribute. - ``inherited`` (`bool`): Whether or not that attribute has been inherited from a base class. - ``eq_key`` and ``order_key`` (`typing.Callable` or `None`): The callables that are used for comparing and ordering objects by this attribute, respectively. These are set by passing a callable to `attr.ib`'s ``eq``, ``order``, or ``cmp`` arguments. See also :ref:`comparison customization `. Instances of this class are frequently used for introspection purposes like: - `fields` returns a tuple of them. - Validators get them passed as the first argument. - The :ref:`field transformer ` hook receives a list of them. .. versionadded:: 20.1.0 *inherited* .. versionadded:: 20.1.0 *on_setattr* .. versionchanged:: 20.2.0 *inherited* is not taken into account for equality checks and hashing anymore. .. versionadded:: 21.1.0 *eq_key* and *order_key* For the full version history of the fields, see `attr.ib`. """ __slots__ = ( "name", "default", "validator", "repr", "eq", "eq_key", "order", "order_key", "hash", "init", "metadata", "type", "converter", "kw_only", "inherited", "on_setattr", ) def __init__( self, name, default, validator, repr, cmp, # XXX: unused, remove along with other cmp code. hash, init, inherited, metadata=None, type=None, converter=None, kw_only=False, eq=None, eq_key=None, order=None, order_key=None, on_setattr=None, ): eq, eq_key, order, order_key = _determine_attrib_eq_order( cmp, eq_key or eq, order_key or order, True ) # Cache this descriptor here to speed things up later. bound_setattr = _obj_setattr.__get__(self, Attribute) # Despite the big red warning, people *do* instantiate `Attribute` # themselves. bound_setattr("name", name) bound_setattr("default", default) bound_setattr("validator", validator) bound_setattr("repr", repr) bound_setattr("eq", eq) bound_setattr("eq_key", eq_key) bound_setattr("order", order) bound_setattr("order_key", order_key) bound_setattr("hash", hash) bound_setattr("init", init) bound_setattr("converter", converter) bound_setattr( "metadata", ( types.MappingProxyType(dict(metadata)) # Shallow copy if metadata else _empty_metadata_singleton ), ) bound_setattr("type", type) bound_setattr("kw_only", kw_only) bound_setattr("inherited", inherited) bound_setattr("on_setattr", on_setattr) def __setattr__(self, name, value): raise FrozenInstanceError() @classmethod def from_counting_attr(cls, name, ca, type=None): # type holds the annotated value. deal with conflicts: if type is None: type = ca.type elif ca.type is not None: raise ValueError( "Type annotation and type argument cannot both be present" ) inst_dict = { k: getattr(ca, k) for k in Attribute.__slots__ if k not in ( "name", "validator", "default", "type", "inherited", ) # exclude methods and deprecated alias } return cls( name=name, validator=ca._validator, default=ca._default, type=type, cmp=None, inherited=False, **inst_dict ) # Don't use attr.evolve since fields(Attribute) doesn't work def evolve(self, **changes): """ Copy *self* and apply *changes*. This works similarly to `attr.evolve` but that function does not work with ``Attribute``. It is mainly meant to be used for `transform-fields`. .. versionadded:: 20.3.0 """ new = copy.copy(self) new._setattrs(changes.items()) return new # Don't use _add_pickle since fields(Attribute) doesn't work def __getstate__(self): """ Play nice with pickle. """ return tuple( getattr(self, name) if name != "metadata" else dict(self.metadata) for name in self.__slots__ ) def __setstate__(self, state): """ Play nice with pickle. """ self._setattrs(zip(self.__slots__, state)) def _setattrs(self, name_values_pairs): bound_setattr = _obj_setattr.__get__(self, Attribute) for name, value in name_values_pairs: if name != "metadata": bound_setattr(name, value) else: bound_setattr( name, types.MappingProxyType(dict(value)) if value else _empty_metadata_singleton, ) _a = [ Attribute( name=name, default=NOTHING, validator=None, repr=True, cmp=None, eq=True, order=False, hash=(name != "metadata"), init=True, inherited=False, ) for name in Attribute.__slots__ ] Attribute = _add_hash( _add_eq( _add_repr(Attribute, attrs=_a), attrs=[a for a in _a if a.name != "inherited"], ), attrs=[a for a in _a if a.hash and a.name != "inherited"], ) class _CountingAttr: """ Intermediate representation of attributes that uses a counter to preserve the order in which the attributes have been defined. *Internal* data structure of the attrs library. Running into is most likely the result of a bug like a forgotten `@attr.s` decorator. """ __slots__ = ( "counter", "_default", "repr", "eq", "eq_key", "order", "order_key", "hash", "init", "metadata", "_validator", "converter", "type", "kw_only", "on_setattr", ) __attrs_attrs__ = tuple( Attribute( name=name, default=NOTHING, validator=None, repr=True, cmp=None, hash=True, init=True, kw_only=False, eq=True, eq_key=None, order=False, order_key=None, inherited=False, on_setattr=None, ) for name in ( "counter", "_default", "repr", "eq", "order", "hash", "init", "on_setattr", ) ) + ( Attribute( name="metadata", default=None, validator=None, repr=True, cmp=None, hash=False, init=True, kw_only=False, eq=True, eq_key=None, order=False, order_key=None, inherited=False, on_setattr=None, ), ) cls_counter = 0 def __init__( self, default, validator, repr, cmp, hash, init, converter, metadata, type, kw_only, eq, eq_key, order, order_key, on_setattr, ): _CountingAttr.cls_counter += 1 self.counter = _CountingAttr.cls_counter self._default = default self._validator = validator self.converter = converter self.repr = repr self.eq = eq self.eq_key = eq_key self.order = order self.order_key = order_key self.hash = hash self.init = init self.metadata = metadata self.type = type self.kw_only = kw_only self.on_setattr = on_setattr def validator(self, meth): """ Decorator that adds *meth* to the list of validators. Returns *meth* unchanged. .. versionadded:: 17.1.0 """ if self._validator is None: self._validator = meth else: self._validator = and_(self._validator, meth) return meth def default(self, meth): """ Decorator that allows to set the default for an attribute. Returns *meth* unchanged. :raises DefaultAlreadySetError: If default has been set before. .. versionadded:: 17.1.0 """ if self._default is not NOTHING: raise DefaultAlreadySetError() self._default = Factory(meth, takes_self=True) return meth _CountingAttr = _add_eq(_add_repr(_CountingAttr)) class Factory: """ Stores a factory callable. If passed as the default value to `attrs.field`, the factory is used to generate a new value. :param callable factory: A callable that takes either none or exactly one mandatory positional argument depending on *takes_self*. :param bool takes_self: Pass the partially initialized instance that is being initialized as a positional argument. .. versionadded:: 17.1.0 *takes_self* """ __slots__ = ("factory", "takes_self") def __init__(self, factory, takes_self=False): """ `Factory` is part of the default machinery so if we want a default value here, we have to implement it ourselves. """ self.factory = factory self.takes_self = takes_self def __getstate__(self): """ Play nice with pickle. """ return tuple(getattr(self, name) for name in self.__slots__) def __setstate__(self, state): """ Play nice with pickle. """ for name, value in zip(self.__slots__, state): setattr(self, name, value) _f = [ Attribute( name=name, default=NOTHING, validator=None, repr=True, cmp=None, eq=True, order=False, hash=True, init=True, inherited=False, ) for name in Factory.__slots__ ] Factory = _add_hash(_add_eq(_add_repr(Factory, attrs=_f), attrs=_f), attrs=_f) def make_class(name, attrs, bases=(object,), **attributes_arguments): """ A quick way to create a new class called *name* with *attrs*. :param str name: The name for the new class. :param attrs: A list of names or a dictionary of mappings of names to attributes. If *attrs* is a list or an ordered dict (`dict` on Python 3.6+, `collections.OrderedDict` otherwise), the order is deduced from the order of the names or attributes inside *attrs*. Otherwise the order of the definition of the attributes is used. :type attrs: `list` or `dict` :param tuple bases: Classes that the new class will subclass. :param attributes_arguments: Passed unmodified to `attr.s`. :return: A new class with *attrs*. :rtype: type .. versionadded:: 17.1.0 *bases* .. versionchanged:: 18.1.0 If *attrs* is ordered, the order is retained. """ if isinstance(attrs, dict): cls_dict = attrs elif isinstance(attrs, (list, tuple)): cls_dict = {a: attrib() for a in attrs} else: raise TypeError("attrs argument must be a dict or a list.") pre_init = cls_dict.pop("__attrs_pre_init__", None) post_init = cls_dict.pop("__attrs_post_init__", None) user_init = cls_dict.pop("__init__", None) body = {} if pre_init is not None: body["__attrs_pre_init__"] = pre_init if post_init is not None: body["__attrs_post_init__"] = post_init if user_init is not None: body["__init__"] = user_init type_ = types.new_class(name, bases, {}, lambda ns: ns.update(body)) # For pickling to work, the __module__ variable needs to be set to the # frame where the class is created. Bypass this step in environments where # sys._getframe is not defined (Jython for example) or sys._getframe is not # defined for arguments greater than 0 (IronPython). try: type_.__module__ = sys._getframe(1).f_globals.get( "__name__", "__main__" ) except (AttributeError, ValueError): pass # We do it here for proper warnings with meaningful stacklevel. cmp = attributes_arguments.pop("cmp", None) ( attributes_arguments["eq"], attributes_arguments["order"], ) = _determine_attrs_eq_order( cmp, attributes_arguments.get("eq"), attributes_arguments.get("order"), True, ) return _attrs(these=cls_dict, **attributes_arguments)(type_) # These are required by within this module so we define them here and merely # import into .validators / .converters. @attrs(slots=True, hash=True) class _AndValidator: """ Compose many validators to a single one. """ _validators = attrib() def __call__(self, inst, attr, value): for v in self._validators: v(inst, attr, value) def and_(*validators): """ A validator that composes multiple validators into one. When called on a value, it runs all wrapped validators. :param callables validators: Arbitrary number of validators. .. versionadded:: 17.1.0 """ vals = [] for validator in validators: vals.extend( validator._validators if isinstance(validator, _AndValidator) else [validator] ) return _AndValidator(tuple(vals)) def pipe(*converters): """ A converter that composes multiple converters into one. When called on a value, it runs all wrapped converters, returning the *last* value. Type annotations will be inferred from the wrapped converters', if they have any. :param callables converters: Arbitrary number of converters. .. versionadded:: 20.1.0 """ def pipe_converter(val): for converter in converters: val = converter(val) return val if not converters: # If the converter list is empty, pipe_converter is the identity. A = typing.TypeVar("A") pipe_converter.__annotations__ = {"val": A, "return": A} else: # Get parameter type from first converter. t = _AnnotationExtractor(converters[0]).get_first_param_type() if t: pipe_converter.__annotations__["val"] = t # Get return type from last converter. rt = _AnnotationExtractor(converters[-1]).get_return_type() if rt: pipe_converter.__annotations__["return"] = rt return pipe_converter ================================================ FILE: lib/spack/spack/vendor/attr/_next_gen.py ================================================ # SPDX-License-Identifier: MIT """ These are Python 3.6+-only and keyword-only APIs that call `attr.s` and `attr.ib` with different default values. """ from functools import partial from . import setters from ._funcs import asdict as _asdict from ._funcs import astuple as _astuple from ._make import ( NOTHING, _frozen_setattrs, _ng_default_on_setattr, attrib, attrs, ) from .exceptions import UnannotatedAttributeError def define( maybe_cls=None, *, these=None, repr=None, hash=None, init=None, slots=True, frozen=False, weakref_slot=True, str=False, auto_attribs=None, kw_only=False, cache_hash=False, auto_exc=True, eq=None, order=False, auto_detect=True, getstate_setstate=None, on_setattr=None, field_transformer=None, match_args=True, ): r""" Define an ``attrs`` class. Differences to the classic `attr.s` that it uses underneath: - Automatically detect whether or not *auto_attribs* should be `True` (c.f. *auto_attribs* parameter). - If *frozen* is `False`, run converters and validators when setting an attribute by default. - *slots=True* .. caution:: Usually this has only upsides and few visible effects in everyday programming. But it *can* lead to some suprising behaviors, so please make sure to read :term:`slotted classes`. - *auto_exc=True* - *auto_detect=True* - *order=False* - Some options that were only relevant on Python 2 or were kept around for backwards-compatibility have been removed. Please note that these are all defaults and you can change them as you wish. :param Optional[bool] auto_attribs: If set to `True` or `False`, it behaves exactly like `attr.s`. If left `None`, `attr.s` will try to guess: 1. If any attributes are annotated and no unannotated `attrs.fields`\ s are found, it assumes *auto_attribs=True*. 2. Otherwise it assumes *auto_attribs=False* and tries to collect `attrs.fields`\ s. For now, please refer to `attr.s` for the rest of the parameters. .. versionadded:: 20.1.0 .. versionchanged:: 21.3.0 Converters are also run ``on_setattr``. """ def do_it(cls, auto_attribs): return attrs( maybe_cls=cls, these=these, repr=repr, hash=hash, init=init, slots=slots, frozen=frozen, weakref_slot=weakref_slot, str=str, auto_attribs=auto_attribs, kw_only=kw_only, cache_hash=cache_hash, auto_exc=auto_exc, eq=eq, order=order, auto_detect=auto_detect, collect_by_mro=True, getstate_setstate=getstate_setstate, on_setattr=on_setattr, field_transformer=field_transformer, match_args=match_args, ) def wrap(cls): """ Making this a wrapper ensures this code runs during class creation. We also ensure that frozen-ness of classes is inherited. """ nonlocal frozen, on_setattr had_on_setattr = on_setattr not in (None, setters.NO_OP) # By default, mutable classes convert & validate on setattr. if frozen is False and on_setattr is None: on_setattr = _ng_default_on_setattr # However, if we subclass a frozen class, we inherit the immutability # and disable on_setattr. for base_cls in cls.__bases__: if base_cls.__setattr__ is _frozen_setattrs: if had_on_setattr: raise ValueError( "Frozen classes can't use on_setattr " "(frozen-ness was inherited)." ) on_setattr = setters.NO_OP break if auto_attribs is not None: return do_it(cls, auto_attribs) try: return do_it(cls, True) except UnannotatedAttributeError: return do_it(cls, False) # maybe_cls's type depends on the usage of the decorator. It's a class # if it's used as `@attrs` but ``None`` if used as `@attrs()`. if maybe_cls is None: return wrap else: return wrap(maybe_cls) mutable = define frozen = partial(define, frozen=True, on_setattr=None) def field( *, default=NOTHING, validator=None, repr=True, hash=None, init=True, metadata=None, converter=None, factory=None, kw_only=False, eq=None, order=None, on_setattr=None, ): """ Identical to `attr.ib`, except keyword-only and with some arguments removed. .. versionadded:: 20.1.0 """ return attrib( default=default, validator=validator, repr=repr, hash=hash, init=init, metadata=metadata, converter=converter, factory=factory, kw_only=kw_only, eq=eq, order=order, on_setattr=on_setattr, ) def asdict(inst, *, recurse=True, filter=None, value_serializer=None): """ Same as `attr.asdict`, except that collections types are always retained and dict is always used as *dict_factory*. .. versionadded:: 21.3.0 """ return _asdict( inst=inst, recurse=recurse, filter=filter, value_serializer=value_serializer, retain_collection_types=True, ) def astuple(inst, *, recurse=True, filter=None): """ Same as `attr.astuple`, except that collections types are always retained and `tuple` is always used as the *tuple_factory*. .. versionadded:: 21.3.0 """ return _astuple( inst=inst, recurse=recurse, filter=filter, retain_collection_types=True ) ================================================ FILE: lib/spack/spack/vendor/attr/_version_info.py ================================================ # SPDX-License-Identifier: MIT from functools import total_ordering from ._funcs import astuple from ._make import attrib, attrs @total_ordering @attrs(eq=False, order=False, slots=True, frozen=True) class VersionInfo: """ A version object that can be compared to tuple of length 1--4: >>> attr.VersionInfo(19, 1, 0, "final") <= (19, 2) True >>> attr.VersionInfo(19, 1, 0, "final") < (19, 1, 1) True >>> vi = attr.VersionInfo(19, 2, 0, "final") >>> vi < (19, 1, 1) False >>> vi < (19,) False >>> vi == (19, 2,) True >>> vi == (19, 2, 1) False .. versionadded:: 19.2 """ year = attrib(type=int) minor = attrib(type=int) micro = attrib(type=int) releaselevel = attrib(type=str) @classmethod def _from_version_string(cls, s): """ Parse *s* and return a _VersionInfo. """ v = s.split(".") if len(v) == 3: v.append("final") return cls( year=int(v[0]), minor=int(v[1]), micro=int(v[2]), releaselevel=v[3] ) def _ensure_tuple(self, other): """ Ensure *other* is a tuple of a valid length. Returns a possibly transformed *other* and ourselves as a tuple of the same length as *other*. """ if self.__class__ is other.__class__: other = astuple(other) if not isinstance(other, tuple): raise NotImplementedError if not (1 <= len(other) <= 4): raise NotImplementedError return astuple(self)[: len(other)], other def __eq__(self, other): try: us, them = self._ensure_tuple(other) except NotImplementedError: return NotImplemented return us == them def __lt__(self, other): try: us, them = self._ensure_tuple(other) except NotImplementedError: return NotImplemented # Since alphabetically "dev0" < "final" < "post1" < "post2", we don't # have to do anything special with releaselevel for now. return us < them ================================================ FILE: lib/spack/spack/vendor/attr/_version_info.pyi ================================================ class VersionInfo: @property def year(self) -> int: ... @property def minor(self) -> int: ... @property def micro(self) -> int: ... @property def releaselevel(self) -> str: ... ================================================ FILE: lib/spack/spack/vendor/attr/converters.py ================================================ # SPDX-License-Identifier: MIT """ Commonly useful converters. """ import typing from ._compat import _AnnotationExtractor from ._make import NOTHING, Factory, pipe __all__ = [ "default_if_none", "optional", "pipe", "to_bool", ] def optional(converter): """ A converter that allows an attribute to be optional. An optional attribute is one which can be set to ``None``. Type annotations will be inferred from the wrapped converter's, if it has any. :param callable converter: the converter that is used for non-``None`` values. .. versionadded:: 17.1.0 """ def optional_converter(val): if val is None: return None return converter(val) xtr = _AnnotationExtractor(converter) t = xtr.get_first_param_type() if t: optional_converter.__annotations__["val"] = typing.Optional[t] rt = xtr.get_return_type() if rt: optional_converter.__annotations__["return"] = typing.Optional[rt] return optional_converter def default_if_none(default=NOTHING, factory=None): """ A converter that allows to replace ``None`` values by *default* or the result of *factory*. :param default: Value to be used if ``None`` is passed. Passing an instance of `attrs.Factory` is supported, however the ``takes_self`` option is *not*. :param callable factory: A callable that takes no parameters whose result is used if ``None`` is passed. :raises TypeError: If **neither** *default* or *factory* is passed. :raises TypeError: If **both** *default* and *factory* are passed. :raises ValueError: If an instance of `attrs.Factory` is passed with ``takes_self=True``. .. versionadded:: 18.2.0 """ if default is NOTHING and factory is None: raise TypeError("Must pass either `default` or `factory`.") if default is not NOTHING and factory is not None: raise TypeError( "Must pass either `default` or `factory` but not both." ) if factory is not None: default = Factory(factory) if isinstance(default, Factory): if default.takes_self: raise ValueError( "`takes_self` is not supported by default_if_none." ) def default_if_none_converter(val): if val is not None: return val return default.factory() else: def default_if_none_converter(val): if val is not None: return val return default return default_if_none_converter def to_bool(val): """ Convert "boolean" strings (e.g., from env. vars.) to real booleans. Values mapping to :code:`True`: - :code:`True` - :code:`"true"` / :code:`"t"` - :code:`"yes"` / :code:`"y"` - :code:`"on"` - :code:`"1"` - :code:`1` Values mapping to :code:`False`: - :code:`False` - :code:`"false"` / :code:`"f"` - :code:`"no"` / :code:`"n"` - :code:`"off"` - :code:`"0"` - :code:`0` :raises ValueError: for any other value. .. versionadded:: 21.3.0 """ if isinstance(val, str): val = val.lower() truthy = {True, "true", "t", "yes", "y", "on", "1", 1} falsy = {False, "false", "f", "no", "n", "off", "0", 0} try: if val in truthy: return True if val in falsy: return False except TypeError: # Raised when "val" is not hashable (e.g., lists) pass raise ValueError("Cannot convert value to bool: {}".format(val)) ================================================ FILE: lib/spack/spack/vendor/attr/converters.pyi ================================================ from typing import Callable, Optional, TypeVar, overload from . import _ConverterType _T = TypeVar("_T") def pipe(*validators: _ConverterType) -> _ConverterType: ... def optional(converter: _ConverterType) -> _ConverterType: ... @overload def default_if_none(default: _T) -> _ConverterType: ... @overload def default_if_none(*, factory: Callable[[], _T]) -> _ConverterType: ... def to_bool(val: str) -> bool: ... ================================================ FILE: lib/spack/spack/vendor/attr/exceptions.py ================================================ # SPDX-License-Identifier: MIT class FrozenError(AttributeError): """ A frozen/immutable instance or attribute have been attempted to be modified. It mirrors the behavior of ``namedtuples`` by using the same error message and subclassing `AttributeError`. .. versionadded:: 20.1.0 """ msg = "can't set attribute" args = [msg] class FrozenInstanceError(FrozenError): """ A frozen instance has been attempted to be modified. .. versionadded:: 16.1.0 """ class FrozenAttributeError(FrozenError): """ A frozen attribute has been attempted to be modified. .. versionadded:: 20.1.0 """ class AttrsAttributeNotFoundError(ValueError): """ An ``attrs`` function couldn't find an attribute that the user asked for. .. versionadded:: 16.2.0 """ class NotAnAttrsClassError(ValueError): """ A non-``attrs`` class has been passed into an ``attrs`` function. .. versionadded:: 16.2.0 """ class DefaultAlreadySetError(RuntimeError): """ A default has been set using ``attr.ib()`` and is attempted to be reset using the decorator. .. versionadded:: 17.1.0 """ class UnannotatedAttributeError(RuntimeError): """ A class with ``auto_attribs=True`` has an ``attr.ib()`` without a type annotation. .. versionadded:: 17.3.0 """ class PythonTooOldError(RuntimeError): """ It was attempted to use an ``attrs`` feature that requires a newer Python version. .. versionadded:: 18.2.0 """ class NotCallableError(TypeError): """ A ``attr.ib()`` requiring a callable has been set with a value that is not callable. .. versionadded:: 19.2.0 """ def __init__(self, msg, value): super(TypeError, self).__init__(msg, value) self.msg = msg self.value = value def __str__(self): return str(self.msg) ================================================ FILE: lib/spack/spack/vendor/attr/exceptions.pyi ================================================ from typing import Any class FrozenError(AttributeError): msg: str = ... class FrozenInstanceError(FrozenError): ... class FrozenAttributeError(FrozenError): ... class AttrsAttributeNotFoundError(ValueError): ... class NotAnAttrsClassError(ValueError): ... class DefaultAlreadySetError(RuntimeError): ... class UnannotatedAttributeError(RuntimeError): ... class PythonTooOldError(RuntimeError): ... class NotCallableError(TypeError): msg: str = ... value: Any = ... def __init__(self, msg: str, value: Any) -> None: ... ================================================ FILE: lib/spack/spack/vendor/attr/filters.py ================================================ # SPDX-License-Identifier: MIT """ Commonly useful filters for `attr.asdict`. """ from ._make import Attribute def _split_what(what): """ Returns a tuple of `frozenset`s of classes and attributes. """ return ( frozenset(cls for cls in what if isinstance(cls, type)), frozenset(cls for cls in what if isinstance(cls, Attribute)), ) def include(*what): """ Include *what*. :param what: What to include. :type what: `list` of `type` or `attrs.Attribute`\\ s :rtype: `callable` """ cls, attrs = _split_what(what) def include_(attribute, value): return value.__class__ in cls or attribute in attrs return include_ def exclude(*what): """ Exclude *what*. :param what: What to exclude. :type what: `list` of classes or `attrs.Attribute`\\ s. :rtype: `callable` """ cls, attrs = _split_what(what) def exclude_(attribute, value): return value.__class__ not in cls and attribute not in attrs return exclude_ ================================================ FILE: lib/spack/spack/vendor/attr/filters.pyi ================================================ from typing import Any, Union from . import Attribute, _FilterType def include(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... def exclude(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... ================================================ FILE: lib/spack/spack/vendor/attr/py.typed ================================================ ================================================ FILE: lib/spack/spack/vendor/attr/setters.py ================================================ # SPDX-License-Identifier: MIT """ Commonly used hooks for on_setattr. """ from . import _config from .exceptions import FrozenAttributeError def pipe(*setters): """ Run all *setters* and return the return value of the last one. .. versionadded:: 20.1.0 """ def wrapped_pipe(instance, attrib, new_value): rv = new_value for setter in setters: rv = setter(instance, attrib, rv) return rv return wrapped_pipe def frozen(_, __, ___): """ Prevent an attribute to be modified. .. versionadded:: 20.1.0 """ raise FrozenAttributeError() def validate(instance, attrib, new_value): """ Run *attrib*'s validator on *new_value* if it has one. .. versionadded:: 20.1.0 """ if _config._run_validators is False: return new_value v = attrib.validator if not v: return new_value v(instance, attrib, new_value) return new_value def convert(instance, attrib, new_value): """ Run *attrib*'s converter -- if it has one -- on *new_value* and return the result. .. versionadded:: 20.1.0 """ c = attrib.converter if c: return c(new_value) return new_value # Sentinel for disabling class-wide *on_setattr* hooks for certain attributes. # autodata stopped working, so the docstring is inlined in the API docs. NO_OP = object() ================================================ FILE: lib/spack/spack/vendor/attr/setters.pyi ================================================ from typing import Any, NewType, NoReturn, TypeVar, cast from . import Attribute, _OnSetAttrType _T = TypeVar("_T") def frozen( instance: Any, attribute: Attribute[Any], new_value: Any ) -> NoReturn: ... def pipe(*setters: _OnSetAttrType) -> _OnSetAttrType: ... def validate(instance: Any, attribute: Attribute[_T], new_value: _T) -> _T: ... # convert is allowed to return Any, because they can be chained using pipe. def convert( instance: Any, attribute: Attribute[Any], new_value: Any ) -> Any: ... _NoOpType = NewType("_NoOpType", object) NO_OP: _NoOpType ================================================ FILE: lib/spack/spack/vendor/attr/validators.py ================================================ # SPDX-License-Identifier: MIT """ Commonly useful validators. """ import operator import re from contextlib import contextmanager from ._config import get_run_validators, set_run_validators from ._make import _AndValidator, and_, attrib, attrs from .exceptions import NotCallableError try: Pattern = re.Pattern except AttributeError: # Python <3.7 lacks a Pattern type. Pattern = type(re.compile("")) __all__ = [ "and_", "deep_iterable", "deep_mapping", "disabled", "ge", "get_disabled", "gt", "in_", "instance_of", "is_callable", "le", "lt", "matches_re", "max_len", "min_len", "optional", "provides", "set_disabled", ] def set_disabled(disabled): """ Globally disable or enable running validators. By default, they are run. :param disabled: If ``True``, disable running all validators. :type disabled: bool .. warning:: This function is not thread-safe! .. versionadded:: 21.3.0 """ set_run_validators(not disabled) def get_disabled(): """ Return a bool indicating whether validators are currently disabled or not. :return: ``True`` if validators are currently disabled. :rtype: bool .. versionadded:: 21.3.0 """ return not get_run_validators() @contextmanager def disabled(): """ Context manager that disables running validators within its context. .. warning:: This context manager is not thread-safe! .. versionadded:: 21.3.0 """ set_run_validators(False) try: yield finally: set_run_validators(True) @attrs(repr=False, slots=True, hash=True) class _InstanceOfValidator: type = attrib() def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if not isinstance(value, self.type): raise TypeError( "'{name}' must be {type!r} (got {value!r} that is a " "{actual!r}).".format( name=attr.name, type=self.type, actual=value.__class__, value=value, ), attr, self.type, value, ) def __repr__(self): return "".format( type=self.type ) def instance_of(type): """ A validator that raises a `TypeError` if the initializer is called with a wrong type for this particular attribute (checks are performed using `isinstance` therefore it's also valid to pass a tuple of types). :param type: The type to check for. :type type: type or tuple of types :raises TypeError: With a human readable error message, the attribute (of type `attrs.Attribute`), the expected type, and the value it got. """ return _InstanceOfValidator(type) @attrs(repr=False, frozen=True, slots=True) class _MatchesReValidator: pattern = attrib() match_func = attrib() def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if not self.match_func(value): raise ValueError( "'{name}' must match regex {pattern!r}" " ({value!r} doesn't)".format( name=attr.name, pattern=self.pattern.pattern, value=value ), attr, self.pattern, value, ) def __repr__(self): return "".format( pattern=self.pattern ) def matches_re(regex, flags=0, func=None): r""" A validator that raises `ValueError` if the initializer is called with a string that doesn't match *regex*. :param regex: a regex string or precompiled pattern to match against :param int flags: flags that will be passed to the underlying re function (default 0) :param callable func: which underlying `re` function to call. Valid options are `re.fullmatch`, `re.search`, and `re.match`; the default ``None`` means `re.fullmatch`. For performance reasons, the pattern is always precompiled using `re.compile`. .. versionadded:: 19.2.0 .. versionchanged:: 21.3.0 *regex* can be a pre-compiled pattern. """ valid_funcs = (re.fullmatch, None, re.search, re.match) if func not in valid_funcs: raise ValueError( "'func' must be one of {}.".format( ", ".join( sorted( e and e.__name__ or "None" for e in set(valid_funcs) ) ) ) ) if isinstance(regex, Pattern): if flags: raise TypeError( "'flags' can only be used with a string pattern; " "pass flags to re.compile() instead" ) pattern = regex else: pattern = re.compile(regex, flags) if func is re.match: match_func = pattern.match elif func is re.search: match_func = pattern.search else: match_func = pattern.fullmatch return _MatchesReValidator(pattern, match_func) @attrs(repr=False, slots=True, hash=True) class _ProvidesValidator: interface = attrib() def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if not self.interface.providedBy(value): raise TypeError( "'{name}' must provide {interface!r} which {value!r} " "doesn't.".format( name=attr.name, interface=self.interface, value=value ), attr, self.interface, value, ) def __repr__(self): return "".format( interface=self.interface ) def provides(interface): """ A validator that raises a `TypeError` if the initializer is called with an object that does not provide the requested *interface* (checks are performed using ``interface.providedBy(value)`` (see `zope.interface `_). :param interface: The interface to check for. :type interface: ``zope.interface.Interface`` :raises TypeError: With a human readable error message, the attribute (of type `attrs.Attribute`), the expected interface, and the value it got. """ return _ProvidesValidator(interface) @attrs(repr=False, slots=True, hash=True) class _OptionalValidator: validator = attrib() def __call__(self, inst, attr, value): if value is None: return self.validator(inst, attr, value) def __repr__(self): return "".format( what=repr(self.validator) ) def optional(validator): """ A validator that makes an attribute optional. An optional attribute is one which can be set to ``None`` in addition to satisfying the requirements of the sub-validator. :param validator: A validator (or a list of validators) that is used for non-``None`` values. :type validator: callable or `list` of callables. .. versionadded:: 15.1.0 .. versionchanged:: 17.1.0 *validator* can be a list of validators. """ if isinstance(validator, list): return _OptionalValidator(_AndValidator(validator)) return _OptionalValidator(validator) @attrs(repr=False, slots=True, hash=True) class _InValidator: options = attrib() def __call__(self, inst, attr, value): try: in_options = value in self.options except TypeError: # e.g. `1 in "abc"` in_options = False if not in_options: raise ValueError( "'{name}' must be in {options!r} (got {value!r})".format( name=attr.name, options=self.options, value=value ), attr, self.options, value, ) def __repr__(self): return "".format( options=self.options ) def in_(options): """ A validator that raises a `ValueError` if the initializer is called with a value that does not belong in the options provided. The check is performed using ``value in options``. :param options: Allowed options. :type options: list, tuple, `enum.Enum`, ... :raises ValueError: With a human readable error message, the attribute (of type `attrs.Attribute`), the expected options, and the value it got. .. versionadded:: 17.1.0 .. versionchanged:: 22.1.0 The ValueError was incomplete until now and only contained the human readable error message. Now it contains all the information that has been promised since 17.1.0. """ return _InValidator(options) @attrs(repr=False, slots=False, hash=True) class _IsCallableValidator: def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if not callable(value): message = ( "'{name}' must be callable " "(got {value!r} that is a {actual!r})." ) raise NotCallableError( msg=message.format( name=attr.name, value=value, actual=value.__class__ ), value=value, ) def __repr__(self): return "" def is_callable(): """ A validator that raises a `attr.exceptions.NotCallableError` if the initializer is called with a value for this particular attribute that is not callable. .. versionadded:: 19.1.0 :raises `attr.exceptions.NotCallableError`: With a human readable error message containing the attribute (`attrs.Attribute`) name, and the value it got. """ return _IsCallableValidator() @attrs(repr=False, slots=True, hash=True) class _DeepIterable: member_validator = attrib(validator=is_callable()) iterable_validator = attrib( default=None, validator=optional(is_callable()) ) def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if self.iterable_validator is not None: self.iterable_validator(inst, attr, value) for member in value: self.member_validator(inst, attr, member) def __repr__(self): iterable_identifier = ( "" if self.iterable_validator is None else " {iterable!r}".format(iterable=self.iterable_validator) ) return ( "" ).format( iterable_identifier=iterable_identifier, member=self.member_validator, ) def deep_iterable(member_validator, iterable_validator=None): """ A validator that performs deep validation of an iterable. :param member_validator: Validator(s) to apply to iterable members :param iterable_validator: Validator to apply to iterable itself (optional) .. versionadded:: 19.1.0 :raises TypeError: if any sub-validators fail """ if isinstance(member_validator, (list, tuple)): member_validator = and_(*member_validator) return _DeepIterable(member_validator, iterable_validator) @attrs(repr=False, slots=True, hash=True) class _DeepMapping: key_validator = attrib(validator=is_callable()) value_validator = attrib(validator=is_callable()) mapping_validator = attrib(default=None, validator=optional(is_callable())) def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if self.mapping_validator is not None: self.mapping_validator(inst, attr, value) for key in value: self.key_validator(inst, attr, key) self.value_validator(inst, attr, value[key]) def __repr__(self): return ( "" ).format(key=self.key_validator, value=self.value_validator) def deep_mapping(key_validator, value_validator, mapping_validator=None): """ A validator that performs deep validation of a dictionary. :param key_validator: Validator to apply to dictionary keys :param value_validator: Validator to apply to dictionary values :param mapping_validator: Validator to apply to top-level mapping attribute (optional) .. versionadded:: 19.1.0 :raises TypeError: if any sub-validators fail """ return _DeepMapping(key_validator, value_validator, mapping_validator) @attrs(repr=False, frozen=True, slots=True) class _NumberValidator: bound = attrib() compare_op = attrib() compare_func = attrib() def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if not self.compare_func(value, self.bound): raise ValueError( "'{name}' must be {op} {bound}: {value}".format( name=attr.name, op=self.compare_op, bound=self.bound, value=value, ) ) def __repr__(self): return "".format( op=self.compare_op, bound=self.bound ) def lt(val): """ A validator that raises `ValueError` if the initializer is called with a number larger or equal to *val*. :param val: Exclusive upper bound for values .. versionadded:: 21.3.0 """ return _NumberValidator(val, "<", operator.lt) def le(val): """ A validator that raises `ValueError` if the initializer is called with a number greater than *val*. :param val: Inclusive upper bound for values .. versionadded:: 21.3.0 """ return _NumberValidator(val, "<=", operator.le) def ge(val): """ A validator that raises `ValueError` if the initializer is called with a number smaller than *val*. :param val: Inclusive lower bound for values .. versionadded:: 21.3.0 """ return _NumberValidator(val, ">=", operator.ge) def gt(val): """ A validator that raises `ValueError` if the initializer is called with a number smaller or equal to *val*. :param val: Exclusive lower bound for values .. versionadded:: 21.3.0 """ return _NumberValidator(val, ">", operator.gt) @attrs(repr=False, frozen=True, slots=True) class _MaxLengthValidator: max_length = attrib() def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if len(value) > self.max_length: raise ValueError( "Length of '{name}' must be <= {max}: {len}".format( name=attr.name, max=self.max_length, len=len(value) ) ) def __repr__(self): return "".format(max=self.max_length) def max_len(length): """ A validator that raises `ValueError` if the initializer is called with a string or iterable that is longer than *length*. :param int length: Maximum length of the string or iterable .. versionadded:: 21.3.0 """ return _MaxLengthValidator(length) @attrs(repr=False, frozen=True, slots=True) class _MinLengthValidator: min_length = attrib() def __call__(self, inst, attr, value): """ We use a callable class to be able to change the ``__repr__``. """ if len(value) < self.min_length: raise ValueError( "Length of '{name}' must be => {min}: {len}".format( name=attr.name, min=self.min_length, len=len(value) ) ) def __repr__(self): return "".format(min=self.min_length) def min_len(length): """ A validator that raises `ValueError` if the initializer is called with a string or iterable that is shorter than *length*. :param int length: Minimum length of the string or iterable .. versionadded:: 22.1.0 """ return _MinLengthValidator(length) ================================================ FILE: lib/spack/spack/vendor/attr/validators.pyi ================================================ from typing import ( Any, AnyStr, Callable, Container, ContextManager, Iterable, List, Mapping, Match, Optional, Pattern, Tuple, Type, TypeVar, Union, overload, ) from . import _ValidatorType from . import _ValidatorArgType _T = TypeVar("_T") _T1 = TypeVar("_T1") _T2 = TypeVar("_T2") _T3 = TypeVar("_T3") _I = TypeVar("_I", bound=Iterable) _K = TypeVar("_K") _V = TypeVar("_V") _M = TypeVar("_M", bound=Mapping) def set_disabled(run: bool) -> None: ... def get_disabled() -> bool: ... def disabled() -> ContextManager[None]: ... # To be more precise on instance_of use some overloads. # If there are more than 3 items in the tuple then we fall back to Any @overload def instance_of(type: Type[_T]) -> _ValidatorType[_T]: ... @overload def instance_of(type: Tuple[Type[_T]]) -> _ValidatorType[_T]: ... @overload def instance_of( type: Tuple[Type[_T1], Type[_T2]] ) -> _ValidatorType[Union[_T1, _T2]]: ... @overload def instance_of( type: Tuple[Type[_T1], Type[_T2], Type[_T3]] ) -> _ValidatorType[Union[_T1, _T2, _T3]]: ... @overload def instance_of(type: Tuple[type, ...]) -> _ValidatorType[Any]: ... def provides(interface: Any) -> _ValidatorType[Any]: ... def optional( validator: Union[_ValidatorType[_T], List[_ValidatorType[_T]]] ) -> _ValidatorType[Optional[_T]]: ... def in_(options: Container[_T]) -> _ValidatorType[_T]: ... def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ... def matches_re( regex: Union[Pattern[AnyStr], AnyStr], flags: int = ..., func: Optional[ Callable[[AnyStr, AnyStr, int], Optional[Match[AnyStr]]] ] = ..., ) -> _ValidatorType[AnyStr]: ... def deep_iterable( member_validator: _ValidatorArgType[_T], iterable_validator: Optional[_ValidatorType[_I]] = ..., ) -> _ValidatorType[_I]: ... def deep_mapping( key_validator: _ValidatorType[_K], value_validator: _ValidatorType[_V], mapping_validator: Optional[_ValidatorType[_M]] = ..., ) -> _ValidatorType[_M]: ... def is_callable() -> _ValidatorType[_T]: ... def lt(val: _T) -> _ValidatorType[_T]: ... def le(val: _T) -> _ValidatorType[_T]: ... def ge(val: _T) -> _ValidatorType[_T]: ... def gt(val: _T) -> _ValidatorType[_T]: ... def max_len(length: int) -> _ValidatorType[_T]: ... def min_len(length: int) -> _ValidatorType[_T]: ... ================================================ FILE: lib/spack/spack/vendor/attrs/LICENSE ================================================ The MIT License (MIT) Copyright (c) 2015 Hynek Schlawack and the attrs contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: lib/spack/spack/vendor/attrs/__init__.py ================================================ # SPDX-License-Identifier: MIT from spack.vendor.attr import ( NOTHING, Attribute, Factory, __author__, __copyright__, __description__, __doc__, __email__, __license__, __title__, __url__, __version__, __version_info__, assoc, cmp_using, define, evolve, field, fields, fields_dict, frozen, has, make_class, mutable, resolve_types, validate, ) from spack.vendor.attr._next_gen import asdict, astuple from . import converters, exceptions, filters, setters, validators __all__ = [ "__author__", "__copyright__", "__description__", "__doc__", "__email__", "__license__", "__title__", "__url__", "__version__", "__version_info__", "asdict", "assoc", "astuple", "Attribute", "cmp_using", "converters", "define", "evolve", "exceptions", "Factory", "field", "fields_dict", "fields", "filters", "frozen", "has", "make_class", "mutable", "NOTHING", "resolve_types", "setters", "validate", "validators", ] ================================================ FILE: lib/spack/spack/vendor/attrs/__init__.pyi ================================================ from typing import ( Any, Callable, Dict, Mapping, Optional, Sequence, Tuple, Type, ) # Because we need to type our own stuff, we have to make everything from # attr explicitly public too. from attr import __author__ as __author__ from attr import __copyright__ as __copyright__ from attr import __description__ as __description__ from attr import __email__ as __email__ from attr import __license__ as __license__ from attr import __title__ as __title__ from attr import __url__ as __url__ from attr import __version__ as __version__ from attr import __version_info__ as __version_info__ from attr import _FilterType from attr import assoc as assoc from attr import Attribute as Attribute from attr import cmp_using as cmp_using from attr import converters as converters from attr import define as define from attr import evolve as evolve from attr import exceptions as exceptions from attr import Factory as Factory from attr import field as field from attr import fields as fields from attr import fields_dict as fields_dict from attr import filters as filters from attr import frozen as frozen from attr import has as has from attr import make_class as make_class from attr import mutable as mutable from attr import NOTHING as NOTHING from attr import resolve_types as resolve_types from attr import setters as setters from attr import validate as validate from attr import validators as validators # TODO: see definition of attr.asdict/astuple def asdict( inst: Any, recurse: bool = ..., filter: Optional[_FilterType[Any]] = ..., dict_factory: Type[Mapping[Any, Any]] = ..., retain_collection_types: bool = ..., value_serializer: Optional[ Callable[[type, Attribute[Any], Any], Any] ] = ..., tuple_keys: bool = ..., ) -> Dict[str, Any]: ... # TODO: add support for returning NamedTuple from the mypy plugin def astuple( inst: Any, recurse: bool = ..., filter: Optional[_FilterType[Any]] = ..., tuple_factory: Type[Sequence[Any]] = ..., retain_collection_types: bool = ..., ) -> Tuple[Any, ...]: ... ================================================ FILE: lib/spack/spack/vendor/attrs/converters.py ================================================ # SPDX-License-Identifier: MIT from spack.vendor.attr.converters import * # noqa ================================================ FILE: lib/spack/spack/vendor/attrs/exceptions.py ================================================ # SPDX-License-Identifier: MIT from spack.vendor.attr.exceptions import * # noqa ================================================ FILE: lib/spack/spack/vendor/attrs/filters.py ================================================ # SPDX-License-Identifier: MIT from spack.vendor.attr.filters import * # noqa ================================================ FILE: lib/spack/spack/vendor/attrs/py.typed ================================================ ================================================ FILE: lib/spack/spack/vendor/attrs/setters.py ================================================ # SPDX-License-Identifier: MIT from spack.vendor.attr.setters import * # noqa ================================================ FILE: lib/spack/spack/vendor/attrs/validators.py ================================================ # SPDX-License-Identifier: MIT from spack.vendor.attr.validators import * # noqa ================================================ FILE: lib/spack/spack/vendor/distro/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: lib/spack/spack/vendor/distro/__init__.py ================================================ from .distro import ( NORMALIZED_DISTRO_ID, NORMALIZED_LSB_ID, NORMALIZED_OS_ID, LinuxDistribution, __version__, build_number, codename, distro_release_attr, distro_release_info, id, info, like, linux_distribution, lsb_release_attr, lsb_release_info, major_version, minor_version, name, os_release_attr, os_release_info, uname_attr, uname_info, version, version_parts, ) __all__ = [ "NORMALIZED_DISTRO_ID", "NORMALIZED_LSB_ID", "NORMALIZED_OS_ID", "LinuxDistribution", "build_number", "codename", "distro_release_attr", "distro_release_info", "id", "info", "like", "linux_distribution", "lsb_release_attr", "lsb_release_info", "major_version", "minor_version", "name", "os_release_attr", "os_release_info", "uname_attr", "uname_info", "version", "version_parts", ] __version__ = __version__ ================================================ FILE: lib/spack/spack/vendor/distro/__main__.py ================================================ from .distro import main if __name__ == "__main__": main() ================================================ FILE: lib/spack/spack/vendor/distro/distro.py ================================================ #!/usr/bin/env python # Copyright 2015,2016,2017 Nir Cohen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ The ``distro`` package (``distro`` stands for Linux Distribution) provides information about the Linux distribution it runs on, such as a reliable machine-readable distro ID, or version information. It is the recommended replacement for Python's original :py:func:`platform.linux_distribution` function, but it provides much more functionality. An alternative implementation became necessary because Python 3.5 deprecated this function, and Python 3.8 removed it altogether. Its predecessor function :py:func:`platform.dist` was already deprecated since Python 2.6 and removed in Python 3.8. Still, there are many cases in which access to OS distribution information is needed. See `Python issue 1322 `_ for more information. """ import argparse import json import logging import os import re import shlex import subprocess import sys import warnings from typing import ( Any, Callable, Dict, Iterable, Optional, Sequence, TextIO, Tuple, Type, ) try: from typing import TypedDict except ImportError: # Python 3.7 TypedDict = dict __version__ = "1.8.0" class VersionDict(TypedDict): major: str minor: str build_number: str class InfoDict(TypedDict): id: str version: str version_parts: VersionDict like: str codename: str _UNIXCONFDIR = os.environ.get("UNIXCONFDIR", "/etc") _UNIXUSRLIBDIR = os.environ.get("UNIXUSRLIBDIR", "/usr/lib") _OS_RELEASE_BASENAME = "os-release" #: Translation table for normalizing the "ID" attribute defined in os-release #: files, for use by the :func:`distro.id` method. #: #: * Key: Value as defined in the os-release file, translated to lower case, #: with blanks translated to underscores. #: #: * Value: Normalized value. NORMALIZED_OS_ID = { "ol": "oracle", # Oracle Linux "opensuse-leap": "opensuse", # Newer versions of OpenSuSE report as opensuse-leap } #: Translation table for normalizing the "Distributor ID" attribute returned by #: the lsb_release command, for use by the :func:`distro.id` method. #: #: * Key: Value as returned by the lsb_release command, translated to lower #: case, with blanks translated to underscores. #: #: * Value: Normalized value. NORMALIZED_LSB_ID = { "enterpriseenterpriseas": "oracle", # Oracle Enterprise Linux 4 "enterpriseenterpriseserver": "oracle", # Oracle Linux 5 "redhatenterpriseworkstation": "rhel", # RHEL 6, 7 Workstation "redhatenterpriseserver": "rhel", # RHEL 6, 7 Server "redhatenterprisecomputenode": "rhel", # RHEL 6 ComputeNode } #: Translation table for normalizing the distro ID derived from the file name #: of distro release files, for use by the :func:`distro.id` method. #: #: * Key: Value as derived from the file name of a distro release file, #: translated to lower case, with blanks translated to underscores. #: #: * Value: Normalized value. NORMALIZED_DISTRO_ID = { "redhat": "rhel", # RHEL 6.x, 7.x } # Pattern for content of distro release file (reversed) _DISTRO_RELEASE_CONTENT_REVERSED_PATTERN = re.compile( r"(?:[^)]*\)(.*)\()? *(?:STL )?([\d.+\-a-z]*\d) *(?:esaeler *)?(.+)" ) # Pattern for base file name of distro release file _DISTRO_RELEASE_BASENAME_PATTERN = re.compile(r"(\w+)[-_](release|version)$") # Base file names to be looked up for if _UNIXCONFDIR is not readable. _DISTRO_RELEASE_BASENAMES = [ "SuSE-release", "arch-release", "base-release", "centos-release", "fedora-release", "gentoo-release", "mageia-release", "mandrake-release", "mandriva-release", "mandrivalinux-release", "manjaro-release", "oracle-release", "redhat-release", "rocky-release", "sl-release", "slackware-version", ] # Base file names to be ignored when searching for distro release file _DISTRO_RELEASE_IGNORE_BASENAMES = ( "debian_version", "lsb-release", "oem-release", _OS_RELEASE_BASENAME, "system-release", "plesk-release", "iredmail-release", ) def linux_distribution(full_distribution_name: bool = True) -> Tuple[str, str, str]: """ .. deprecated:: 1.6.0 :func:`distro.linux_distribution()` is deprecated. It should only be used as a compatibility shim with Python's :py:func:`platform.linux_distribution()`. Please use :func:`distro.id`, :func:`distro.version` and :func:`distro.name` instead. Return information about the current OS distribution as a tuple ``(id_name, version, codename)`` with items as follows: * ``id_name``: If *full_distribution_name* is false, the result of :func:`distro.id`. Otherwise, the result of :func:`distro.name`. * ``version``: The result of :func:`distro.version`. * ``codename``: The extra item (usually in parentheses) after the os-release version number, or the result of :func:`distro.codename`. The interface of this function is compatible with the original :py:func:`platform.linux_distribution` function, supporting a subset of its parameters. The data it returns may not exactly be the same, because it uses more data sources than the original function, and that may lead to different data if the OS distribution is not consistent across multiple data sources it provides (there are indeed such distributions ...). Another reason for differences is the fact that the :func:`distro.id` method normalizes the distro ID string to a reliable machine-readable value for a number of popular OS distributions. """ warnings.warn( "distro.linux_distribution() is deprecated. It should only be used as a " "compatibility shim with Python's platform.linux_distribution(). Please use " "distro.id(), distro.version() and distro.name() instead.", DeprecationWarning, stacklevel=2, ) return _distro.linux_distribution(full_distribution_name) def id() -> str: """ Return the distro ID of the current distribution, as a machine-readable string. For a number of OS distributions, the returned distro ID value is *reliable*, in the sense that it is documented and that it does not change across releases of the distribution. This package maintains the following reliable distro ID values: ============== ========================================= Distro ID Distribution ============== ========================================= "ubuntu" Ubuntu "debian" Debian "rhel" RedHat Enterprise Linux "centos" CentOS "fedora" Fedora "sles" SUSE Linux Enterprise Server "opensuse" openSUSE "amzn" Amazon Linux "arch" Arch Linux "buildroot" Buildroot "cloudlinux" CloudLinux OS "exherbo" Exherbo Linux "gentoo" GenToo Linux "ibm_powerkvm" IBM PowerKVM "kvmibm" KVM for IBM z Systems "linuxmint" Linux Mint "mageia" Mageia "mandriva" Mandriva Linux "parallels" Parallels "pidora" Pidora "raspbian" Raspbian "oracle" Oracle Linux (and Oracle Enterprise Linux) "scientific" Scientific Linux "slackware" Slackware "xenserver" XenServer "openbsd" OpenBSD "netbsd" NetBSD "freebsd" FreeBSD "midnightbsd" MidnightBSD "rocky" Rocky Linux "aix" AIX "guix" Guix System ============== ========================================= If you have a need to get distros for reliable IDs added into this set, or if you find that the :func:`distro.id` function returns a different distro ID for one of the listed distros, please create an issue in the `distro issue tracker`_. **Lookup hierarchy and transformations:** First, the ID is obtained from the following sources, in the specified order. The first available and non-empty value is used: * the value of the "ID" attribute of the os-release file, * the value of the "Distributor ID" attribute returned by the lsb_release command, * the first part of the file name of the distro release file, The so determined ID value then passes the following transformations, before it is returned by this method: * it is translated to lower case, * blanks (which should not be there anyway) are translated to underscores, * a normalization of the ID is performed, based upon `normalization tables`_. The purpose of this normalization is to ensure that the ID is as reliable as possible, even across incompatible changes in the OS distributions. A common reason for an incompatible change is the addition of an os-release file, or the addition of the lsb_release command, with ID values that differ from what was previously determined from the distro release file name. """ return _distro.id() def name(pretty: bool = False) -> str: """ Return the name of the current OS distribution, as a human-readable string. If *pretty* is false, the name is returned without version or codename. (e.g. "CentOS Linux") If *pretty* is true, the version and codename are appended. (e.g. "CentOS Linux 7.1.1503 (Core)") **Lookup hierarchy:** The name is obtained from the following sources, in the specified order. The first available and non-empty value is used: * If *pretty* is false: - the value of the "NAME" attribute of the os-release file, - the value of the "Distributor ID" attribute returned by the lsb_release command, - the value of the "" field of the distro release file. * If *pretty* is true: - the value of the "PRETTY_NAME" attribute of the os-release file, - the value of the "Description" attribute returned by the lsb_release command, - the value of the "" field of the distro release file, appended with the value of the pretty version ("" and "" fields) of the distro release file, if available. """ return _distro.name(pretty) def version(pretty: bool = False, best: bool = False) -> str: """ Return the version of the current OS distribution, as a human-readable string. If *pretty* is false, the version is returned without codename (e.g. "7.0"). If *pretty* is true, the codename in parenthesis is appended, if the codename is non-empty (e.g. "7.0 (Maipo)"). Some distributions provide version numbers with different precisions in the different sources of distribution information. Examining the different sources in a fixed priority order does not always yield the most precise version (e.g. for Debian 8.2, or CentOS 7.1). Some other distributions may not provide this kind of information. In these cases, an empty string would be returned. This behavior can be observed with rolling releases distributions (e.g. Arch Linux). The *best* parameter can be used to control the approach for the returned version: If *best* is false, the first non-empty version number in priority order of the examined sources is returned. If *best* is true, the most precise version number out of all examined sources is returned. **Lookup hierarchy:** In all cases, the version number is obtained from the following sources. If *best* is false, this order represents the priority order: * the value of the "VERSION_ID" attribute of the os-release file, * the value of the "Release" attribute returned by the lsb_release command, * the version number parsed from the "" field of the first line of the distro release file, * the version number parsed from the "PRETTY_NAME" attribute of the os-release file, if it follows the format of the distro release files. * the version number parsed from the "Description" attribute returned by the lsb_release command, if it follows the format of the distro release files. """ return _distro.version(pretty, best) def version_parts(best: bool = False) -> Tuple[str, str, str]: """ Return the version of the current OS distribution as a tuple ``(major, minor, build_number)`` with items as follows: * ``major``: The result of :func:`distro.major_version`. * ``minor``: The result of :func:`distro.minor_version`. * ``build_number``: The result of :func:`distro.build_number`. For a description of the *best* parameter, see the :func:`distro.version` method. """ return _distro.version_parts(best) def major_version(best: bool = False) -> str: """ Return the major version of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The major version is the first part of the dot-separated version string. For a description of the *best* parameter, see the :func:`distro.version` method. """ return _distro.major_version(best) def minor_version(best: bool = False) -> str: """ Return the minor version of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The minor version is the second part of the dot-separated version string. For a description of the *best* parameter, see the :func:`distro.version` method. """ return _distro.minor_version(best) def build_number(best: bool = False) -> str: """ Return the build number of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The build number is the third part of the dot-separated version string. For a description of the *best* parameter, see the :func:`distro.version` method. """ return _distro.build_number(best) def like() -> str: """ Return a space-separated list of distro IDs of distributions that are closely related to the current OS distribution in regards to packaging and programming interfaces, for example distributions the current distribution is a derivative from. **Lookup hierarchy:** This information item is only provided by the os-release file. For details, see the description of the "ID_LIKE" attribute in the `os-release man page `_. """ return _distro.like() def codename() -> str: """ Return the codename for the release of the current OS distribution, as a string. If the distribution does not have a codename, an empty string is returned. Note that the returned codename is not always really a codename. For example, openSUSE returns "x86_64". This function does not handle such cases in any special way and just returns the string it finds, if any. **Lookup hierarchy:** * the codename within the "VERSION" attribute of the os-release file, if provided, * the value of the "Codename" attribute returned by the lsb_release command, * the value of the "" field of the distro release file. """ return _distro.codename() def info(pretty: bool = False, best: bool = False) -> InfoDict: """ Return certain machine-readable information items about the current OS distribution in a dictionary, as shown in the following example: .. sourcecode:: python { 'id': 'rhel', 'version': '7.0', 'version_parts': { 'major': '7', 'minor': '0', 'build_number': '' }, 'like': 'fedora', 'codename': 'Maipo' } The dictionary structure and keys are always the same, regardless of which information items are available in the underlying data sources. The values for the various keys are as follows: * ``id``: The result of :func:`distro.id`. * ``version``: The result of :func:`distro.version`. * ``version_parts -> major``: The result of :func:`distro.major_version`. * ``version_parts -> minor``: The result of :func:`distro.minor_version`. * ``version_parts -> build_number``: The result of :func:`distro.build_number`. * ``like``: The result of :func:`distro.like`. * ``codename``: The result of :func:`distro.codename`. For a description of the *pretty* and *best* parameters, see the :func:`distro.version` method. """ return _distro.info(pretty, best) def os_release_info() -> Dict[str, str]: """ Return a dictionary containing key-value pairs for the information items from the os-release file data source of the current OS distribution. See `os-release file`_ for details about these information items. """ return _distro.os_release_info() def lsb_release_info() -> Dict[str, str]: """ Return a dictionary containing key-value pairs for the information items from the lsb_release command data source of the current OS distribution. See `lsb_release command output`_ for details about these information items. """ return _distro.lsb_release_info() def distro_release_info() -> Dict[str, str]: """ Return a dictionary containing key-value pairs for the information items from the distro release file data source of the current OS distribution. See `distro release file`_ for details about these information items. """ return _distro.distro_release_info() def uname_info() -> Dict[str, str]: """ Return a dictionary containing key-value pairs for the information items from the distro release file data source of the current OS distribution. """ return _distro.uname_info() def os_release_attr(attribute: str) -> str: """ Return a single named information item from the os-release file data source of the current OS distribution. Parameters: * ``attribute`` (string): Key of the information item. Returns: * (string): Value of the information item, if the item exists. The empty string, if the item does not exist. See `os-release file`_ for details about these information items. """ return _distro.os_release_attr(attribute) def lsb_release_attr(attribute: str) -> str: """ Return a single named information item from the lsb_release command output data source of the current OS distribution. Parameters: * ``attribute`` (string): Key of the information item. Returns: * (string): Value of the information item, if the item exists. The empty string, if the item does not exist. See `lsb_release command output`_ for details about these information items. """ return _distro.lsb_release_attr(attribute) def distro_release_attr(attribute: str) -> str: """ Return a single named information item from the distro release file data source of the current OS distribution. Parameters: * ``attribute`` (string): Key of the information item. Returns: * (string): Value of the information item, if the item exists. The empty string, if the item does not exist. See `distro release file`_ for details about these information items. """ return _distro.distro_release_attr(attribute) def uname_attr(attribute: str) -> str: """ Return a single named information item from the distro release file data source of the current OS distribution. Parameters: * ``attribute`` (string): Key of the information item. Returns: * (string): Value of the information item, if the item exists. The empty string, if the item does not exist. """ return _distro.uname_attr(attribute) try: from functools import cached_property except ImportError: # Python < 3.8 class cached_property: # type: ignore """A version of @property which caches the value. On access, it calls the underlying function and sets the value in `__dict__` so future accesses will not re-call the property. """ def __init__(self, f: Callable[[Any], Any]) -> None: self._fname = f.__name__ self._f = f def __get__(self, obj: Any, owner: Type[Any]) -> Any: assert obj is not None, f"call {self._fname} on an instance" ret = obj.__dict__[self._fname] = self._f(obj) return ret class LinuxDistribution: """ Provides information about a OS distribution. This package creates a private module-global instance of this class with default initialization arguments, that is used by the `consolidated accessor functions`_ and `single source accessor functions`_. By using default initialization arguments, that module-global instance returns data about the current OS distribution (i.e. the distro this package runs on). Normally, it is not necessary to create additional instances of this class. However, in situations where control is needed over the exact data sources that are used, instances of this class can be created with a specific distro release file, or a specific os-release file, or without invoking the lsb_release command. """ def __init__( self, include_lsb: Optional[bool] = None, os_release_file: str = "", distro_release_file: str = "", include_uname: Optional[bool] = None, root_dir: Optional[str] = None, include_oslevel: Optional[bool] = None, ) -> None: """ The initialization method of this class gathers information from the available data sources, and stores that in private instance attributes. Subsequent access to the information items uses these private instance attributes, so that the data sources are read only once. Parameters: * ``include_lsb`` (bool): Controls whether the `lsb_release command output`_ is included as a data source. If the lsb_release command is not available in the program execution path, the data source for the lsb_release command will be empty. * ``os_release_file`` (string): The path name of the `os-release file`_ that is to be used as a data source. An empty string (the default) will cause the default path name to be used (see `os-release file`_ for details). If the specified or defaulted os-release file does not exist, the data source for the os-release file will be empty. * ``distro_release_file`` (string): The path name of the `distro release file`_ that is to be used as a data source. An empty string (the default) will cause a default search algorithm to be used (see `distro release file`_ for details). If the specified distro release file does not exist, or if no default distro release file can be found, the data source for the distro release file will be empty. * ``include_uname`` (bool): Controls whether uname command output is included as a data source. If the uname command is not available in the program execution path the data source for the uname command will be empty. * ``root_dir`` (string): The absolute path to the root directory to use to find distro-related information files. Note that ``include_*`` parameters must not be enabled in combination with ``root_dir``. * ``include_oslevel`` (bool): Controls whether (AIX) oslevel command output is included as a data source. If the oslevel command is not available in the program execution path the data source will be empty. Public instance attributes: * ``os_release_file`` (string): The path name of the `os-release file`_ that is actually used as a data source. The empty string if no distro release file is used as a data source. * ``distro_release_file`` (string): The path name of the `distro release file`_ that is actually used as a data source. The empty string if no distro release file is used as a data source. * ``include_lsb`` (bool): The result of the ``include_lsb`` parameter. This controls whether the lsb information will be loaded. * ``include_uname`` (bool): The result of the ``include_uname`` parameter. This controls whether the uname information will be loaded. * ``include_oslevel`` (bool): The result of the ``include_oslevel`` parameter. This controls whether (AIX) oslevel information will be loaded. * ``root_dir`` (string): The result of the ``root_dir`` parameter. The absolute path to the root directory to use to find distro-related information files. Raises: * :py:exc:`ValueError`: Initialization parameters combination is not supported. * :py:exc:`OSError`: Some I/O issue with an os-release file or distro release file. * :py:exc:`UnicodeError`: A data source has unexpected characters or uses an unexpected encoding. """ self.root_dir = root_dir self.etc_dir = os.path.join(root_dir, "etc") if root_dir else _UNIXCONFDIR self.usr_lib_dir = ( os.path.join(root_dir, "usr/lib") if root_dir else _UNIXUSRLIBDIR ) if os_release_file: self.os_release_file = os_release_file else: etc_dir_os_release_file = os.path.join(self.etc_dir, _OS_RELEASE_BASENAME) usr_lib_os_release_file = os.path.join( self.usr_lib_dir, _OS_RELEASE_BASENAME ) # NOTE: The idea is to respect order **and** have it set # at all times for API backwards compatibility. if os.path.isfile(etc_dir_os_release_file) or not os.path.isfile( usr_lib_os_release_file ): self.os_release_file = etc_dir_os_release_file else: self.os_release_file = usr_lib_os_release_file self.distro_release_file = distro_release_file or "" # updated later is_root_dir_defined = root_dir is not None if is_root_dir_defined and (include_lsb or include_uname or include_oslevel): raise ValueError( "Including subprocess data sources from specific root_dir is disallowed" " to prevent false information" ) self.include_lsb = ( include_lsb if include_lsb is not None else not is_root_dir_defined ) self.include_uname = ( include_uname if include_uname is not None else not is_root_dir_defined ) self.include_oslevel = ( include_oslevel if include_oslevel is not None else not is_root_dir_defined ) def __repr__(self) -> str: """Return repr of all info""" return ( "LinuxDistribution(" "os_release_file={self.os_release_file!r}, " "distro_release_file={self.distro_release_file!r}, " "include_lsb={self.include_lsb!r}, " "include_uname={self.include_uname!r}, " "include_oslevel={self.include_oslevel!r}, " "root_dir={self.root_dir!r}, " "_os_release_info={self._os_release_info!r}, " "_lsb_release_info={self._lsb_release_info!r}, " "_distro_release_info={self._distro_release_info!r}, " "_uname_info={self._uname_info!r}, " "_oslevel_info={self._oslevel_info!r})".format(self=self) ) def linux_distribution( self, full_distribution_name: bool = True ) -> Tuple[str, str, str]: """ Return information about the OS distribution that is compatible with Python's :func:`platform.linux_distribution`, supporting a subset of its parameters. For details, see :func:`distro.linux_distribution`. """ return ( self.name() if full_distribution_name else self.id(), self.version(), self._os_release_info.get("release_codename") or self.codename(), ) def id(self) -> str: """Return the distro ID of the OS distribution, as a string. For details, see :func:`distro.id`. """ def normalize(distro_id: str, table: Dict[str, str]) -> str: distro_id = distro_id.lower().replace(" ", "_") return table.get(distro_id, distro_id) distro_id = self.os_release_attr("id") if distro_id: return normalize(distro_id, NORMALIZED_OS_ID) distro_id = self.lsb_release_attr("distributor_id") if distro_id: return normalize(distro_id, NORMALIZED_LSB_ID) distro_id = self.distro_release_attr("id") if distro_id: return normalize(distro_id, NORMALIZED_DISTRO_ID) distro_id = self.uname_attr("id") if distro_id: return normalize(distro_id, NORMALIZED_DISTRO_ID) return "" def name(self, pretty: bool = False) -> str: """ Return the name of the OS distribution, as a string. For details, see :func:`distro.name`. """ name = ( self.os_release_attr("name") or self.lsb_release_attr("distributor_id") or self.distro_release_attr("name") or self.uname_attr("name") ) if pretty: name = self.os_release_attr("pretty_name") or self.lsb_release_attr( "description" ) if not name: name = self.distro_release_attr("name") or self.uname_attr("name") version = self.version(pretty=True) if version: name = f"{name} {version}" return name or "" def version(self, pretty: bool = False, best: bool = False) -> str: """ Return the version of the OS distribution, as a string. For details, see :func:`distro.version`. """ versions = [ self.os_release_attr("version_id"), self.lsb_release_attr("release"), self.distro_release_attr("version_id"), self._parse_distro_release_content(self.os_release_attr("pretty_name")).get( "version_id", "" ), self._parse_distro_release_content( self.lsb_release_attr("description") ).get("version_id", ""), self.uname_attr("release"), ] if self.uname_attr("id").startswith("aix"): # On AIX platforms, prefer oslevel command output. versions.insert(0, self.oslevel_info()) elif self.id() == "debian" or "debian" in self.like().split(): # On Debian-like, add debian_version file content to candidates list. versions.append(self._debian_version) version = "" if best: # This algorithm uses the last version in priority order that has # the best precision. If the versions are not in conflict, that # does not matter; otherwise, using the last one instead of the # first one might be considered a surprise. for v in versions: if v.count(".") > version.count(".") or version == "": version = v else: for v in versions: if v != "": version = v break if pretty and version and self.codename(): version = f"{version} ({self.codename()})" return version def version_parts(self, best: bool = False) -> Tuple[str, str, str]: """ Return the version of the OS distribution, as a tuple of version numbers. For details, see :func:`distro.version_parts`. """ version_str = self.version(best=best) if version_str: version_regex = re.compile(r"(\d+)\.?(\d+)?\.?(\d+)?") matches = version_regex.match(version_str) if matches: major, minor, build_number = matches.groups() return major, minor or "", build_number or "" return "", "", "" def major_version(self, best: bool = False) -> str: """ Return the major version number of the current distribution. For details, see :func:`distro.major_version`. """ return self.version_parts(best)[0] def minor_version(self, best: bool = False) -> str: """ Return the minor version number of the current distribution. For details, see :func:`distro.minor_version`. """ return self.version_parts(best)[1] def build_number(self, best: bool = False) -> str: """ Return the build number of the current distribution. For details, see :func:`distro.build_number`. """ return self.version_parts(best)[2] def like(self) -> str: """ Return the IDs of distributions that are like the OS distribution. For details, see :func:`distro.like`. """ return self.os_release_attr("id_like") or "" def codename(self) -> str: """ Return the codename of the OS distribution. For details, see :func:`distro.codename`. """ try: # Handle os_release specially since distros might purposefully set # this to empty string to have no codename return self._os_release_info["codename"] except KeyError: return ( self.lsb_release_attr("codename") or self.distro_release_attr("codename") or "" ) def info(self, pretty: bool = False, best: bool = False) -> InfoDict: """ Return certain machine-readable information about the OS distribution. For details, see :func:`distro.info`. """ return dict( id=self.id(), version=self.version(pretty, best), version_parts=dict( major=self.major_version(best), minor=self.minor_version(best), build_number=self.build_number(best), ), like=self.like(), codename=self.codename(), ) def os_release_info(self) -> Dict[str, str]: """ Return a dictionary containing key-value pairs for the information items from the os-release file data source of the OS distribution. For details, see :func:`distro.os_release_info`. """ return self._os_release_info def lsb_release_info(self) -> Dict[str, str]: """ Return a dictionary containing key-value pairs for the information items from the lsb_release command data source of the OS distribution. For details, see :func:`distro.lsb_release_info`. """ return self._lsb_release_info def distro_release_info(self) -> Dict[str, str]: """ Return a dictionary containing key-value pairs for the information items from the distro release file data source of the OS distribution. For details, see :func:`distro.distro_release_info`. """ return self._distro_release_info def uname_info(self) -> Dict[str, str]: """ Return a dictionary containing key-value pairs for the information items from the uname command data source of the OS distribution. For details, see :func:`distro.uname_info`. """ return self._uname_info def oslevel_info(self) -> str: """ Return AIX' oslevel command output. """ return self._oslevel_info def os_release_attr(self, attribute: str) -> str: """ Return a single named information item from the os-release file data source of the OS distribution. For details, see :func:`distro.os_release_attr`. """ return self._os_release_info.get(attribute, "") def lsb_release_attr(self, attribute: str) -> str: """ Return a single named information item from the lsb_release command output data source of the OS distribution. For details, see :func:`distro.lsb_release_attr`. """ return self._lsb_release_info.get(attribute, "") def distro_release_attr(self, attribute: str) -> str: """ Return a single named information item from the distro release file data source of the OS distribution. For details, see :func:`distro.distro_release_attr`. """ return self._distro_release_info.get(attribute, "") def uname_attr(self, attribute: str) -> str: """ Return a single named information item from the uname command output data source of the OS distribution. For details, see :func:`distro.uname_attr`. """ return self._uname_info.get(attribute, "") @cached_property def _os_release_info(self) -> Dict[str, str]: """ Get the information items from the specified os-release file. Returns: A dictionary containing all information items. """ if os.path.isfile(self.os_release_file): with open(self.os_release_file, encoding="utf-8") as release_file: return self._parse_os_release_content(release_file) return {} @staticmethod def _parse_os_release_content(lines: TextIO) -> Dict[str, str]: """ Parse the lines of an os-release file. Parameters: * lines: Iterable through the lines in the os-release file. Each line must be a unicode string or a UTF-8 encoded byte string. Returns: A dictionary containing all information items. """ props = {} lexer = shlex.shlex(lines, posix=True) lexer.whitespace_split = True tokens = list(lexer) for token in tokens: # At this point, all shell-like parsing has been done (i.e. # comments processed, quotes and backslash escape sequences # processed, multi-line values assembled, trailing newlines # stripped, etc.), so the tokens are now either: # * variable assignments: var=value # * commands or their arguments (not allowed in os-release) # Ignore any tokens that are not variable assignments if "=" in token: k, v = token.split("=", 1) props[k.lower()] = v if "version" in props: # extract release codename (if any) from version attribute match = re.search(r"\((\D+)\)|,\s*(\D+)", props["version"]) if match: release_codename = match.group(1) or match.group(2) props["codename"] = props["release_codename"] = release_codename if "version_codename" in props: # os-release added a version_codename field. Use that in # preference to anything else Note that some distros purposefully # do not have code names. They should be setting # version_codename="" props["codename"] = props["version_codename"] elif "ubuntu_codename" in props: # Same as above but a non-standard field name used on older Ubuntus props["codename"] = props["ubuntu_codename"] return props @cached_property def _lsb_release_info(self) -> Dict[str, str]: """ Get the information items from the lsb_release command output. Returns: A dictionary containing all information items. """ if not self.include_lsb: return {} try: cmd = ("lsb_release", "-a") stdout = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) # Command not found or lsb_release returned error except (OSError, subprocess.CalledProcessError): return {} content = self._to_str(stdout).splitlines() return self._parse_lsb_release_content(content) @staticmethod def _parse_lsb_release_content(lines: Iterable[str]) -> Dict[str, str]: """ Parse the output of the lsb_release command. Parameters: * lines: Iterable through the lines of the lsb_release output. Each line must be a unicode string or a UTF-8 encoded byte string. Returns: A dictionary containing all information items. """ props = {} for line in lines: kv = line.strip("\n").split(":", 1) if len(kv) != 2: # Ignore lines without colon. continue k, v = kv props.update({k.replace(" ", "_").lower(): v.strip()}) return props @cached_property def _uname_info(self) -> Dict[str, str]: if not self.include_uname: return {} try: cmd = ("uname", "-rs") stdout = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) except OSError: return {} content = self._to_str(stdout).splitlines() return self._parse_uname_content(content) @cached_property def _oslevel_info(self) -> str: if not self.include_oslevel: return "" try: stdout = subprocess.check_output("oslevel", stderr=subprocess.DEVNULL) except (OSError, subprocess.CalledProcessError): return "" return self._to_str(stdout).strip() @cached_property def _debian_version(self) -> str: try: with open( os.path.join(self.etc_dir, "debian_version"), encoding="ascii" ) as fp: return fp.readline().rstrip() except FileNotFoundError: return "" @staticmethod def _parse_uname_content(lines: Sequence[str]) -> Dict[str, str]: if not lines: return {} props = {} match = re.search(r"^([^\s]+)\s+([\d\.]+)", lines[0].strip()) if match: name, version = match.groups() # This is to prevent the Linux kernel version from # appearing as the 'best' version on otherwise # identifiable distributions. if name == "Linux": return {} props["id"] = name.lower() props["name"] = name props["release"] = version return props @staticmethod def _to_str(bytestring: bytes) -> str: encoding = sys.getfilesystemencoding() return bytestring.decode(encoding) @cached_property def _distro_release_info(self) -> Dict[str, str]: """ Get the information items from the specified distro release file. Returns: A dictionary containing all information items. """ if self.distro_release_file: # If it was specified, we use it and parse what we can, even if # its file name or content does not match the expected pattern. distro_info = self._parse_distro_release_file(self.distro_release_file) basename = os.path.basename(self.distro_release_file) # The file name pattern for user-specified distro release files # is somewhat more tolerant (compared to when searching for the # file), because we want to use what was specified as best as # possible. match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename) else: try: with os.scandir(self.etc_dir) as it: etc_files = [ p.path for p in it if p.is_file() and p.name not in _DISTRO_RELEASE_IGNORE_BASENAMES ] # We sort for repeatability in cases where there are multiple # distro specific files; e.g. CentOS, Oracle, Enterprise all # containing `redhat-release` on top of their own. etc_files.sort() except OSError: # This may occur when /etc is not readable but we can't be # sure about the *-release files. Check common entries of # /etc for information. If they turn out to not be there the # error is handled in `_parse_distro_release_file()`. etc_files = [ os.path.join(self.etc_dir, basename) for basename in _DISTRO_RELEASE_BASENAMES ] for filepath in etc_files: match = _DISTRO_RELEASE_BASENAME_PATTERN.match(os.path.basename(filepath)) if match is None: continue distro_info = self._parse_distro_release_file(filepath) # The name is always present if the pattern matches. if "name" not in distro_info: continue self.distro_release_file = filepath break else: # the loop didn't "break": no candidate. return {} if match is not None: distro_info["id"] = match.group(1) # CloudLinux < 7: manually enrich info with proper id. if "cloudlinux" in distro_info.get("name", "").lower(): distro_info["id"] = "cloudlinux" return distro_info def _parse_distro_release_file(self, filepath: str) -> Dict[str, str]: """ Parse a distro release file. Parameters: * filepath: Path name of the distro release file. Returns: A dictionary containing all information items. """ try: with open(filepath, encoding="utf-8") as fp: # Only parse the first line. For instance, on SLES there # are multiple lines. We don't want them... return self._parse_distro_release_content(fp.readline()) except OSError: # Ignore not being able to read a specific, seemingly version # related file. # See https://github.com/python-distro/distro/issues/162 return {} @staticmethod def _parse_distro_release_content(line: str) -> Dict[str, str]: """ Parse a line from a distro release file. Parameters: * line: Line from the distro release file. Must be a unicode string or a UTF-8 encoded byte string. Returns: A dictionary containing all information items. """ matches = _DISTRO_RELEASE_CONTENT_REVERSED_PATTERN.match(line.strip()[::-1]) distro_info = {} if matches: # regexp ensures non-None distro_info["name"] = matches.group(3)[::-1] if matches.group(2): distro_info["version_id"] = matches.group(2)[::-1] if matches.group(1): distro_info["codename"] = matches.group(1)[::-1] elif line: distro_info["name"] = line.strip() return distro_info _distro = LinuxDistribution() def main() -> None: logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) parser = argparse.ArgumentParser(description="OS distro info tool") parser.add_argument( "--json", "-j", help="Output in machine readable format", action="store_true" ) parser.add_argument( "--root-dir", "-r", type=str, dest="root_dir", help="Path to the root filesystem directory (defaults to /)", ) args = parser.parse_args() if args.root_dir: dist = LinuxDistribution( include_lsb=False, include_uname=False, include_oslevel=False, root_dir=args.root_dir, ) else: dist = _distro if args.json: logger.info(json.dumps(dist.info(), indent=4, sort_keys=True)) else: logger.info("Name: %s", dist.name(pretty=True)) distribution_version = dist.version(pretty=True) logger.info("Version: %s", distribution_version) distribution_codename = dist.codename() logger.info("Codename: %s", distribution_codename) if __name__ == "__main__": main() ================================================ FILE: lib/spack/spack/vendor/distro/py.typed ================================================ ================================================ FILE: lib/spack/spack/vendor/jinja2/LICENSE.rst ================================================ Copyright 2007 Pallets Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: lib/spack/spack/vendor/jinja2/__init__.py ================================================ """Jinja is a template engine written in pure Python. It provides a non-XML syntax that supports inline expressions and an optional sandboxed environment. """ from .bccache import BytecodeCache as BytecodeCache from .bccache import FileSystemBytecodeCache as FileSystemBytecodeCache from .bccache import MemcachedBytecodeCache as MemcachedBytecodeCache from .environment import Environment as Environment from .environment import Template as Template from .exceptions import TemplateAssertionError as TemplateAssertionError from .exceptions import TemplateError as TemplateError from .exceptions import TemplateNotFound as TemplateNotFound from .exceptions import TemplateRuntimeError as TemplateRuntimeError from .exceptions import TemplatesNotFound as TemplatesNotFound from .exceptions import TemplateSyntaxError as TemplateSyntaxError from .exceptions import UndefinedError as UndefinedError from .filters import contextfilter from .filters import environmentfilter from .filters import evalcontextfilter from .loaders import BaseLoader as BaseLoader from .loaders import ChoiceLoader as ChoiceLoader from .loaders import DictLoader as DictLoader from .loaders import FileSystemLoader as FileSystemLoader from .loaders import FunctionLoader as FunctionLoader from .loaders import ModuleLoader as ModuleLoader from .loaders import PackageLoader as PackageLoader from .loaders import PrefixLoader as PrefixLoader from .runtime import ChainableUndefined as ChainableUndefined from .runtime import DebugUndefined as DebugUndefined from .runtime import make_logging_undefined as make_logging_undefined from .runtime import StrictUndefined as StrictUndefined from .runtime import Undefined as Undefined from .utils import clear_caches as clear_caches from .utils import contextfunction from .utils import environmentfunction from .utils import escape from .utils import evalcontextfunction from .utils import is_undefined as is_undefined from .utils import Markup from .utils import pass_context as pass_context from .utils import pass_environment as pass_environment from .utils import pass_eval_context as pass_eval_context from .utils import select_autoescape as select_autoescape __version__ = "3.0.3" ================================================ FILE: lib/spack/spack/vendor/jinja2/_identifier.py ================================================ import re # generated by scripts/generate_identifier_pattern.py pattern = re.compile( r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 ) ================================================ FILE: lib/spack/spack/vendor/jinja2/async_utils.py ================================================ import inspect import typing as t from functools import wraps from .utils import _PassArg from .utils import pass_eval_context V = t.TypeVar("V") def async_variant(normal_func): # type: ignore def decorator(async_func): # type: ignore pass_arg = _PassArg.from_obj(normal_func) need_eval_context = pass_arg is None if pass_arg is _PassArg.environment: def is_async(args: t.Any) -> bool: return t.cast(bool, args[0].is_async) else: def is_async(args: t.Any) -> bool: return t.cast(bool, args[0].environment.is_async) @wraps(normal_func) def wrapper(*args, **kwargs): # type: ignore b = is_async(args) if need_eval_context: args = args[1:] if b: return async_func(*args, **kwargs) return normal_func(*args, **kwargs) if need_eval_context: wrapper = pass_eval_context(wrapper) wrapper.jinja_async_variant = True return wrapper return decorator _common_primitives = {int, float, bool, str, list, dict, tuple, type(None)} async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V": # Avoid a costly call to isawaitable if type(value) in _common_primitives: return t.cast("V", value) if inspect.isawaitable(value): return await t.cast("t.Awaitable[V]", value) return t.cast("V", value) async def auto_aiter( iterable: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", ) -> "t.AsyncIterator[V]": if hasattr(iterable, "__aiter__"): async for item in t.cast("t.AsyncIterable[V]", iterable): yield item else: for item in t.cast("t.Iterable[V]", iterable): yield item async def auto_to_list( value: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", ) -> t.List["V"]: return [x async for x in auto_aiter(value)] ================================================ FILE: lib/spack/spack/vendor/jinja2/bccache.py ================================================ """The optional bytecode cache system. This is useful if you have very complex template situations and the compilation of all those templates slows down your application too much. Situations where this is useful are often forking web applications that are initialized on the first request. """ import errno import fnmatch import marshal import os import pickle import stat import sys import tempfile import typing as t from hashlib import sha1 from io import BytesIO from types import CodeType if t.TYPE_CHECKING: import spack.vendor.typing_extensions as te from .environment import Environment class _MemcachedClient(te.Protocol): def get(self, key: str) -> bytes: ... def set(self, key: str, value: bytes, timeout: t.Optional[int] = None) -> None: ... bc_version = 5 # Magic bytes to identify Jinja bytecode cache files. Contains the # Python major and minor version to avoid loading incompatible bytecode # if a project upgrades its Python version. bc_magic = ( b"j2" + pickle.dumps(bc_version, 2) + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2) ) class Bucket: """Buckets are used to store the bytecode for one template. It's created and initialized by the bytecode cache and passed to the loading functions. The buckets get an internal checksum from the cache assigned and use this to automatically reject outdated cache material. Individual bytecode cache subclasses don't have to care about cache invalidation. """ def __init__(self, environment: "Environment", key: str, checksum: str) -> None: self.environment = environment self.key = key self.checksum = checksum self.reset() def reset(self) -> None: """Resets the bucket (unloads the bytecode).""" self.code: t.Optional[CodeType] = None def load_bytecode(self, f: t.BinaryIO) -> None: """Loads bytecode from a file or file like object.""" # make sure the magic header is correct magic = f.read(len(bc_magic)) if magic != bc_magic: self.reset() return # the source code of the file changed, we need to reload checksum = pickle.load(f) if self.checksum != checksum: self.reset() return # if marshal_load fails then we need to reload try: self.code = marshal.load(f) except (EOFError, ValueError, TypeError): self.reset() return def write_bytecode(self, f: t.BinaryIO) -> None: """Dump the bytecode into the file or file like object passed.""" if self.code is None: raise TypeError("can't write empty bucket") f.write(bc_magic) pickle.dump(self.checksum, f, 2) marshal.dump(self.code, f) def bytecode_from_string(self, string: bytes) -> None: """Load bytecode from bytes.""" self.load_bytecode(BytesIO(string)) def bytecode_to_string(self) -> bytes: """Return the bytecode as bytes.""" out = BytesIO() self.write_bytecode(out) return out.getvalue() class BytecodeCache: """To implement your own bytecode cache you have to subclass this class and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of these methods are passed a :class:`~spack.vendor.jinja2.bccache.Bucket`. A very basic bytecode cache that saves the bytecode on the file system:: from os import path class MyCache(BytecodeCache): def __init__(self, directory): self.directory = directory def load_bytecode(self, bucket): filename = path.join(self.directory, bucket.key) if path.exists(filename): with open(filename, 'rb') as f: bucket.load_bytecode(f) def dump_bytecode(self, bucket): filename = path.join(self.directory, bucket.key) with open(filename, 'wb') as f: bucket.write_bytecode(f) A more advanced version of a filesystem based bytecode cache is part of Jinja. """ def load_bytecode(self, bucket: Bucket) -> None: """Subclasses have to override this method to load bytecode into a bucket. If they are not able to find code in the cache for the bucket, it must not do anything. """ raise NotImplementedError() def dump_bytecode(self, bucket: Bucket) -> None: """Subclasses have to override this method to write the bytecode from a bucket back to the cache. If it unable to do so it must not fail silently but raise an exception. """ raise NotImplementedError() def clear(self) -> None: """Clears the cache. This method is not used by Jinja but should be implemented to allow applications to clear the bytecode cache used by a particular environment. """ def get_cache_key( self, name: str, filename: t.Optional[t.Union[str]] = None ) -> str: """Returns the unique hash key for this template name.""" hash = sha1(name.encode("utf-8")) if filename is not None: hash.update(f"|{filename}".encode()) return hash.hexdigest() def get_source_checksum(self, source: str) -> str: """Returns a checksum for the source.""" return sha1(source.encode("utf-8")).hexdigest() def get_bucket( self, environment: "Environment", name: str, filename: t.Optional[str], source: str, ) -> Bucket: """Return a cache bucket for the given template. All arguments are mandatory but filename may be `None`. """ key = self.get_cache_key(name, filename) checksum = self.get_source_checksum(source) bucket = Bucket(environment, key, checksum) self.load_bytecode(bucket) return bucket def set_bucket(self, bucket: Bucket) -> None: """Put the bucket into the cache.""" self.dump_bytecode(bucket) class FileSystemBytecodeCache(BytecodeCache): """A bytecode cache that stores bytecode on the filesystem. It accepts two arguments: The directory where the cache items are stored and a pattern string that is used to build the filename. If no directory is specified a default cache directory is selected. On Windows the user's temp directory is used, on UNIX systems a directory is created for the user in the system temp directory. The pattern can be used to have multiple separate caches operate on the same directory. The default pattern is ``'__spack.vendor.jinja2_%s.cache'``. ``%s`` is replaced with the cache key. >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') This bytecode cache supports clearing of the cache using the clear method. """ def __init__( self, directory: t.Optional[str] = None, pattern: str = "__spack.vendor.jinja2_%s.cache" ) -> None: if directory is None: directory = self._get_default_cache_dir() self.directory = directory self.pattern = pattern def _get_default_cache_dir(self) -> str: def _unsafe_dir() -> "te.NoReturn": raise RuntimeError( "Cannot determine safe temp directory. You " "need to explicitly provide one." ) tmpdir = tempfile.gettempdir() # On windows the temporary directory is used specific unless # explicitly forced otherwise. We can just use that. if os.name == "nt": return tmpdir if not hasattr(os, "getuid"): _unsafe_dir() dirname = f"_spack.vendor.jinja2-cache-{os.getuid()}" actual_dir = os.path.join(tmpdir, dirname) try: os.mkdir(actual_dir, stat.S_IRWXU) except OSError as e: if e.errno != errno.EEXIST: raise try: os.chmod(actual_dir, stat.S_IRWXU) actual_dir_stat = os.lstat(actual_dir) if ( actual_dir_stat.st_uid != os.getuid() or not stat.S_ISDIR(actual_dir_stat.st_mode) or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU ): _unsafe_dir() except OSError as e: if e.errno != errno.EEXIST: raise actual_dir_stat = os.lstat(actual_dir) if ( actual_dir_stat.st_uid != os.getuid() or not stat.S_ISDIR(actual_dir_stat.st_mode) or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU ): _unsafe_dir() return actual_dir def _get_cache_filename(self, bucket: Bucket) -> str: return os.path.join(self.directory, self.pattern % (bucket.key,)) def load_bytecode(self, bucket: Bucket) -> None: filename = self._get_cache_filename(bucket) if os.path.exists(filename): with open(filename, "rb") as f: bucket.load_bytecode(f) def dump_bytecode(self, bucket: Bucket) -> None: with open(self._get_cache_filename(bucket), "wb") as f: bucket.write_bytecode(f) def clear(self) -> None: # imported lazily here because google app-engine doesn't support # write access on the file system and the function does not exist # normally. from os import remove files = fnmatch.filter(os.listdir(self.directory), self.pattern % ("*",)) for filename in files: try: remove(os.path.join(self.directory, filename)) except OSError: pass class MemcachedBytecodeCache(BytecodeCache): """This class implements a bytecode cache that uses a memcache cache for storing the information. It does not enforce a specific memcache library (tummy's memcache or cmemcache) but will accept any class that provides the minimal interface required. Libraries compatible with this class: - `cachelib `_ - `python-memcached `_ (Unfortunately the django cache interface is not compatible because it does not support storing binary data, only text. You can however pass the underlying cache client to the bytecode cache which is available as `django.core.cache.cache._client`.) The minimal interface for the client passed to the constructor is this: .. class:: MinimalClientInterface .. method:: set(key, value[, timeout]) Stores the bytecode in the cache. `value` is a string and `timeout` the timeout of the key. If timeout is not provided a default timeout or no timeout should be assumed, if it's provided it's an integer with the number of seconds the cache item should exist. .. method:: get(key) Returns the value for the cache key. If the item does not exist in the cache the return value must be `None`. The other arguments to the constructor are the prefix for all keys that is added before the actual cache key and the timeout for the bytecode in the cache system. We recommend a high (or no) timeout. This bytecode cache does not support clearing of used items in the cache. The clear method is a no-operation function. .. versionadded:: 2.7 Added support for ignoring memcache errors through the `ignore_memcache_errors` parameter. """ def __init__( self, client: "_MemcachedClient", prefix: str = "spack.vendor.jinja2/bytecode/", timeout: t.Optional[int] = None, ignore_memcache_errors: bool = True, ): self.client = client self.prefix = prefix self.timeout = timeout self.ignore_memcache_errors = ignore_memcache_errors def load_bytecode(self, bucket: Bucket) -> None: try: code = self.client.get(self.prefix + bucket.key) except Exception: if not self.ignore_memcache_errors: raise else: bucket.bytecode_from_string(code) def dump_bytecode(self, bucket: Bucket) -> None: key = self.prefix + bucket.key value = bucket.bytecode_to_string() try: if self.timeout is not None: self.client.set(key, value, self.timeout) else: self.client.set(key, value) except Exception: if not self.ignore_memcache_errors: raise ================================================ FILE: lib/spack/spack/vendor/jinja2/compiler.py ================================================ """Compiles nodes from the parser into Python code.""" import typing as t from contextlib import contextmanager from functools import update_wrapper from io import StringIO from itertools import chain from keyword import iskeyword as is_python_keyword from spack.vendor.markupsafe import escape from spack.vendor.markupsafe import Markup from . import nodes from .exceptions import TemplateAssertionError from .idtracking import Symbols from .idtracking import VAR_LOAD_ALIAS from .idtracking import VAR_LOAD_PARAMETER from .idtracking import VAR_LOAD_RESOLVE from .idtracking import VAR_LOAD_UNDEFINED from .nodes import EvalContext from .optimizer import Optimizer from .utils import _PassArg from .utils import concat from .visitor import NodeVisitor if t.TYPE_CHECKING: import spack.vendor.typing_extensions as te from .environment import Environment F = t.TypeVar("F", bound=t.Callable[..., t.Any]) operators = { "eq": "==", "ne": "!=", "gt": ">", "gteq": ">=", "lt": "<", "lteq": "<=", "in": "in", "notin": "not in", } def optimizeconst(f: F) -> F: def new_func( self: "CodeGenerator", node: nodes.Expr, frame: "Frame", **kwargs: t.Any ) -> t.Any: # Only optimize if the frame is not volatile if self.optimizer is not None and not frame.eval_ctx.volatile: new_node = self.optimizer.visit(node, frame.eval_ctx) if new_node != node: return self.visit(new_node, frame) return f(self, node, frame, **kwargs) return update_wrapper(t.cast(F, new_func), f) def _make_binop(op: str) -> t.Callable[["CodeGenerator", nodes.BinExpr, "Frame"], None]: @optimizeconst def visitor(self: "CodeGenerator", node: nodes.BinExpr, frame: Frame) -> None: if ( self.environment.sandboxed and op in self.environment.intercepted_binops # type: ignore ): self.write(f"environment.call_binop(context, {op!r}, ") self.visit(node.left, frame) self.write(", ") self.visit(node.right, frame) else: self.write("(") self.visit(node.left, frame) self.write(f" {op} ") self.visit(node.right, frame) self.write(")") return visitor def _make_unop( op: str, ) -> t.Callable[["CodeGenerator", nodes.UnaryExpr, "Frame"], None]: @optimizeconst def visitor(self: "CodeGenerator", node: nodes.UnaryExpr, frame: Frame) -> None: if ( self.environment.sandboxed and op in self.environment.intercepted_unops # type: ignore ): self.write(f"environment.call_unop(context, {op!r}, ") self.visit(node.node, frame) else: self.write("(" + op) self.visit(node.node, frame) self.write(")") return visitor def generate( node: nodes.Template, environment: "Environment", name: t.Optional[str], filename: t.Optional[str], stream: t.Optional[t.TextIO] = None, defer_init: bool = False, optimized: bool = True, ) -> t.Optional[str]: """Generate the python source for a node tree.""" if not isinstance(node, nodes.Template): raise TypeError("Can't compile non template nodes") generator = environment.code_generator_class( environment, name, filename, stream, defer_init, optimized ) generator.visit(node) if stream is None: return generator.stream.getvalue() # type: ignore return None def has_safe_repr(value: t.Any) -> bool: """Does the node have a safe representation?""" if value is None or value is NotImplemented or value is Ellipsis: return True if type(value) in {bool, int, float, complex, range, str, Markup}: return True if type(value) in {tuple, list, set, frozenset}: return all(has_safe_repr(v) for v in value) if type(value) is dict: return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items()) return False def find_undeclared( nodes: t.Iterable[nodes.Node], names: t.Iterable[str] ) -> t.Set[str]: """Check if the names passed are accessed undeclared. The return value is a set of all the undeclared names from the sequence of names found. """ visitor = UndeclaredNameVisitor(names) try: for node in nodes: visitor.visit(node) except VisitorExit: pass return visitor.undeclared class MacroRef: def __init__(self, node: t.Union[nodes.Macro, nodes.CallBlock]) -> None: self.node = node self.accesses_caller = False self.accesses_kwargs = False self.accesses_varargs = False class Frame: """Holds compile time information for us.""" def __init__( self, eval_ctx: EvalContext, parent: t.Optional["Frame"] = None, level: t.Optional[int] = None, ) -> None: self.eval_ctx = eval_ctx # the parent of this frame self.parent = parent if parent is None: self.symbols = Symbols(level=level) # in some dynamic inheritance situations the compiler needs to add # write tests around output statements. self.require_output_check = False # inside some tags we are using a buffer rather than yield statements. # this for example affects {% filter %} or {% macro %}. If a frame # is buffered this variable points to the name of the list used as # buffer. self.buffer: t.Optional[str] = None # the name of the block we're in, otherwise None. self.block: t.Optional[str] = None else: self.symbols = Symbols(parent.symbols, level=level) self.require_output_check = parent.require_output_check self.buffer = parent.buffer self.block = parent.block # a toplevel frame is the root + soft frames such as if conditions. self.toplevel = False # the root frame is basically just the outermost frame, so no if # conditions. This information is used to optimize inheritance # situations. self.rootlevel = False # variables set inside of loops and blocks should not affect outer frames, # but they still needs to be kept track of as part of the active context. self.loop_frame = False self.block_frame = False # track whether the frame is being used in an if-statement or conditional # expression as it determines which errors should be raised during runtime # or compile time. self.soft_frame = False def copy(self) -> "Frame": """Create a copy of the current one.""" rv = t.cast(Frame, object.__new__(self.__class__)) rv.__dict__.update(self.__dict__) rv.symbols = self.symbols.copy() return rv def inner(self, isolated: bool = False) -> "Frame": """Return an inner frame.""" if isolated: return Frame(self.eval_ctx, level=self.symbols.level + 1) return Frame(self.eval_ctx, self) def soft(self) -> "Frame": """Return a soft frame. A soft frame may not be modified as standalone thing as it shares the resources with the frame it was created of, but it's not a rootlevel frame any longer. This is only used to implement if-statements and conditional expressions. """ rv = self.copy() rv.rootlevel = False rv.soft_frame = True return rv __copy__ = copy class VisitorExit(RuntimeError): """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" class DependencyFinderVisitor(NodeVisitor): """A visitor that collects filter and test calls.""" def __init__(self) -> None: self.filters: t.Set[str] = set() self.tests: t.Set[str] = set() def visit_Filter(self, node: nodes.Filter) -> None: self.generic_visit(node) self.filters.add(node.name) def visit_Test(self, node: nodes.Test) -> None: self.generic_visit(node) self.tests.add(node.name) def visit_Block(self, node: nodes.Block) -> None: """Stop visiting at blocks.""" class UndeclaredNameVisitor(NodeVisitor): """A visitor that checks if a name is accessed without being declared. This is different from the frame visitor as it will not stop at closure frames. """ def __init__(self, names: t.Iterable[str]) -> None: self.names = set(names) self.undeclared: t.Set[str] = set() def visit_Name(self, node: nodes.Name) -> None: if node.ctx == "load" and node.name in self.names: self.undeclared.add(node.name) if self.undeclared == self.names: raise VisitorExit() else: self.names.discard(node.name) def visit_Block(self, node: nodes.Block) -> None: """Stop visiting a blocks.""" class CompilerExit(Exception): """Raised if the compiler encountered a situation where it just doesn't make sense to further process the code. Any block that raises such an exception is not further processed. """ class CodeGenerator(NodeVisitor): def __init__( self, environment: "Environment", name: t.Optional[str], filename: t.Optional[str], stream: t.Optional[t.TextIO] = None, defer_init: bool = False, optimized: bool = True, ) -> None: if stream is None: stream = StringIO() self.environment = environment self.name = name self.filename = filename self.stream = stream self.created_block_context = False self.defer_init = defer_init self.optimizer: t.Optional[Optimizer] = None if optimized: self.optimizer = Optimizer(environment) # aliases for imports self.import_aliases: t.Dict[str, str] = {} # a registry for all blocks. Because blocks are moved out # into the global python scope they are registered here self.blocks: t.Dict[str, nodes.Block] = {} # the number of extends statements so far self.extends_so_far = 0 # some templates have a rootlevel extends. In this case we # can safely assume that we're a child template and do some # more optimizations. self.has_known_extends = False # the current line number self.code_lineno = 1 # registry of all filters and tests (global, not block local) self.tests: t.Dict[str, str] = {} self.filters: t.Dict[str, str] = {} # the debug information self.debug_info: t.List[t.Tuple[int, int]] = [] self._write_debug_info: t.Optional[int] = None # the number of new lines before the next write() self._new_lines = 0 # the line number of the last written statement self._last_line = 0 # true if nothing was written so far. self._first_write = True # used by the `temporary_identifier` method to get new # unique, temporary identifier self._last_identifier = 0 # the current indentation self._indentation = 0 # Tracks toplevel assignments self._assign_stack: t.List[t.Set[str]] = [] # Tracks parameter definition blocks self._param_def_block: t.List[t.Set[str]] = [] # Tracks the current context. self._context_reference_stack = ["context"] @property def optimized(self) -> bool: return self.optimizer is not None # -- Various compilation helpers def fail(self, msg: str, lineno: int) -> "te.NoReturn": """Fail with a :exc:`TemplateAssertionError`.""" raise TemplateAssertionError(msg, lineno, self.name, self.filename) def temporary_identifier(self) -> str: """Get a new unique identifier.""" self._last_identifier += 1 return f"t_{self._last_identifier}" def buffer(self, frame: Frame) -> None: """Enable buffering for the frame from that point onwards.""" frame.buffer = self.temporary_identifier() self.writeline(f"{frame.buffer} = []") def return_buffer_contents( self, frame: Frame, force_unescaped: bool = False ) -> None: """Return the buffer contents of the frame.""" if not force_unescaped: if frame.eval_ctx.volatile: self.writeline("if context.eval_ctx.autoescape:") self.indent() self.writeline(f"return Markup(concat({frame.buffer}))") self.outdent() self.writeline("else:") self.indent() self.writeline(f"return concat({frame.buffer})") self.outdent() return elif frame.eval_ctx.autoescape: self.writeline(f"return Markup(concat({frame.buffer}))") return self.writeline(f"return concat({frame.buffer})") def indent(self) -> None: """Indent by one.""" self._indentation += 1 def outdent(self, step: int = 1) -> None: """Outdent by step.""" self._indentation -= step def start_write(self, frame: Frame, node: t.Optional[nodes.Node] = None) -> None: """Yield or write into the frame buffer.""" if frame.buffer is None: self.writeline("yield ", node) else: self.writeline(f"{frame.buffer}.append(", node) def end_write(self, frame: Frame) -> None: """End the writing process started by `start_write`.""" if frame.buffer is not None: self.write(")") def simple_write( self, s: str, frame: Frame, node: t.Optional[nodes.Node] = None ) -> None: """Simple shortcut for start_write + write + end_write.""" self.start_write(frame, node) self.write(s) self.end_write(frame) def blockvisit(self, nodes: t.Iterable[nodes.Node], frame: Frame) -> None: """Visit a list of nodes as block in a frame. If the current frame is no buffer a dummy ``if 0: yield None`` is written automatically. """ try: self.writeline("pass") for node in nodes: self.visit(node, frame) except CompilerExit: pass def write(self, x: str) -> None: """Write a string into the output stream.""" if self._new_lines: if not self._first_write: self.stream.write("\n" * self._new_lines) self.code_lineno += self._new_lines if self._write_debug_info is not None: self.debug_info.append((self._write_debug_info, self.code_lineno)) self._write_debug_info = None self._first_write = False self.stream.write(" " * self._indentation) self._new_lines = 0 self.stream.write(x) def writeline( self, x: str, node: t.Optional[nodes.Node] = None, extra: int = 0 ) -> None: """Combination of newline and write.""" self.newline(node, extra) self.write(x) def newline(self, node: t.Optional[nodes.Node] = None, extra: int = 0) -> None: """Add one or more newlines before the next write.""" self._new_lines = max(self._new_lines, 1 + extra) if node is not None and node.lineno != self._last_line: self._write_debug_info = node.lineno self._last_line = node.lineno def signature( self, node: t.Union[nodes.Call, nodes.Filter, nodes.Test], frame: Frame, extra_kwargs: t.Optional[t.Mapping[str, t.Any]] = None, ) -> None: """Writes a function call to the stream for the current node. A leading comma is added automatically. The extra keyword arguments may not include python keywords otherwise a syntax error could occur. The extra keyword arguments should be given as python dict. """ # if any of the given keyword arguments is a python keyword # we have to make sure that no invalid call is created. kwarg_workaround = any( is_python_keyword(t.cast(str, k)) for k in chain((x.key for x in node.kwargs), extra_kwargs or ()) ) for arg in node.args: self.write(", ") self.visit(arg, frame) if not kwarg_workaround: for kwarg in node.kwargs: self.write(", ") self.visit(kwarg, frame) if extra_kwargs is not None: for key, value in extra_kwargs.items(): self.write(f", {key}={value}") if node.dyn_args: self.write(", *") self.visit(node.dyn_args, frame) if kwarg_workaround: if node.dyn_kwargs is not None: self.write(", **dict({") else: self.write(", **{") for kwarg in node.kwargs: self.write(f"{kwarg.key!r}: ") self.visit(kwarg.value, frame) self.write(", ") if extra_kwargs is not None: for key, value in extra_kwargs.items(): self.write(f"{key!r}: {value}, ") if node.dyn_kwargs is not None: self.write("}, **") self.visit(node.dyn_kwargs, frame) self.write(")") else: self.write("}") elif node.dyn_kwargs is not None: self.write(", **") self.visit(node.dyn_kwargs, frame) def pull_dependencies(self, nodes: t.Iterable[nodes.Node]) -> None: """Find all filter and test names used in the template and assign them to variables in the compiled namespace. Checking that the names are registered with the environment is done when compiling the Filter and Test nodes. If the node is in an If or CondExpr node, the check is done at runtime instead. .. versionchanged:: 3.0 Filters and tests in If and CondExpr nodes are checked at runtime instead of compile time. """ visitor = DependencyFinderVisitor() for node in nodes: visitor.visit(node) for id_map, names, dependency in (self.filters, visitor.filters, "filters"), ( self.tests, visitor.tests, "tests", ): for name in sorted(names): if name not in id_map: id_map[name] = self.temporary_identifier() # add check during runtime that dependencies used inside of executed # blocks are defined, as this step may be skipped during compile time self.writeline("try:") self.indent() self.writeline(f"{id_map[name]} = environment.{dependency}[{name!r}]") self.outdent() self.writeline("except KeyError:") self.indent() self.writeline("@internalcode") self.writeline(f"def {id_map[name]}(*unused):") self.indent() self.writeline( f'raise TemplateRuntimeError("No {dependency[:-1]}' f' named {name!r} found.")' ) self.outdent() self.outdent() def enter_frame(self, frame: Frame) -> None: undefs = [] for target, (action, param) in frame.symbols.loads.items(): if action == VAR_LOAD_PARAMETER: pass elif action == VAR_LOAD_RESOLVE: self.writeline(f"{target} = {self.get_resolve_func()}({param!r})") elif action == VAR_LOAD_ALIAS: self.writeline(f"{target} = {param}") elif action == VAR_LOAD_UNDEFINED: undefs.append(target) else: raise NotImplementedError("unknown load instruction") if undefs: self.writeline(f"{' = '.join(undefs)} = missing") def leave_frame(self, frame: Frame, with_python_scope: bool = False) -> None: if not with_python_scope: undefs = [] for target in frame.symbols.loads: undefs.append(target) if undefs: self.writeline(f"{' = '.join(undefs)} = missing") def choose_async(self, async_value: str = "async ", sync_value: str = "") -> str: return async_value if self.environment.is_async else sync_value def func(self, name: str) -> str: return f"{self.choose_async()}def {name}" def macro_body( self, node: t.Union[nodes.Macro, nodes.CallBlock], frame: Frame ) -> t.Tuple[Frame, MacroRef]: """Dump the function def of a macro or call block.""" frame = frame.inner() frame.symbols.analyze_node(node) macro_ref = MacroRef(node) explicit_caller = None skip_special_params = set() args = [] for idx, arg in enumerate(node.args): if arg.name == "caller": explicit_caller = idx if arg.name in ("kwargs", "varargs"): skip_special_params.add(arg.name) args.append(frame.symbols.ref(arg.name)) undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) if "caller" in undeclared: # In older Jinja versions there was a bug that allowed caller # to retain the special behavior even if it was mentioned in # the argument list. However thankfully this was only really # working if it was the last argument. So we are explicitly # checking this now and error out if it is anywhere else in # the argument list. if explicit_caller is not None: try: node.defaults[explicit_caller - len(node.args)] except IndexError: self.fail( "When defining macros or call blocks the " 'special "caller" argument must be omitted ' "or be given a default.", node.lineno, ) else: args.append(frame.symbols.declare_parameter("caller")) macro_ref.accesses_caller = True if "kwargs" in undeclared and "kwargs" not in skip_special_params: args.append(frame.symbols.declare_parameter("kwargs")) macro_ref.accesses_kwargs = True if "varargs" in undeclared and "varargs" not in skip_special_params: args.append(frame.symbols.declare_parameter("varargs")) macro_ref.accesses_varargs = True # macros are delayed, they never require output checks frame.require_output_check = False frame.symbols.analyze_node(node) self.writeline(f"{self.func('macro')}({', '.join(args)}):", node) self.indent() self.buffer(frame) self.enter_frame(frame) self.push_parameter_definitions(frame) for idx, arg in enumerate(node.args): ref = frame.symbols.ref(arg.name) self.writeline(f"if {ref} is missing:") self.indent() try: default = node.defaults[idx - len(node.args)] except IndexError: self.writeline( f'{ref} = undefined("parameter {arg.name!r} was not provided",' f" name={arg.name!r})" ) else: self.writeline(f"{ref} = ") self.visit(default, frame) self.mark_parameter_stored(ref) self.outdent() self.pop_parameter_definitions() self.blockvisit(node.body, frame) self.return_buffer_contents(frame, force_unescaped=True) self.leave_frame(frame, with_python_scope=True) self.outdent() return frame, macro_ref def macro_def(self, macro_ref: MacroRef, frame: Frame) -> None: """Dump the macro definition for the def created by macro_body.""" arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) name = getattr(macro_ref.node, "name", None) if len(macro_ref.node.args) == 1: arg_tuple += "," self.write( f"Macro(environment, macro, {name!r}, ({arg_tuple})," f" {macro_ref.accesses_kwargs!r}, {macro_ref.accesses_varargs!r}," f" {macro_ref.accesses_caller!r}, context.eval_ctx.autoescape)" ) def position(self, node: nodes.Node) -> str: """Return a human readable position for the node.""" rv = f"line {node.lineno}" if self.name is not None: rv = f"{rv} in {self.name!r}" return rv def dump_local_context(self, frame: Frame) -> str: items_kv = ", ".join( f"{name!r}: {target}" for name, target in frame.symbols.dump_stores().items() ) return f"{{{items_kv}}}" def write_commons(self) -> None: """Writes a common preamble that is used by root and block functions. Primarily this sets up common local helpers and enforces a generator through a dead branch. """ self.writeline("resolve = context.resolve_or_missing") self.writeline("undefined = environment.undefined") # always use the standard Undefined class for the implicit else of # conditional expressions self.writeline("cond_expr_undefined = Undefined") self.writeline("if 0: yield None") def push_parameter_definitions(self, frame: Frame) -> None: """Pushes all parameter targets from the given frame into a local stack that permits tracking of yet to be assigned parameters. In particular this enables the optimization from `visit_Name` to skip undefined expressions for parameters in macros as macros can reference otherwise unbound parameters. """ self._param_def_block.append(frame.symbols.dump_param_targets()) def pop_parameter_definitions(self) -> None: """Pops the current parameter definitions set.""" self._param_def_block.pop() def mark_parameter_stored(self, target: str) -> None: """Marks a parameter in the current parameter definitions as stored. This will skip the enforced undefined checks. """ if self._param_def_block: self._param_def_block[-1].discard(target) def push_context_reference(self, target: str) -> None: self._context_reference_stack.append(target) def pop_context_reference(self) -> None: self._context_reference_stack.pop() def get_context_ref(self) -> str: return self._context_reference_stack[-1] def get_resolve_func(self) -> str: target = self._context_reference_stack[-1] if target == "context": return "resolve" return f"{target}.resolve" def derive_context(self, frame: Frame) -> str: return f"{self.get_context_ref()}.derived({self.dump_local_context(frame)})" def parameter_is_undeclared(self, target: str) -> bool: """Checks if a given target is an undeclared parameter.""" if not self._param_def_block: return False return target in self._param_def_block[-1] def push_assign_tracking(self) -> None: """Pushes a new layer for assignment tracking.""" self._assign_stack.append(set()) def pop_assign_tracking(self, frame: Frame) -> None: """Pops the topmost level for assignment tracking and updates the context variables if necessary. """ vars = self._assign_stack.pop() if ( not frame.block_frame and not frame.loop_frame and not frame.toplevel or not vars ): return public_names = [x for x in vars if x[:1] != "_"] if len(vars) == 1: name = next(iter(vars)) ref = frame.symbols.ref(name) if frame.loop_frame: self.writeline(f"_loop_vars[{name!r}] = {ref}") return if frame.block_frame: self.writeline(f"_block_vars[{name!r}] = {ref}") return self.writeline(f"context.vars[{name!r}] = {ref}") else: if frame.loop_frame: self.writeline("_loop_vars.update({") elif frame.block_frame: self.writeline("_block_vars.update({") else: self.writeline("context.vars.update({") for idx, name in enumerate(vars): if idx: self.write(", ") ref = frame.symbols.ref(name) self.write(f"{name!r}: {ref}") self.write("})") if not frame.block_frame and not frame.loop_frame and public_names: if len(public_names) == 1: self.writeline(f"context.exported_vars.add({public_names[0]!r})") else: names_str = ", ".join(map(repr, public_names)) self.writeline(f"context.exported_vars.update(({names_str}))") # -- Statement Visitors def visit_Template( self, node: nodes.Template, frame: t.Optional[Frame] = None ) -> None: assert frame is None, "no root frame allowed" eval_ctx = EvalContext(self.environment, self.name) from .runtime import exported, async_exported if self.environment.is_async: exported_names = sorted(exported + async_exported) else: exported_names = sorted(exported) self.writeline("from __future__ import generator_stop") # Python < 3.7 self.writeline("from spack.vendor.jinja2.runtime import " + ", ".join(exported_names)) # if we want a deferred initialization we cannot move the # environment into a local name envenv = "" if self.defer_init else ", environment=environment" # do we have an extends tag at all? If not, we can save some # overhead by just not processing any inheritance code. have_extends = node.find(nodes.Extends) is not None # find all blocks for block in node.find_all(nodes.Block): if block.name in self.blocks: self.fail(f"block {block.name!r} defined twice", block.lineno) self.blocks[block.name] = block # find all imports and import them for import_ in node.find_all(nodes.ImportedName): if import_.importname not in self.import_aliases: imp = import_.importname self.import_aliases[imp] = alias = self.temporary_identifier() if "." in imp: module, obj = imp.rsplit(".", 1) self.writeline(f"from {module} import {obj} as {alias}") else: self.writeline(f"import {imp} as {alias}") # add the load name self.writeline(f"name = {self.name!r}") # generate the root render function. self.writeline( f"{self.func('root')}(context, missing=missing{envenv}):", extra=1 ) self.indent() self.write_commons() # process the root frame = Frame(eval_ctx) if "self" in find_undeclared(node.body, ("self",)): ref = frame.symbols.declare_parameter("self") self.writeline(f"{ref} = TemplateReference(context)") frame.symbols.analyze_node(node) frame.toplevel = frame.rootlevel = True frame.require_output_check = have_extends and not self.has_known_extends if have_extends: self.writeline("parent_template = None") self.enter_frame(frame) self.pull_dependencies(node.body) self.blockvisit(node.body, frame) self.leave_frame(frame, with_python_scope=True) self.outdent() # make sure that the parent root is called. if have_extends: if not self.has_known_extends: self.indent() self.writeline("if parent_template is not None:") self.indent() if not self.environment.is_async: self.writeline("yield from parent_template.root_render_func(context)") else: self.writeline( "async for event in parent_template.root_render_func(context):" ) self.indent() self.writeline("yield event") self.outdent() self.outdent(1 + (not self.has_known_extends)) # at this point we now have the blocks collected and can visit them too. for name, block in self.blocks.items(): self.writeline( f"{self.func('block_' + name)}(context, missing=missing{envenv}):", block, 1, ) self.indent() self.write_commons() # It's important that we do not make this frame a child of the # toplevel template. This would cause a variety of # interesting issues with identifier tracking. block_frame = Frame(eval_ctx) block_frame.block_frame = True undeclared = find_undeclared(block.body, ("self", "super")) if "self" in undeclared: ref = block_frame.symbols.declare_parameter("self") self.writeline(f"{ref} = TemplateReference(context)") if "super" in undeclared: ref = block_frame.symbols.declare_parameter("super") self.writeline(f"{ref} = context.super({name!r}, block_{name})") block_frame.symbols.analyze_node(block) block_frame.block = name self.writeline("_block_vars = {}") self.enter_frame(block_frame) self.pull_dependencies(block.body) self.blockvisit(block.body, block_frame) self.leave_frame(block_frame, with_python_scope=True) self.outdent() blocks_kv_str = ", ".join(f"{x!r}: block_{x}" for x in self.blocks) self.writeline(f"blocks = {{{blocks_kv_str}}}", extra=1) debug_kv_str = "&".join(f"{k}={v}" for k, v in self.debug_info) self.writeline(f"debug_info = {debug_kv_str!r}") def visit_Block(self, node: nodes.Block, frame: Frame) -> None: """Call a block and register it for the template.""" level = 0 if frame.toplevel: # if we know that we are a child template, there is no need to # check if we are one if self.has_known_extends: return if self.extends_so_far > 0: self.writeline("if parent_template is None:") self.indent() level += 1 if node.scoped: context = self.derive_context(frame) else: context = self.get_context_ref() if node.required: self.writeline(f"if len(context.blocks[{node.name!r}]) <= 1:", node) self.indent() self.writeline( f'raise TemplateRuntimeError("Required block {node.name!r} not found")', node, ) self.outdent() if not self.environment.is_async and frame.buffer is None: self.writeline( f"yield from context.blocks[{node.name!r}][0]({context})", node ) else: self.writeline( f"{self.choose_async()}for event in" f" context.blocks[{node.name!r}][0]({context}):", node, ) self.indent() self.simple_write("event", frame) self.outdent() self.outdent(level) def visit_Extends(self, node: nodes.Extends, frame: Frame) -> None: """Calls the extender.""" if not frame.toplevel: self.fail("cannot use extend from a non top-level scope", node.lineno) # if the number of extends statements in general is zero so # far, we don't have to add a check if something extended # the template before this one. if self.extends_so_far > 0: # if we have a known extends we just add a template runtime # error into the generated code. We could catch that at compile # time too, but i welcome it not to confuse users by throwing the # same error at different times just "because we can". if not self.has_known_extends: self.writeline("if parent_template is not None:") self.indent() self.writeline('raise TemplateRuntimeError("extended multiple times")') # if we have a known extends already we don't need that code here # as we know that the template execution will end here. if self.has_known_extends: raise CompilerExit() else: self.outdent() self.writeline("parent_template = environment.get_template(", node) self.visit(node.template, frame) self.write(f", {self.name!r})") self.writeline("for name, parent_block in parent_template.blocks.items():") self.indent() self.writeline("context.blocks.setdefault(name, []).append(parent_block)") self.outdent() # if this extends statement was in the root level we can take # advantage of that information and simplify the generated code # in the top level from this point onwards if frame.rootlevel: self.has_known_extends = True # and now we have one more self.extends_so_far += 1 def visit_Include(self, node: nodes.Include, frame: Frame) -> None: """Handles includes.""" if node.ignore_missing: self.writeline("try:") self.indent() func_name = "get_or_select_template" if isinstance(node.template, nodes.Const): if isinstance(node.template.value, str): func_name = "get_template" elif isinstance(node.template.value, (tuple, list)): func_name = "select_template" elif isinstance(node.template, (nodes.Tuple, nodes.List)): func_name = "select_template" self.writeline(f"template = environment.{func_name}(", node) self.visit(node.template, frame) self.write(f", {self.name!r})") if node.ignore_missing: self.outdent() self.writeline("except TemplateNotFound:") self.indent() self.writeline("pass") self.outdent() self.writeline("else:") self.indent() skip_event_yield = False if node.with_context: self.writeline( f"{self.choose_async()}for event in template.root_render_func(" "template.new_context(context.get_all(), True," f" {self.dump_local_context(frame)})):" ) elif self.environment.is_async: self.writeline( "for event in (await template._get_default_module_async())" "._body_stream:" ) else: self.writeline("yield from template._get_default_module()._body_stream") skip_event_yield = True if not skip_event_yield: self.indent() self.simple_write("event", frame) self.outdent() if node.ignore_missing: self.outdent() def _import_common( self, node: t.Union[nodes.Import, nodes.FromImport], frame: Frame ) -> None: self.write(f"{self.choose_async('await ')}environment.get_template(") self.visit(node.template, frame) self.write(f", {self.name!r}).") if node.with_context: f_name = f"make_module{self.choose_async('_async')}" self.write( f"{f_name}(context.get_all(), True, {self.dump_local_context(frame)})" ) else: self.write(f"_get_default_module{self.choose_async('_async')}(context)") def visit_Import(self, node: nodes.Import, frame: Frame) -> None: """Visit regular imports.""" self.writeline(f"{frame.symbols.ref(node.target)} = ", node) if frame.toplevel: self.write(f"context.vars[{node.target!r}] = ") self._import_common(node, frame) if frame.toplevel and not node.target.startswith("_"): self.writeline(f"context.exported_vars.discard({node.target!r})") def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None: """Visit named imports.""" self.newline(node) self.write("included_template = ") self._import_common(node, frame) var_names = [] discarded_names = [] for name in node.names: if isinstance(name, tuple): name, alias = name else: alias = name self.writeline( f"{frame.symbols.ref(alias)} =" f" getattr(included_template, {name!r}, missing)" ) self.writeline(f"if {frame.symbols.ref(alias)} is missing:") self.indent() message = ( "the template {included_template.__name__!r}" f" (imported on {self.position(node)})" f" does not export the requested name {name!r}" ) self.writeline( f"{frame.symbols.ref(alias)} = undefined(f{message!r}, name={name!r})" ) self.outdent() if frame.toplevel: var_names.append(alias) if not alias.startswith("_"): discarded_names.append(alias) if var_names: if len(var_names) == 1: name = var_names[0] self.writeline(f"context.vars[{name!r}] = {frame.symbols.ref(name)}") else: names_kv = ", ".join( f"{name!r}: {frame.symbols.ref(name)}" for name in var_names ) self.writeline(f"context.vars.update({{{names_kv}}})") if discarded_names: if len(discarded_names) == 1: self.writeline(f"context.exported_vars.discard({discarded_names[0]!r})") else: names_str = ", ".join(map(repr, discarded_names)) self.writeline( f"context.exported_vars.difference_update(({names_str}))" ) def visit_For(self, node: nodes.For, frame: Frame) -> None: loop_frame = frame.inner() loop_frame.loop_frame = True test_frame = frame.inner() else_frame = frame.inner() # try to figure out if we have an extended loop. An extended loop # is necessary if the loop is in recursive mode if the special loop # variable is accessed in the body if the body is a scoped block. extended_loop = ( node.recursive or "loop" in find_undeclared(node.iter_child_nodes(only=("body",)), ("loop",)) or any(block.scoped for block in node.find_all(nodes.Block)) ) loop_ref = None if extended_loop: loop_ref = loop_frame.symbols.declare_parameter("loop") loop_frame.symbols.analyze_node(node, for_branch="body") if node.else_: else_frame.symbols.analyze_node(node, for_branch="else") if node.test: loop_filter_func = self.temporary_identifier() test_frame.symbols.analyze_node(node, for_branch="test") self.writeline(f"{self.func(loop_filter_func)}(fiter):", node.test) self.indent() self.enter_frame(test_frame) self.writeline(self.choose_async("async for ", "for ")) self.visit(node.target, loop_frame) self.write(" in ") self.write(self.choose_async("auto_aiter(fiter)", "fiter")) self.write(":") self.indent() self.writeline("if ", node.test) self.visit(node.test, test_frame) self.write(":") self.indent() self.writeline("yield ") self.visit(node.target, loop_frame) self.outdent(3) self.leave_frame(test_frame, with_python_scope=True) # if we don't have an recursive loop we have to find the shadowed # variables at that point. Because loops can be nested but the loop # variable is a special one we have to enforce aliasing for it. if node.recursive: self.writeline( f"{self.func('loop')}(reciter, loop_render_func, depth=0):", node ) self.indent() self.buffer(loop_frame) # Use the same buffer for the else frame else_frame.buffer = loop_frame.buffer # make sure the loop variable is a special one and raise a template # assertion error if a loop tries to write to loop if extended_loop: self.writeline(f"{loop_ref} = missing") for name in node.find_all(nodes.Name): if name.ctx == "store" and name.name == "loop": self.fail( "Can't assign to special loop variable in for-loop target", name.lineno, ) if node.else_: iteration_indicator = self.temporary_identifier() self.writeline(f"{iteration_indicator} = 1") self.writeline(self.choose_async("async for ", "for "), node) self.visit(node.target, loop_frame) if extended_loop: self.write(f", {loop_ref} in {self.choose_async('Async')}LoopContext(") else: self.write(" in ") if node.test: self.write(f"{loop_filter_func}(") if node.recursive: self.write("reciter") else: if self.environment.is_async and not extended_loop: self.write("auto_aiter(") self.visit(node.iter, frame) if self.environment.is_async and not extended_loop: self.write(")") if node.test: self.write(")") if node.recursive: self.write(", undefined, loop_render_func, depth):") else: self.write(", undefined):" if extended_loop else ":") self.indent() self.enter_frame(loop_frame) self.writeline("_loop_vars = {}") self.blockvisit(node.body, loop_frame) if node.else_: self.writeline(f"{iteration_indicator} = 0") self.outdent() self.leave_frame( loop_frame, with_python_scope=node.recursive and not node.else_ ) if node.else_: self.writeline(f"if {iteration_indicator}:") self.indent() self.enter_frame(else_frame) self.blockvisit(node.else_, else_frame) self.leave_frame(else_frame) self.outdent() # if the node was recursive we have to return the buffer contents # and start the iteration code if node.recursive: self.return_buffer_contents(loop_frame) self.outdent() self.start_write(frame, node) self.write(f"{self.choose_async('await ')}loop(") if self.environment.is_async: self.write("auto_aiter(") self.visit(node.iter, frame) if self.environment.is_async: self.write(")") self.write(", loop)") self.end_write(frame) # at the end of the iteration, clear any assignments made in the # loop from the top level if self._assign_stack: self._assign_stack[-1].difference_update(loop_frame.symbols.stores) def visit_If(self, node: nodes.If, frame: Frame) -> None: if_frame = frame.soft() self.writeline("if ", node) self.visit(node.test, if_frame) self.write(":") self.indent() self.blockvisit(node.body, if_frame) self.outdent() for elif_ in node.elif_: self.writeline("elif ", elif_) self.visit(elif_.test, if_frame) self.write(":") self.indent() self.blockvisit(elif_.body, if_frame) self.outdent() if node.else_: self.writeline("else:") self.indent() self.blockvisit(node.else_, if_frame) self.outdent() def visit_Macro(self, node: nodes.Macro, frame: Frame) -> None: macro_frame, macro_ref = self.macro_body(node, frame) self.newline() if frame.toplevel: if not node.name.startswith("_"): self.write(f"context.exported_vars.add({node.name!r})") self.writeline(f"context.vars[{node.name!r}] = ") self.write(f"{frame.symbols.ref(node.name)} = ") self.macro_def(macro_ref, macro_frame) def visit_CallBlock(self, node: nodes.CallBlock, frame: Frame) -> None: call_frame, macro_ref = self.macro_body(node, frame) self.writeline("caller = ") self.macro_def(macro_ref, call_frame) self.start_write(frame, node) self.visit_Call(node.call, frame, forward_caller=True) self.end_write(frame) def visit_FilterBlock(self, node: nodes.FilterBlock, frame: Frame) -> None: filter_frame = frame.inner() filter_frame.symbols.analyze_node(node) self.enter_frame(filter_frame) self.buffer(filter_frame) self.blockvisit(node.body, filter_frame) self.start_write(frame, node) self.visit_Filter(node.filter, filter_frame) self.end_write(frame) self.leave_frame(filter_frame) def visit_With(self, node: nodes.With, frame: Frame) -> None: with_frame = frame.inner() with_frame.symbols.analyze_node(node) self.enter_frame(with_frame) for target, expr in zip(node.targets, node.values): self.newline() self.visit(target, with_frame) self.write(" = ") self.visit(expr, frame) self.blockvisit(node.body, with_frame) self.leave_frame(with_frame) def visit_ExprStmt(self, node: nodes.ExprStmt, frame: Frame) -> None: self.newline(node) self.visit(node.node, frame) class _FinalizeInfo(t.NamedTuple): const: t.Optional[t.Callable[..., str]] src: t.Optional[str] @staticmethod def _default_finalize(value: t.Any) -> t.Any: """The default finalize function if the environment isn't configured with one. Or, if the environment has one, this is called on that function's output for constants. """ return str(value) _finalize: t.Optional[_FinalizeInfo] = None def _make_finalize(self) -> _FinalizeInfo: """Build the finalize function to be used on constants and at runtime. Cached so it's only created once for all output nodes. Returns a ``namedtuple`` with the following attributes: ``const`` A function to finalize constant data at compile time. ``src`` Source code to output around nodes to be evaluated at runtime. """ if self._finalize is not None: return self._finalize finalize: t.Optional[t.Callable[..., t.Any]] finalize = default = self._default_finalize src = None if self.environment.finalize: src = "environment.finalize(" env_finalize = self.environment.finalize pass_arg = { _PassArg.context: "context", _PassArg.eval_context: "context.eval_ctx", _PassArg.environment: "environment", }.get( _PassArg.from_obj(env_finalize) # type: ignore ) finalize = None if pass_arg is None: def finalize(value: t.Any) -> t.Any: return default(env_finalize(value)) else: src = f"{src}{pass_arg}, " if pass_arg == "environment": def finalize(value: t.Any) -> t.Any: return default(env_finalize(self.environment, value)) self._finalize = self._FinalizeInfo(finalize, src) return self._finalize def _output_const_repr(self, group: t.Iterable[t.Any]) -> str: """Given a group of constant values converted from ``Output`` child nodes, produce a string to write to the template module source. """ return repr(concat(group)) def _output_child_to_const( self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo ) -> str: """Try to optimize a child of an ``Output`` node by trying to convert it to constant, finalized data at compile time. If :exc:`Impossible` is raised, the node is not constant and will be evaluated at runtime. Any other exception will also be evaluated at runtime for easier debugging. """ const = node.as_const(frame.eval_ctx) if frame.eval_ctx.autoescape: const = escape(const) # Template data doesn't go through finalize. if isinstance(node, nodes.TemplateData): return str(const) return finalize.const(const) # type: ignore def _output_child_pre( self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo ) -> None: """Output extra source code before visiting a child of an ``Output`` node. """ if frame.eval_ctx.volatile: self.write("(escape if context.eval_ctx.autoescape else str)(") elif frame.eval_ctx.autoescape: self.write("escape(") else: self.write("str(") if finalize.src is not None: self.write(finalize.src) def _output_child_post( self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo ) -> None: """Output extra source code after visiting a child of an ``Output`` node. """ self.write(")") if finalize.src is not None: self.write(")") def visit_Output(self, node: nodes.Output, frame: Frame) -> None: # If an extends is active, don't render outside a block. if frame.require_output_check: # A top-level extends is known to exist at compile time. if self.has_known_extends: return self.writeline("if parent_template is None:") self.indent() finalize = self._make_finalize() body: t.List[t.Union[t.List[t.Any], nodes.Expr]] = [] # Evaluate constants at compile time if possible. Each item in # body will be either a list of static data or a node to be # evaluated at runtime. for child in node.nodes: try: if not ( # If the finalize function requires runtime context, # constants can't be evaluated at compile time. finalize.const # Unless it's basic template data that won't be # finalized anyway. or isinstance(child, nodes.TemplateData) ): raise nodes.Impossible() const = self._output_child_to_const(child, frame, finalize) except (nodes.Impossible, Exception): # The node was not constant and needs to be evaluated at # runtime. Or another error was raised, which is easier # to debug at runtime. body.append(child) continue if body and isinstance(body[-1], list): body[-1].append(const) else: body.append([const]) if frame.buffer is not None: if len(body) == 1: self.writeline(f"{frame.buffer}.append(") else: self.writeline(f"{frame.buffer}.extend((") self.indent() for item in body: if isinstance(item, list): # A group of constant data to join and output. val = self._output_const_repr(item) if frame.buffer is None: self.writeline("yield " + val) else: self.writeline(val + ",") else: if frame.buffer is None: self.writeline("yield ", item) else: self.newline(item) # A node to be evaluated at runtime. self._output_child_pre(item, frame, finalize) self.visit(item, frame) self._output_child_post(item, frame, finalize) if frame.buffer is not None: self.write(",") if frame.buffer is not None: self.outdent() self.writeline(")" if len(body) == 1 else "))") if frame.require_output_check: self.outdent() def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: self.push_assign_tracking() self.newline(node) self.visit(node.target, frame) self.write(" = ") self.visit(node.node, frame) self.pop_assign_tracking(frame) def visit_AssignBlock(self, node: nodes.AssignBlock, frame: Frame) -> None: self.push_assign_tracking() block_frame = frame.inner() # This is a special case. Since a set block always captures we # will disable output checks. This way one can use set blocks # toplevel even in extended templates. block_frame.require_output_check = False block_frame.symbols.analyze_node(node) self.enter_frame(block_frame) self.buffer(block_frame) self.blockvisit(node.body, block_frame) self.newline(node) self.visit(node.target, frame) self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") if node.filter is not None: self.visit_Filter(node.filter, block_frame) else: self.write(f"concat({block_frame.buffer})") self.write(")") self.pop_assign_tracking(frame) self.leave_frame(block_frame) # -- Expression Visitors def visit_Name(self, node: nodes.Name, frame: Frame) -> None: if node.ctx == "store" and ( frame.toplevel or frame.loop_frame or frame.block_frame ): if self._assign_stack: self._assign_stack[-1].add(node.name) ref = frame.symbols.ref(node.name) # If we are looking up a variable we might have to deal with the # case where it's undefined. We can skip that case if the load # instruction indicates a parameter which are always defined. if node.ctx == "load": load = frame.symbols.find_load(ref) if not ( load is not None and load[0] == VAR_LOAD_PARAMETER and not self.parameter_is_undeclared(ref) ): self.write( f"(undefined(name={node.name!r}) if {ref} is missing else {ref})" ) return self.write(ref) def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: # NSRefs can only be used to store values; since they use the normal # `foo.bar` notation they will be parsed as a normal attribute access # when used anywhere but in a `set` context ref = frame.symbols.ref(node.name) self.writeline(f"if not isinstance({ref}, Namespace):") self.indent() self.writeline( "raise TemplateRuntimeError" '("cannot assign attribute on non-namespace object")' ) self.outdent() self.writeline(f"{ref}[{node.attr!r}]") def visit_Const(self, node: nodes.Const, frame: Frame) -> None: val = node.as_const(frame.eval_ctx) if isinstance(val, float): self.write(str(val)) else: self.write(repr(val)) def visit_TemplateData(self, node: nodes.TemplateData, frame: Frame) -> None: try: self.write(repr(node.as_const(frame.eval_ctx))) except nodes.Impossible: self.write( f"(Markup if context.eval_ctx.autoescape else identity)({node.data!r})" ) def visit_Tuple(self, node: nodes.Tuple, frame: Frame) -> None: self.write("(") idx = -1 for idx, item in enumerate(node.items): if idx: self.write(", ") self.visit(item, frame) self.write(",)" if idx == 0 else ")") def visit_List(self, node: nodes.List, frame: Frame) -> None: self.write("[") for idx, item in enumerate(node.items): if idx: self.write(", ") self.visit(item, frame) self.write("]") def visit_Dict(self, node: nodes.Dict, frame: Frame) -> None: self.write("{") for idx, item in enumerate(node.items): if idx: self.write(", ") self.visit(item.key, frame) self.write(": ") self.visit(item.value, frame) self.write("}") visit_Add = _make_binop("+") visit_Sub = _make_binop("-") visit_Mul = _make_binop("*") visit_Div = _make_binop("/") visit_FloorDiv = _make_binop("//") visit_Pow = _make_binop("**") visit_Mod = _make_binop("%") visit_And = _make_binop("and") visit_Or = _make_binop("or") visit_Pos = _make_unop("+") visit_Neg = _make_unop("-") visit_Not = _make_unop("not ") @optimizeconst def visit_Concat(self, node: nodes.Concat, frame: Frame) -> None: if frame.eval_ctx.volatile: func_name = "(markup_join if context.eval_ctx.volatile else str_join)" elif frame.eval_ctx.autoescape: func_name = "markup_join" else: func_name = "str_join" self.write(f"{func_name}((") for arg in node.nodes: self.visit(arg, frame) self.write(", ") self.write("))") @optimizeconst def visit_Compare(self, node: nodes.Compare, frame: Frame) -> None: self.write("(") self.visit(node.expr, frame) for op in node.ops: self.visit(op, frame) self.write(")") def visit_Operand(self, node: nodes.Operand, frame: Frame) -> None: self.write(f" {operators[node.op]} ") self.visit(node.expr, frame) @optimizeconst def visit_Getattr(self, node: nodes.Getattr, frame: Frame) -> None: if self.environment.is_async: self.write("(await auto_await(") self.write("environment.getattr(") self.visit(node.node, frame) self.write(f", {node.attr!r})") if self.environment.is_async: self.write("))") @optimizeconst def visit_Getitem(self, node: nodes.Getitem, frame: Frame) -> None: # slices bypass the environment getitem method. if isinstance(node.arg, nodes.Slice): self.visit(node.node, frame) self.write("[") self.visit(node.arg, frame) self.write("]") else: if self.environment.is_async: self.write("(await auto_await(") self.write("environment.getitem(") self.visit(node.node, frame) self.write(", ") self.visit(node.arg, frame) self.write(")") if self.environment.is_async: self.write("))") def visit_Slice(self, node: nodes.Slice, frame: Frame) -> None: if node.start is not None: self.visit(node.start, frame) self.write(":") if node.stop is not None: self.visit(node.stop, frame) if node.step is not None: self.write(":") self.visit(node.step, frame) @contextmanager def _filter_test_common( self, node: t.Union[nodes.Filter, nodes.Test], frame: Frame, is_filter: bool ) -> t.Iterator[None]: if self.environment.is_async: self.write("await auto_await(") if is_filter: self.write(f"{self.filters[node.name]}(") func = self.environment.filters.get(node.name) else: self.write(f"{self.tests[node.name]}(") func = self.environment.tests.get(node.name) # When inside an If or CondExpr frame, allow the filter to be # undefined at compile time and only raise an error if it's # actually called at runtime. See pull_dependencies. if func is None and not frame.soft_frame: type_name = "filter" if is_filter else "test" self.fail(f"No {type_name} named {node.name!r}.", node.lineno) pass_arg = { _PassArg.context: "context", _PassArg.eval_context: "context.eval_ctx", _PassArg.environment: "environment", }.get( _PassArg.from_obj(func) # type: ignore ) if pass_arg is not None: self.write(f"{pass_arg}, ") # Back to the visitor function to handle visiting the target of # the filter or test. yield self.signature(node, frame) self.write(")") if self.environment.is_async: self.write(")") @optimizeconst def visit_Filter(self, node: nodes.Filter, frame: Frame) -> None: with self._filter_test_common(node, frame, True): # if the filter node is None we are inside a filter block # and want to write to the current buffer if node.node is not None: self.visit(node.node, frame) elif frame.eval_ctx.volatile: self.write( f"(Markup(concat({frame.buffer}))" f" if context.eval_ctx.autoescape else concat({frame.buffer}))" ) elif frame.eval_ctx.autoescape: self.write(f"Markup(concat({frame.buffer}))") else: self.write(f"concat({frame.buffer})") @optimizeconst def visit_Test(self, node: nodes.Test, frame: Frame) -> None: with self._filter_test_common(node, frame, False): self.visit(node.node, frame) @optimizeconst def visit_CondExpr(self, node: nodes.CondExpr, frame: Frame) -> None: frame = frame.soft() def write_expr2() -> None: if node.expr2 is not None: self.visit(node.expr2, frame) return self.write( f'cond_expr_undefined("the inline if-expression on' f" {self.position(node)} evaluated to false and no else" f' section was defined.")' ) self.write("(") self.visit(node.expr1, frame) self.write(" if ") self.visit(node.test, frame) self.write(" else ") write_expr2() self.write(")") @optimizeconst def visit_Call( self, node: nodes.Call, frame: Frame, forward_caller: bool = False ) -> None: if self.environment.is_async: self.write("await auto_await(") if self.environment.sandboxed: self.write("environment.call(context, ") else: self.write("context.call(") self.visit(node.node, frame) extra_kwargs = {"caller": "caller"} if forward_caller else None loop_kwargs = {"_loop_vars": "_loop_vars"} if frame.loop_frame else {} block_kwargs = {"_block_vars": "_block_vars"} if frame.block_frame else {} if extra_kwargs: extra_kwargs.update(loop_kwargs, **block_kwargs) elif loop_kwargs or block_kwargs: extra_kwargs = dict(loop_kwargs, **block_kwargs) self.signature(node, frame, extra_kwargs) self.write(")") if self.environment.is_async: self.write(")") def visit_Keyword(self, node: nodes.Keyword, frame: Frame) -> None: self.write(node.key + "=") self.visit(node.value, frame) # -- Unused nodes for extensions def visit_MarkSafe(self, node: nodes.MarkSafe, frame: Frame) -> None: self.write("Markup(") self.visit(node.expr, frame) self.write(")") def visit_MarkSafeIfAutoescape( self, node: nodes.MarkSafeIfAutoescape, frame: Frame ) -> None: self.write("(Markup if context.eval_ctx.autoescape else identity)(") self.visit(node.expr, frame) self.write(")") def visit_EnvironmentAttribute( self, node: nodes.EnvironmentAttribute, frame: Frame ) -> None: self.write("environment." + node.name) def visit_ExtensionAttribute( self, node: nodes.ExtensionAttribute, frame: Frame ) -> None: self.write(f"environment.extensions[{node.identifier!r}].{node.name}") def visit_ImportedName(self, node: nodes.ImportedName, frame: Frame) -> None: self.write(self.import_aliases[node.importname]) def visit_InternalName(self, node: nodes.InternalName, frame: Frame) -> None: self.write(node.name) def visit_ContextReference( self, node: nodes.ContextReference, frame: Frame ) -> None: self.write("context") def visit_DerivedContextReference( self, node: nodes.DerivedContextReference, frame: Frame ) -> None: self.write(self.derive_context(frame)) def visit_Continue(self, node: nodes.Continue, frame: Frame) -> None: self.writeline("continue", node) def visit_Break(self, node: nodes.Break, frame: Frame) -> None: self.writeline("break", node) def visit_Scope(self, node: nodes.Scope, frame: Frame) -> None: scope_frame = frame.inner() scope_frame.symbols.analyze_node(node) self.enter_frame(scope_frame) self.blockvisit(node.body, scope_frame) self.leave_frame(scope_frame) def visit_OverlayScope(self, node: nodes.OverlayScope, frame: Frame) -> None: ctx = self.temporary_identifier() self.writeline(f"{ctx} = {self.derive_context(frame)}") self.writeline(f"{ctx}.vars = ") self.visit(node.context, frame) self.push_context_reference(ctx) scope_frame = frame.inner(isolated=True) scope_frame.symbols.analyze_node(node) self.enter_frame(scope_frame) self.blockvisit(node.body, scope_frame) self.leave_frame(scope_frame) self.pop_context_reference() def visit_EvalContextModifier( self, node: nodes.EvalContextModifier, frame: Frame ) -> None: for keyword in node.options: self.writeline(f"context.eval_ctx.{keyword.key} = ") self.visit(keyword.value, frame) try: val = keyword.value.as_const(frame.eval_ctx) except nodes.Impossible: frame.eval_ctx.volatile = True else: setattr(frame.eval_ctx, keyword.key, val) def visit_ScopedEvalContextModifier( self, node: nodes.ScopedEvalContextModifier, frame: Frame ) -> None: old_ctx_name = self.temporary_identifier() saved_ctx = frame.eval_ctx.save() self.writeline(f"{old_ctx_name} = context.eval_ctx.save()") self.visit_EvalContextModifier(node, frame) for child in node.body: self.visit(child, frame) frame.eval_ctx.revert(saved_ctx) self.writeline(f"context.eval_ctx.revert({old_ctx_name})") ================================================ FILE: lib/spack/spack/vendor/jinja2/constants.py ================================================ #: list of lorem ipsum words used by the lipsum() helper function LOREM_IPSUM_WORDS = """\ a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at auctor augue bibendum blandit class commodo condimentum congue consectetuer consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque penatibus per pharetra phasellus placerat platea porta porttitor posuere potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus viverra volutpat vulputate""" ================================================ FILE: lib/spack/spack/vendor/jinja2/debug.py ================================================ import platform import sys import typing as t from types import CodeType from types import TracebackType from .exceptions import TemplateSyntaxError from .utils import internal_code from .utils import missing if t.TYPE_CHECKING: from .runtime import Context def rewrite_traceback_stack(source: t.Optional[str] = None) -> BaseException: """Rewrite the current exception to replace any tracebacks from within compiled template code with tracebacks that look like they came from the template source. This must be called within an ``except`` block. :param source: For ``TemplateSyntaxError``, the original source if known. :return: The original exception with the rewritten traceback. """ _, exc_value, tb = sys.exc_info() exc_value = t.cast(BaseException, exc_value) tb = t.cast(TracebackType, tb) if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: exc_value.translated = True exc_value.source = source # Remove the old traceback, otherwise the frames from the # compiler still show up. exc_value.with_traceback(None) # Outside of runtime, so the frame isn't executing template # code, but it still needs to point at the template. tb = fake_traceback( exc_value, None, exc_value.filename or "", exc_value.lineno ) else: # Skip the frame for the render function. tb = tb.tb_next stack = [] # Build the stack of traceback object, replacing any in template # code with the source file and line information. while tb is not None: # Skip frames decorated with @internalcode. These are internal # calls that aren't useful in template debugging output. if tb.tb_frame.f_code in internal_code: tb = tb.tb_next continue template = tb.tb_frame.f_globals.get("__jinja_template__") if template is not None: lineno = template.get_corresponding_lineno(tb.tb_lineno) fake_tb = fake_traceback(exc_value, tb, template.filename, lineno) stack.append(fake_tb) else: stack.append(tb) tb = tb.tb_next tb_next = None # Assign tb_next in reverse to avoid circular references. for tb in reversed(stack): tb_next = tb_set_next(tb, tb_next) return exc_value.with_traceback(tb_next) def fake_traceback( # type: ignore exc_value: BaseException, tb: t.Optional[TracebackType], filename: str, lineno: int ) -> TracebackType: """Produce a new traceback object that looks like it came from the template source instead of the compiled code. The filename, line number, and location name will point to the template, and the local variables will be the current template context. :param exc_value: The original exception to be re-raised to create the new traceback. :param tb: The original traceback to get the local variables and code info from. :param filename: The template filename. :param lineno: The line number in the template source. """ if tb is not None: # Replace the real locals with the context that would be # available at that point in the template. locals = get_template_locals(tb.tb_frame.f_locals) locals.pop("__jinja_exception__", None) else: locals = {} globals = { "__name__": filename, "__file__": filename, "__jinja_exception__": exc_value, } # Raise an exception at the correct line number. code: CodeType = compile( "\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec" ) # Build a new code object that points to the template file and # replaces the location with a block name. location = "template" if tb is not None: function = tb.tb_frame.f_code.co_name if function == "root": location = "top-level template code" elif function.startswith("block_"): location = f"block {function[6:]!r}" if sys.version_info >= (3, 8): code = code.replace(co_name=location) else: code = CodeType( code.co_argcount, code.co_kwonlyargcount, code.co_nlocals, code.co_stacksize, code.co_flags, code.co_code, code.co_consts, code.co_names, code.co_varnames, code.co_filename, location, code.co_firstlineno, code.co_lnotab, code.co_freevars, code.co_cellvars, ) # Execute the new code, which is guaranteed to raise, and return # the new traceback without this frame. try: exec(code, globals, locals) except BaseException: return sys.exc_info()[2].tb_next # type: ignore def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> t.Dict[str, t.Any]: """Based on the runtime locals, get the context that would be available at that point in the template. """ # Start with the current template context. ctx: "t.Optional[Context]" = real_locals.get("context") if ctx is not None: data: t.Dict[str, t.Any] = ctx.get_all().copy() else: data = {} # Might be in a derived context that only sets local variables # rather than pushing a context. Local variables follow the scheme # l_depth_name. Find the highest-depth local that has a value for # each name. local_overrides: t.Dict[str, t.Tuple[int, t.Any]] = {} for name, value in real_locals.items(): if not name.startswith("l_") or value is missing: # Not a template variable, or no longer relevant. continue try: _, depth_str, name = name.split("_", 2) depth = int(depth_str) except ValueError: continue cur_depth = local_overrides.get(name, (-1,))[0] if cur_depth < depth: local_overrides[name] = (depth, value) # Modify the context with any derived context. for name, (_, value) in local_overrides.items(): if value is missing: data.pop(name, None) else: data[name] = value return data if sys.version_info >= (3, 7): # tb_next is directly assignable as of Python 3.7 def tb_set_next( tb: TracebackType, tb_next: t.Optional[TracebackType] ) -> TracebackType: tb.tb_next = tb_next return tb elif platform.python_implementation() == "PyPy": # PyPy might have special support, and won't work with ctypes. try: import tputil # type: ignore except ImportError: # Without tproxy support, use the original traceback. def tb_set_next( tb: TracebackType, tb_next: t.Optional[TracebackType] ) -> TracebackType: return tb else: # With tproxy support, create a proxy around the traceback that # returns the new tb_next. def tb_set_next( tb: TracebackType, tb_next: t.Optional[TracebackType] ) -> TracebackType: def controller(op): # type: ignore if op.opname == "__getattribute__" and op.args[0] == "tb_next": return tb_next return op.delegate() return tputil.make_proxy(controller, obj=tb) # type: ignore else: # Use ctypes to assign tb_next at the C level since it's read-only # from Python. import ctypes class _CTraceback(ctypes.Structure): _fields_ = [ # Extra PyObject slots when compiled with Py_TRACE_REFS. ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), # Only care about tb_next as an object, not a traceback. ("tb_next", ctypes.py_object), ] def tb_set_next( tb: TracebackType, tb_next: t.Optional[TracebackType] ) -> TracebackType: c_tb = _CTraceback.from_address(id(tb)) # Clear out the old tb_next. if tb.tb_next is not None: c_tb_next = ctypes.py_object(tb.tb_next) c_tb.tb_next = ctypes.py_object() ctypes.pythonapi.Py_DecRef(c_tb_next) # Assign the new tb_next. if tb_next is not None: c_tb_next = ctypes.py_object(tb_next) ctypes.pythonapi.Py_IncRef(c_tb_next) c_tb.tb_next = c_tb_next return tb ================================================ FILE: lib/spack/spack/vendor/jinja2/defaults.py ================================================ import typing as t from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 from .tests import TESTS as DEFAULT_TESTS # noqa: F401 from .utils import Cycler from .utils import generate_lorem_ipsum from .utils import Joiner from .utils import Namespace if t.TYPE_CHECKING: import spack.vendor.typing_extensions as te # defaults for the parser / lexer BLOCK_START_STRING = "{%" BLOCK_END_STRING = "%}" VARIABLE_START_STRING = "{{" VARIABLE_END_STRING = "}}" COMMENT_START_STRING = "{#" COMMENT_END_STRING = "#}" LINE_STATEMENT_PREFIX: t.Optional[str] = None LINE_COMMENT_PREFIX: t.Optional[str] = None TRIM_BLOCKS = False LSTRIP_BLOCKS = False NEWLINE_SEQUENCE: "te.Literal['\\n', '\\r\\n', '\\r']" = "\n" KEEP_TRAILING_NEWLINE = False # default filters, tests and namespace DEFAULT_NAMESPACE = { "range": range, "dict": dict, "lipsum": generate_lorem_ipsum, "cycler": Cycler, "joiner": Joiner, "namespace": Namespace, } # default policies DEFAULT_POLICIES: t.Dict[str, t.Any] = { "compiler.ascii_str": True, "urlize.rel": "noopener", "urlize.target": None, "urlize.extra_schemes": None, "truncate.leeway": 5, "json.dumps_function": None, "json.dumps_kwargs": {"sort_keys": True}, "ext.i18n.trimmed": False, } ================================================ FILE: lib/spack/spack/vendor/jinja2/environment.py ================================================ """Classes for managing templates and their runtime and compile time options. """ import os import sys import typing import typing as t import weakref from collections import ChainMap from functools import lru_cache from functools import partial from functools import reduce from types import CodeType from spack.vendor.markupsafe import Markup from . import nodes from .compiler import CodeGenerator from .compiler import generate from .defaults import BLOCK_END_STRING from .defaults import BLOCK_START_STRING from .defaults import COMMENT_END_STRING from .defaults import COMMENT_START_STRING from .defaults import DEFAULT_FILTERS from .defaults import DEFAULT_NAMESPACE from .defaults import DEFAULT_POLICIES from .defaults import DEFAULT_TESTS from .defaults import KEEP_TRAILING_NEWLINE from .defaults import LINE_COMMENT_PREFIX from .defaults import LINE_STATEMENT_PREFIX from .defaults import LSTRIP_BLOCKS from .defaults import NEWLINE_SEQUENCE from .defaults import TRIM_BLOCKS from .defaults import VARIABLE_END_STRING from .defaults import VARIABLE_START_STRING from .exceptions import TemplateNotFound from .exceptions import TemplateRuntimeError from .exceptions import TemplatesNotFound from .exceptions import TemplateSyntaxError from .exceptions import UndefinedError from .lexer import get_lexer from .lexer import Lexer from .lexer import TokenStream from .nodes import EvalContext from .parser import Parser from .runtime import Context from .runtime import new_context from .runtime import Undefined from .utils import _PassArg from .utils import concat from .utils import consume from .utils import import_string from .utils import internalcode from .utils import LRUCache from .utils import missing if t.TYPE_CHECKING: import spack.vendor.typing_extensions as te from .bccache import BytecodeCache from .ext import Extension from .loaders import BaseLoader _env_bound = t.TypeVar("_env_bound", bound="Environment") # for direct template usage we have up to ten living environments @lru_cache(maxsize=10) def get_spontaneous_environment(cls: t.Type[_env_bound], *args: t.Any) -> _env_bound: """Return a new spontaneous environment. A spontaneous environment is used for templates created directly rather than through an existing environment. :param cls: Environment class to create. :param args: Positional arguments passed to environment. """ env = cls(*args) env.shared = True return env def create_cache( size: int, ) -> t.Optional[t.MutableMapping[t.Tuple[weakref.ref, str], "Template"]]: """Return the cache class for the given size.""" if size == 0: return None if size < 0: return {} return LRUCache(size) # type: ignore def copy_cache( cache: t.Optional[t.MutableMapping], ) -> t.Optional[t.MutableMapping[t.Tuple[weakref.ref, str], "Template"]]: """Create an empty copy of the given cache.""" if cache is None: return None if type(cache) is dict: return {} return LRUCache(cache.capacity) # type: ignore def load_extensions( environment: "Environment", extensions: t.Sequence[t.Union[str, t.Type["Extension"]]], ) -> t.Dict[str, "Extension"]: """Load the extensions from the list and bind it to the environment. Returns a dict of instantiated extensions. """ result = {} for extension in extensions: if isinstance(extension, str): extension = t.cast(t.Type["Extension"], import_string(extension)) result[extension.identifier] = extension(environment) return result def _environment_config_check(environment: "Environment") -> "Environment": """Perform a sanity check on the environment.""" assert issubclass( environment.undefined, Undefined ), "'undefined' must be a subclass of 'spack.vendor.jinja2.Undefined'." assert ( environment.block_start_string != environment.variable_start_string != environment.comment_start_string ), "block, variable and comment start strings must be different." assert environment.newline_sequence in { "\r", "\r\n", "\n", }, "'newline_sequence' must be one of '\\n', '\\r\\n', or '\\r'." return environment class Environment: r"""The core component of Jinja is the `Environment`. It contains important shared variables like configuration, filters, tests, globals and others. Instances of this class may be modified if they are not shared and if no template was loaded so far. Modifications on environments after the first template was loaded will lead to surprising effects and undefined behavior. Here are the possible initialization parameters: `block_start_string` The string marking the beginning of a block. Defaults to ``'{%'``. `block_end_string` The string marking the end of a block. Defaults to ``'%}'``. `variable_start_string` The string marking the beginning of a print statement. Defaults to ``'{{'``. `variable_end_string` The string marking the end of a print statement. Defaults to ``'}}'``. `comment_start_string` The string marking the beginning of a comment. Defaults to ``'{#'``. `comment_end_string` The string marking the end of a comment. Defaults to ``'#}'``. `line_statement_prefix` If given and a string, this will be used as prefix for line based statements. See also :ref:`line-statements`. `line_comment_prefix` If given and a string, this will be used as prefix for line based comments. See also :ref:`line-statements`. .. versionadded:: 2.2 `trim_blocks` If this is set to ``True`` the first newline after a block is removed (block, not variable tag!). Defaults to `False`. `lstrip_blocks` If this is set to ``True`` leading spaces and tabs are stripped from the start of a line to a block. Defaults to `False`. `newline_sequence` The sequence that starts a newline. Must be one of ``'\r'``, ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a useful default for Linux and OS X systems as well as web applications. `keep_trailing_newline` Preserve the trailing newline when rendering templates. The default is ``False``, which causes a single newline, if present, to be stripped from the end of the template. .. versionadded:: 2.7 `extensions` List of Jinja extensions to use. This can either be import paths as strings or extension classes. For more information have a look at :ref:`the extensions documentation `. `optimized` should the optimizer be enabled? Default is ``True``. `undefined` :class:`Undefined` or a subclass of it that is used to represent undefined values in the template. `finalize` A callable that can be used to process the result of a variable expression before it is output. For example one can convert ``None`` implicitly into an empty string here. `autoescape` If set to ``True`` the XML/HTML autoescaping feature is enabled by default. For more details about autoescaping see :class:`~spack.vendor.markupsafe.Markup`. As of Jinja 2.4 this can also be a callable that is passed the template name and has to return ``True`` or ``False`` depending on autoescape should be enabled by default. .. versionchanged:: 2.4 `autoescape` can now be a function `loader` The template loader for this environment. `cache_size` The size of the cache. Per default this is ``400`` which means that if more than 400 templates are loaded the loader will clean out the least recently used template. If the cache size is set to ``0`` templates are recompiled all the time, if the cache size is ``-1`` the cache will not be cleaned. .. versionchanged:: 2.8 The cache size was increased to 400 from a low 50. `auto_reload` Some loaders load templates from locations where the template sources may change (ie: file system or database). If ``auto_reload`` is set to ``True`` (default) every time a template is requested the loader checks if the source changed and if yes, it will reload the template. For higher performance it's possible to disable that. `bytecode_cache` If set to a bytecode cache object, this object will provide a cache for the internal Jinja bytecode so that templates don't have to be parsed if they were not changed. See :ref:`bytecode-cache` for more information. `enable_async` If set to true this enables async template execution which allows using async functions and generators. """ #: if this environment is sandboxed. Modifying this variable won't make #: the environment sandboxed though. For a real sandboxed environment #: have a look at spack.vendor.jinja2.sandbox. This flag alone controls the code #: generation by the compiler. sandboxed = False #: True if the environment is just an overlay overlayed = False #: the environment this environment is linked to if it is an overlay linked_to: t.Optional["Environment"] = None #: shared environments have this set to `True`. A shared environment #: must not be modified shared = False #: the class that is used for code generation. See #: :class:`~spack.vendor.jinja2.compiler.CodeGenerator` for more information. code_generator_class: t.Type["CodeGenerator"] = CodeGenerator #: the context class that is used for templates. See #: :class:`~spack.vendor.jinja2.runtime.Context` for more information. context_class: t.Type[Context] = Context template_class: t.Type["Template"] def __init__( self, block_start_string: str = BLOCK_START_STRING, block_end_string: str = BLOCK_END_STRING, variable_start_string: str = VARIABLE_START_STRING, variable_end_string: str = VARIABLE_END_STRING, comment_start_string: str = COMMENT_START_STRING, comment_end_string: str = COMMENT_END_STRING, line_statement_prefix: t.Optional[str] = LINE_STATEMENT_PREFIX, line_comment_prefix: t.Optional[str] = LINE_COMMENT_PREFIX, trim_blocks: bool = TRIM_BLOCKS, lstrip_blocks: bool = LSTRIP_BLOCKS, newline_sequence: "te.Literal['\\n', '\\r\\n', '\\r']" = NEWLINE_SEQUENCE, keep_trailing_newline: bool = KEEP_TRAILING_NEWLINE, extensions: t.Sequence[t.Union[str, t.Type["Extension"]]] = (), optimized: bool = True, undefined: t.Type[Undefined] = Undefined, finalize: t.Optional[t.Callable[..., t.Any]] = None, autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = False, loader: t.Optional["BaseLoader"] = None, cache_size: int = 400, auto_reload: bool = True, bytecode_cache: t.Optional["BytecodeCache"] = None, enable_async: bool = False, ): # !!Important notice!! # The constructor accepts quite a few arguments that should be # passed by keyword rather than position. However it's important to # not change the order of arguments because it's used at least # internally in those cases: # - spontaneous environments (i18n extension and Template) # - unittests # If parameter changes are required only add parameters at the end # and don't change the arguments (or the defaults!) of the arguments # existing already. # lexer / parser information self.block_start_string = block_start_string self.block_end_string = block_end_string self.variable_start_string = variable_start_string self.variable_end_string = variable_end_string self.comment_start_string = comment_start_string self.comment_end_string = comment_end_string self.line_statement_prefix = line_statement_prefix self.line_comment_prefix = line_comment_prefix self.trim_blocks = trim_blocks self.lstrip_blocks = lstrip_blocks self.newline_sequence = newline_sequence self.keep_trailing_newline = keep_trailing_newline # runtime information self.undefined: t.Type[Undefined] = undefined self.optimized = optimized self.finalize = finalize self.autoescape = autoescape # defaults self.filters = DEFAULT_FILTERS.copy() self.tests = DEFAULT_TESTS.copy() self.globals = DEFAULT_NAMESPACE.copy() # set the loader provided self.loader = loader self.cache = create_cache(cache_size) self.bytecode_cache = bytecode_cache self.auto_reload = auto_reload # configurable policies self.policies = DEFAULT_POLICIES.copy() # load extensions self.extensions = load_extensions(self, extensions) self.is_async = enable_async _environment_config_check(self) def add_extension(self, extension: t.Union[str, t.Type["Extension"]]) -> None: """Adds an extension after the environment was created. .. versionadded:: 2.5 """ self.extensions.update(load_extensions(self, [extension])) def extend(self, **attributes: t.Any) -> None: """Add the items to the instance of the environment if they do not exist yet. This is used by :ref:`extensions ` to register callbacks and configuration values without breaking inheritance. """ for key, value in attributes.items(): if not hasattr(self, key): setattr(self, key, value) def overlay( self, block_start_string: str = missing, block_end_string: str = missing, variable_start_string: str = missing, variable_end_string: str = missing, comment_start_string: str = missing, comment_end_string: str = missing, line_statement_prefix: t.Optional[str] = missing, line_comment_prefix: t.Optional[str] = missing, trim_blocks: bool = missing, lstrip_blocks: bool = missing, extensions: t.Sequence[t.Union[str, t.Type["Extension"]]] = missing, optimized: bool = missing, undefined: t.Type[Undefined] = missing, finalize: t.Optional[t.Callable[..., t.Any]] = missing, autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = missing, loader: t.Optional["BaseLoader"] = missing, cache_size: int = missing, auto_reload: bool = missing, bytecode_cache: t.Optional["BytecodeCache"] = missing, ) -> "Environment": """Create a new overlay environment that shares all the data with the current environment except for cache and the overridden attributes. Extensions cannot be removed for an overlayed environment. An overlayed environment automatically gets all the extensions of the environment it is linked to plus optional extra extensions. Creating overlays should happen after the initial environment was set up completely. Not all attributes are truly linked, some are just copied over so modifications on the original environment may not shine through. """ args = dict(locals()) del args["self"], args["cache_size"], args["extensions"] rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) rv.overlayed = True rv.linked_to = self for key, value in args.items(): if value is not missing: setattr(rv, key, value) if cache_size is not missing: rv.cache = create_cache(cache_size) else: rv.cache = copy_cache(self.cache) rv.extensions = {} for key, value in self.extensions.items(): rv.extensions[key] = value.bind(rv) if extensions is not missing: rv.extensions.update(load_extensions(rv, extensions)) return _environment_config_check(rv) @property def lexer(self) -> Lexer: """The lexer for this environment.""" return get_lexer(self) def iter_extensions(self) -> t.Iterator["Extension"]: """Iterates over the extensions by priority.""" return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) def getitem( self, obj: t.Any, argument: t.Union[str, t.Any] ) -> t.Union[t.Any, Undefined]: """Get an item or attribute of an object but prefer the item.""" try: return obj[argument] except (AttributeError, TypeError, LookupError): if isinstance(argument, str): try: attr = str(argument) except Exception: pass else: try: return getattr(obj, attr) except AttributeError: pass return self.undefined(obj=obj, name=argument) def getattr(self, obj: t.Any, attribute: str) -> t.Any: """Get an item or attribute of an object but prefer the attribute. Unlike :meth:`getitem` the attribute *must* be a string. """ try: return getattr(obj, attribute) except AttributeError: pass try: return obj[attribute] except (TypeError, LookupError, AttributeError): return self.undefined(obj=obj, name=attribute) def _filter_test_common( self, name: t.Union[str, Undefined], value: t.Any, args: t.Optional[t.Sequence[t.Any]], kwargs: t.Optional[t.Mapping[str, t.Any]], context: t.Optional[Context], eval_ctx: t.Optional[EvalContext], is_filter: bool, ) -> t.Any: if is_filter: env_map = self.filters type_name = "filter" else: env_map = self.tests type_name = "test" func = env_map.get(name) # type: ignore if func is None: msg = f"No {type_name} named {name!r}." if isinstance(name, Undefined): try: name._fail_with_undefined_error() except Exception as e: msg = f"{msg} ({e}; did you forget to quote the callable name?)" raise TemplateRuntimeError(msg) args = [value, *(args if args is not None else ())] kwargs = kwargs if kwargs is not None else {} pass_arg = _PassArg.from_obj(func) if pass_arg is _PassArg.context: if context is None: raise TemplateRuntimeError( f"Attempted to invoke a context {type_name} without context." ) args.insert(0, context) elif pass_arg is _PassArg.eval_context: if eval_ctx is None: if context is not None: eval_ctx = context.eval_ctx else: eval_ctx = EvalContext(self) args.insert(0, eval_ctx) elif pass_arg is _PassArg.environment: args.insert(0, self) return func(*args, **kwargs) def call_filter( self, name: str, value: t.Any, args: t.Optional[t.Sequence[t.Any]] = None, kwargs: t.Optional[t.Mapping[str, t.Any]] = None, context: t.Optional[Context] = None, eval_ctx: t.Optional[EvalContext] = None, ) -> t.Any: """Invoke a filter on a value the same way the compiler does. This might return a coroutine if the filter is running from an environment in async mode and the filter supports async execution. It's your responsibility to await this if needed. .. versionadded:: 2.7 """ return self._filter_test_common( name, value, args, kwargs, context, eval_ctx, True ) def call_test( self, name: str, value: t.Any, args: t.Optional[t.Sequence[t.Any]] = None, kwargs: t.Optional[t.Mapping[str, t.Any]] = None, context: t.Optional[Context] = None, eval_ctx: t.Optional[EvalContext] = None, ) -> t.Any: """Invoke a test on a value the same way the compiler does. This might return a coroutine if the test is running from an environment in async mode and the test supports async execution. It's your responsibility to await this if needed. .. versionchanged:: 3.0 Tests support ``@pass_context``, etc. decorators. Added the ``context`` and ``eval_ctx`` parameters. .. versionadded:: 2.7 """ return self._filter_test_common( name, value, args, kwargs, context, eval_ctx, False ) @internalcode def parse( self, source: str, name: t.Optional[str] = None, filename: t.Optional[str] = None, ) -> nodes.Template: """Parse the sourcecode and return the abstract syntax tree. This tree of nodes is used by the compiler to convert the template into executable source- or bytecode. This is useful for debugging or to extract information from templates. If you are :ref:`developing Jinja extensions ` this gives you a good overview of the node tree generated. """ try: return self._parse(source, name, filename) except TemplateSyntaxError: self.handle_exception(source=source) def _parse( self, source: str, name: t.Optional[str], filename: t.Optional[str] ) -> nodes.Template: """Internal parsing function used by `parse` and `compile`.""" return Parser(self, source, name, filename).parse() def lex( self, source: str, name: t.Optional[str] = None, filename: t.Optional[str] = None, ) -> t.Iterator[t.Tuple[int, str, str]]: """Lex the given sourcecode and return a generator that yields tokens as tuples in the form ``(lineno, token_type, value)``. This can be useful for :ref:`extension development ` and debugging templates. This does not perform preprocessing. If you want the preprocessing of the extensions to be applied you have to filter source through the :meth:`preprocess` method. """ source = str(source) try: return self.lexer.tokeniter(source, name, filename) except TemplateSyntaxError: self.handle_exception(source=source) def preprocess( self, source: str, name: t.Optional[str] = None, filename: t.Optional[str] = None, ) -> str: """Preprocesses the source with all extensions. This is automatically called for all parsing and compiling methods but *not* for :meth:`lex` because there you usually only want the actual source tokenized. """ return reduce( lambda s, e: e.preprocess(s, name, filename), self.iter_extensions(), str(source), ) def _tokenize( self, source: str, name: t.Optional[str], filename: t.Optional[str] = None, state: t.Optional[str] = None, ) -> TokenStream: """Called by the parser to do the preprocessing and filtering for all the extensions. Returns a :class:`~spack.vendor.jinja2.lexer.TokenStream`. """ source = self.preprocess(source, name, filename) stream = self.lexer.tokenize(source, name, filename, state) for ext in self.iter_extensions(): stream = ext.filter_stream(stream) # type: ignore if not isinstance(stream, TokenStream): stream = TokenStream(stream, name, filename) # type: ignore return stream def _generate( self, source: nodes.Template, name: t.Optional[str], filename: t.Optional[str], defer_init: bool = False, ) -> str: """Internal hook that can be overridden to hook a different generate method in. .. versionadded:: 2.5 """ return generate( # type: ignore source, self, name, filename, defer_init=defer_init, optimized=self.optimized, ) def _compile(self, source: str, filename: str) -> CodeType: """Internal hook that can be overridden to hook a different compile method in. .. versionadded:: 2.5 """ return compile(source, filename, "exec") # type: ignore @typing.overload def compile( # type: ignore self, source: t.Union[str, nodes.Template], name: t.Optional[str] = None, filename: t.Optional[str] = None, raw: "te.Literal[False]" = False, defer_init: bool = False, ) -> CodeType: ... @typing.overload def compile( self, source: t.Union[str, nodes.Template], name: t.Optional[str] = None, filename: t.Optional[str] = None, raw: "te.Literal[True]" = ..., defer_init: bool = False, ) -> str: ... @internalcode def compile( self, source: t.Union[str, nodes.Template], name: t.Optional[str] = None, filename: t.Optional[str] = None, raw: bool = False, defer_init: bool = False, ) -> t.Union[str, CodeType]: """Compile a node or template source code. The `name` parameter is the load name of the template after it was joined using :meth:`join_path` if necessary, not the filename on the file system. the `filename` parameter is the estimated filename of the template on the file system. If the template came from a database or memory this can be omitted. The return value of this method is a python code object. If the `raw` parameter is `True` the return value will be a string with python code equivalent to the bytecode returned otherwise. This method is mainly used internally. `defer_init` is use internally to aid the module code generator. This causes the generated code to be able to import without the global environment variable to be set. .. versionadded:: 2.4 `defer_init` parameter added. """ source_hint = None try: if isinstance(source, str): source_hint = source source = self._parse(source, name, filename) source = self._generate(source, name, filename, defer_init=defer_init) if raw: return source if filename is None: filename = "